From 1e50a2a5f7824335212ee8f3d8bbdf25297f8dfd Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:39:42 +0000 Subject: [PATCH 001/118] CLI and API robustness (S1, S2, S3, S4, S5, S6, S10 CLI side) - S1: main() reconfigures stdout to UTF-8 and stderr to backslashreplace (guarded by hasattr(..., "reconfigure")); stdin is read as bytes and decoded as UTF-8. Czech text no longer crashes or is corrupted on cp1252. - S4: a non-UTF-8 input file or stdin is "cannot read NAME: not valid UTF-8 (byte 0x.. at offset N)", exit 1 (offset is in the file, not a buffer). A non-UTF-8 style file is a StyleError (style/loader.py _read only). - S5: an OSError writing -o is "cannot write PATH: ...", exit 1. - S6: -o with an extension no writer knows (x.docx, .pdf, .md) and no -f is a usage error (exit 2); no extension keeps the html default; -f wins. - S2: StyleError is defined in errors.py as StyleError(RenderError, ValueError) and re-exported from style/loader.py, legaldown_render.style and the package. An unknown format raises the new OptionsError(RenderError, ValueError), exported from the package. - S3: RenderRefused (now storing .reason) and StyleError define __reduce__ so they round-trip through pickle. - S10 (CLI side): the argparse type _setting turns StyleError into ArgumentTypeError so its message reaches the user. Docs: README (exit codes, encoding, -o extension, error classes), docs/architecture.md (errors). Tests: tests/test_wp2_cli.py (new); no existing test or golden changed. --- README.md | 15 +- docs/architecture.md | 8 +- src/legaldown_render/__init__.py | 5 +- src/legaldown_render/api.py | 9 +- src/legaldown_render/cli.py | 68 +++++++- src/legaldown_render/errors.py | 35 +++- src/legaldown_render/style/loader.py | 17 +- tests/test_wp2_cli.py | 246 +++++++++++++++++++++++++++ 8 files changed, 369 insertions(+), 34 deletions(-) create mode 100644 tests/test_wp2_cli.py diff --git a/README.md b/README.md index c7f12d7..1eeed6a 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,12 @@ legaldown-render --print-style --style continental # every setting, Diagnostics go to stderr, one per line with its rule id. The exit code is 0 when the document rendered, even if it has errors (they show as markers), 1 when rendering was refused (strict mode, or a template that cannot be assembled with its answers) or the document or answers file cannot be -read, 2 for an invalid style, setting, or usage, and 70 for an internal error (a bug to report). +read (a document that is not UTF-8 included) or the output file cannot be written, 2 for an invalid +style, setting, or usage, and 70 for an internal error (a bug to report). + +Documents are read as UTF-8, and standard output is written as UTF-8 whatever the console's encoding. +`-o` takes its format from the file extension (`.html`, `.htm`, `.txt`, `.text`); a name with no +extension is HTML, and any other extension (`.docx`, `.pdf`, `.md`) needs an explicit `--format`. ## Python @@ -83,9 +88,11 @@ open("contract.html", "w").write(result.output) ``` `render()` also accepts a `RenderOptions`. Pass `answers={...}` to render a template filled in. -Problems in the document never raise. Only an invalid style (`StyleError`), an unreadable document -(`DocumentError`), strict mode, or a template that cannot be assembled with its answers -(`RenderRefused`) do. +Problems in the document never raise. Only an invalid style (`StyleError`), an unknown output +format (`OptionsError`), an unreadable document (`DocumentError`), strict mode, or a template that +cannot be assembled with its answers (`RenderRefused`) do. All of them are `RenderError`s, and +`StyleError` and `OptionsError` are also `ValueError`s. They can be pickled, so they cross process +boundaries (`multiprocessing`, `concurrent.futures`) intact. ## Styles and preferences diff --git a/docs/architecture.md b/docs/architecture.md index f8a25fb..832b53c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -182,7 +182,7 @@ src/legaldown_render/ ├── __init__.py public API, __version__, SPEC_VERSION, CONFORMANCE_LEVEL ├── api.py render(), render_file(), RenderOptions, RenderResult ├── cli.py the legaldown-render command -├── errors.py RenderError, DocumentError, RenderRefused, InternalError +├── errors.py RenderError, StyleError, OptionsError, DocumentError, RenderRefused, InternalError ├── tree.py render tree nodes (source and resolved inlines, blocks, document) ├── build.py stage 3 ├── resolve/ @@ -220,8 +220,10 @@ turns into an error. - **Document problems** render as failure markers and are listed in `diagnostics`. `strict=True` raises `RenderRefused` instead, carrying the diagnostics. -- **Configuration problems** (an invalid style, setting, locale, or format) raise before - rendering, with every problem named. +- **Configuration problems** raise before rendering, with every problem named: an invalid style, + setting, or locale raises `StyleError`; an unknown output format raises `OptionsError`. Both are + `RenderError`s (and `ValueError`s, as they always were). Every renderer error pickles, so it + survives `multiprocessing`. - **Internal inconsistencies** raise `InternalError`, and the CLI exits with 70. Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can evaluate: diff --git a/src/legaldown_render/__init__.py b/src/legaldown_render/__init__.py index acbd0a8..8845c22 100644 --- a/src/legaldown_render/__init__.py +++ b/src/legaldown_render/__init__.py @@ -18,8 +18,8 @@ from __future__ import annotations from .api import RenderOptions, RenderResult, render, render_file -from .errors import DocumentError, InternalError, RenderError, RenderRefused -from .style import Style, StyleError, builtin_styles, dump_style, load_style +from .errors import DocumentError, InternalError, OptionsError, RenderError, RenderRefused, StyleError +from .style import Style, builtin_styles, dump_style, load_style __version__ = "0.3.0" @@ -43,6 +43,7 @@ "dump_style", "builtin_styles", "RenderError", + "OptionsError", "DocumentError", "RenderRefused", "InternalError", diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index 3d58fc7..0fb88e7 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -21,9 +21,9 @@ from legaldown import AssemblyError, Diagnostic, Document, ValidationResult, parse, parse_template, validate from .build import build_tree, normalize_source -from .errors import DocumentError, RenderRefused +from .errors import DocumentError, OptionsError, RenderRefused, StyleError from .resolve import Formatter, parse_locale, resolve_tree -from .style import Style, StyleError, load_style +from .style import Style, load_style from .tree import RenderTree, assert_resolved from .writers import FORMATS, HtmlWriter, TextWriter @@ -86,7 +86,8 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any Problems in the document never raise: they render as failure markers and are listed in ``diagnostics``. Raises :class:`StyleError` for an invalid - style or setting, :class:`DocumentError` for a document that cannot be + style or setting, :class:`OptionsError` for an unknown output format, + :class:`DocumentError` for a document that cannot be read, and :class:`RenderRefused` in strict mode when the document has errors, or when a template cannot be assembled with ``answers``. """ @@ -95,7 +96,7 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any elif settings: raise TypeError("pass either a RenderOptions or keyword settings, not both") if options.format not in FORMATS: - raise ValueError(f"unknown output format '{options.format}' (known: {', '.join(FORMATS)})") + raise OptionsError(f"unknown output format '{options.format}' (known: {', '.join(FORMATS)})") overrides = dict(options.overrides) if options.locale: overrides["locale"] = options.locale diff --git a/src/legaldown_render/cli.py b/src/legaldown_render/cli.py index 9906985..8509c08 100644 --- a/src/legaldown_render/cli.py +++ b/src/legaldown_render/cli.py @@ -2,12 +2,17 @@ Exit codes: 0 — rendered (document errors, if any, are shown as markers and listed on stderr); 1 — not rendered: refused under ``--strict``, or the -document cannot be read; 2 — a usage or style problem; 70 — an internal -error (a bug to report). +document cannot be read (including input that is not UTF-8) or the output cannot +be written; 2 — a usage or style problem; 70 — an internal error (a bug to +report). + +Documents and output are always UTF-8 (the HTML says ````), +never the console's or the locale's encoding. """ from __future__ import annotations import argparse +import contextlib import sys from pathlib import Path from typing import Any @@ -16,8 +21,8 @@ from . import __version__ from .api import RenderOptions, render -from .errors import DocumentError, InternalError, RenderRefused -from .style import StyleError, builtin_styles, dump_style, load_style, parse_override +from .errors import DocumentError, InternalError, RenderRefused, StyleError +from .style import builtin_styles, dump_style, load_style, parse_override from .writers import FORMATS, format_for_path @@ -26,7 +31,11 @@ def _setting(text: str) -> tuple[str, Any]: if not separator or not key.strip(): raise argparse.ArgumentTypeError(f"expected KEY=VALUE, got '{text}'") # Typed by the setting: text stays as written, anything else is YAML. - return key.strip(), parse_override(key.strip(), value) + try: + return key.strip(), parse_override(key.strip(), value) + except StyleError as error: + # argparse shows only ArgumentTypeError (and TypeError/ValueError, with a generic message). + raise argparse.ArgumentTypeError(str(error)) from None def _parser() -> argparse.ArgumentParser: @@ -63,7 +72,34 @@ def _parser() -> argparse.ArgumentParser: return parser +def _configure_streams() -> None: + """Make the standard streams safe for any content, whatever the console's + encoding: the document and the style are UTF-8, so what is written to + standard output is UTF-8 as well, and a diagnostic that quotes characters + the console cannot show escapes them (``\\u2192``) instead of failing with a + UnicodeEncodeError. Replaced streams without ``reconfigure`` (a test + capture, a StringIO) are left alone.""" + if hasattr(sys.stdout, "reconfigure"): + with contextlib.suppress(ValueError, OSError): + sys.stdout.reconfigure(encoding="utf-8") + if hasattr(sys.stderr, "reconfigure"): + with contextlib.suppress(ValueError, OSError): + sys.stderr.reconfigure(errors="backslashreplace") + + +def _read_source(path: str) -> str: + """The document at *path*, or standard input for ``-``, decoded as UTF-8 + from its bytes (never in the locale's encoding). Raises OSError, or + UnicodeDecodeError for input that is not UTF-8.""" + if path == "-": + buffer = getattr(sys.stdin, "buffer", None) + # A replaced stdin without bytes (StringIO) already holds text. + return buffer.read().decode("utf-8") if buffer is not None else sys.stdin.read() + return Path(path).read_bytes().decode("utf-8") + + def main(argv: list[str] | None = None) -> int: + _configure_streams() parser = _parser() args = parser.parse_args(argv) overrides = dict(args.overrides) @@ -83,13 +119,25 @@ def main(argv: list[str] | None = None) -> int: if not args.input: parser.error("an input file is required") - output_format = args.format or (format_for_path(args.output) if args.output else None) or "html" + output_format = args.format + if not output_format and args.output: + file_name = Path(args.output).name + output_format = format_for_path(file_name) + if not output_format and "." in file_name.rstrip("."): + # A name with an extension we do not write (x.docx, x.pdf, x.md): HTML under that + # name would mislead. A name without one keeps the html default. + parser.error(f"cannot tell the output format from '{args.output}'; use --format html or text") + output_format = output_format or "html" name = "" if args.input == "-" else args.input try: - source = sys.stdin.read() if args.input == "-" else Path(args.input).read_text(encoding="utf-8") + source = _read_source(args.input) except OSError as error: print(f"legaldown-render: cannot read {name}: {error.strerror or error}", file=sys.stderr) return 1 + except UnicodeDecodeError as error: + print(f"legaldown-render: cannot read {name}: not valid UTF-8 " + f"(byte 0x{error.object[error.start]:02x} at offset {error.start})", file=sys.stderr) + return 1 answers = None if args.answers: @@ -124,7 +172,11 @@ def main(argv: list[str] | None = None) -> int: _report(name, result.diagnostics, quiet=args.quiet) if args.output: - Path(args.output).write_text(result.output, encoding="utf-8") + try: + Path(args.output).write_text(result.output, encoding="utf-8") + except OSError as error: + print(f"legaldown-render: cannot write {args.output}: {error.strerror or error}", file=sys.stderr) + return 1 else: sys.stdout.write(result.output) return 0 diff --git a/src/legaldown_render/errors.py b/src/legaldown_render/errors.py index 8ba21b5..ff9526a 100644 --- a/src/legaldown_render/errors.py +++ b/src/legaldown_render/errors.py @@ -10,6 +10,33 @@ class RenderError(Exception): """Base class for every error the renderer raises.""" +class StyleError(RenderError, ValueError): + """A style template or override is invalid. ``problems`` lists each + problem with its key path. + + It is a :class:`ValueError` as well as a :class:`RenderError`, so code + that caught it as a ``ValueError`` keeps working. It lives here, with the + other errors, and is re-exported by :mod:`legaldown_render.style`.""" + + def __init__(self, source: str, problems: list[str]) -> None: + self.source = source + self.problems = problems + details = "\n".join(f" - {problem}" for problem in problems) + super().__init__(f"Invalid style {source}:\n{details}") + + def __reduce__(self) -> tuple[type[StyleError], tuple[str, list[str]]]: + # ``Exception`` pickles ``args``, which here is the one formatted message, + # and the two-argument constructor would then fail to rebuild it + # (multiprocessing would raise BrokenProcessPool instead of the error). + return type(self), (self.source, self.problems) + + +class OptionsError(RenderError, ValueError): + """The render options are invalid, e.g. an unknown output format. It is + a :class:`ValueError` as well, as the bare ``ValueError`` it replaces + was.""" + + class DocumentError(RenderError): """The document cannot be read at all, e.g. its frontmatter is not valid YAML.""" @@ -18,13 +45,19 @@ class DocumentError(RenderError): class RenderRefused(RenderError): """Rendering was refused: the document has errors and strict mode is on, or a template could not be assembled with its answers (§15.7). - ``diagnostics`` holds every finding.""" + ``diagnostics`` holds every finding; ``reason`` is the message given when + it was raised, or None for the strict-mode default.""" def __init__(self, diagnostics: list[Diagnostic], reason: str | None = None) -> None: self.diagnostics = diagnostics + self.reason = reason errors = sum(1 for diagnostic in diagnostics if diagnostic.level == "error") super().__init__(reason or f"Rendering refused in strict mode: the document has {errors} error(s).") + def __reduce__(self) -> tuple[type[RenderRefused], tuple[list[Diagnostic], str | None]]: + # See StyleError.__reduce__: the constructor does not take the message. + return type(self), (self.diagnostics, self.reason) + class InternalError(RenderError): """The renderer reached a state that is a bug, such as a marker the diff --git a/src/legaldown_render/style/loader.py b/src/legaldown_render/style/loader.py index c3239ac..3c48b67 100644 --- a/src/legaldown_render/style/loader.py +++ b/src/legaldown_render/style/loader.py @@ -24,23 +24,13 @@ import yaml +from ..errors import StyleError from . import model from .model import Style STYLE_FORMAT_VERSION = 1 -class StyleError(ValueError): - """A style template or override is invalid. ``problems`` lists each - problem with its key path.""" - - def __init__(self, source: str, problems: list[str]) -> None: - self.source = source - self.problems = problems - details = "\n".join(f" - {problem}" for problem in problems) - super().__init__(f"Invalid style {source}:\n{details}") - - def builtin_styles() -> list[str]: """Names of the style templates shipped with the package.""" folder = resources.files(__package__).joinpath("builtin") @@ -119,9 +109,12 @@ def _read(spec: str | Path, relative_to: Path | None) -> tuple[dict, str, Path | if relative_to is not None and not path.is_absolute(): path = relative_to / path try: - text = path.read_text("utf-8") + text = path.read_bytes().decode("utf-8") except OSError as error: raise StyleError(str(path), [f"cannot read the file: {error.strerror or error}"]) from None + except UnicodeDecodeError as error: + raise StyleError(str(path), [f"cannot read the file: not valid UTF-8 (byte 0x{error.object[error.start]:02x} " + f"at offset {error.start})"]) from None raw = _parse(text, str(path)) raw.setdefault("name", path.stem) return raw, str(path), path.parent diff --git a/tests/test_wp2_cli.py b/tests/test_wp2_cli.py new file mode 100644 index 0000000..8b864fd --- /dev/null +++ b/tests/test_wp2_cli.py @@ -0,0 +1,246 @@ +"""CLI and API robustness: stream encodings (S1), non-UTF-8 input (S4), an +unwritable output (S5), an unknown output extension (S6), the error +hierarchy (S2), picklable errors (S3), and the ``--set`` error path (S10, CLI +side).""" +from __future__ import annotations + +import io +import os +import pickle +import subprocess +import sys +from pathlib import Path + +import pytest +from conftest import DOCUMENTS +from legaldown import Diagnostic + +import legaldown_render +import legaldown_render.cli as cli +from legaldown_render import ( + OptionsError, + RenderError, + RenderOptions, + RenderRefused, + StyleError, + load_style, + render, +) +from legaldown_render.cli import main + +SRC = Path(__file__).resolve().parents[1] / "src" +FEATURES = str(DOCUMENTS / "features.lgd") +CZECH = "---\nlegaldown: '0.2'\ntitle: Smlouva o dílo\nlanguage: cs\n---\n\n# Příloha\n\nŽluťoučký kůň úpěl ďábelské ódy – “citát” → konec, ōsaka.\n" + + +def _run(args: list[str], stdin: bytes | None = None, *, narrow: bool = True) -> subprocess.CompletedProcess[bytes]: + env = dict(os.environ, PYTHONPATH=str(SRC)) + if narrow: + # A Windows-like console: neither stream is UTF-8. + env.update(PYTHONIOENCODING="cp1252", PYTHONUTF8="0") + return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *args], input=stdin, + capture_output=True, env=env, check=False) + + +def test_the_package_under_test_is_this_worktree() -> None: + assert Path(legaldown_render.__file__).resolve().is_relative_to(SRC) + + +# --- S1: encodings --------------------------------------------------------------------------- + +@pytest.mark.parametrize("fmt", ["html", "text"]) +def test_stdin_to_stdout_is_utf8_whatever_the_console_encoding(fmt: str) -> None: + expected = render(CZECH, format=fmt).output.encode("utf-8") + done = _run(["-", "-f", fmt, "-q"], CZECH.encode("utf-8")) + assert done.returncode == 0, done.stderr + assert done.stdout == expected + assert "Příloha".encode() in done.stdout + + +def test_file_to_stdout_is_utf8_whatever_the_console_encoding(tmp_path: Path) -> None: + path = tmp_path / "smlouva.lgd" + path.write_bytes(CZECH.encode("utf-8")) + done = _run([str(path), "-f", "text", "-q"]) + assert done.returncode == 0, done.stderr + assert done.stdout == render(CZECH, format="text").output.encode("utf-8") + + +def test_print_style_survives_a_narrow_console() -> None: + done = _run(["--print-style", "--style", "continental", "--set", "placeholders.blank=…"]) + assert done.returncode == 0, done.stderr + assert "…".encode() in done.stdout + + +def test_stderr_never_crashes_on_a_narrow_console() -> None: + done = _run(["žlutý→.lgd"]) + assert done.returncode == 1 + assert b"Traceback" not in done.stderr + assert b"cannot read" in done.stderr + assert b"\\u2192" in done.stderr # not encodable in cp1252: backslash-escaped, not an error + + +# --- S4: not UTF-8 --------------------------------------------------------------------------- + +def test_latin1_input_file_is_a_clean_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + path = tmp_path / "latin1.lgd" + path.write_bytes("# Příloha\n\nCaf\xe9\n".encode("latin-1", errors="replace")) + assert main([str(path)]) == 1 + err = capsys.readouterr().err + assert err.startswith(f"legaldown-render: cannot read {path}: not valid UTF-8 (byte 0x") + assert "at offset" in err + assert "Traceback" not in err + + +def test_the_offset_is_in_the_file_not_in_a_buffer(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + path = tmp_path / "late.lgd" + path.write_bytes(b"a" * 20000 + b"\xe9") + assert main([str(path)]) == 1 + assert "(byte 0xe9 at offset 20000)" in capsys.readouterr().err + + +def test_latin1_stdin_is_a_clean_error(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(sys, "stdin", io.TextIOWrapper(io.BytesIO(b"# Caf\xe9\n"), encoding="cp1252")) + assert main(["-"]) == 1 + assert "legaldown-render: cannot read : not valid UTF-8 (byte 0xe9 at offset 5)" in capsys.readouterr().err + + +def test_stdin_is_decoded_as_utf8_not_as_the_locale(monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(sys, "stdin", io.TextIOWrapper(io.BytesIO(CZECH.encode("utf-8")), encoding="cp1252")) + assert main(["-", "-f", "text", "-q"]) == 0 + assert "Žluťoučký kůň" in capsys.readouterr().out + + +def test_non_utf8_style_file_is_a_style_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + path = tmp_path / "bad.yaml" + path.write_bytes(b"version: 1\nname: Caf\xe9\n") + with pytest.raises(StyleError, match="not valid UTF-8"): + load_style(path) + assert main([FEATURES, "--style", str(path)]) == 2 + err = capsys.readouterr().err + assert "not valid UTF-8" in err + assert "Traceback" not in err + + +# --- S5: unwritable output ------------------------------------------------------------------- + +def test_unwritable_output_is_a_clean_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + target = tmp_path / "missing-folder" / "out.html" + assert main([FEATURES, "-q", "-o", str(target)]) == 1 + err = capsys.readouterr().err + assert err.startswith(f"legaldown-render: cannot write {target}: ") + assert "Traceback" not in err + + +def test_output_that_is_a_directory(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + assert main([FEATURES, "-q", "-f", "html", "-o", str(tmp_path)]) == 1 + assert "cannot write" in capsys.readouterr().err + + +# --- S6: unknown extension ------------------------------------------------------------------- + +@pytest.mark.parametrize("name", ["x.docx", "x.pdf", "x.md", "x.DOCX"]) +def test_unknown_output_extension_without_format_is_a_usage_error( + tmp_path: Path, capsys: pytest.CaptureFixture[str], name: str) -> None: + target = tmp_path / name + with pytest.raises(SystemExit) as exit_info: + main([FEATURES, "-q", "-o", str(target)]) + assert exit_info.value.code == 2 + err = capsys.readouterr().err + assert f"cannot tell the output format from '{target}'" in err + assert "--format html or text" in err + assert not target.exists() + + +def test_explicit_format_is_honoured_whatever_the_extension(tmp_path: Path) -> None: + target = tmp_path / "x.docx" + assert main([FEATURES, "-q", "-f", "text", "-o", str(target)]) == 0 + assert target.read_text(encoding="utf-8").startswith("Services Agreement") + + +def test_output_without_an_extension_keeps_the_html_default(tmp_path: Path) -> None: + for target in (tmp_path / "out", tmp_path / "dir.d" / "out"): + target.parent.mkdir(exist_ok=True) + assert main([FEATURES, "-q", "-o", str(target)]) == 0 + assert target.read_text(encoding="utf-8").startswith("") + + +# --- S2: error hierarchy --------------------------------------------------------------------- + +def test_style_error_is_a_render_error_and_still_a_value_error() -> None: + assert issubclass(StyleError, RenderError) + assert issubclass(StyleError, ValueError) + + +def test_style_error_is_importable_from_where_it_always_was() -> None: + from legaldown_render import errors + from legaldown_render.style import StyleError as from_style + from legaldown_render.style.loader import StyleError as from_loader + + assert errors.StyleError is StyleError is from_style is from_loader + + +def test_unknown_format_raises_options_error() -> None: + assert issubclass(OptionsError, RenderError) + assert issubclass(OptionsError, ValueError) + with pytest.raises(OptionsError, match="unknown output format 'pdf'"): + render("# x\n", RenderOptions(format="pdf")) + with pytest.raises(OptionsError): + render("# x\n", format="docx") + with pytest.raises(RenderError): + render("# x\n", format="docx") + + +def test_invalid_style_through_the_api_is_a_render_error() -> None: + with pytest.raises(RenderError): + render("# x\n", style="no-such-style") + + +# --- S3: pickling ---------------------------------------------------------------------------- + +def test_render_refused_round_trips_through_pickle() -> None: + diagnostics = [Diagnostic(rule="ref-broken", level="error", message="No such target", line=3)] + for reason in (None, "Assembly refused: 1 error(s)."): + error = RenderRefused(diagnostics, reason) + copy = pickle.loads(pickle.dumps(error)) + assert type(copy) is RenderRefused + assert copy.diagnostics == diagnostics + assert str(copy) == str(error) + assert copy.reason == error.reason + + +def test_a_real_refusal_round_trips_through_pickle() -> None: + with pytest.raises(RenderRefused) as caught: + render(Path(FEATURES).read_text(encoding="utf-8"), strict=True) + copy = pickle.loads(pickle.dumps(caught.value)) + assert copy.diagnostics == caught.value.diagnostics + assert str(copy) == str(caught.value) + + +def test_style_error_round_trips_through_pickle() -> None: + error = StyleError("overrides", ["a.b: wrong", "c: also wrong"]) + copy = pickle.loads(pickle.dumps(error)) + assert type(copy) is StyleError + assert (copy.source, copy.problems) == ("overrides", ["a.b: wrong", "c: also wrong"]) + assert str(copy) == str(error) + + +def test_options_error_round_trips_through_pickle() -> None: + error = OptionsError("unknown output format 'pdf'") + assert str(pickle.loads(pickle.dumps(error))) == str(error) + + +# --- S10 (CLI side): a StyleError from --set reaches the user --------------------------------- + +def test_set_style_error_is_shown_by_argparse(monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str]) -> None: + def refuse(key: str, text: str) -> object: + raise StyleError("overrides", [f"{key}: the value is no good"]) + + monkeypatch.setattr(cli, "parse_override", refuse) + with pytest.raises(SystemExit) as exit_info: + main([FEATURES, "--set", "numbering.scheme=oops"]) + assert exit_info.value.code == 2 + err = capsys.readouterr().err + assert "numbering.scheme: the value is no good" in err + assert "--set" in err From 6a65d836d76e738f94c8f7c3f2461aa537bd82d8 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:40:07 +0000 Subject: [PATCH 002/118] Values: exact long amounts, faithful durations, no raw unit ids, spaced symbols (R1-R4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R2: Babel calls for number/money/duration run in a decimal context whose precision is max(28, len(value) + 8). Amounts of 27+ digits no longer raise InvalidOperation and fractions over 28 digits are no longer rounded (§10.3). R3: durations are formatted like number() (every written digit, locale separators) and the plural category is taken from the displayed value (Locale.plural_form on the Decimal: "1.0 days", cs "2,5 roku"). Babel's format_unit, which rounds to 3 decimals, is no longer used. R1: the CLDR pattern is looked up through one helper (Locale._data is the only access; Babel 2.x has no public API): requested length, other lengths, the language-only locale, then English. A raw "duration-day" is never printed. Formatter.format_duration returns DurationText(text, english_fallback); the resolver emits one render-locale-fallback Warning per (locale, unit) when English stands in. Formatter.duration keeps returning str. R4: an alphabetic currency symbol or code touching a digit on either side gets U+00A0 in symbol and code mode ("CZK 5.00", "CHF 1.00"); "$5.00", "R$ 5,00", "CA$5.00" and cs "5,00 Kč" are unchanged. New: tests/test_wp1_values.py. No existing test or golden changed. Docs: ADR 0005 and docs/architecture.md. --- docs/architecture.md | 4 +- docs/decisions/0005-locale-formatting.md | 13 +- src/legaldown_render/resolve/resolver.py | 8 +- src/legaldown_render/resolve/values.py | 125 +++++++++++++++-- tests/test_wp1_values.py | 170 +++++++++++++++++++++++ 5 files changed, 307 insertions(+), 13 deletions(-) create mode 100644 tests/test_wp1_values.py diff --git a/docs/architecture.md b/docs/architecture.md index f8a25fb..13e7d6a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -137,7 +137,7 @@ Guessing at lost structure was tried, and it traded each gap for new bugs. | `{{ref:}}` | `CrossRef(designation, anchor)` (§13.3). A ref to an unnumbered item or paragraph falls back to its section's designation, with `ref-not-enumerated` | `[BROKEN REF: id]` | | `{{term:}}` | `TermRef(label or term, def anchor)` (§7.3, §13.4) | `[UNDEFINED: id]` | | `"Term" {{def:}}` | `DefinedTerm` without quotation marks, anchored at `def:` (§7.2) | n/a | -| `{{date:}}` `{{money:}}` `{{duration:}}` | `Value`, formatted for the locale by Babel (ADR 0005) | `[INVALID DATE: …]`, `[INVALID AMOUNT: …]`, `[UNKNOWN CURRENCY: …]`, `[INVALID DURATION: …]`, `[INVALID DURATION UNIT: …]` | +| `{{date:}}` `{{money:}}` `{{duration:}}` | `Value`, formatted for the locale by Babel (ADR 0005): amounts and durations keep every written digit, and a duration without a pattern in the locale falls back to another length, the language alone, then English with `render-locale-fallback` | `[INVALID DATE: …]`, `[INVALID AMOUNT: …]`, `[UNKNOWN CURRENCY: …]`, `[INVALID DURATION: …]`, `[INVALID DURATION UNIT: …]` | | `{{party:}}` `{{side:}}` | `Value(label or display name)` (§3.6) | `[INVALID PARTY: …]`, `[UNKNOWN PARTY: …]`, and the side forms | | `{{field:}}` | `Value(raw value)` | `[INVALID FIELD: …]` | | `{{placeholder:}}` | `Blank`, with the question's prompt if the style asks for it | `[INVALID PLACEHOLDER: …]`, `[UNKNOWN CURRENCY: …]`, `[INVALID DURATION UNIT: …]` | @@ -226,7 +226,7 @@ turns into an error. Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can evaluate: `ref-not-enumerated`, a specification rule id, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-locale-fallback`, and `render-ref-ambiguous`. The validator's +`render-not-processed`, `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), and `render-ref-ambiguous`. The validator's diagnostics carry their line (§16.9), and the CLI prints it as `file:line:`. The renderer's own have none yet: they are about a directive within a block, and the validator makes only a block's line public (`Document.line_of`, U2). diff --git a/docs/decisions/0005-locale-formatting.md b/docs/decisions/0005-locale-formatting.md index b313de1..794646e 100644 --- a/docs/decisions/0005-locale-formatting.md +++ b/docs/decisions/0005-locale-formatting.md @@ -19,8 +19,17 @@ Use [Babel](https://babel.pocoo.org/). It is pure Python and ships CLDR data. - `format_date` for dates - `format_currency` for money, padded to the currency's minor units and never rounded (§10.3). - Values are handled as `Decimal` from the source string. -- `format_unit` for durations, with CLDR plural rules + Values are handled as `Decimal` from the source string, in a decimal context with the precision + of the source (at least 28 digits, plus a margin), so that no amount of 28 digits or more is + rounded or raises. An alphabetic symbol or code is separated from the digits by U+00A0, as CLDR's + currency spacing says ("CZK 5.00"); Babel does not do that itself. +- CLDR unit patterns for durations, with CLDR plural rules. Babel's `format_unit` is not used: it + rounds the number to 3 decimals, chooses the plural from the unrounded value, and prints the raw + unit id ("30 duration-day") where a length has no pattern. The renderer formats the number as it + does any amount, chooses the plural category from the digits it shows ("1.0" is plural in English, + "2,5" is `many` in Czech), and looks the pattern up through one helper in `resolve/values.py`: + requested length, then the other lengths, then the language alone, then English, with a + `render-locale-fallback` Warning when English stands in. - CLDR's ISO 4217 data to check currency codes All formatting goes through the `locale/` module, so the dependency sits behind one interface. diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index cd44dc4..0301967 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -620,7 +620,13 @@ def _duration(self, directive: Directive, value: str) -> Inline: return FailureMarker(f"[INVALID DURATION: {value}]") if unit not in DURATION_UNITS: return FailureMarker(f"[INVALID DURATION UNIT: {unit}]" if unit else "[INVALID DURATION UNIT]") - return Value("duration", self.formatter.duration(value, unit)) + shown = self.formatter.format_duration(value, unit) + if shown.english_fallback: + # One Warning per locale and unit: _warn drops a repeat. A value + # is never shown as a raw CLDR unit id (§10.5). + self._warn("render-locale-fallback", + f"No CLDR pattern for '{DURATION_UNITS[unit]}' in '{self.formatter.tag}'; English used.") + return Value("duration", shown.text) def _field(self, directive: Directive, value: str) -> Inline: field_type = directive.params.get("type") diff --git a/src/legaldown_render/resolve/values.py b/src/legaldown_render/resolve/values.py index 6e96538..f15ec9c 100644 --- a/src/legaldown_render/resolve/values.py +++ b/src/legaldown_render/resolve/values.py @@ -5,19 +5,24 @@ a period — and only their display varies by locale. Amounts are handled as ``Decimal`` from the source text and are never rounded or truncated (§10.3): money is padded to its currency's minor units, and any further digits written -in the source are kept. +in the source are kept. A duration keeps every digit it is written with too, +and its plural follows the digits it displays (§10.5). """ from __future__ import annotations import datetime +import decimal import re +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass from decimal import Decimal from babel import Locale, UnknownLocaleError from babel.dates import format_date from babel.numbers import format_currency, parse_pattern -from babel.units import format_unit +from ..errors import InternalError from ..style.model import Values #: §10.5 duration units → CLDR unit ids. @@ -31,6 +36,16 @@ "Y": "duration-year", } +#: CLDR unit-pattern lengths, in the order a missing length is replaced. +_LENGTHS = ("long", "short", "narrow") + +#: A letter directly touching a digit, on either side. CLDR's currency +#: spacing puts a no-break space between an alphabetic currency symbol and +#: the digits ("CZK 5.00", "5,00 Kč"); Babel's patterns do not apply it, so +#: it is applied here. A symbol that ends in a sign ("$", "R$", "CA$") has no +#: letter next to the digits and is left alone. +_LETTER_DIGIT = re.compile(r"(?<=[^\W\d_])(?=\d)|(?<=\d)(?=[^\W\d_])") + def parse_locale(tag: str) -> Locale | None: """The Babel locale for a BCP 47 *tag* (``cs-CZ``, ``en``), or None.""" @@ -40,6 +55,84 @@ def parse_locale(tag: str) -> Locale | None: return None +@dataclass(frozen=True) +class DurationText: + """A formatted duration, and whether the locale had no duration pattern + for its unit at all, so that English is shown. The formatter stays free + of diagnostics: the resolver reports the fallback (``render-locale-fallback``).""" + + text: str + english_fallback: bool + + +@contextmanager +def _exact(value: str) -> Iterator[None]: + """A decimal context in which the amount *value* is never rounded (§10.3). + + The default context carries 28 significant digits: Babel's quantizing then + rounds a longer fraction and raises ``InvalidOperation`` for an integer of + 28 digits or more. ``Decimal(value)`` itself is always exact; the + precision here covers every digit of the source plus a margin for the + digits formatting adds (padding to a currency's minor units). With it no + operation on a valid amount can signal, so no trap is relaxed: one that + fired would be a bug, and surfaces as ``InternalError``. + """ + precision = max(28, len(value) + 8) + context = decimal.Context(prec=precision, rounding=decimal.ROUND_HALF_EVEN, + Emin=decimal.MIN_EMIN, Emax=decimal.MAX_EMAX) + with decimal.localcontext(context): + yield + + +def _unit_patterns(locale: Locale, unit_id: str, length: str) -> dict[str, str]: + """The CLDR patterns of *unit_id* at *length* in *locale*, by plural + category (``one``, ``few``, ``other``, …), empty when there are none. The + "per unit" form (``per``) is not a plural category and is left out. + + Babel 2.x has no public API for a pattern by plural category: ``format_unit`` + formats the number itself, rounding it to 3 decimals, and prints the unit + id when a length has no pattern. ``Locale._data`` is the one place the + patterns can be read, and it is read only here. Babel has already merged + the data a regional locale inherits into it, so no parent is looked up. + """ + lengths = locale._data.get("unit_patterns", {}).get(unit_id, {}) # noqa: SLF001 + patterns = lengths.get(length, {}) + return {category: pattern for category, pattern in patterns.items() if category != "per"} + + +def _duration_pattern(locale: Locale, unit_id: str, length: str, value: Decimal) -> tuple[str, bool]: + """The CLDR pattern for a duration of *value*, and whether it is English + for a locale that is not. + + A raw unit id (``duration-day``) is never the answer. The lookup goes: + the requested length, then the other lengths (long, short, narrow); the + same for the locale's language alone; then English. In each set of + patterns the plural category of *value* is used, or ``other`` when the set + has no pattern for that category. The category comes from the locale + whose pattern is used, because it is that locale's grammar. + """ + lengths = (length, *(other for other in _LENGTHS if other != length)) + candidates = [locale] + language = parse_locale(locale.language) + if language is not None and str(language) != str(locale): + candidates.append(language) + for candidate in candidates: + category = candidate.plural_form(value) + for each in lengths: + patterns = _unit_patterns(candidate, unit_id, each) + pattern = patterns.get(category) or patterns.get("other") + if pattern: + return pattern, False + english = Locale("en") + category = english.plural_form(value) + for each in lengths: + patterns = _unit_patterns(english, unit_id, each) + pattern = patterns.get(category) or patterns.get("other") + if pattern: + return pattern, locale.language != "en" + raise InternalError(f"Babel has no English pattern for the unit '{unit_id}'. This is a bug.") + + class Formatter: """Formats values for one locale and one style's ``values`` settings.""" @@ -58,6 +151,10 @@ def date(self, value: str) -> str: def number(self, value: str) -> str: """A decimal amount with the locale's separators, keeping every digit written after the decimal point.""" + with _exact(value): + return self._number(value) + + def _number(self, value: str) -> str: decimals = len(value.partition(".")[2]) pattern = parse_pattern(self.locale.decimal_formats[None].pattern) pattern.frac_prec = (decimals, decimals) @@ -67,6 +164,10 @@ def money(self, amount: str, currency: str | None) -> str: """A money amount (already validated), with its currency when given.""" if not currency: return self.number(amount) + with _exact(amount): + return self._money(amount, currency) + + def _money(self, amount: str, currency: str) -> str: value = Decimal(amount) if self.values.money == "name": return format_currency(value, currency, locale=self.locale, format_type="name", @@ -76,12 +177,20 @@ def money(self, amount: str, currency: str | None) -> str: standard = self.locale.currency_formats["standard"].pattern pattern = standard.replace("¤", "¤¤") if "¤¤" not in standard else standard text = format_currency(value, currency, format=pattern, locale=self.locale, decimal_quantization=False) - if self.values.money == "code": - # CLDR spaces a letter code from the digits ("USD 10,000.00"); - # Babel does not apply that rule, so it is applied here. - text = re.sub(rf"^{currency}(?=\d)", f"{currency}\u00a0", text) - return text + # CLDR spaces an alphabetic symbol or code from the digits, on + # whichever side it stands ("USD 10,000.00", "10 000,00 CZK"). + return _LETTER_DIGIT.sub(" ", text) def duration(self, value: str, unit: str) -> str: """A duration (already validated) such as "12 months" or "12 měsíců".""" - return format_unit(Decimal(value), DURATION_UNITS[unit], length=self.values.duration, locale=self.locale) + return self.format_duration(value, unit).text + + def format_duration(self, value: str, unit: str) -> DurationText: + """A duration with every digit as written ("0.0001 days"), its plural + chosen from those digits as CLDR defines it ("1.0 days" in English, + "2,5 roku" in Czech), and whether English had to stand in for a + locale without a pattern (see :func:`_duration_pattern`).""" + with _exact(value): + pattern, english = _duration_pattern( + self.locale, DURATION_UNITS[unit], self.values.duration, Decimal(value)) + return DurationText(pattern.replace("{0}", self._number(value)), english) diff --git a/tests/test_wp1_values.py b/tests/test_wp1_values.py new file mode 100644 index 0000000..e19fbc8 --- /dev/null +++ b/tests/test_wp1_values.py @@ -0,0 +1,170 @@ +"""Value formatting (§10, §13.5): exact amounts, durations that keep their +written digits, no raw CLDR unit id, and spaced alphabetic currency symbols.""" +from __future__ import annotations + +import pytest + +from legaldown_render import render +from legaldown_render.resolve import DURATION_UNITS, Formatter, parse_locale +from legaldown_render.resolve.values import DurationText +from legaldown_render.style.model import Values + + +def formatter(tag: str, **values: str) -> Formatter: + locale = parse_locale(tag) + assert locale is not None + return Formatter(locale, Values(**values)) + + +# -- R2: no crash, no rounding on long amounts (§10.3) ----------------------- + +@pytest.mark.parametrize("digits", [29, 40, 60]) +def test_long_integers_format_exactly(digits: int) -> None: + value = "1234567890" * 6 + value = value[:digits] + grouped = f"{int(value):,}" + en = formatter("en-US") + assert en.number(value) == grouped + assert en.money(value, "USD") == f"${grouped}.00" + assert formatter("en-US", money="code").money(value, "USD") == f"USD\xa0{grouped}.00" + assert formatter("en-US", money="name").money(value, "USD") == f"{grouped}.00 US dollars" + + +def test_long_fraction_keeps_every_digit() -> None: + fraction = "1234567891" * 4 + en = formatter("en-US") + assert en.number("0." + fraction) == "0." + fraction + assert en.money("1." + fraction, "EUR") == "€1." + fraction + assert en.number("1" * 29 + ".5") == f"{int('1' * 29):,}.5" + + +def test_long_duration_keeps_every_digit() -> None: + assert formatter("en-US").duration("1" * 40, "D") == f"{int('1' * 40):,} days" + assert formatter("en-US").duration("0." + "1" * 40, "D") == "0." + "1" * 40 + " days" + + +def test_long_amount_renders_end_to_end() -> None: + source = "---\ntitle: T\n---\nPay {{money: " + "9" * 29 + "}} {{money: 12345678901234567890123456789.1234567890123456789}}.\n" + output = render(source, format="text", locale="en-US").output + assert f"{int('9' * 29):,}" in output + assert "12,345,678,901,234,567,890,123,456,789.1234567890123456789" in output + + +# -- R3: durations keep their digits; the plural follows the displayed value - + +@pytest.mark.parametrize(("tag", "value", "unit", "expected"), [ + ("en", "0.0001", "D", "0.0001 days"), + ("en", "1.23456", "Y", "1.23456 years"), + ("en", "1", "D", "1 day"), + ("en", "1.0", "D", "1.0 days"), + ("en", "12", "MO", "12 months"), + ("cs", "1", "Y", "1 rok"), + ("cs", "2", "Y", "2 roky"), + ("cs", "5", "Y", "5 let"), + ("cs", "2.5", "Y", "2,5 roku"), + ("de", "1.50", "W", "1,50 Wochen"), +]) +def test_duration_digits_and_plural(tag: str, value: str, unit: str, expected: str) -> None: + assert formatter(tag).duration(value, unit) == expected + + +# -- R1: never a raw CLDR unit id --------------------------------------------- + +@pytest.mark.parametrize("tag", ["pt-BR", "ja", "zh", "ko", "it", "tr", "nl", "cs", "de", "sk", "pl", "en"]) +@pytest.mark.parametrize("length", ["long", "short", "narrow"]) +@pytest.mark.parametrize("value", ["1", "2", "5", "1.5"]) +def test_no_cldr_unit_id_is_ever_printed(tag: str, length: str, value: str) -> None: + f = formatter(tag, duration=length) + for unit in DURATION_UNITS: + text = f.duration(value, unit) + assert "duration-" not in text + assert value.replace(".", f.locale.number_symbols["latn"]["decimal"]) in text + + +def test_pt_br_long_falls_back_to_the_short_pattern() -> None: + pt = formatter("pt-BR") + assert pt.duration("30", "D") == "30 dias" + assert pt.duration("12", "MO") == "12 meses" + assert pt.duration("1", "Y") == "1 ano" + + +def test_fallback_goes_to_the_other_length_first() -> None: + # cs has a long and a narrow pattern: narrow is asked, narrow is used. + assert formatter("cs", duration="narrow").duration("30", "D") == "30 d." + # zh has only "short": long and narrow fall back to it, still Chinese. + assert formatter("zh", duration="long").duration("30", "D") == "30天" + assert formatter("zh", duration="narrow").duration("30", "D") == "30天" + + +def test_a_fallback_inside_the_locale_is_not_signalled() -> None: + # pt-BR long has no pattern but short has: Portuguese, no English. + assert formatter("pt-BR").format_duration("30", "D") == DurationText("30 dias", False) + assert formatter("en").format_duration("1", "D") == DurationText("1 day", False) + + +def test_english_is_the_last_resort_and_is_signalled(monkeypatch: pytest.MonkeyPatch) -> None: + from legaldown_render.resolve import values + + real = values._unit_patterns + # A Czech locale without any duration pattern (Babel's data has none such). + monkeypatch.setattr(values, "_unit_patterns", + lambda locale, unit, length: {} if locale.language == "cs" else real(locale, unit, length)) + f = formatter("cs") + assert f.format_duration("5", "D") == DurationText("5 days", True) + assert f.duration("1", "D") == "1 day" + assert f.duration("1.5", "Y") == "1,5 years" # the number keeps the locale's separator + + +def test_fallback_warning_is_emitted_once_by_the_resolver(monkeypatch: pytest.MonkeyPatch) -> None: + from legaldown_render.resolve import values + + real = values._unit_patterns + monkeypatch.setattr(values, "_unit_patterns", + lambda locale, unit, length: {} if locale.language == "cs" else real(locale, unit, length)) + source = ("---\ntitle: T\nlanguage: cs\n---\n" + "Lhůta {{duration: 5, unit=D}} a {{duration: 6, unit=D}} a {{duration: 2, unit=MO}}.\n") + result = render(source, format="text") + fallbacks = [d for d in result.diagnostics if d.rule == "render-locale-fallback"] + assert [d.message for d in fallbacks] == [ + "No CLDR pattern for 'duration-day' in 'cs'; English used.", + "No CLDR pattern for 'duration-month' in 'cs'; English used.", + ] + assert all(d.level == "warning" for d in fallbacks) + assert "5 days" in result.output + + +def test_normal_locales_emit_no_fallback_warning() -> None: + for language in ("cs", "pt-BR", "ja", "en"): + source = f"---\ntitle: T\nlanguage: {language}\n---\nX {{{{duration: 5, unit=D}}}}.\n" + result = render(source, format="text") + assert not [d for d in result.diagnostics if d.rule == "render-locale-fallback"], language + + +# -- R4: an alphabetic symbol is spaced from the digits ----------------------- + +@pytest.mark.parametrize(("tag", "currency", "expected"), [ + ("en", "CZK", "CZK\xa05.00"), + ("en", "CHF", "CHF\xa05.00"), + ("en", "USD", "$5.00"), + ("en", "EUR", "€5.00"), + ("pt-BR", "BRL", "R$\xa05,00"), + ("en", "CAD", "CA$5.00"), + ("cs", "CZK", "5,00\xa0Kč"), +]) +def test_symbol_mode_spacing(tag: str, currency: str, expected: str) -> None: + assert formatter(tag).money("5", currency) == expected + + +def test_symbol_after_the_digits_is_spaced_too() -> None: + # Whatever the locale puts after the amount, a letter never touches a digit. + import re + + for tag in ("en", "de", "fr", "cs", "pl", "sv", "ja", "ru", "pt-PT", "es", "nl", "it"): + for currency in ("CZK", "CHF", "USD", "EUR", "SEK", "PLN", "XAF"): + text = formatter(tag).money("5", currency) + assert not re.search(r"[^\W\d_]\d|\d[^\W\d_]", text), (tag, currency, text) + + +def test_code_mode_still_spaced() -> None: + assert formatter("en", money="code").money("5", "CZK") == "CZK\xa05.00" + assert formatter("de", money="code").money("5", "CZK") == "5,00\xa0CZK" From e5196f8e7e299a50f67a1a46dc25615db2fd58dc Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:40:38 +0000 Subject: [PATCH 003/118] Build fidelity: comment-only items and quotes, emphasis next to directives, dropped-comment spacing, {{term:}} text (B2, B3, B4, B8) - B2: a list item whose model blocks are all comments/raw HTML (kind "html") is left out, after the whole list is traversed so item numbering and PlacedMarker.item stay aligned; a list left with no items and an ordinary quote with no built blocks are left out. A truly empty item and an empty drafting note keep their place. (build.py _list, list, quote) - B3: every directive sentinel is wrapped in U+2E31 (category Po, accepted by markdown-it's isPunctChar) on both sides, as the braces it replaces are punctuation, so `_text_{{ref: a}}`, `{{ref: a}}_text_`, `__b__{{ref: a}}` and `*t*{{ref: a}}` parse as with `{x}`. sentinel_re, encoded_sentinel_re and _restore consume the wrappers; the hidden form's "h" tag is no longer needed. (build.py) - B4: _merge_text strips the spaces before a dropped inline when a line break follows, so no double space is left. - B8: Resolver._survey collects definition id -> plain rendered term text (first definition wins, via _title_text); {{term:}} uses it, falling back to the validator's definition_lookup. Existing test changed: test_empty_list_items_keep_their_place is extended (an empty item still keeps its place; a comment-only one goes). Goldens are unchanged. Docs: CONFORMANCE.md, docs/architecture.md. --- CONFORMANCE.md | 9 + docs/architecture.md | 13 +- src/legaldown_render/build.py | 79 +++++--- src/legaldown_render/resolve/resolver.py | 21 +- tests/test_render.py | 5 + tests/test_wp4_build.py | 246 +++++++++++++++++++++++ 6 files changed, 343 insertions(+), 30 deletions(-) create mode 100644 tests/test_wp4_build.py diff --git a/CONFORMANCE.md b/CONFORMANCE.md index dd09b8d..f8b7774 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -54,6 +54,15 @@ It is checked against the specification's own examples and fixtures corpus (`tes quote's content is its text. The renderer follows it, and refuses (`DocumentError`) a document whose lists, quotes, and inline formatting (emphasis, links) together nest more than 100 levels, while building it and before resolving or writing it, so no document can exhaust the stack. +- **Comment-only items and quotes.** A list item whose blocks are all comments or raw HTML is + left out (§8.6: the author removed it), and so is a list left with no items and a quote that + holds only comments. Numbers and designations are those of the items that remain. An item with + no content at all keeps its place (CommonMark), and so does an empty drafting note, whose label + is the template construct (§15.6). +- **`{{term:}}` text.** A `{{term:}}` shows the term as rendered at its first definition in the + body, with its Markdown resolved (`"A *b* c" {{def: abc}}` is "A b c"), not the term's raw + source; `label=` overrides it. A definition not in the body (a template's other alternative) + falls back to the validator's term text. - **Ambiguous references.** Each list starts again at its first number, as contracts are usually drafted, so two lists in one section can both have an item "(a)", and a style can number a paragraph or an item like a subsection ("1.1"). A `{{ref:}}` to a section, paragraph, or list diff --git a/docs/architecture.md b/docs/architecture.md index f8a25fb..4fe9320 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -96,13 +96,18 @@ Markdown cannot reinterpret directive syntax, such as an underscore in an identi random for each build, so no text in a document, whether written literally, as an entity, or percent-encoded, can pass for a sentinel. Emphasis that wraps only a defined term is dropped with it, because the style decides how defined terms look (§7.2). A `{{def:}}` becomes a sentinel -that renders nothing, led by a punctuation character so that an emphasis closer just before it -still closes. A defined term in a link title or alt text shows its term but is not the +that renders nothing. Every sentinel is wrapped in a punctuation character (U+2E31, Unicode +category Po, which markdown-it's `isPunctChar` accepts) on both sides, as the `{{` and `}}` it +replaces are punctuation, so that CommonMark's emphasis flanking rules see the same neighbours as +in the source: `_text_{{ref: a}}` closes its emphasis. A defined term in a link title or alt text shows its term but is not the definition's anchor. An HTML block in the model (`kind="html"`), a comment block included, renders nothing (§8.6). Within a block's text, inline comments and tags are dropped; where one stood between two spaces, -one space stays. The validator reports raw HTML other than comments (`raw-html`, §8.7). A heading +one space stays, and before a line break none does. A list item whose blocks are all comments, +and a quote that holds only comments, are dropped (with a list left empty), after the whole list +is traversed so that the validator's item numbers stay aligned with the markers it placed. The +validator reports raw HTML other than comments (`raw-html`, §8.7). A heading in a list item or a quote (`kind="heading"`) is not a section (§4.1) and shows as a bold line. The builder never works around the validator's model. Where the model loses or misreads @@ -135,7 +140,7 @@ Guessing at lost structure was tried, and it traded each gap for new bugs. | Directive | Resolves to | Failure markers | |---|---|---| | `{{ref:}}` | `CrossRef(designation, anchor)` (§13.3). A ref to an unnumbered item or paragraph falls back to its section's designation, with `ref-not-enumerated` | `[BROKEN REF: id]` | -| `{{term:}}` | `TermRef(label or term, def anchor)` (§7.3, §13.4) | `[UNDEFINED: id]` | +| `{{term:}}` | `TermRef(label or the definition's rendered term text, def anchor)` (§7.3, §13.4); the text is collected in `Resolver._survey` | `[UNDEFINED: id]` | | `"Term" {{def:}}` | `DefinedTerm` without quotation marks, anchored at `def:` (§7.2) | n/a | | `{{date:}}` `{{money:}}` `{{duration:}}` | `Value`, formatted for the locale by Babel (ADR 0005) | `[INVALID DATE: …]`, `[INVALID AMOUNT: …]`, `[UNKNOWN CURRENCY: …]`, `[INVALID DURATION: …]`, `[INVALID DURATION UNIT: …]` | | `{{party:}}` `{{side:}}` | `Value(label or display name)` (§3.6) | `[INVALID PARTY: …]`, `[UNKNOWN PARTY: …]`, and the side forms | diff --git a/src/legaldown_render/build.py b/src/legaldown_render/build.py index d99bdd4..2d8e568 100644 --- a/src/legaldown_render/build.py +++ b/src/legaldown_render/build.py @@ -14,7 +14,10 @@ ``\\ue000:\\ue001``, so Markdown can never reinterpret directive syntax, and put back as tree nodes afterwards. The nonce is random for every build, so no text in a document — written out, as an entity, or -percent-encoded — can pass for a sentinel. +percent-encoded — can pass for a sentinel. Each sentinel is wrapped in +Unicode punctuation on both sides, as the ``{{`` and ``}}`` it replaces are +punctuation, so that Markdown's emphasis rules (CommonMark's left- and +right-flanking delimiter runs) see the same neighbours as in the source. Lists and quotes are built as the validator's model holds them: list items hold blocks, nested lists included, and a quote's content is the blocks the @@ -74,10 +77,17 @@ ) _OPEN, _CLOSE = "\ue000", "\ue001" -# Leads the sentinel of source that renders nothing (a {{def:}}). It is -# Unicode punctuation, so an emphasis closer just before it still counts as -# right-flanking and closes (CommonMark); a bare sentinel reads like a letter. -_HIDDEN_LEAD = "\u2e31" +# Wraps every sentinel, before and after. A directive's braces are Unicode +# punctuation, and CommonMark decides whether an emphasis delimiter run opens +# or closes from the punctuation and whitespace next to it; a bare sentinel +# (private use, category Co) reads like a letter and would change that: +# ``_text_{{ref: a}}`` would not close its emphasis. U+2E31 is category Po — +# markdown-it-py's ``isPunctChar`` accepts only categories P* — and too rare +# in a document to matter, and the pairs the builder adds are the only ones +# the sentinel patterns consume. +_WRAP = "\u2e31" +_WRAPPERS = (_WRAP,) +_ENCODED_WRAP = "".join(f"%{byte:02X}" for byte in _WRAP.encode()) # Stand where inline HTML or a comment was dropped, until _merge_text joins # the text around them. A dropped line break (
) still parts two words. _DROPPED = Text("") @@ -181,12 +191,12 @@ def __init__(self, document: Document, result: ValidationResult) -> None: self.lex = cache(lex) self.depth = 0 # quotes and list items the builder is inside nonce = secrets.token_hex(8) - # A hidden sentinel has its own form ("h"), so the lead character is - # taken only with one, never from the source before another sentinel. - self.sentinel_re = re.compile(f"(?:{_HIDDEN_LEAD}{_OPEN}{nonce}h|{_OPEN}{nonce}):(\\d+){_CLOSE}") + # A sentinel with its wrappers; only a complete one matches, so a + # wrapper character in the source is never taken. + self.sentinel_re = re.compile(f"{_WRAP}{_OPEN}{nonce}:(\\d+){_CLOSE}{_WRAP}") # How markdown-it percent-encodes a sentinel inside a URL. self.encoded_sentinel_re = re.compile( - f"(?:%E2%B8%B1%EE%80%80{nonce}h|%EE%80%80{nonce}):(\\d+)%EE%80%81", re.IGNORECASE) + f"{_ENCODED_WRAP}%EE%80%80{nonce}:(\\d+)%EE%80%81{_ENCODED_WRAP}", re.IGNORECASE) self.nonce = nonce # -- sentinels ------------------------------------------------------------ @@ -194,9 +204,7 @@ def __init__(self, document: Document, result: ValidationResult) -> None: def _sentinel(self, payload: _Payload) -> str: self.payloads.append(payload) index = len(self.payloads) - 1 - if isinstance(payload, _Hidden): - return f"{_HIDDEN_LEAD}{_OPEN}{self.nonce}h:{index}{_CLOSE}" - return f"{_OPEN}{self.nonce}:{index}{_CLOSE}" + return f"{_WRAP}{_OPEN}{self.nonce}:{index}{_CLOSE}{_WRAP}" def _protect(self, body: str) -> str: """*body* — one block's text — with every directive replaced by a @@ -392,16 +400,24 @@ def block(self, block: ModelBlock, placed: list[PlacedMarker], *, top_level: boo return Paragraph((Strong(inlines),)) if inlines else None raise InternalError(f"unexpected block kind '{block.kind}' in the validator's model") - def list(self, block: ModelBlock, placed: list[PlacedMarker]) -> List: + def list(self, block: ModelBlock, placed: list[PlacedMarker]) -> List | None: """A list, nested as the validator's model holds it: items that hold blocks. A placed marker names the item it marks by its pre-order - number among all the list's items, as the validator numbers them.""" + number among all the list's items, as the validator numbers them. + None for a list whose every item is only a comment.""" markers = {marker.item: marker for marker in placed} return self._list(block, markers, count()) - def _list(self, block: ModelBlock, markers: dict[int | None, PlacedMarker], numbers: Iterator[int]) -> List: + def _list(self, block: ModelBlock, markers: dict[int | None, PlacedMarker], numbers: Iterator[int]) -> List | None: """*block*'s items, numbered from *numbers* in pre-order, to find the - items *markers* belong to.""" + items *markers* belong to. + + An item whose blocks are all comments or raw HTML is the author + removing it (§8.6), and is left out — after the whole list is + traversed, so the numbers (and with them the markers) stay those of + the validator's list. An item with no blocks at all is an empty item + (CommonMark) and keeps its place, so later items keep their numbers. + A list left with no items is left out.""" self._check_depth(1) self.depth += 1 try: @@ -416,15 +432,19 @@ def _list(self, block: ModelBlock, markers: dict[int | None, PlacedMarker], numb blocks: list[Block] = [] for child in children: if child.kind in ("ordered_list", "unordered_list"): - blocks.append(self._list(child, markers, numbers)) + nested = self._list(child, markers, numbers) + if nested is not None: + blocks.append(nested) elif child.kind == "html": continue # never emitted (§8.6, §8.7) elif (built := self.block(child, [], top_level=False)) is not None: blocks.append(built) + if item.blocks and all(child.kind == "html" for child in item.blocks): + continue # only a comment: the item is removed items.append(ListItem(blocks=tuple(blocks), anchor_id=identifier, condition=condition)) finally: self.depth -= 1 - return List(ordered=block.kind == "ordered_list", items=tuple(items)) + return List(ordered=block.kind == "ordered_list", items=tuple(items)) if items else None def _check_depth(self, more: int) -> None: """Refuse a document that nests lists, quotes, and inline formatting @@ -433,13 +453,15 @@ def _check_depth(self, more: int) -> None: raise DocumentError(f"The document nests lists, quotes, and inline formatting more than " f"{MAX_NESTING} levels deep, which is not rendered.") - def quote(self, block: ModelBlock) -> Block: + def quote(self, block: ModelBlock) -> Block | None: """A block quote, its content the blocks the validator reads in it (``quote_blocks``). Past the validator's quote depth, it reads the quote's text as one paragraph, and so does the builder. A drafting note is decided by the validator's own test (§15.6), and its content is the validator's reading without the ``[!DRAFTING]`` marker - (``drafting_note_blocks``).""" + (``drafting_note_blocks``). A quote that holds nothing but comments + is left out (None); an empty drafting note stays, since its label is + the template construct the reader sees (§15.6).""" drafting = is_drafting_note(block) self._check_depth(1) # The quotes and list items the quote is in, before entering it. @@ -450,7 +472,9 @@ def quote(self, block: ModelBlock) -> Block: blocks = self.blocks(children, None, markers=False) finally: self.depth -= 1 - return DraftingNote(blocks) if drafting else Quote(blocks) + if drafting: + return DraftingNote(blocks) + return Quote(blocks) if blocks else None # -- document ------------------------------------------------------------- @@ -520,7 +544,8 @@ def _trim(inlines: tuple[Inline, ...]) -> tuple[Inline, ...]: def _merge_text(inlines: list[Inline]) -> tuple[Inline, ...]: """*inlines* with adjacent text joined. Where HTML or a comment was dropped between two spaces, one of them goes: ``a b`` reads - "a b". A dropped ``
`` between two words leaves one space.""" + "a b". A dropped ``
`` between two words leaves one space. Before a + line break, the spaces before a dropped inline all go.""" merged: list[Inline] = [] dropped = parted = False for inline in inlines: @@ -529,7 +554,15 @@ def _merge_text(inlines: list[Inline]) -> tuple[Inline, ...]: parted = parted or inline is _DROPPED_BREAK continue before = merged[-1] if merged else None - if dropped and isinstance(inline, Text) and isinstance(before, Text) and before.text.endswith(" "): + if dropped and isinstance(inline, SoftBreak | HardBreak) and isinstance(before, Text): + # The comment ended its line: the space before it goes, or the + # line would end in one (and "a \nb" read "a b"). + stripped = before.text.rstrip(" ") + merged.pop() + if stripped: + merged.append(Text(stripped)) + before = merged[-1] if merged else None + elif dropped and isinstance(inline, Text) and isinstance(before, Text) and before.text.endswith(" "): inline = Text(inline.text.lstrip(" ")) elif parted and before is not None and not isinstance(inline, SoftBreak | HardBreak): if isinstance(before, Text): diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index cd44dc4..cc5e055 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -147,6 +147,9 @@ def __init__( questions = self.metadata.questions self.questions: dict[str, Any] = questions if isinstance(questions, dict) else {} self.inconsistent_placeholders: set[str] = set() + # A definition id -> the plain text of its term as rendered (Markdown + # resolved), from the first definition in the body (see _survey). + self.term_texts: dict[str, str] = {} self.attachment_anchors: dict[str, str | None] = {} self.section_presences: list[Presence] = [] @@ -238,10 +241,19 @@ def _settle_term_target(self, inline: Inline) -> Inline: def _survey(self) -> None: """One pass over the tree before resolving: which placeholder ids - are used with conflicting types (§10.7).""" + are used with conflicting types (§10.7), and what each defined term + reads as, for ``{{term:}}`` (§7.3).""" types: dict[str, set[str]] = {} for inline in iter_tree_inlines(self.tree): - if isinstance(inline, DirectiveSource): + if isinstance(inline, DefinitionSource): + # The first definition of an id is the one the validator's + # lookup and the anchor keep. The term is the text the reader + # sees at the definition: its Markdown is resolved, not the + # validator's raw ``anchor.term``. + identifier = self._definition_id(inline) + if identifier not in self.term_texts: + self.term_texts[identifier] = self._title_text(inline.children) + elif isinstance(inline, DirectiveSource): directive = inline.directive if directive.name == "placeholder" and directive.positional and not directive.malformed: types.setdefault(directive.positional, set()).add(self._placeholder_type(directive)) @@ -580,7 +592,10 @@ def _term(self, directive: Directive, definition_id: str) -> Inline: term = self.result.index.definition_lookup.get(definition_id) if term is None: return FailureMarker(f"[UNDEFINED: {definition_id}]") - text = directive.params.get("label") or self._plain_value(term) + # The term as rendered at its definition; the validator's own text + # (an empty term reads as its id, a definition not in the body) is + # the fallback. + text = directive.params.get("label") or self.term_texts.get(definition_id) or self._plain_value(term) # Settled once everything is resolved (see _settle_term_target). target = DEFINITION_ANCHOR_PREFIX + definition_id return TermRef(text, target, self.style.definitions.term_style) diff --git a/tests/test_render.py b/tests/test_render.py index d3c8f7b..8aac3c0 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -922,6 +922,11 @@ def test_empty_list_items_keep_their_place() -> None: output = text("# A\n\n- \n - \n- b {#b}\n\nSee {{ref: b}}.\n") assert "(a)\n (i)\n(b) b" in output assert "See 1(b)." in output + # An item that holds only a comment is the author removing it (§8.6): + # it is left out, and the numbers are still the validator's. + output = text("# A\n\n- \n- \n- b {#b}\n\nSee {{ref: b}}.\n") + assert "(a)\n(b) b" in output + assert "See 1(b)." in output @pytest.mark.parametrize("output_format", ["html", "text"]) diff --git a/tests/test_wp4_build.py b/tests/test_wp4_build.py new file mode 100644 index 0000000..f21ae17 --- /dev/null +++ b/tests/test_wp4_build.py @@ -0,0 +1,246 @@ +"""Build fidelity (WP4): what the builder keeps of a comment-only item or +quote (B2), how emphasis next to a directive parses (B3), the spacing where +an inline comment was dropped (B4), and what ``{{term:}}`` shows (B8).""" +from __future__ import annotations + +import html as html_escape +import unicodedata + +import pytest +from legaldown import parse, validate +from markdown_it import MarkdownIt +from markdown_it.common.utils import isPunctChar + +from legaldown_render import render +from legaldown_render.build import _WRAPPERS, build_tree +from legaldown_render.tree import ( + DirectiveSource, + DraftingNote, + Emphasis, + Paragraph, + Quote, + Strong, + Text, +) + +FRONT = "---\ntitle: T\n---\n" + + +def text(body: str, **settings: object) -> str: + return render(FRONT + body, format="text", **settings).output + + +def html(body: str, **settings: object) -> str: + return render(FRONT + body, format="html", **settings).output + + +def built(body: str): + document = parse(FRONT + body) + return build_tree(document, validate(document)) + + +def blocks_of(body: str): + return built(body).sections[0].blocks + + +# -- B2: comment-only list items and quotes ---------------------------------- + + +def test_a_list_item_holding_only_a_comment_is_dropped() -> None: + output = text("# A\n\n- a\n- \n- c\n") + assert "(a) a\n(b) c" in output + assert "(c)" not in output + + +def test_dropping_a_comment_only_item_keeps_markers_on_their_items() -> None: + # Numbering and the marker-to-item mapping are those of the validator's + # (unfiltered) list; only the output drops the item. + body = "# A\n\n- \n- b {#b}\n - \n - bb {#bb}\n- c {#c}\n\nSee {{ref: b}}, {{ref: bb}}, {{ref: c}}.\n" + output = text(body) + assert "(a) b\n (i) bb\n(b) c" in output + assert "See 1(a), 1(a)(i), 1(b)." in output + + +def test_a_list_left_with_no_items_is_dropped() -> None: + output = text("# A\n\nBefore.\n\n- \n- \n\nAfter.\n") + assert "(a)" not in output + assert "Before.\n\nAfter." in output + assert "\n") + assert "\n") + + +def test_an_empty_item_keeps_its_place_but_a_comment_only_one_does_not() -> None: + # The validator keeps an empty item (CommonMark); a comment is the + # author's way to remove one. + output = text("# A\n\n- \n- \n- b\n") + assert "(a)\n(b) b" in output + + +def test_an_item_with_a_comment_and_text_stays() -> None: + assert "(a) kept" in text("# A\n\n- kept \n") + + +def test_an_item_with_a_nested_list_stays_even_when_its_own_text_is_a_comment() -> None: + output = text("# A\n\n- \n - inner\n") + assert "inner" in output + + +def test_a_comment_only_quote_is_dropped() -> None: + assert " \n") + assert blocks_of("# A\n\n> \n") == () + assert [type(b) for b in blocks_of("# A\n\n> \n\nText.\n")] == [Paragraph] + + +def test_a_quote_with_text_stays() -> None: + assert [type(b) for b in blocks_of("# A\n\n> said \n")] == [Quote] + + +def test_an_empty_drafting_note_keeps_its_label() -> None: + blocks = blocks_of("# A\n\n> [!DRAFTING]\n> \n") + assert [type(b) for b in blocks] == [DraftingNote] + assert "Drafting note" in html("# A\n\n> [!DRAFTING]\n> \n") or "ld-drafting" in html( + "# A\n\n> [!DRAFTING]\n> \n") + + +def test_a_quote_nested_in_a_quote_that_is_comment_only_is_dropped_with_it() -> None: + assert blocks_of("# A\n\n> > \n") == () + + +def test_a_comment_only_item_does_not_leave_an_empty_list_in_an_item() -> None: + output = text("# A\n\n- a\n - \n- b\n") + assert "(a) a\n(b) b" in output + + +# -- B3: emphasis flanking next to a directive ------------------------------- + + +def _signature(inlines) -> str: + out = [] + for inline in inlines: + match inline: + case Text(text=value): + out.append(html_escape.escape(value, quote=False)) + case DirectiveSource(): + out.append("{x}") + case Emphasis(children=children): + out.append(f"{_signature(children)}") + case Strong(children=children): + out.append(f"{_signature(children)}") + case _: + raise AssertionError(inline) + return "".join(out) + + +@pytest.mark.parametrize("source", [ + "_text_{{ref: a}}", + "{{ref: a}}_text_", + "__b__{{ref: a}}", + "{{ref: a}}__b__", + "*t*{{ref: a}}", + "{{ref: a}}*t*", + "x _text_{{ref: a}} y", + "x{{ref: a}}_text_y", + "*a{{ref: a}}* b", + "**a{{ref: a}}**b", + "_{{ref: a}}_", + "_a_{{ref: a}}_b_", + "foo_{{ref: a}}_bar", +]) +def test_emphasis_next_to_a_directive_parses_as_it_does_next_to_a_brace(source: str) -> None: + body = f"# A {{#a}}\n\n{source}\n" + paragraphs = [b for b in blocks_of(body) if isinstance(b, Paragraph)] + expected = MarkdownIt("commonmark").renderInline(source.replace("{{ref: a}}", "{x}")) + assert _signature(paragraphs[0].inlines) == expected + + +def test_every_sentinel_wrapper_is_punctuation_to_markdown_it() -> None: + wrappers = _WRAPPERS + assert wrappers + for char in wrappers: + assert isPunctChar(char), hex(ord(char)) + assert unicodedata.category(char).startswith("P") + + +def test_a_directive_next_to_emphasis_renders() -> None: + output = html("# A {#a}\n\n_text_{{ref: a}} and {{ref: a}}_more_\n") + assert "text" in output and "more" in output + + +def test_wrappers_do_not_leak_into_code_urls_or_alt_text() -> None: + output = text("# A {#a}\n\n`{{ref: a}}` [x](http://e.com/{{ref: a}}) ![{{ref: a}}](p.png)\n") + assert "{{ref: a}}" in output + for char in _WRAPPERS: + assert char not in output + + +# -- B4: spacing where an inline comment was dropped ------------------------- + + +@pytest.mark.parametrize(("body", "expected"), [ + ("one \ntwo\n", "one two"), + ("one \ntwo\n", "one\ntwo"), + ("one \\\ntwo\n", "one\ntwo"), + ("one \ntwo\n", "one two"), + ("one
\ntwo\n", "one two"), +]) +def test_no_double_space_before_a_line_break(body: str, expected: str) -> None: + assert text("# A\n\n" + body).endswith("\n\n" + expected + "\n") + + +def test_no_trailing_space_when_a_dropped_comment_ends_the_block() -> None: + paragraph = blocks_of("# A\n\nword \n")[0] + assert [i.text for i in paragraph.inlines] == ["word"] + + +def test_a_space_still_parts_words_around_a_dropped_comment() -> None: + assert "a b" in text("# A\n\na b\n") + assert "a b" in text("# A\n\na
b\n") + + +def test_a_dropped_comment_between_text_and_a_hard_break_leaves_no_space() -> None: + paragraph = blocks_of("# A\n\none \\\ntwo\n")[0] + first = paragraph.inlines[0] + assert isinstance(first, Text) and first.text == "one" + + +# -- B8: what {{term:}} shows ------------------------------------------------ + + +def test_a_term_reference_shows_the_rendered_text_of_the_term() -> None: + output = text('# A\n\n"A *b* c" {{def: abc}} means it.\n\nSee {{term: abc}}.\n') + assert "See A b c." in output + assert "*" not in output.split("See", 1)[1] + + +def test_a_term_reference_label_still_wins() -> None: + output = text('# A\n\n"A *b* c" {{def: abc}} means it.\n\nSee {{term: abc, label=the thing}}.\n') + assert "See the thing." in output + + +def test_a_term_reference_resolves_directives_inside_the_term() -> None: + body = '# A {#a}\n\n"Fee **under** {{ref: a}}" {{def: fee}} means it.\n\nSee {{term: fee}}.\n' + output = text(body) + assert "See Fee under" in output + + +def test_a_term_reference_uses_the_first_definition() -> None: + body = '# A\n\n"First *one*" {{def: t}} here.\n\n"Second" {{def: t}} there.\n\nSee {{term: t}}.\n' + assert "See First one." in text(body) + + +def test_a_term_reference_before_its_definition() -> None: + body = '# A\n\nSee {{term: abc}} first.\n\n"A *b* c" {{def: abc}} means it.\n' + assert "See A b c first." in text(body) + + +def test_a_plain_term_reference_is_unchanged() -> None: + assert "See Fee." in text('# A\n\n"Fee" {{def: fee}} means it.\n\nSee {{term: fee}}.\n') + + +def test_an_empty_term_falls_back_to_the_validator() -> None: + output = text('# A\n\n"" {{def: late-fee}} means it.\n\nSee {{term: late-fee}}.\n') + assert "See Late Fee." in output + + +def test_an_undefined_term_is_still_reported() -> None: + assert "[UNDEFINED: nope]" in text("# A\n\nSee {{term: nope}}.\n") From ef856757e5d44ff7e4ec3df1fd3c258349fcb19b Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:40:44 +0000 Subject: [PATCH 004/118] render_file(): a non-UTF-8 file raises DocumentError (S4, API side) render_file() read the file with read_text, so a non-UTF-8 file raised a bare UnicodeDecodeError. It now reads bytes and raises DocumentError with the CLI's wording ("not valid UTF-8 (byte 0x.. at offset N)"). Behaviour change: DocumentError is not a ValueError, so code that caught UnicodeDecodeError/ValueError from render_file for this case must catch DocumentError (or RenderError). OSError for unreadable files is unchanged. Test: tests/test_wp2_cli.py::test_render_file_raises_document_error_for_non_utf8. --- src/legaldown_render/api.py | 13 +++++++++++-- tests/test_wp2_cli.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index 0fb88e7..508ce04 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -123,8 +123,17 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any def render_file(path: str | Path, options: RenderOptions | None = None, /, **settings: Any) -> RenderResult: - """Render the LegalDown file at *path* (see :func:`render`).""" - return render(Path(path).read_text(encoding="utf-8"), options, **settings) + """Render the LegalDown file at *path* (see :func:`render`). Raises + :class:`DocumentError` when the file is not valid UTF-8, as for any + document that cannot be read at all; an unreadable file raises + :class:`OSError`.""" + data = Path(path).read_bytes() + try: + source = data.decode("utf-8") + except UnicodeDecodeError as error: + raise DocumentError(f"The document cannot be read: {path} is not valid UTF-8 " + f"(byte 0x{data[error.start]:02x} at offset {error.start})") from None + return render(source, options, **settings) def _assemble(source: str, answers: Mapping[str, Any]) -> tuple[str, list[Diagnostic]]: diff --git a/tests/test_wp2_cli.py b/tests/test_wp2_cli.py index 8b864fd..949fcff 100644 --- a/tests/test_wp2_cli.py +++ b/tests/test_wp2_cli.py @@ -18,6 +18,7 @@ import legaldown_render import legaldown_render.cli as cli from legaldown_render import ( + DocumentError, OptionsError, RenderError, RenderOptions, @@ -25,6 +26,7 @@ StyleError, load_style, render, + render_file, ) from legaldown_render.cli import main @@ -122,6 +124,16 @@ def test_non_utf8_style_file_is_a_style_error(tmp_path: Path, capsys: pytest.Cap assert "Traceback" not in err +def test_render_file_raises_document_error_for_non_utf8(tmp_path: Path) -> None: + path = tmp_path / "latin1.lgd" + path.write_bytes(b"# Caf\xe9\n") + with pytest.raises(DocumentError, match=r"not valid UTF-8 \(byte 0xe9 at offset 5\)"): + render_file(path) + good = tmp_path / "good.lgd" + good.write_bytes(CZECH.encode("utf-8")) + assert "Příloha" in render_file(good, format="text").output + + # --- S5: unwritable output ------------------------------------------------------------------- def test_unwritable_output_is_a_clean_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: From 5dd564bc69f34610967a85ef3f3edc49fe04d1d7 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:41:36 +0000 Subject: [PATCH 005/118] Money keeps every written fraction digit (R2 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit money with a currency showed the larger of the currency's minor units and the written digits only when the last written digit was non-zero: "1.500" EUR gave "€1.50" and "1.1230" gave "€1.123". The fraction digit count is now fixed in the pattern as max(minor units, written decimals) with currency_digits off, in symbol, code and name mode, under _exact. "5" EUR is still "€5.00", JPY "5" is "¥5", JPY "0.125" is "¥0.125". Tests added to tests/test_wp1_values.py; no existing test or golden changed. --- src/legaldown_render/resolve/values.py | 35 ++++++++++++++++--------- tests/test_wp1_values.py | 36 ++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/legaldown_render/resolve/values.py b/src/legaldown_render/resolve/values.py index f15ec9c..2e20bbc 100644 --- a/src/legaldown_render/resolve/values.py +++ b/src/legaldown_render/resolve/values.py @@ -4,8 +4,8 @@ Values are stored canonically in the source — ISO dates, decimal amounts with a period — and only their display varies by locale. Amounts are handled as ``Decimal`` from the source text and are never rounded or truncated (§10.3): -money is padded to its currency's minor units, and any further digits written -in the source are kept. A duration keeps every digit it is written with too, +money is padded to its currency's minor units, and every digit written in +the source is kept (the fraction shows the larger of the two). A duration keeps every digit it is written with too, and its plural follows the digits it displays (§10.5). """ from __future__ import annotations @@ -20,7 +20,7 @@ from babel import Locale, UnknownLocaleError from babel.dates import format_date -from babel.numbers import format_currency, parse_pattern +from babel.numbers import format_currency, get_currency_precision, parse_pattern from ..errors import InternalError from ..style.model import Values @@ -169,17 +169,28 @@ def money(self, amount: str, currency: str | None) -> str: def _money(self, amount: str, currency: str) -> str: value = Decimal(amount) - if self.values.money == "name": - return format_currency(value, currency, locale=self.locale, format_type="name", - decimal_quantization=False) - pattern = None - if self.values.money == "code": - standard = self.locale.currency_formats["standard"].pattern - pattern = standard.replace("¤", "¤¤") if "¤¤" not in standard else standard - text = format_currency(value, currency, format=pattern, locale=self.locale, decimal_quantization=False) + mode = self.values.money + if mode == "name": + source = self.locale.decimal_formats[None].pattern + else: + source = self.locale.currency_formats["standard"].pattern + if mode == "code" and "¤¤" not in source: + source = source.replace("¤", "¤¤") + # Fraction digits shown: the currency's minor units, or every digit + # written when there are more (§10.3). Babel's own quantizing would + # drop a trailing zero beyond the minor units ("1.500" EUR), so the + # digit count is fixed in the pattern, and currency_digits is off. + pattern = parse_pattern(source) + digits = max(get_currency_precision(currency), len(amount.partition(".")[2])) + pattern.frac_prec = (digits, digits) + text = format_currency(value, currency, format=pattern, locale=self.locale, currency_digits=False, + format_type="name" if mode == "name" else "standard", + decimal_quantization=False) + if mode == "name": + return text # CLDR spaces an alphabetic symbol or code from the digits, on # whichever side it stands ("USD 10,000.00", "10 000,00 CZK"). - return _LETTER_DIGIT.sub(" ", text) + return _LETTER_DIGIT.sub("\u00a0", text) def duration(self, value: str, unit: str) -> str: """A duration (already validated) such as "12 months" or "12 měsíců".""" diff --git a/tests/test_wp1_values.py b/tests/test_wp1_values.py index e19fbc8..8549a9c 100644 --- a/tests/test_wp1_values.py +++ b/tests/test_wp1_values.py @@ -168,3 +168,39 @@ def test_symbol_after_the_digits_is_spaced_too() -> None: def test_code_mode_still_spaced() -> None: assert formatter("en", money="code").money("5", "CZK") == "CZK\xa05.00" assert formatter("de", money="code").money("5", "CZK") == "5,00\xa0CZK" + + +# -- money keeps every written fraction digit, padded to the minor units ------ + +@pytest.mark.parametrize(("amount", "currency", "expected"), [ + ("1.1230", "EUR", "€1.1230"), + ("1.500", "EUR", "€1.500"), + ("1.50", "EUR", "€1.50"), + ("5", "EUR", "€5.00"), + ("5.5", "EUR", "€5.50"), + ("5", "JPY", "¥5"), + ("0.125", "JPY", "¥0.125"), + ("5.0", "JPY", "¥5.0"), + ("1234.5670", "USD", "$1,234.5670"), +]) +def test_money_fraction_digits_symbol(amount: str, currency: str, expected: str) -> None: + assert formatter("en-US").money(amount, currency) == expected + + +def test_money_fraction_digits_code_and_name() -> None: + code = formatter("en-US", money="code") + name = formatter("en-US", money="name") + assert code.money("1.500", "EUR") == "EUR\xa01.500" + assert code.money("5", "EUR") == "EUR\xa05.00" + assert code.money("0.125", "JPY") == "JPY\xa00.125" + assert code.money("5", "JPY") == "JPY\xa05" + assert name.money("1.500", "EUR") == "1.500 euros" + assert name.money("5", "EUR") == "5.00 euros" + assert name.money("0.125", "JPY") == "0.125 Japanese yen" + assert name.money("5", "JPY") == "5 Japanese yen" + + +def test_money_trailing_zeros_in_other_locales_and_long_amounts() -> None: + assert formatter("cs-CZ").money("1.500", "CZK") == "1,500\xa0Kč" + assert formatter("de", money="code").money("1.1230", "EUR") == "1,1230\xa0EUR" + assert formatter("en-US").money("1" * 40 + ".1230", "EUR") == f"€{int('1' * 40):,}.1230" From 90346706b7516a8be30042c5d761837ad793ef48 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:42:46 +0000 Subject: [PATCH 006/118] Terms that refer to themselves render instead of crashing (B8 follow-up) `"Self {{term: s}}" {{def: s}}`, and cycles through several definitions, raised RecursionError in Resolver._term (already so before WP4, through _plain_value of the validator's raw term). The term-text computation now tracks the definitions it has open (the _survey pre-pass and the _plain_value fallback, _plain_open): a {{term:}} to an id already open shows its label=, else the definition id, and goes no further. The validator 0.4.0 reports nothing for such documents; the spec's definition-circular Warning is not emitted by it, and no diagnostic is added here (CONFORMANCE.md notes it). Tests: direct and indirect cycles, at the definition site and at reference sites, in the text and HTML writers. --- CONFORMANCE.md | 4 +++ src/legaldown_render/resolve/resolver.py | 33 +++++++++++++++++---- tests/test_wp4_build.py | 37 ++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index f8b7774..542eca8 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -63,6 +63,10 @@ It is checked against the specification's own examples and fixtures corpus (`tes body, with its Markdown resolved (`"A *b* c" {{def: abc}}` is "A b c"), not the term's raw source; `label=` overrides it. A definition not in the body (a template's other alternative) falls back to the validator's term text. + A term that refers to itself, directly or through other terms (`"A {{term: a}}" {{def: a}}`), is + rendered without looping: the inner reference shows its `label=`, else the definition id. The + validator (0.4.0) does not report these cycles; the specification's `definition-circular` + Warning (§16) is not emitted yet. - **Ambiguous references.** Each list starts again at its first number, as contracts are usually drafted, so two lists in one section can both have an item "(a)", and a style can number a paragraph or an item like a subsection ("1.1"). A `{{ref:}}` to a section, paragraph, or list diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index cc5e055..34ad16c 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -150,6 +150,10 @@ def __init__( # A definition id -> the plain text of its term as rendered (Markdown # resolved), from the first definition in the body (see _survey). self.term_texts: dict[str, str] = {} + # The definitions whose term text is being worked out, so that a + # {{term:}} inside a term, directly or through other terms, cannot + # send the computation round in a circle. + self._terms_open: list[str] = [] self.attachment_anchors: dict[str, str | None] = {} self.section_presences: list[Presence] = [] @@ -252,7 +256,11 @@ def _survey(self) -> None: # validator's raw ``anchor.term``. identifier = self._definition_id(inline) if identifier not in self.term_texts: - self.term_texts[identifier] = self._title_text(inline.children) + self._terms_open.append(identifier) + try: + self.term_texts[identifier] = self._title_text(inline.children) + finally: + self._terms_open.pop() elif isinstance(inline, DirectiveSource): directive = inline.directive if directive.name == "placeholder" and directive.positional and not directive.malformed: @@ -592,14 +600,29 @@ def _term(self, directive: Directive, definition_id: str) -> Inline: term = self.result.index.definition_lookup.get(definition_id) if term is None: return FailureMarker(f"[UNDEFINED: {definition_id}]") - # The term as rendered at its definition; the validator's own text - # (an empty term reads as its id, a definition not in the body) is - # the fallback. - text = directive.params.get("label") or self.term_texts.get(definition_id) or self._plain_value(term) + label = directive.params.get("label") + if definition_id in self._terms_open: + # A term that refers to itself, directly or through others: the + # reference shows its label, else the id, and goes no further. + text = label or definition_id + else: + # The term as rendered at its definition; the validator's own + # text (an empty term reads as its id, a definition not in the + # body) is the fallback. + text = label or self.term_texts.get(definition_id) or self._plain_open(definition_id, term) # Settled once everything is resolved (see _settle_term_target). target = DEFINITION_ANCHOR_PREFIX + definition_id return TermRef(text, target, self.style.definitions.term_style) + def _plain_open(self, definition_id: str, term: str) -> str: + """The validator's *term* text of a definition as plain text, with + the definition marked open while its directives resolve.""" + self._terms_open.append(definition_id) + try: + return self._plain_value(term) + finally: + self._terms_open.pop() + def _date(self, directive: Directive, value: str) -> Inline: if not is_valid_iso_date(value): return FailureMarker(f"[INVALID DATE: {value}]") diff --git a/tests/test_wp4_build.py b/tests/test_wp4_build.py index f21ae17..396bb69 100644 --- a/tests/test_wp4_build.py +++ b/tests/test_wp4_build.py @@ -244,3 +244,40 @@ def test_an_empty_term_falls_back_to_the_validator() -> None: def test_an_undefined_term_is_still_reported() -> None: assert "[UNDEFINED: nope]" in text("# A\n\nSee {{term: nope}}.\n") + + +# -- B8: a term that refers to itself ---------------------------------------- + +SELF = '# A\n\n"Self {{term: s}}" {{def: s}} means it.\n\nSee {{term: s}} and {{term: s, label=the label}}.\n' +INDIRECT = ('# A\n\n"A {{term: b}}" {{def: a}} x.\n\n"B {{term: a}}" {{def: b}} y.\n\n' + "See {{term: a}}, {{term: b}} and {{term: a, label=the label}}.\n") + + +@pytest.mark.parametrize("output_format", ["text", "html"]) +@pytest.mark.parametrize("body", [SELF, INDIRECT], ids=["direct", "indirect"]) +def test_a_cyclic_term_renders_in_both_writers(body: str, output_format: str) -> None: + result = render(FRONT + body, format=output_format) + assert "the label" in result.output + assert "[UNDEFINED" not in result.output + + +def test_a_term_inside_its_own_definition_shows_its_id() -> None: + output = text(SELF) + # At the definition and at a reference alike: the inner reference shows + # the id (its label when it has one), and goes no further. + assert "Self Self s means it." in output + assert "See Self s and the label." in output + + +def test_a_term_reference_in_a_cycle_shows_its_label_else_its_id() -> None: + output = text(INDIRECT) + assert output.rstrip().endswith("and the label.") + # No term shows its own reference as raw directive source. + assert "{{" not in output + + +def test_a_cycle_through_a_reference_site_inside_another_terms_definition() -> None: + body = ('# A\n\n"A" {{def: a}} x.\n\n"B {{term: a}} and {{term: c}}" {{def: b}} y.\n\n' + '"C {{term: b}}" {{def: c}} z.\n\nSee {{term: b}}.\n') + assert "See B A and" in text(body) + assert "B A and" in html(body) From 64be9dfa29e1a72d9a4a4697103e4dea4ebe2b59 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:42:46 +0000 Subject: [PATCH 007/118] B2 guard: an item a marker belongs to never vanishes A comment-only list item is now dropped only when the markers give it no anchor identifier and no condition. The validator does not place a marker on such an item, so this is defensive; the test drives the builder with a made-up marker. --- src/legaldown_render/build.py | 7 +++++-- tests/test_wp4_build.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/legaldown_render/build.py b/src/legaldown_render/build.py index 2d8e568..a07daa4 100644 --- a/src/legaldown_render/build.py +++ b/src/legaldown_render/build.py @@ -439,8 +439,11 @@ def _list(self, block: ModelBlock, markers: dict[int | None, PlacedMarker], numb continue # never emitted (§8.6, §8.7) elif (built := self.block(child, [], top_level=False)) is not None: blocks.append(built) - if item.blocks and all(child.kind == "html" for child in item.blocks): - continue # only a comment: the item is removed + # Only a comment: the item is removed. Defensive: an item a + # marker belongs to (an anchor or a condition) never vanishes. + if item.blocks and all(child.kind == "html" for child in item.blocks) \ + and not identifier and not condition: + continue items.append(ListItem(blocks=tuple(blocks), anchor_id=identifier, condition=condition)) finally: self.depth -= 1 diff --git a/tests/test_wp4_build.py b/tests/test_wp4_build.py index 396bb69..6862f29 100644 --- a/tests/test_wp4_build.py +++ b/tests/test_wp4_build.py @@ -281,3 +281,21 @@ def test_a_cycle_through_a_reference_site_inside_another_terms_definition() -> N '"C {{term: b}}" {{def: c}} z.\n\nSee {{term: b}}.\n') assert "See B A and" in text(body) assert "B A and" in html(body) + + +def test_a_comment_only_item_with_a_marker_is_kept() -> None: + # Defensive (B2): an item a marker belongs to never vanishes. The + # validator does not place a marker on a comment-only item, so a marker + # is made up here to drive the builder's guard. + from legaldown import PlacedMarker + + from legaldown_render.build import _Builder + + document = parse(FRONT + "# A\n\n- \n- b\n") + result = validate(document) + marker = PlacedMarker(section=0, block=0, fragment=0, offset=0, source="", + identifier="x", condition="", item=0) + out = _Builder(document, result).list(document.sections[0].blocks[0], [marker]) + assert out is not None and [item.anchor_id for item in out.items] == ["x", ""] + plain = _Builder(document, result).list(document.sections[0].blocks[0], []) + assert plain is not None and len(plain.items) == 1 From 14b6977f0c12908b818a5aa6ba39156a9ca6ef05 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:46:45 +0000 Subject: [PATCH 008/118] Style validation: blank fallback, extends cycles, flat dumps, overrides, template fields (R6, S7-S11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - R6: Resolver._placeholder renders [TBD: id] when the effective placeholders.blank is empty or whitespace (spec §13.5 rule 6); no StyleError. - S7: extends cycles are detected on resolved paths (Path.resolve), so ../dir/a.yaml, a second spelling, or a symlink no longer recurses until "File name too long". - S8: dump_style drops `extends`, so a --print-style dump is self-contained and reloads from any directory. - S9: overriding `version` or `extends` is a StyleError (it was silently ignored, and --print-style could print an unloadable style). - S10 (loader side): a YAML error in a --set value raises StyleError naming the key and the YAML problem (with line/column) instead of resurfacing as a misleading type error; reaches stderr through WP2's argparse wrapper. - S11: model.TemplateFields / template_settings inventory the {field} names of every text template setting (verified against numbering.fill, Resolver._ref, fill_condition); the loader reports unknown identifier-shaped fields and missing required ones, naming the setting (labels.condition must contain {condition}, §15.8). Other braces stay literal. Built-in styles and labels pass. Existing test changed: tests/test_style.py::test_dump_round_trips now compares with extends=None, because the flattened dump no longer records what the style extended (S8). Docs: style-templates.md (field table, flat dump, overrides), CONFORMANCE.md, architecture.md. --- CONFORMANCE.md | 2 + docs/architecture.md | 3 +- docs/style-templates.md | 39 +++- src/legaldown_render/resolve/resolver.py | 4 +- src/legaldown_render/style/loader.py | 118 ++++++++-- src/legaldown_render/style/model.py | 61 ++++- tests/test_style.py | 4 +- tests/test_wp6_style.py | 282 +++++++++++++++++++++++ 8 files changed, 486 insertions(+), 27 deletions(-) create mode 100644 tests/test_wp6_style.py diff --git a/CONFORMANCE.md b/CONFORMANCE.md index dd09b8d..8a5c0ee 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -35,10 +35,12 @@ It is checked against the specification's own examples and fixtures corpus (`tes | Style templates in a separate file (§13.7) | ✅ YAML, layered and validated; see [docs/style-templates.md](docs/style-templates.md) | | Signature blocks (§2.2) | ✅ One per party, with `legal_name` and each representative | | Labels following the document language (§13.7) | ✅ Built in for `en`, `cs`, `de`, `fr`, `pl`, and `sk`; any label can be set by a style | +| A conditional unit's condition is shown (§15.8, MUST) | ✅ Also for a custom style: `labels.condition` must contain `{condition}`, or the style is refused | | `ref-not-enumerated` Warning (§6.3, §16.3) | ✅ | | `raw-html` Warning; raw HTML never emitted (§8.7) | ✅ Nothing is emitted: an HTML block is dropped whole, and an inline tag is dropped with its text kept. The Warning is the validator's, once per block or text that holds raw HTML other than comments | | Automatic term recognition (§7.4, MAY) | ❌ Not supported | | Extended tables through raw HTML (§9.2, MAY) | ❌ Not supported; the HTML table is dropped with the Warning | +| Visual blank for a placeholder (§13.5 rule 6) | ✅ `placeholders.blank`; a blank that is empty or only whitespace renders `[TBD: id]` | | Question prompts next to blanks (§15.8, MAY) | ✅ `placeholders.show_prompt` | | Final check (§15.9, SHOULD) | ✅ `--final` / `final=True`: the validator's `placeholder-unfilled` and `template-construct-present` Errors; with `--strict`, the document is refused | diff --git a/docs/architecture.md b/docs/architecture.md index c9d5128..11bc3c5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -221,7 +221,8 @@ turns into an error. - **Document problems** render as failure markers and are listed in `diagnostics`. `strict=True` raises `RenderRefused` instead, carrying the diagnostics. - **Configuration problems** raise before rendering, with every problem named: an invalid style, - setting, or locale raises `StyleError`; an unknown output format raises `OptionsError`. Both are + setting, or locale raises `StyleError` (this includes a `{field}` that a text template does not + accept or lacks, a `--set` value that is not valid YAML, and an `extends` cycle); an unknown output format raises `OptionsError`. Both are `RenderError`s (and `ValueError`s, as they always were). Every renderer error pickles, so it survives `multiprocessing`. - **Internal inconsistencies** raise `InternalError`, and the CLI exits with 70. diff --git a/docs/style-templates.md b/docs/style-templates.md index 88ee6f5..c0027e1 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -25,7 +25,8 @@ Values are layered. Each layer overrides the one before it, **field by field**: 1. **Built-in defaults.** Every setting has one, defined in `legaldown_render/style/model.py`. The `default` style is exactly these defaults. 2. **The `extends` chain.** A style names the style it builds on, which can be a built-in name or - a path relative to the file. Chains are resolved base first, and cycles are rejected. + a path relative to the file. Chains are resolved base first, and cycles are rejected, however + the files are spelled (`../dir/a.yaml`, a symbolic link). 3. **The chosen style**: `--style NAME|PATH`, or `RenderOptions(style=...)`. 4. **Per-job overrides**: `--set numbering.scheme=mixed` or `overrides={"numbering.scheme": "mixed"}`, plus `--locale`. On the command line, text settings take the value as written @@ -34,7 +35,9 @@ Values are layered. Each layer overrides the one before it, **field by field**: or `C:\new` stay exactly as written. To keep quotation marks around a value, wrap it in the other kind (`--set 'references.format="{designation}"'` gives `{designation}`, and `--set "references.format='\"{designation}\"'"` gives - `"{designation}"`). Other settings are read as YAML (`true`, `2`, `[...]`). + `"{designation}"`). Other settings are read as YAML (`true`, `2`, `[...]`); a value that is not + valid YAML is an error naming the setting and the YAML problem. `version` and `extends` belong + to the template file and cannot be overridden: choose the template with `--style`. Mappings merge field by field, so `headings: {1: {align: center}}` keeps level 1's size and weight. Lists, such as `enumeration.levels`, replace the list they override. @@ -51,7 +54,8 @@ legaldown-render: Invalid style 'default': To see the effective style, with every setting after all layers are applied, use `legaldown-render --print-style --style continental --set locale=cs-CZ`. The output is itself a -valid style file. +valid style file, and a self-contained one: every setting is written out and there is no `extends`, +so it loads from any directory. ## Built-in styles @@ -100,7 +104,7 @@ values: duration: long # long (12 months) | short | narrow placeholders: - blank: "[_____]" + blank: "[_____]" # empty or whitespace only: renders [TBD: id] (§13.5 rule 6) show_prompt: false # show the question's prompt next to the blank (§15.8) title_block: { parties: true, effective_date: true, version: false } @@ -152,6 +156,33 @@ example, decimal headings use `label: "{path}"` and `ref: ".{n}"`. Legal enumera referenced as "2.3(b)". Under the `none` scheme, designations are heading text, and item pieces are set off by a space ("Termination (a)", §13.3). +### Text templates and their fields + +A text setting that holds `{field}` names is checked when the style is loaded. A field is a name +in braces that is an identifier (`{n}`, `{designation}`); any other braces (`{}`, `{1}`, `{a b}`) +are literal text. An identifier the setting does not accept is an error naming the setting (a +typo would otherwise be shown literally), and so is a missing required field. + +| Setting | Accepted fields | Required | +|---|---|---| +| `numbering.levels[].label` | `{n}`, `{path}` | none | +| `numbering.levels[].ref` | `{n}` | `{n}` | +| `enumeration.levels[].label`, `paragraphs.format.label` | `{n}`, `{path}`, `{section}` | none | +| `enumeration.levels[].ref`, `paragraphs.format.ref` | `{n}`, `{section}` | `{n}` | +| `references.format` | `{designation}` | `{designation}`, or every reference reads the same | +| `labels.condition` | `{condition}` | `{condition}`, as §15.8 requires the condition to be shown | +| `labels.attachment_file` | `{file}` | none | +| every other label | none | none | + +``` +$ legaldown-render contract.lgd --set references.format=nope +legaldown-render: Invalid style 'default': + - references.format: must contain {designation} — every reference would read the same (§13.3) +``` + +A `ref` is one piece of a designation, so it cannot show the whole `{path}`. A label left unset +takes the built-in one for the document language, which always passes. + ### Built-in labels Labels follow the document's `language`. English (`en`), Czech (`cs`), German (`de`), French diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 0301967..bec7d29 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -661,7 +661,9 @@ def _placeholder(self, directive: Directive, placeholder_id: str) -> Inline: prompt = "" if self.style.placeholders.show_prompt and isinstance(declared, dict): prompt = str(declared.get("prompt") or "") - return Blank(placeholder_id, self.style.placeholders.blank, prompt) + # §13.5 rule 6: a style whose blank is empty offers no visual blank. + blank = self.style.placeholders.blank + return Blank(placeholder_id, blank if blank.strip() else f"[TBD: {placeholder_id}]", prompt) def _attach(self, directive: Directive, attachment_id: str) -> Inline: title = self.result.index.attachment_lookup.get(attachment_id) diff --git a/src/legaldown_render/style/loader.py b/src/legaldown_render/style/loader.py index 3c48b67..56f3498 100644 --- a/src/legaldown_render/style/loader.py +++ b/src/legaldown_render/style/loader.py @@ -16,6 +16,8 @@ import copy import dataclasses import difflib +import os +import re import types from collections.abc import Mapping from importlib import resources @@ -26,7 +28,7 @@ from ..errors import StyleError from . import model -from .model import Style +from .model import Style, template_settings STYLE_FORMAT_VERSION = 1 @@ -54,7 +56,9 @@ def load_style( else: data, source = _load_chain(style if style is not None else "default", seen=()) if overrides: - data = _merge(data, _expand_dotted(overrides)) + expanded = _expand_dotted(overrides) + _refuse_meta_overrides(expanded) + data = _merge(data, expanded) return _build(data, source) @@ -64,8 +68,14 @@ def style_to_dict(style: Style) -> dict[str, Any]: def dump_style(style: Style) -> str: - """*style* as a complete YAML style template.""" - return yaml.safe_dump(style_to_dict(style), sort_keys=False, allow_unicode=True) + """*style* as a complete, self-contained YAML style template. + + Every setting is written out, so the template extends nothing: ``extends`` + names a file relative to the one that holds it, which a dump written + elsewhere (``--print-style > house.yaml``) could not find again.""" + data = style_to_dict(style) + data.pop("extends", None) + return yaml.safe_dump(data, sort_keys=False, allow_unicode=True) # --------------------------------------------------------------------------- @@ -73,29 +83,38 @@ def dump_style(style: Style) -> str: # --------------------------------------------------------------------------- -def _load_chain(spec: str | Path, *, seen: tuple[str, ...], relative_to: Path | None = None) -> tuple[dict, str]: - """The merged raw mapping for *spec* and every template it extends.""" - raw, source, folder = _read(spec, relative_to) - if source in seen: - chain = " -> ".join((*seen, source)) +def _load_chain( + spec: str | Path, *, seen: tuple[tuple[str, str], ...], relative_to: Path | None = None +) -> tuple[dict, str]: + """The merged raw mapping for *spec* and every template it extends. + + *seen* holds the (identity, label) of each template on the chain so far. + A file's identity is its resolved path, so ``../dir/a.yaml`` and + ``a.yaml`` — or a symbolic link — are one template and a cycle through + them is found.""" + raw, source, folder, identity = _read(spec, relative_to) + if any(identity == earlier for earlier, _ in seen): + chain = " -> ".join((*(label for _, label in seen), source)) raise StyleError(source, [f"extends: circular chain {chain}"]) + here = (*seen, (identity, source)) parent = raw.get("extends") if parent is None: if source == "'default'": # Start from every built-in default, so that a partial mapping — # one heading level's alignment, say — merges field by field. return _merge(style_to_dict(Style()), raw), source - base, _ = _load_chain("default", seen=(*seen, source)) + base, _ = _load_chain("default", seen=here) elif isinstance(parent, str) and parent: - base, _ = _load_chain(parent, seen=(*seen, source), relative_to=folder) + base, _ = _load_chain(parent, seen=here, relative_to=folder) else: raise StyleError(source, ["extends: must be a style name or a path"]) return _merge(base, raw), source -def _read(spec: str | Path, relative_to: Path | None) -> tuple[dict, str, Path | None]: - """The raw mapping in *spec*, a label for messages, and the folder that - relative ``extends`` paths are resolved against.""" +def _read(spec: str | Path, relative_to: Path | None) -> tuple[dict, str, Path | None, str]: + """The raw mapping in *spec*, a label for messages, the folder that + relative ``extends`` paths are resolved against, and the template's + identity (its resolved path, or the quoted name of a built-in).""" name = str(spec) is_path = isinstance(spec, Path) or name.endswith((".yaml", ".yml")) or "/" in name or "\\" in name if not is_path: @@ -104,7 +123,7 @@ def _read(spec: str | Path, relative_to: Path | None) -> tuple[dict, str, Path | raise StyleError(f"'{name}'", [f"no built-in style named '{name}' (built-in: {known}); " f"to load a file, give a path ending in .yaml"]) text = resources.files(__package__).joinpath("builtin", f"{name}.yaml").read_text("utf-8") - return _parse(text, f"'{name}'"), f"'{name}'", None + return _parse(text, f"'{name}'"), f"'{name}'", None, f"'{name}'" path = Path(spec) if relative_to is not None and not path.is_absolute(): path = relative_to / path @@ -117,7 +136,16 @@ def _read(spec: str | Path, relative_to: Path | None) -> tuple[dict, str, Path | f"at offset {error.start})"]) from None raw = _parse(text, str(path)) raw.setdefault("name", path.stem) - return raw, str(path), path.parent + return raw, str(path), path.parent, _identity(path) + + +def _identity(path: Path) -> str: + """*path* made absolute, with ``..`` and symbolic links followed, so two + spellings of one file compare equal.""" + try: + return str(path.resolve()) + except (OSError, RuntimeError): # a symbolic link loop + return os.path.abspath(path) def _parse(text: str, source: str) -> dict: @@ -167,6 +195,23 @@ def _expand_dotted(overrides: Mapping[str, Any]) -> dict: return expanded +#: Top-level settings that describe the template file, not how a document +#: looks; a job's overrides cannot change them. +_META_KEYS = { + "version": "the style format version belongs to the template file", + "extends": "choose the template with --style (or RenderOptions.style) instead", +} + + +def _refuse_meta_overrides(expanded: Mapping) -> None: + """Overrides are applied after the ``extends`` chain is resolved and the + version checked, so an override of either would be silently ignored — or, + for ``--print-style``, written into a style that no longer loads.""" + problems = [f"{key}: cannot be overridden; {reason}" for key, reason in _META_KEYS.items() if key in expanded] + if problems: + raise StyleError("overrides", problems) + + def _copy(value: Any) -> Any: """A deep copy of *value* with every mapping made a plain dict.""" if isinstance(value, Mapping): @@ -184,7 +229,8 @@ def parse_override(key: str, text: str) -> Any: keep quotation marks around a value, wrap it in the other kind (``'"{designation}"'``). Other settings read it as YAML, so that ``true``, ``2``, or ``[...]`` mean what they say. ``null`` clears a - setting that may be unset, such as ``locale``. + setting that may be unset, such as ``locale``. A value that is not valid + YAML raises :class:`StyleError`, naming *key* and the YAML problem. """ hint = _hint_for(key.split(".")) if hint is not None and _is_text(hint): @@ -197,8 +243,19 @@ def parse_override(key: str, text: str) -> Any: return text try: return yaml.safe_load(text) if text.strip() else "" - except yaml.YAMLError: - return text + except yaml.YAMLError as error: + raise StyleError("overrides", [f"{key}: the value is not valid YAML: {_yaml_problem(error)}"]) from None + + +def _yaml_problem(error: yaml.YAMLError) -> str: + """What is wrong in a YAML text, on one line: the problem and where it is.""" + problem = getattr(error, "problem", None) + if not problem: + return " ".join(str(error).split()) + context = getattr(error, "context", None) + mark = getattr(error, "problem_mark", None) + where = f" (line {mark.line + 1}, column {mark.column + 1})" if mark is not None else "" + return f"{context}, {problem}{where}" if context else f"{problem}{where}" def _hint_for(parts: list[str]) -> Any: @@ -234,14 +291,37 @@ def _is_text(hint: Any) -> bool: # --------------------------------------------------------------------------- +#: ``{name}`` as the resolver's ``fill`` and label substitution read it. +_FIELD_RE = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}") + + def _build(data: Mapping, source: str) -> Style: problems: list[str] = [] style = _convert(data, Style, "", problems) + problems.extend(_template_problems(style)) if problems: raise StyleError(source, problems) return style +def _template_problems(style: Style) -> list[str]: + """What is wrong with the ``{field}`` names in the style's text templates + (:func:`~.model.template_settings`): an identifier-shaped field the + setting does not accept (a typo would be shown literally), or a required + field it leaves out (``labels.condition`` without ``{condition}`` would + hide the condition, which §15.8 requires to be shown).""" + problems = [] + for key, template, fields in template_settings(style): + found = set(_FIELD_RE.findall(template)) + accepted = ", ".join(f"{{{name}}}" for name in sorted(fields.accepted)) + for name in sorted(found - fields.accepted): + takes = f"accepts {accepted}" if accepted else "takes no fields" + problems.append(f"{key}: unknown field {{{name}}}; this setting {takes}") + for name in sorted(fields.required - found): + problems.append(f"{key}: must contain {{{name}}}" + (f" — {fields.why}" if fields.why else "")) + return problems + + def _convert(value: Any, hint: Any, path: str, problems: list[str]) -> Any: label = path or "(root)" origin = get_origin(hint) diff --git a/src/legaldown_render/style/model.py b/src/legaldown_render/style/model.py index 1a0f464..6903be2 100644 --- a/src/legaldown_render/style/model.py +++ b/src/legaldown_render/style/model.py @@ -15,7 +15,8 @@ """ from __future__ import annotations -from dataclasses import dataclass, field +from collections.abc import Iterator +from dataclasses import dataclass, field, fields from typing import Literal Counter = Literal["decimal", "lower-alpha", "upper-alpha", "lower-roman", "upper-roman"] @@ -283,3 +284,61 @@ class Style: headings: dict[int, HeadingStyle] = field(default_factory=_default_headings) page: Page = field(default_factory=Page) html: Html = field(default_factory=Html) + + +# --------------------------------------------------------------------------- +# Text templates +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class TemplateFields: + """The ``{field}`` names a text template setting accepts and requires. + + Only identifier-shaped braces (``{n}``, ``{designation}``) are fields; any + other braces (``{}``, ``{1}``, ``{a b}``) are literal text, as the + resolver keeps them. ``why`` says what a required field is for, in the + message about a template that leaves it out.""" + + accepted: frozenset[str] = frozenset() + required: frozenset[str] = frozenset() + why: str = "" + + +#: The designation-building settings are read in :mod:`..resolve.numbering` +#: (``fill``) and :mod:`..resolve.resolver`. A heading's ``label`` sees its own +#: counter and its designation; its ``ref`` is one piece of a designation, so +#: it sees only the counter. +HEADING_LABEL = TemplateFields(frozenset({"n", "path"})) +HEADING_REF = TemplateFields(frozenset({"n"}), frozenset({"n"}), "each level must tell its sections apart (§13.3)") +#: List items and paragraphs also see the containing section's designation. +ITEM_LABEL = TemplateFields(frozenset({"n", "path", "section"})) +ITEM_REF = TemplateFields(frozenset({"n", "section"}), frozenset({"n"}), "each item must tell itself apart (§13.3)") +REFERENCE_FORMAT = TemplateFields( + frozenset({"designation"}), frozenset({"designation"}), "every reference would read the same (§13.3)" +) +#: Labels are plain words, except for the two that carry a value. +LABEL_FIELDS: dict[str, TemplateFields] = { + "condition": TemplateFields( + frozenset({"condition"}), frozenset({"condition"}), "a conditional unit's condition must be shown (§15.8)" + ), + "attachment_file": TemplateFields(frozenset({"file"})), +} + + +def template_settings(style: Style) -> Iterator[tuple[str, str, TemplateFields]]: + """Every text template setting of *style* that has been set, as + ``(key path, template, fields)``. A label left unset (``None``) takes a + built-in one, which is correct by construction.""" + for name, kind_label, kind_ref in (("numbering", HEADING_LABEL, HEADING_REF), + ("enumeration", ITEM_LABEL, ITEM_REF)): + for index, level in enumerate(getattr(style, name).levels): + yield f"{name}.levels[{index}].label", level.label, kind_label + yield f"{name}.levels[{index}].ref", level.ref, kind_ref + yield "paragraphs.format.label", style.paragraphs.format.label, ITEM_LABEL + yield "paragraphs.format.ref", style.paragraphs.format.ref, ITEM_REF + yield "references.format", style.references.format, REFERENCE_FORMAT + for item in fields(Labels): + template = getattr(style.labels, item.name) + if template is not None: + yield f"labels.{item.name}", template, LABEL_FIELDS.get(item.name, TemplateFields()) diff --git a/tests/test_style.py b/tests/test_style.py index 44f71bd..c0e5495 100644 --- a/tests/test_style.py +++ b/tests/test_style.py @@ -1,5 +1,6 @@ from __future__ import annotations +from dataclasses import replace from pathlib import Path import pytest @@ -68,7 +69,8 @@ def test_dump_round_trips(tmp_path: Path) -> None: style = load_style("continental", overrides={"locale": "de-DE"}) path = tmp_path / "dumped.yaml" path.write_text(dump_style(style)) - assert load_style(path) == style + # The dump is flattened (S8): the same style, without the record of what it extended. + assert load_style(path) == replace(style, extends=None) def test_command_line_values_are_typed_by_their_setting() -> None: diff --git a/tests/test_wp6_style.py b/tests/test_wp6_style.py new file mode 100644 index 0000000..870e752 --- /dev/null +++ b/tests/test_wp6_style.py @@ -0,0 +1,282 @@ +"""Style validation: an empty blank falls back to ``[TBD: id]`` (R6), ``extends`` +cycles are found on resolved paths (S7), ``--print-style`` is self-contained +(S8), ``version`` and ``extends`` cannot be overridden (S9), a YAML error in a +``--set`` value is reported (S10, loader side), and the ``{field}`` +placeholders of every text template are validated (S11).""" +from __future__ import annotations + +import dataclasses +import os +import subprocess +import sys +from dataclasses import replace +from pathlib import Path + +import pytest + +from legaldown_render import RenderOptions, StyleError, builtin_styles, dump_style, load_style, render +from legaldown_render.style import parse_override + +SRC = Path(__file__).resolve().parents[1] / "src" +DOC = "---\nlegaldown: '0.2'\ntitle: T\n---\n\n# One {#one}\n\nAmount {{placeholder: fee}} and {{ref: one}}.\n" + + +def _cli(*args: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]: + env = dict(os.environ, PYTHONPATH=str(SRC), PYTHONUTF8="1") + return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *args], input=stdin, + capture_output=True, text=True, env=env, check=False) + + +# --- R6: an empty blank -------------------------------------------------------------------------- + +@pytest.mark.parametrize("blank", ["", " ", "\t\n"]) +@pytest.mark.parametrize("fmt", ["html", "text"]) +def test_an_empty_blank_falls_back_to_tbd(blank: str, fmt: str) -> None: + """§13.5 rule 6: no visual blank means ``[TBD: id]``.""" + result = render(DOC, RenderOptions(format=fmt, overrides={"placeholders.blank": blank})) + assert "[TBD: fee]" in result.output + + +def test_a_visible_blank_is_kept() -> None: + result = render(DOC, RenderOptions(format="text", overrides={"placeholders.blank": "[__]"})) + assert "[__]" in result.output + assert "TBD" not in result.output + + +# --- S7: extends cycles ---------------------------------------------------------------------------- + +def test_a_cycle_through_a_parent_directory_is_found(tmp_path: Path) -> None: + (tmp_path / "dir").mkdir() + (tmp_path / "dir" / "a.yaml").write_text("version: 1\nextends: ../dir/a.yaml\n") + with pytest.raises(StyleError, match="circular"): + load_style(tmp_path / "dir" / "a.yaml") + + +def test_a_cycle_through_two_spellings_of_one_file_is_found(tmp_path: Path) -> None: + (tmp_path / "x").mkdir() + (tmp_path / "x" / "a.yaml").write_text("version: 1\nextends: ../x/b.yaml\n") + (tmp_path / "x" / "b.yaml").write_text("version: 1\nextends: ./a.yaml\n") + with pytest.raises(StyleError, match="circular"): + load_style(tmp_path / "x" / "../x/a.yaml") + + +def test_a_cycle_through_a_symlink_is_found(tmp_path: Path) -> None: + (tmp_path / "a.yaml").write_text("version: 1\nextends: link.yaml\n") + try: + (tmp_path / "link.yaml").symlink_to(tmp_path / "a.yaml") + except OSError: + pytest.skip("symlinks are unavailable") + with pytest.raises(StyleError, match="circular"): + load_style(tmp_path / "a.yaml") + + +def test_the_same_base_used_twice_is_not_a_cycle(tmp_path: Path) -> None: + (tmp_path / "base.yaml").write_text("version: 1\nnumbering: {scheme: none}\n") + (tmp_path / "mid.yaml").write_text("version: 1\nextends: base.yaml\n") + (tmp_path / "top.yaml").write_text("version: 1\nextends: mid.yaml\n") + assert load_style(tmp_path / "top.yaml").numbering.scheme == "none" + + +# --- S8: a dump is self-contained ---------------------------------------------------------------------- + +def test_a_dump_does_not_extend_anything(tmp_path: Path) -> None: + (tmp_path / "base.yaml").write_text("version: 1\nnumbering: {scheme: none}\n") + (tmp_path / "child.yaml").write_text("version: 1\nextends: base.yaml\nlocale: cs-CZ\n") + style = load_style(tmp_path / "child.yaml") + text = dump_style(style) + assert "extends" not in text + elsewhere = tmp_path / "elsewhere" / "deep" + elsewhere.mkdir(parents=True) + dumped = elsewhere / "dumped.yaml" + dumped.write_text(text) + reloaded = load_style(dumped) + assert reloaded.numbering.scheme == "none" + assert reloaded.locale == "cs-CZ" + assert reloaded.name == style.name + + +@pytest.mark.parametrize("name", builtin_styles()) +def test_a_dumped_builtin_reloads_equal(name: str, tmp_path: Path) -> None: + style = load_style(name) + path = tmp_path / "dump.yaml" + path.write_text(dump_style(style)) + assert "extends" not in path.read_text() + # The dump is flattened: it is the same style, minus the record of what it extended. + assert load_style(path) == replace(style, extends=None) + + +def test_print_style_of_a_relative_extends_reloads_from_anywhere(tmp_path: Path) -> None: + (tmp_path / "base.yaml").write_text("version: 1\nplaceholders: {blank: '[__]'}\n") + (tmp_path / "child.yaml").write_text("version: 1\nextends: base.yaml\n") + done = _cli("--print-style", "--style", str(tmp_path / "child.yaml")) + assert done.returncode == 0, done.stderr + assert "extends" not in done.stdout + elsewhere = tmp_path / "other" + elsewhere.mkdir() + (elsewhere / "dump.yaml").write_text(done.stdout) + assert load_style(elsewhere / "dump.yaml").placeholders.blank == "[__]" + + +# --- S9: version and extends are not overridable ------------------------------------------------------------ + +@pytest.mark.parametrize("key", ["version", "extends"]) +def test_version_and_extends_cannot_be_overridden(key: str) -> None: + with pytest.raises(StyleError, match=key): + load_style("default", overrides={key: "2" if key == "version" else "continental"}) + + +def test_a_nested_override_of_version_is_refused_too() -> None: + with pytest.raises(StyleError, match="version"): + load_style("default", overrides={"version": 1}) + + +@pytest.mark.parametrize("setting", ["version=2", "extends=continental"]) +@pytest.mark.parametrize("print_style", [True, False]) +def test_the_cli_refuses_them(setting: str, print_style: bool) -> None: + args = ["--print-style"] if print_style else ["-", "-q"] + done = _cli(*args, "--set", setting, stdin=DOC) + assert done.returncode == 2, done.stdout + assert setting.split("=")[0] in done.stderr + assert "cannot be overridden" in done.stderr + assert "Traceback" not in done.stderr + + +# --- S10: a YAML error in a --set value ------------------------------------------------------------------- + +def test_a_yaml_error_in_an_override_names_the_key() -> None: + with pytest.raises(StyleError) as caught: + parse_override("numbering.levels", "[{counter: decimal") + message = str(caught.value) + assert "numbering.levels" in message + assert "YAML" in message + assert "flow" in message or "expected" in message or "while" in message # the YAML problem itself + + +def test_a_yaml_error_in_an_override_reaches_stderr() -> None: + done = _cli("--print-style", "--set", "numbering.levels=[{counter: decimal") + assert done.returncode == 2 + assert "numbering.levels" in done.stderr + assert "YAML" in done.stderr + assert "Traceback" not in done.stderr + assert "must be a list" not in done.stderr # not the misleading type error + + +def test_a_yaml_error_in_a_render_run_reaches_stderr() -> None: + done = _cli("-", "-q", "--set", "enumeration.levels=[{a", stdin=DOC) + assert done.returncode != 0 + assert "enumeration.levels" in done.stderr + assert "YAML" in done.stderr + assert "Traceback" not in done.stderr + + +def test_valid_yaml_overrides_still_parse() -> None: + assert parse_override("title_block.version", "true") is True + assert parse_override("contents.depth", "3") == 3 + assert parse_override("numbering.levels", "[{counter: decimal}]") == [{"counter": "decimal"}] + + +# --- S11: template fields ------------------------------------------------------------------------------------ + +@pytest.mark.parametrize("name", builtin_styles()) +def test_every_builtin_style_passes(name: str) -> None: + load_style(name) + + +@pytest.mark.parametrize("language", ["en", "cs", "de", "fr", "pl", "sk"]) +def test_every_builtin_label_passes(language: str) -> None: + from legaldown_render.style import BUILTIN_LABELS + labels = BUILTIN_LABELS[language] + overrides = {f"labels.{item.name}": getattr(labels, item.name) for item in dataclasses.fields(labels)} + assert load_style("default", overrides=overrides).labels == labels + + +def _problems(overrides: dict[str, object]) -> list[str]: + with pytest.raises(StyleError) as caught: + load_style("default", overrides=overrides) + return caught.value.problems + + +def test_labels_condition_must_show_the_condition() -> None: + """§15.8: a conditional unit's condition MUST be shown.""" + problems = _problems({"labels.condition": "Conditional"}) + assert any(p.startswith("labels.condition:") and "{condition}" in p for p in problems) + + +def test_references_format_must_show_the_designation() -> None: + problems = _problems({"references.format": "nope"}) + assert any(p.startswith("references.format:") and "{designation}" in p for p in problems) + + +def test_a_typo_in_a_field_is_named() -> None: + problems = _problems({"references.format": "{desgination}"}) + assert any(p.startswith("references.format:") and "{desgination}" in p and "{designation}" in p for p in problems) + + +@pytest.mark.parametrize(("setting", "value"), [ + ("labels.condition", "Only if {condition} {n}"), + ("labels.attachment_file", "{condition}"), + ("labels.drafting_note", "{note}"), + ("labels.colon", "{n}"), + ("references.format", "{n}{designation}"), + ("numbering.levels", [{"label": "{n}", "ref": "{path}"}]), + ("numbering.levels", [{"label": "{section}", "ref": "{n}"}]), + ("numbering.levels", [{"label": "{n}", "ref": ".x"}]), + ("enumeration.levels", [{"label": "{wrong}", "ref": "{n}"}]), + ("enumeration.levels", [{"label": "({n})", "ref": "{path}{n}"}]), + ("paragraphs.format", {"label": "{n}", "ref": "{foo}"}), + ("paragraphs.format", {"label": "{n}", "ref": "{section}"}), +]) +def test_invalid_templates_are_refused(setting: str, value: object) -> None: + problems = _problems({setting: value}) + assert any(p.startswith(setting) for p in problems), problems + + +def test_the_problem_names_the_level() -> None: + problems = _problems({"numbering.levels": [{"ref": "{n}"}, {"ref": "x"}]}) + assert any(p.startswith("numbering.levels[1].ref:") for p in problems) + + +@pytest.mark.parametrize(("setting", "value"), [ + ("labels.condition", "{condition}"), + ("labels.condition", "Only if: {condition} (see {condition})"), + ("labels.condition", "[{condition}] {x y} { } {} {1} {{ }} {-n}"), + ("labels.attachment_file", "no field at all"), + ("labels.attachment_file", "{file}"), + ("labels.drafting_note", "Note {1} {} {a b}"), + ("labels.colon", ""), + ("references.format", "Section {designation}"), + ("references.format", "{designation} {a.b} {0}"), + ("numbering.levels", [{"label": "{path}.", "ref": "{n}"}]), + ("numbering.levels", [{"label": "Article {n} ({path})", "ref": ".{n}"}]), + ("enumeration.levels", [{"label": "({n})", "ref": "{section}({n})"}]), + ("enumeration.levels", [{"label": "{section}.{path}.{n}", "ref": "({n})"}]), + ("paragraphs.format", {"label": "{section}.{n}", "ref": ".{n}"}), + ("paragraphs.format", {"label": "{path} {n} {section}", "ref": "{section}.{n}"}), +]) +def test_valid_templates_are_accepted(setting: str, value: object) -> None: + load_style("default", overrides={setting: value}) + + +def test_unset_labels_are_not_checked() -> None: + assert load_style("default", overrides={"labels.condition": None}).labels.condition is None + + +def test_braces_that_are_not_identifiers_stay_literal() -> None: + result = render(DOC, RenderOptions(format="text", overrides={"references.format": "{{{designation}}} {0} {}"})) + assert "{{1}} {0} {}" in result.output + + +def test_a_style_object_is_checked_too() -> None: + from dataclasses import replace + + from legaldown_render.style import References + style = replace(load_style("default"), references=References(format="oops")) + with pytest.raises(StyleError, match="references.format"): + load_style(style) + + +def test_all_template_problems_are_reported_together() -> None: + problems = _problems({"references.format": "x", "labels.condition": "y", "placeholders.blank": ["x"]}) + assert any(p.startswith("references.format") for p in problems) + assert any(p.startswith("labels.condition") for p in problems) + assert any(p.startswith("placeholders.blank") for p in problems) From e035a29ebc7fdadb8e8b3437db273b303af674df Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:48:03 +0000 Subject: [PATCH 009/118] Style templates: an empty labels.condition keeps its render-time fallback (S11 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The required-field check no longer refuses an empty template for a setting the resolver already replaces when empty: `labels.condition or "{condition}"` (Resolver._condition) and `labels.attachment_file or "{file}"`. An empty labels.condition shows the bare condition, which meets §15.8, and refusing it would break existing styles for no gain. TemplateFields.empty_falls_back marks exactly these two settings. Settings with no fallback (references.format and the numbering/enumeration/paragraph refs) still require {designation}/{n}; unknown fields are checked in every non-empty template. Tests: new tests for the empty condition label (renders the bare condition) and for empty templates without a fallback. docs/style-templates.md updated. --- docs/style-templates.md | 2 +- src/legaldown_render/style/loader.py | 7 ++++++- src/legaldown_render/style/model.py | 12 +++++++++--- tests/test_wp6_style.py | 16 ++++++++++++++++ 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/docs/style-templates.md b/docs/style-templates.md index c0027e1..ac76ff4 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -170,7 +170,7 @@ typo would otherwise be shown literally), and so is a missing required field. | `enumeration.levels[].label`, `paragraphs.format.label` | `{n}`, `{path}`, `{section}` | none | | `enumeration.levels[].ref`, `paragraphs.format.ref` | `{n}`, `{section}` | `{n}` | | `references.format` | `{designation}` | `{designation}`, or every reference reads the same | -| `labels.condition` | `{condition}` | `{condition}`, as §15.8 requires the condition to be shown | +| `labels.condition` | `{condition}` | `{condition}`, as §15.8 requires the condition to be shown; an empty value is allowed and shows the bare condition | | `labels.attachment_file` | `{file}` | none | | every other label | none | none | diff --git a/src/legaldown_render/style/loader.py b/src/legaldown_render/style/loader.py index 56f3498..d53f58a 100644 --- a/src/legaldown_render/style/loader.py +++ b/src/legaldown_render/style/loader.py @@ -309,7 +309,10 @@ def _template_problems(style: Style) -> list[str]: (:func:`~.model.template_settings`): an identifier-shaped field the setting does not accept (a typo would be shown literally), or a required field it leaves out (``labels.condition`` without ``{condition}`` would - hide the condition, which §15.8 requires to be shown).""" + hide the condition, which §15.8 requires to be shown). An empty template + is exempt from the required fields where the resolver falls back to the + bare field for it (``labels.condition``): it already shows the condition. + Where there is no such fallback, an empty template lacks its fields.""" problems = [] for key, template, fields in template_settings(style): found = set(_FIELD_RE.findall(template)) @@ -317,6 +320,8 @@ def _template_problems(style: Style) -> list[str]: for name in sorted(found - fields.accepted): takes = f"accepts {accepted}" if accepted else "takes no fields" problems.append(f"{key}: unknown field {{{name}}}; this setting {takes}") + if fields.empty_falls_back and not template: + continue for name in sorted(fields.required - found): problems.append(f"{key}: must contain {{{name}}}" + (f" — {fields.why}" if fields.why else "")) return problems diff --git a/src/legaldown_render/style/model.py b/src/legaldown_render/style/model.py index 6903be2..8032faf 100644 --- a/src/legaldown_render/style/model.py +++ b/src/legaldown_render/style/model.py @@ -298,11 +298,15 @@ class TemplateFields: Only identifier-shaped braces (``{n}``, ``{designation}``) are fields; any other braces (``{}``, ``{1}``, ``{a b}``) are literal text, as the resolver keeps them. ``why`` says what a required field is for, in the - message about a template that leaves it out.""" + message about a template that leaves it out. ``empty_falls_back`` marks + a setting the resolver replaces with the bare field when it is empty + (``labels.condition or "{condition}"``), so an empty template is not + missing its required field: it shows the condition as written.""" accepted: frozenset[str] = frozenset() required: frozenset[str] = frozenset() why: str = "" + empty_falls_back: bool = False #: The designation-building settings are read in :mod:`..resolve.numbering` @@ -320,9 +324,11 @@ class TemplateFields: #: Labels are plain words, except for the two that carry a value. LABEL_FIELDS: dict[str, TemplateFields] = { "condition": TemplateFields( - frozenset({"condition"}), frozenset({"condition"}), "a conditional unit's condition must be shown (§15.8)" + frozenset({"condition"}), frozenset({"condition"}), "a conditional unit's condition must be shown (§15.8)", + empty_falls_back=True, # Resolver._condition: ``self.labels.condition or "{condition}"`` ), - "attachment_file": TemplateFields(frozenset({"file"})), + # Resolver: ``self.labels.attachment_file or "{file}"``; it requires nothing, so the fallback is moot. + "attachment_file": TemplateFields(frozenset({"file"}), empty_falls_back=True), } diff --git a/tests/test_wp6_style.py b/tests/test_wp6_style.py index 870e752..f0cdfd5 100644 --- a/tests/test_wp6_style.py +++ b/tests/test_wp6_style.py @@ -257,6 +257,22 @@ def test_valid_templates_are_accepted(setting: str, value: object) -> None: load_style("default", overrides={setting: value}) +def test_an_empty_condition_label_keeps_its_render_time_fallback() -> None: + """The resolver shows the bare condition for an empty ``labels.condition`` + (§15.8 is met), so the style is not refused.""" + style = load_style("default", overrides={"labels.condition": ""}) + assert style.labels.condition == "" + doc = "---\nlegaldown: '0.2'\ntitle: T\n---\n\n# One {#one when=forum:courts}\n\nText.\n" + assert "forum:courts" in render(doc, RenderOptions(format="text", overrides={"labels.condition": ""})).output + + +def test_an_empty_template_without_a_fallback_still_lacks_its_field() -> None: + for setting in ("references.format", "numbering.levels", "paragraphs.format"): + value: object = {"label": "{n}", "ref": ""} if setting == "paragraphs.format" else ( + [{"ref": ""}] if setting == "numbering.levels" else "") + assert any(p.startswith(setting) for p in _problems({setting: value})) + + def test_unset_labels_are_not_checked() -> None: assert load_style("default", overrides={"labels.condition": None}).labels.condition is None From 493c924b74ccfa1e9c55f974d0e7fbcf9747623a Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:48:41 +0000 Subject: [PATCH 010/118] Links and images: relative image paths only, deny-list for links, empty links (B6, W3, B9/W8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B6 (§2.3): an image whose path has a URI scheme (data: included), starts with / or \ (//host too) or is a Windows drive path is replaced by its alt text in the resolver (Resolver._resolve_inline) and reported as the Core Error path-not-relative, which --strict refuses. The validator 0.4 does not check image paths; the renderer's report is skipped when the validator names the same path. ../ stays allowed (no default document root). The builder now keeps javascript:/file:/data: links and images as nodes (markdown-it validateLink off) so they can be reported and shown as text instead of as literal Markdown. W3 (§8.7): the HTML allow-list becomes a deny-list (javascript, vbscript, file, data) after the WHATWG normalisation; new urls.py shared by resolver and both writers so they agree. A refused link is its text only in both writers. B9/W8: an empty href is just the link text; a link with no text shows its URL; the text writer omits when it equals the text minus mailto:/tel:. Changed goldens: features.{default,continental,outline}.txt and features.default.html: the "[an unsafe one](javascript:alert(1))" source text is now "an unsafe one" (the link is refused, its text kept). --- CONFORMANCE.md | 6 + docs/architecture.md | 21 ++- docs/concepts.md | 5 +- src/legaldown_render/build.py | 7 + src/legaldown_render/resolve/resolver.py | 44 ++++- src/legaldown_render/tree.py | 9 + src/legaldown_render/urls.py | 62 +++++++ src/legaldown_render/writers/html.py | 31 ++-- src/legaldown_render/writers/text.py | 23 ++- tests/golden/features.continental.txt | 2 +- tests/golden/features.default.html | 2 +- tests/golden/features.default.txt | 2 +- tests/golden/features.outline.txt | 2 +- tests/test_wp3_links.py | 216 +++++++++++++++++++++++ 14 files changed, 397 insertions(+), 35 deletions(-) create mode 100644 src/legaldown_render/urls.py create mode 100644 tests/test_wp3_links.py diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 542eca8..d8ff65d 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -38,6 +38,8 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `ref-not-enumerated` Warning (§6.3, §16.3) | ✅ | | `raw-html` Warning; raw HTML never emitted (§8.7) | ✅ Nothing is emitted: an HTML block is dropped whole, and an inline tag is dropped with its text kept. The Warning is the validator's, once per block or text that holds raw HTML other than comments | | Automatic term recognition (§7.4, MAY) | ❌ Not supported | +| Links (§8.7) | ✅ Every link is a hyperlink in HTML and shows `text ` in plain text, whatever its scheme (`ftp:`, `sip:`, `urn:`, ...), except one whose URL is empty or uses `javascript:`, `vbscript:`, `file:`, or `data:`, which shows its text alone. A link with no text shows its URL; the text writer omits `` for an autolink | +| Images and file-reference paths (§8.7, §2.3) | ✅ A relative path (`../` included, as §2.3 defines no default document root) renders `` in HTML and its alt text in plain text. A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is never fetched or embedded: the alt text is shown and the Error `path-not-relative` is reported, which `--strict` refuses | | Extended tables through raw HTML (§9.2, MAY) | ❌ Not supported; the HTML table is dropped with the Warning | | Question prompts next to blanks (§15.8, MAY) | ✅ `placeholders.show_prompt` | | Final check (§15.9, SHOULD) | ✅ `--final` / `final=True`: the validator's `placeholder-unfilled` and `template-construct-present` Errors; with `--strict`, the document is refused | @@ -50,6 +52,10 @@ It is checked against the specification's own examples and fixtures corpus (`tes - **Line numbers.** The validator's diagnostics name their line, and the CLI prints it. The renderer's own (`render-ref-ambiguous`, `ref-not-enumerated`, `render-not-processed`) do not yet ([validator#93](https://github.com/ForLegalAI/legaldown-validator/issues/93)). +- **`path-not-relative` for images.** The renderer reports the Core rule `path-not-relative` + (§2.3, §16.2) for an image path, because the validator (0.4) does not check image paths; it + leaves its own report out when the validator names the same path. The Error has no line number. + The containment rule (`path-outside-root`) is not checked: no document root is configured. - **Nesting limit.** The validator reads lists 64 levels deep and quotes 16; deeper, an item's or quote's content is its text. The renderer follows it, and refuses (`DocumentError`) a document whose lists, quotes, and inline formatting (emphasis, links) together nest more than 100 levels, diff --git a/docs/architecture.md b/docs/architecture.md index 7d7d03d..afddb73 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -232,7 +232,8 @@ turns into an error. - **Internal inconsistencies** raise `InternalError`, and the CLI exits with 70. Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can evaluate: -`ref-not-enumerated`, a specification rule id, plus renderer-specific ids prefixed `render-`: +`ref-not-enumerated` and `path-not-relative` (an Error for an image path that is not relative, §2.3, +reported here because validator 0.4 does not check image paths), specification rule ids, plus renderer-specific ids prefixed `render-`: `render-not-processed`, `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), and `render-ref-ambiguous`. The validator's diagnostics carry their line (§16.9), and the CLI prints it as `file:line:`. The renderer's own have none yet: they are about a directive within a block, and the validator makes only a block's @@ -242,8 +243,21 @@ line public (`Document.line_of`, U2). - Raw HTML in the source is never emitted (§8.7). - Every text node and attribute is escaped. -- Links and images whose URL has a scheme other than `http`, `https`, `mailto`, or `tel` render - as text. markdown-it-py also refuses `javascript:`-style URLs when parsing. +- A link is a hyperlink (§8.7) unless its URL is empty or its scheme is `javascript`, `vbscript`, + `file`, or `data` (a deny-list, `urls.DENIED_SCHEMES`, judged as a browser reads the URL: tabs, + newlines, controls, and case are ignored). Every other scheme (`ftp:`, `sip:`, `urn:`, ...) is + linked. A link that is not made renders as its text alone, in HTML and in text alike. The + builder keeps such links as nodes (it turns off markdown-it-py's own `validateLink`, which + would leave them as literal Markdown), so the writers' check is the one that applies; + `is_safe_href` stays as defence in depth for a tree built another way. A link with no text shows + its URL; the text writer leaves out ` ` when the URL, less a `mailto:` or `tel:` prefix, + is the text. +- An image path must be relative with no URI scheme (§2.3), so a remote resource is never fetched + or embedded. Resolution, not a writer, enforces it: an image whose path has a scheme (`https:`, + `file:`, `data:`, a Windows drive such as `C:\x`) or starts with `/` or `\` (`//host`) is + replaced by its alt text and reported as the Core Error `path-not-relative`, which `--strict` + refuses. `../` stays allowed: §2.3 defines no default document root. The writers' image check + is a second line. - At Rendering level, the only files read are the document, the style template, and the answers file the CLI reads with `--answers` (the Python API takes answers as a mapping). Attachment files and includes are never opened. @@ -256,6 +270,7 @@ line public (`Document.line_of`, U2). |---|---| | `tests/test_golden.py` | The test documents under each built-in style, compared byte for byte with `tests/golden/`. Regenerate with `pytest --update-golden` and review the diff | | `tests/test_render.py` | Numbering schemes, designations, fallbacks, every failure marker, template view, labels by language, HTML safety, unique anchors, determinism, strict mode, locales | +| `tests/test_wp3_links.py` | Which images are shown and which links are links: non-relative image paths, the link scheme deny-list, empty URLs and empty link text | | `tests/test_style.py`, `tests/test_values.py` | Style layering and validation; counters and locale formatting | | `tests/test_cli.py` | The command | | `tests/conformance/` | Every example and fixture in the specification repository renders, every internal link has a target, and each invalid fixture renders its rule's failure marker. Needs `$LEGALDOWN_SPEC_DIR` or a sibling `../LegalDown` checkout | diff --git a/docs/concepts.md b/docs/concepts.md index 4240eac..1509192 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -83,8 +83,9 @@ no random ids, stable anchor names, stable ordering. This makes golden-file test lets rendered output be diffed in Git just like the source. **6. It is safe with documents you did not write.** Raw HTML in the source is never emitted -(§8.7). All text is escaped for the target format. Link URLs are checked against an allow-list of -schemes. At Rendering level the renderer reads no files other than the document, the chosen +(§8.7). All text is escaped for the target format. A link whose URL is empty or uses a script, +`file:`, or `data:` scheme is shown as its text, and an image whose path is not relative is shown as +its alt text and reported (§2.3). At Rendering level the renderer reads no files other than the document, the chosen style template, and the answers file the CLI is given with `--answers`. ## Vocabulary diff --git a/src/legaldown_render/build.py b/src/legaldown_render/build.py index a07daa4..c9aa8e5 100644 --- a/src/legaldown_render/build.py +++ b/src/legaldown_render/build.py @@ -180,6 +180,13 @@ def __init__(self, document: Document, result: ValidationResult) -> None: self.language = document.metadata.language or "en" # Inline parsing only: block structure is the validator's. self.md = MarkdownIt("commonmark", {"html": True}) + # markdown-it would leave ``[x](javascript:...)`` as literal text and + # ``![x](file:///...)`` unparsed, so neither could be told apart from + # prose. The builder keeps the node instead: the resolver reports an + # image path that is not relative (§2.3) and replaces the image with + # its alt text, and the writers show a refused link as its text + # (`urls.DENIED_SCHEMES`), which is the safety markdown-it was for. + self.md.validateLink = lambda url: True self.env: dict = {} self.payloads: list[_Payload] = [] # The markers the validator placed, by (section index or None for diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index da229c4..6753141 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -16,13 +16,16 @@ The validator has already reported every document-level Error for the failures shown here, so the resolver only adds the diagnostics that need a style template or a renderer: ``ref-not-enumerated``, ``render-ref-ambiguous``, -and ``render-not-processed`` for constructs beyond the Rendering level (§17.5). +and ``render-not-processed`` for constructs beyond the Rendering level (§17.5), +and ``path-not-relative`` for an image whose path is not relative (§2.3), which +the validator does not check yet. """ from __future__ import annotations from collections.abc import Callable from dataclasses import dataclass, replace from typing import Any, NamedTuple +from urllib.parse import unquote from legaldown import Diagnostic, Document, ValidationResult from legaldown.grammar import ( @@ -81,6 +84,7 @@ plain_text, unlinked, ) +from ..urls import is_relative_path from .numbering import RENUMBERED, extend, fill, format_counter, heading_levels from .values import DURATION_UNITS, Formatter @@ -510,11 +514,13 @@ def _resolve_inlines(self, inlines: tuple[Inline, ...]) -> tuple[Inline, ...]: out: list[Inline] = [] for inline in inlines: resolved = self._resolve_inline(inline) - if resolved is not None: + if isinstance(resolved, tuple): + out.extend(resolved) + elif resolved is not None: out.append(resolved) return tuple(out) - def _resolve_inline(self, inline: Inline) -> Inline | None: + def _resolve_inline(self, inline: Inline) -> Inline | tuple[Inline, ...] | None: match inline: case DirectiveSource(directive=directive): return self._directive(directive) @@ -530,9 +536,16 @@ def _resolve_inline(self, inline: Inline) -> Inline | None: case Link(children=children, title=title): return replace(inline, children=self._resolve_inlines(children), title=self._resolve_inlines(_attribute_text(title))) - case Image(children=children, title=title): - return replace(inline, children=self._resolve_inlines(_attribute_text(children)), - title=self._resolve_inlines(_attribute_text(title))) + case Image(src=src, children=children, title=title): + alt = self._resolve_inlines(_attribute_text(children)) + if not is_relative_path(src): + # §2.3: an image path is relative and has no URI scheme, so a + # remote or absolute image is never fetched or embedded. Its + # alt text stands in; the validator (0.4) does not check + # image paths, so the renderer reports the Core rule. + self._not_relative(src) + return alt + return replace(inline, children=alt, title=self._resolve_inlines(_attribute_text(title))) case _: return inline @@ -838,6 +851,25 @@ def _warn(self, rule: str, message: str) -> None: if diagnostic not in self.diagnostics: self.diagnostics.append(diagnostic) + def _not_relative(self, path: str) -> None: + """Report an image *path* that is not relative as the Core Error + ``path-not-relative`` (§2.3, §16.2), unless the validator has + reported it. The renderer has no line for an image, so a path the + validator named is the one it has covered. markdown-it has + percent-encoded the path ("C:\\x" reads "C:%5Cx"), so it is decoded + to name the path as written.""" + path = unquote(path) + if any(d.rule == "path-not-relative" and path in d.message for d in self.result.diagnostics): + return + diagnostic = Diagnostic( + rule="path-not-relative", + level="error", + message=f"Image path '{path}' is not relative (§2.3): an image path is a relative path with no URI " + f"scheme, and a remote resource is never fetched. Its alt text is shown instead.", + ) + if diagnostic not in self.diagnostics: + self.diagnostics.append(diagnostic) + _HANDLERS: dict[str, Callable[[Resolver, Directive, str], Inline | None]] = { "ref": Resolver._ref, diff --git a/src/legaldown_render/tree.py b/src/legaldown_render/tree.py index edce701..7e88575 100644 --- a/src/legaldown_render/tree.py +++ b/src/legaldown_render/tree.py @@ -509,3 +509,12 @@ def plain_text(inlines: tuple[Inline, ...]) -> str: case _ if getattr(inline, "children", None) is not None: parts.append(plain_text(inline.children)) return "".join(parts) + + +def has_visible_content(inlines: tuple[Inline, ...]) -> bool: + """True if resolved *inlines* show anything: text other than white space, + or an image (which has a picture even when its alt text is empty).""" + if plain_text(inlines).strip(): + return True + return any(isinstance(inline, Image) or has_visible_content(getattr(inline, "children", None) or ()) + for inline in inlines) diff --git a/src/legaldown_render/urls.py b/src/legaldown_render/urls.py new file mode 100644 index 0000000..c0282d3 --- /dev/null +++ b/src/legaldown_render/urls.py @@ -0,0 +1,62 @@ +"""Reading URLs the way the renderer needs to: which schemes a link may use +(§8.7), and whether an image path is a relative one (§2.3). + +The checks are syntactic, like the validator's (§2.3). They are shared by the +resolver, which decides what an image is, and by both writers, which decide +what a link is, so that the outputs agree. +""" +from __future__ import annotations + +import re +from urllib.parse import unquote + +#: Schemes whose links are never made: script (``javascript:``, ``vbscript:``), +#: local files (``file:``), and ``data:``. markdown-it refuses the same +#: schemes while parsing (``validateLink``, which lets some ``data:`` images +#: through; a link may not); this list is the second line, for a URL that +#: reaches a writer by another way. Every other scheme (``ftp:``, ``sip:``, +#: ``urn:``, ...) is a hyperlink, since §8.7 requires links to be rendered as +#: hyperlinks. +DENIED_SCHEMES = frozenset({"javascript", "vbscript", "file", "data"}) + +_SCHEME_RE = re.compile(r"^([a-zA-Z][a-zA-Z0-9+.-]*):") +# What the WHATWG URL parser ignores before it reads a scheme: tabs and +# newlines anywhere, and C0 controls and spaces at either end. Checking the +# URL as a browser will read it means "java\tscript:" cannot slip through. +_URL_IGNORED_RE = re.compile(r"[\t\n\r]") +_URL_TRIMMED = "".join(chr(code) for code in range(0x21)) + + +def _as_read(url: str) -> str: + """*url* as a browser reads it, before the scheme.""" + return _URL_IGNORED_RE.sub("", url).strip(_URL_TRIMMED) + + +def url_scheme(url: str) -> str | None: + """The lower-cased scheme of *url* as a browser reads it, or None.""" + scheme = _SCHEME_RE.match(_as_read(url)) + return scheme.group(1).lower() if scheme else None + + +def is_safe_href(href: str) -> bool: + """True unless *href* uses a scheme in :data:`DENIED_SCHEMES`, judged as a + browser parses the URL.""" + return url_scheme(href) not in DENIED_SCHEMES + + +def is_link_target(href: str) -> bool: + """True if a link to *href* is made: it names something, and is safe. + An empty destination would be a link to the page itself.""" + return bool(_as_read(href)) and is_safe_href(href) + + +def is_relative_path(path: str) -> bool: + """True if *path*, an image's, is a relative path (§2.3). Not relative: a + path with a URI scheme (``https://``, ``file://``, ``data:``), which + includes a Windows drive path (``C:\\x``, ``C:/x``, a scheme as far as a + parser can tell), and one that starts with ``/`` or ``\\`` (so also + ``//host``). A leading ``../`` is relative: §2.3 defines no default + document root. A percent-encoded character counts as the character, since + a path may be encoded by the time it is read.""" + read = _as_read(unquote(path)) + return not read or (_SCHEME_RE.match(read) is None and read[0] not in "/\\") diff --git a/src/legaldown_render/writers/html.py b/src/legaldown_render/writers/html.py index b365e6f..b6989a6 100644 --- a/src/legaldown_render/writers/html.py +++ b/src/legaldown_render/writers/html.py @@ -9,7 +9,12 @@ Safety (docs/architecture.md): every text node and attribute is escaped, raw HTML from the source never reaches this writer (§8.7), and a link whose URL -uses a scheme outside :data:`SAFE_SCHEMES` renders as plain text. +is empty or uses a script, ``file:``, or ``data:`` scheme +(:data:`~legaldown_render.urls.DENIED_SCHEMES`, judged as a browser reads the +URL) renders as its text alone. Every other scheme is a hyperlink (§8.7). +markdown-it refuses such URLs first, while parsing; this is the second line. +An image path that is not relative (§2.3) never gets here: resolution has +replaced the image with its alt text and reported it. """ from __future__ import annotations @@ -47,17 +52,12 @@ TermRef, Text, Value, + has_visible_content, plain_text, ) +from ..urls import is_link_target, is_safe_href -SAFE_SCHEMES = frozenset({"http", "https", "mailto", "tel"}) -_SCHEME_RE = re.compile(r"^([a-zA-Z][a-zA-Z0-9+.-]*):") -# What the WHATWG URL parser ignores before it reads a scheme: tabs and -# newlines anywhere, and C0 controls and spaces at either end. Checking the -# URL as a browser will read it means "java\tscript:" cannot slip through. -_URL_IGNORED_RE = re.compile(r"[\t\n\r]") -_URL_TRIMMED = "".join(chr(code) for code in range(0x21)) - +__all__ = ["HtmlWriter", "is_safe_href", "stylesheet"] def _text(value: str) -> str: return escape(value, quote=False) @@ -67,13 +67,6 @@ def _attr(value: str) -> str: return escape(value, quote=True) -def is_safe_href(href: str) -> bool: - """True for relative URLs, fragments, and the schemes in SAFE_SCHEMES, - judged as a browser parses the URL.""" - scheme = _SCHEME_RE.match(_URL_IGNORED_RE.sub("", href).strip(_URL_TRIMMED)) - return scheme is None or scheme.group(1).lower() in SAFE_SCHEMES - - class HtmlWriter: """Writes a resolved tree as HTML.""" @@ -319,11 +312,13 @@ def inline(self, inline: Inline) -> str: case Code(text=text): return f"{_text(text)}" case Link(href=href, children=children, title=title): - if not is_safe_href(href): + # A link with no text shows its URL (§8.7: it is still a hyperlink). + shown = children if has_visible_content(children) else (Text(href.strip()),) + if not is_link_target(href): return self.inlines(children) title_attr = f' title="{_attr(plain_text(title))}"' if title else "" outer, self._in_link = self._in_link, True - content = self.inlines(children) + content = self.inlines(shown) self._in_link = outer return f'{content}' case Image(src=src, children=children, title=title): diff --git a/src/legaldown_render/writers/text.py b/src/legaldown_render/writers/text.py index 40167d4..e8fb00c 100644 --- a/src/legaldown_render/writers/text.py +++ b/src/legaldown_render/writers/text.py @@ -35,8 +35,28 @@ Text, Value, ) +from ..urls import is_link_target _INDENT = " " +_ADDRESS_SCHEMES = ("mailto:", "tel:") + + +def _link_text(href: str, text: str) -> str: + """A link as plain text: its text, followed by ```` unless the URL is + no more than the text says. The same links are links as in HTML + (:func:`~legaldown_render.urls.is_link_target`); one that is not shows its + text alone, and one with no text shows its URL.""" + if not is_link_target(href): + return text + url = href.strip() + if not text.strip(): + return url + address = url + for prefix in _ADDRESS_SCHEMES: + if url.lower().startswith(prefix): + address = url[len(prefix):] + # An internal link has no URL worth printing; an autolink's text is its URL. + return text if url.startswith("#") or text in (url, address) else f"{text} <{url}>" def inline_text(inlines: tuple[Inline, ...]) -> str: @@ -55,8 +75,7 @@ def inline_text(inlines: tuple[Inline, ...]) -> str: case Emphasis(children=children) | Strong(children=children) | DefinedTerm(children=children): parts.append(inline_text(children)) case Link(href=href, children=children): - text = inline_text(children) - parts.append(text if href.startswith("#") or href == text else f"{text} <{href}>") + parts.append(_link_text(href, inline_text(children))) case Image(children=children): parts.append(inline_text(children)) case Choice(options=options, separator=separator): diff --git a/tests/golden/features.continental.txt b/tests/golden/features.continental.txt index 5c995c3..0299a7e 100644 --- a/tests/golden/features.continental.txt +++ b/tests/golden/features.continental.txt @@ -63,7 +63,7 @@ This Services Agreement (this Agreement) is made between Acme and Beta Industrie (1) These are deliberately wrong: [BROKEN REF: nowhere], [UNDEFINED: nothing], [INVALID DATE: 2026-13-45], [INVALID AMOUNT], [UNKNOWN CURRENCY: XYZ], [UNKNOWN PARTY: nobody], [INVALID PARTY: Bad_Name], [UNKNOWN SIDE: nobody], [INVALID DURATION: -1], [INVALID DURATION UNIT: M], [INVALID FIELD: value], [UNKNOWN DIRECTIVE: trem], [UNKNOWN ATTACHMENT: schedule-z], [INVALID PLACEHOLDER: fee], and [NOT PROCESSED: include includes/extra.lgd]. -(2) Raw HTML is dropped. A safe link stays; [an unsafe one](javascript:alert(1)) does not. +(2) Raw HTML is dropped. A safe link stays; an unsafe one does not. --- diff --git a/tests/golden/features.default.html b/tests/golden/features.default.html index a6d53fb..a6610a2 100644 --- a/tests/golden/features.default.html +++ b/tests/golden/features.default.html @@ -169,7 +169,7 @@

4. [INVALID FIELD: value], [UNKNOWN DIRECTIVE: trem], [UNKNOWN ATTACHMENT: schedule-z], [INVALID PLACEHOLDER: fee], and [NOT PROCESSED: include includes/extra.lgd].

Raw HTML is dropped. A safe link stays; -[an unsafe one](javascript:alert(1)) does not.

+an unsafe one does not.

Attachments

diff --git a/tests/golden/features.default.txt b/tests/golden/features.default.txt index b302f63..4610d64 100644 --- a/tests/golden/features.default.txt +++ b/tests/golden/features.default.txt @@ -63,7 +63,7 @@ Inline code keeps {{term: services}} literal. These are deliberately wrong: [BROKEN REF: nowhere], [UNDEFINED: nothing], [INVALID DATE: 2026-13-45], [INVALID AMOUNT], [UNKNOWN CURRENCY: XYZ], [UNKNOWN PARTY: nobody], [INVALID PARTY: Bad_Name], [UNKNOWN SIDE: nobody], [INVALID DURATION: -1], [INVALID DURATION UNIT: M], [INVALID FIELD: value], [UNKNOWN DIRECTIVE: trem], [UNKNOWN ATTACHMENT: schedule-z], [INVALID PLACEHOLDER: fee], and [NOT PROCESSED: include includes/extra.lgd]. -Raw HTML is dropped. A safe link stays; [an unsafe one](javascript:alert(1)) does not. +Raw HTML is dropped. A safe link stays; an unsafe one does not. --- diff --git a/tests/golden/features.outline.txt b/tests/golden/features.outline.txt index 7f5255d..52d599e 100644 --- a/tests/golden/features.outline.txt +++ b/tests/golden/features.outline.txt @@ -63,7 +63,7 @@ IV. Broken References These are deliberately wrong: [BROKEN REF: nowhere], [UNDEFINED: nothing], [INVALID DATE: 2026-13-45], [INVALID AMOUNT], [UNKNOWN CURRENCY: XYZ], [UNKNOWN PARTY: nobody], [INVALID PARTY: Bad_Name], [UNKNOWN SIDE: nobody], [INVALID DURATION: -1], [INVALID DURATION UNIT: M], [INVALID FIELD: value], [UNKNOWN DIRECTIVE: trem], [UNKNOWN ATTACHMENT: schedule-z], [INVALID PLACEHOLDER: fee], and [NOT PROCESSED: include includes/extra.lgd]. -Raw HTML is dropped. A safe link stays; [an unsafe one](javascript:alert(1)) does not. +Raw HTML is dropped. A safe link stays; an unsafe one does not. --- diff --git a/tests/test_wp3_links.py b/tests/test_wp3_links.py new file mode 100644 index 0000000..b848b52 --- /dev/null +++ b/tests/test_wp3_links.py @@ -0,0 +1,216 @@ +"""Links and images (WP3): which image paths are shown (B6, §2.3), which links +are links (W3, §8.7), and links with no URL or no text (B9/W8).""" +from __future__ import annotations + +import pytest + +from legaldown_render import RenderRefused, render +from legaldown_render.writers.html import is_safe_href + +FRONT = "---\ntitle: T\n---\n" + + +def text(body: str, **settings: object) -> str: + return render(FRONT + body, format="text", **settings).output + + +def html(body: str, **settings: object) -> str: + return render(FRONT + body, format="html", **settings).output + + +# -- B6: image paths (§2.3) --------------------------------------------------- + +NON_RELATIVE = [ + "https://example.com/p.png", + "HTTP://example.com/p.png", + "ftp://example.com/p.png", + "file:///etc/passwd", + "data:image/png;base64,AAAA", + "data:text/html,x", + "javascript:alert(1)", + "tel:123", + "/etc/passwd", + "/abs/p.png", + "//host/p.png", + "\\\\host\\share\\p.png", + "\\abs.png", + "C:\\x\\p.png", + "C:/x/p.png", + "c:p.png", +] + + +@pytest.mark.parametrize("path", NON_RELATIVE) +def test_a_non_relative_image_shows_its_alt_text(path: str) -> None: + source = f"# A\n\nBefore ![the logo](<{path}>) after.\n" + for fmt in ("html", "text"): + result = render(FRONT + source, format=fmt) + assert " None: + result = render(FRONT + "# A\n\n![x](data:image/png;base64,iVBORw0KGgo=)\n", format="html") + assert "data:" not in result.output + assert " None: + result = render(FRONT + f"# A\n\n![alt](<{path}> \"cap\")\n", format="html") + assert f'alt' in result.output + assert not [d for d in result.diagnostics if d.rule == "path-not-relative"] + assert result.ok + + +def test_a_disguised_scheme_is_still_a_scheme() -> None: + # WHATWG ignores tabs and newlines, and a percent-encoded backslash is a backslash. + for path in ("ht\ttps://example.com/p.png", "%5Chost%5Cp.png", "%5C%5Chost%5Cp.png"): + result = render(FRONT + f"# A\n\n![alt](<{path}>)\n", format="html") + assert " None: + result = render(FRONT + "# A {#a}\n\nSee ![logo of **Acme** ({{ref: a}})](https://x/p.png).\n", format="html") + assert "Acme (" in result.output + + +def test_an_image_inside_a_link_keeps_the_link() -> None: + result = render(FRONT + "# A\n\n[![logo](https://x/p.png)](https://example.com)\n", format="html") + assert 'logo' in result.output + + +def test_the_same_image_path_is_reported_once() -> None: + result = render(FRONT + "# A\n\n![a](https://x/p.png) ![b](https://x/p.png) ![c](/q.png)\n") + errors = [d for d in result.diagnostics if d.rule == "path-not-relative"] + assert len(errors) == 2 # the same path twice is one finding + + +def test_strict_refuses_a_non_relative_image() -> None: + with pytest.raises(RenderRefused) as refused: + render(FRONT + "# A\n\n![a](https://x/p.png)\n", strict=True) + assert "path-not-relative" in {d.rule for d in refused.value.diagnostics} + render(FRONT + "# A\n\n![a](img/p.png)\n", strict=True) + + +def test_the_validators_own_report_is_not_repeated(monkeypatch: pytest.MonkeyPatch) -> None: + from legaldown import Diagnostic + from legaldown import validate as real_validate + + import legaldown_render.api as api + + def validate(document, **kwargs): # type: ignore[no-untyped-def] + result = real_validate(document, **kwargs) + found = Diagnostic(rule="path-not-relative", level="error", line=5, + message="Image path 'https://x/p.png' is not relative (§2.3).") + result.diagnostics.append(found) + return result + + monkeypatch.setattr(api, "validate", validate) + result = render(FRONT + "# A\n\n![a](https://x/p.png) ![b](/other.png)\n") + errors = [d for d in result.diagnostics if d.rule == "path-not-relative"] + assert len(errors) == 2 # the validator's, and ours for the path it did not name + assert any(d.line == 5 for d in errors) + assert any("/other.png" in d.message for d in errors) + + +# -- W3: which links are links (§8.7) ----------------------------------------- + + +@pytest.mark.parametrize("href", [ + "ftp://example.com/f", "sip:alice@example.com", "urn:isbn:0451450523", "https://example.com", + "mailto:a@example.com", "tel:+420123456789", "magnet:?xt=urn:btih:x", "attachments/a.pdf", "../a.pdf", "#top", +]) +def test_every_link_but_the_dangerous_ones_is_a_hyperlink(href: str) -> None: + output = html(f"# A\n\nSee [the link](<{href}>).\n") + assert f'the link' in output + + +@pytest.mark.parametrize("href", [ + "javascript:alert(1)", "JavaScript:alert(1)", " javascript:alert(1)", "java\tscript:alert(1)", + "jav\nascript:x", "vbscript:x", "file:///etc/passwd", "data:text/html,x", "data:image/png;base64,AAAA", +]) +def test_a_refused_link_is_its_text_in_both_writers(href: str) -> None: + # The builder is handed the URL directly: markdown-it refuses some of these + # itself when parsing (§8.7), which is why is_safe_href is only the second line. + from legaldown_render.style import load_style + from legaldown_render.tree import Link, Text + from legaldown_render.writers.html import HtmlWriter + from legaldown_render.writers.text import inline_text + + inlines = (Link(href, (Text("click"),)),) + assert HtmlWriter(load_style(None)).inlines(inlines) == "click" + assert inline_text(inlines) == "click" + assert not is_safe_href(href) + + +def test_the_deny_list_is_judged_as_a_browser_reads_the_url() -> None: + for href in ("java\tscript:alert(1)", "jav\nascript:x", "\x01javascript:x", " JAVASCRIPT:x", "data:text/html,x", + "VBScript:x", "FILE:///x"): + assert not is_safe_href(href), repr(href) + for href in ("https://example.com", "#section", "attachments/a.pdf", "mailto:a@example.com", "ftp://x/y", + "sip:a@b", "urn:x:y", "a:b", "x/data:y", "javascript"): + assert is_safe_href(href), href + + +def test_a_refused_link_in_a_document_is_its_text() -> None: + # markdown-it would leave these as literal Markdown; the builder keeps them + # as links so that both writers can show their text alone. + body = "# A\n\nSee [click](javascript:alert(1)) and [the file](file:///x) and [d](data:text/html,x).\n" + assert "See click and the file and d." in text(body) + output = html(body) + assert "See click and the file and d." in output + assert "href=" not in output.split("")[1] + assert "alert" not in output + + +def test_a_link_with_an_unusual_scheme_shows_its_url_in_text() -> None: + assert "the file " in text("# A\n\nGet [the file](ftp://example.com/f).\n") + + +# -- B9/W8: links without a URL or without text ----------------------------- + + +def test_a_link_with_no_url_is_its_text() -> None: + assert "")[1] + assert 'href=""' not in html("# A\n\nSee [x]().\n") + assert "See x." in text("# A\n\nSee [x]().\n") + assert "x <>" not in text("# A\n\nSee [x]().\n") + assert "See x." in text("# A\n\nSee [x](<>).\n") + assert 'href=""' not in html("# A\n\nSee [x](< >).\n") + + +def test_a_link_with_no_url_and_no_text_is_nothing() -> None: + assert "See ." in text("# A\n\nSee [](<>).\n") + assert ").\n").split("")[1] + + +def test_a_link_with_no_text_shows_its_url() -> None: + assert 'http://z.y' in html("# A\n\nSee [](http://z.y).\n") + assert "See http://z.y." in text("# A\n\nSee [](http://z.y).\n") + + +def test_an_autolink_is_its_address_alone_in_text() -> None: + assert "Mail a@b.c now" in text("# A\n\nMail now\n") + assert "Call tel:+1 now" in text("# A\n\nCall [tel:+1](tel:+1) now\n") + assert "Call +1 now" in text("# A\n\nCall [+1](tel:+1) now\n") + assert "See https://x.y/z." in text("# A\n\nSee .\n") + assert 'href="mailto:a@b.c"' in html("# A\n\nMail now\n") + + +def test_a_link_text_that_differs_from_its_url_shows_both_in_text() -> None: + assert "the site " in text("# A\n\nSee [the site](https://x.y/z)\n") + assert "Mail a " in text("# A\n\nMail [a](mailto:a@b.c)\n") + + +def test_a_link_around_a_picture_keeps_the_picture_not_its_url() -> None: + output = html("# A\n\n[![](img/p.png)](https://example.com)\n") + assert '' in output From 750cba4bf79f50bd013201c7c01ba8448d4a6ea8 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:50:15 +0000 Subject: [PATCH 011/118] Empty image path shows its alt text; deny-list comments say it is the gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An image with an empty or whitespace-only path is replaced by its alt text in the resolver (no , which requests the page itself) and reports nothing: §2.3 has no rule for an empty path. Tests for both writers. urls.py, html.py and docs/architecture.md no longer say markdown-it refuses unsafe schemes while parsing: its validateLink is off in the builder, and the deny-list is the gate for every href/src the HTML writer emits (Link, Image, Value). --- CONFORMANCE.md | 2 +- docs/architecture.md | 10 ++++++---- src/legaldown_render/resolve/resolver.py | 6 +++++- src/legaldown_render/urls.py | 18 ++++++++++++------ src/legaldown_render/writers/html.py | 3 ++- tests/test_wp3_links.py | 10 ++++++++++ 6 files changed, 36 insertions(+), 13 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index d8ff65d..a0dd3ed 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -39,7 +39,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `raw-html` Warning; raw HTML never emitted (§8.7) | ✅ Nothing is emitted: an HTML block is dropped whole, and an inline tag is dropped with its text kept. The Warning is the validator's, once per block or text that holds raw HTML other than comments | | Automatic term recognition (§7.4, MAY) | ❌ Not supported | | Links (§8.7) | ✅ Every link is a hyperlink in HTML and shows `text ` in plain text, whatever its scheme (`ftp:`, `sip:`, `urn:`, ...), except one whose URL is empty or uses `javascript:`, `vbscript:`, `file:`, or `data:`, which shows its text alone. A link with no text shows its URL; the text writer omits `` for an autolink | -| Images and file-reference paths (§8.7, §2.3) | ✅ A relative path (`../` included, as §2.3 defines no default document root) renders `` in HTML and its alt text in plain text. A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is never fetched or embedded: the alt text is shown and the Error `path-not-relative` is reported, which `--strict` refuses | +| Images and file-reference paths (§8.7, §2.3) | ✅ A relative path (`../` included, as §2.3 defines no default document root) renders `` in HTML and its alt text in plain text. A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is never fetched or embedded: the alt text is shown and the Error `path-not-relative` is reported, which `--strict` refuses. An empty path shows the alt text with no diagnostic | | Extended tables through raw HTML (§9.2, MAY) | ❌ Not supported; the HTML table is dropped with the Warning | | Question prompts next to blanks (§15.8, MAY) | ✅ `placeholders.show_prompt` | | Final check (§15.9, SHOULD) | ✅ `--final` / `final=True`: the validator's `placeholder-unfilled` and `template-construct-present` Errors; with `--strict`, the document is refused | diff --git a/docs/architecture.md b/docs/architecture.md index afddb73..c72fe12 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -249,14 +249,16 @@ line public (`Document.line_of`, U2). linked. A link that is not made renders as its text alone, in HTML and in text alike. The builder keeps such links as nodes (it turns off markdown-it-py's own `validateLink`, which would leave them as literal Markdown), so the writers' check is the one that applies; - `is_safe_href` stays as defence in depth for a tree built another way. A link with no text shows - its URL; the text writer leaves out ` ` when the URL, less a `mailto:` or `tel:` prefix, - is the text. + that deny-list is the gate for every `href` and `src` the HTML writer emits (links, images, + value links), and markdown-it-py's check is off so that the resolver can report §2.3 paths. + A link with no text shows its URL; the text writer leaves out ` ` when the URL, less a + `mailto:` or `tel:` prefix, is the text. - An image path must be relative with no URI scheme (§2.3), so a remote resource is never fetched or embedded. Resolution, not a writer, enforces it: an image whose path has a scheme (`https:`, `file:`, `data:`, a Windows drive such as `C:\x`) or starts with `/` or `\` (`//host`) is replaced by its alt text and reported as the Core Error `path-not-relative`, which `--strict` - refuses. `../` stays allowed: §2.3 defines no default document root. The writers' image check + refuses. An image with an empty path shows its alt text too, with no diagnostic (§2.3 has no + rule for it). `../` stays allowed: §2.3 defines no default document root. The writers' image check is a second line. - At Rendering level, the only files read are the document, the style template, and the answers file the CLI reads with `--answers` (the Python API takes answers as a mapping). Attachment diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 6753141..dfe80e9 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -84,7 +84,7 @@ plain_text, unlinked, ) -from ..urls import is_relative_path +from ..urls import is_empty_path, is_relative_path from .numbering import RENUMBERED, extend, fill, format_counter, heading_levels from .values import DURATION_UNITS, Formatter @@ -538,6 +538,10 @@ def _resolve_inline(self, inline: Inline) -> Inline | tuple[Inline, ...] | None: title=self._resolve_inlines(_attribute_text(title))) case Image(src=src, children=children, title=title): alt = self._resolve_inlines(_attribute_text(children)) + if is_empty_path(src): + # No path, no picture: would request the page + # itself. §2.3 says nothing of an empty path, so no rule fires. + return alt if not is_relative_path(src): # §2.3: an image path is relative and has no URI scheme, so a # remote or absolute image is never fetched or embedded. Its diff --git a/src/legaldown_render/urls.py b/src/legaldown_render/urls.py index c0282d3..a76ef7c 100644 --- a/src/legaldown_render/urls.py +++ b/src/legaldown_render/urls.py @@ -11,12 +11,12 @@ from urllib.parse import unquote #: Schemes whose links are never made: script (``javascript:``, ``vbscript:``), -#: local files (``file:``), and ``data:``. markdown-it refuses the same -#: schemes while parsing (``validateLink``, which lets some ``data:`` images -#: through; a link may not); this list is the second line, for a URL that -#: reaches a writer by another way. Every other scheme (``ftp:``, ``sip:``, -#: ``urn:``, ...) is a hyperlink, since §8.7 requires links to be rendered as -#: hyperlinks. +#: local files (``file:``), and ``data:``. This list is the gate: markdown-it's +#: own ``validateLink`` is turned off in the builder, so that the resolver can +#: see and report an image path that is not relative (§2.3), and the HTML +#: writer applies this list to every ``href`` and ``src`` it emits (a link, an +#: image, a value's link). Every other scheme (``ftp:``, ``sip:``, ``urn:``, +#: ...) is a hyperlink, since §8.7 requires links to be rendered as hyperlinks. DENIED_SCHEMES = frozenset({"javascript", "vbscript", "file", "data"}) _SCHEME_RE = re.compile(r"^([a-zA-Z][a-zA-Z0-9+.-]*):") @@ -50,6 +50,12 @@ def is_link_target(href: str) -> bool: return bool(_as_read(href)) and is_safe_href(href) +def is_empty_path(path: str) -> bool: + """True if *path* names nothing: empty, or only what a browser ignores. As + an image's source it would request the page itself.""" + return not _as_read(unquote(path)) + + def is_relative_path(path: str) -> bool: """True if *path*, an image's, is a relative path (§2.3). Not relative: a path with a URI scheme (``https://``, ``file://``, ``data:``), which diff --git a/src/legaldown_render/writers/html.py b/src/legaldown_render/writers/html.py index b6989a6..bf04ff2 100644 --- a/src/legaldown_render/writers/html.py +++ b/src/legaldown_render/writers/html.py @@ -12,7 +12,8 @@ is empty or uses a script, ``file:``, or ``data:`` scheme (:data:`~legaldown_render.urls.DENIED_SCHEMES`, judged as a browser reads the URL) renders as its text alone. Every other scheme is a hyperlink (§8.7). -markdown-it refuses such URLs first, while parsing; this is the second line. +This check is the gate for every ``href`` and ``src`` the writer emits (a link, +an image, a value's link): the builder turns markdown-it's own check off. An image path that is not relative (§2.3) never gets here: resolution has replaced the image with its alt text and reported it. """ diff --git a/tests/test_wp3_links.py b/tests/test_wp3_links.py index b848b52..e745292 100644 --- a/tests/test_wp3_links.py +++ b/tests/test_wp3_links.py @@ -214,3 +214,13 @@ def test_a_link_text_that_differs_from_its_url_shows_both_in_text() -> None: def test_a_link_around_a_picture_keeps_the_picture_not_its_url() -> None: output = html("# A\n\n[![](img/p.png)](https://example.com)\n") assert '' in output + + +@pytest.mark.parametrize("empty", ["", " ", "%20"]) +def test_an_image_with_no_path_is_its_alt_text(empty: str) -> None: + for fmt in ("html", "text"): + result = render(FRONT + f"# A\n\nBefore ![the logo](<{empty}>) after.\n", format=fmt, strict=True) + assert " Date: Tue, 27 Oct 2026 17:54:36 +0000 Subject: [PATCH 012/118] Text writer fidelity: attachment separators, tables, quotes, list items, blank labels (W1, W4-W7) W1: TextWriter takes the style's attachments.separator (api._writer hands it over; TextWriter() keeps the default style's "page-break"). "none" prints no separator; "rule" and "page-break" both print "---" before EACH attachment (plain text has no pages). The "---" that used to precede the "Attachments" heading is gone; the heading now stands directly after the body. W4: "|" in a table cell is escaped as "\|"; the separator row carries alignment colons from Table.align (":---", ":---:", "---:"), unpadded. W5: _prefix gives a blank line the right-trimmed prefix, so one quote stays one quote (">" and "> >" on blank lines). Blocks of a list item are separated by a blank line; a nested list directly after a paragraph (or another list) stays tight as before. W6: an item's content indent is len(label or bullet) + 1, whatever its condition label. W7 (text only): a style-blanked label drops the "label: " prefix in the title block and party details. HTML keeps its empty
. Existing tests changed because of W5: - test_nested_lists_line_up_with_their_item_s_text: blank line between the item's paragraphs. - test_a_later_paragraph_in_a_list_item_is_the_item_s: blank line before the item's second paragraph. - test_a_heading_in_a_quote_or_item_is_a_bold_line_not_a_section: the two-block quotes printed as two quotes; now one quote with a ">" line. Goldens changed (features.default/continental/outline.txt, each hunk read): - W1: "---" moved from before the "Attachments" heading to before each of the two attachments. - W4: the table separator row is now "|:---|---:|:---:|" (the document's table is left/right/center aligned). template.default.txt is unchanged. --- CONFORMANCE.md | 1 + docs/architecture.md | 2 +- docs/style-templates.md | 2 +- src/legaldown_render/api.py | 2 +- src/legaldown_render/writers/text.py | 82 +++++++++----- tests/golden/features.continental.txt | 8 +- tests/golden/features.default.txt | 8 +- tests/golden/features.outline.txt | 8 +- tests/test_render.py | 8 +- tests/test_wp5_text.py | 150 ++++++++++++++++++++++++++ 10 files changed, 231 insertions(+), 40 deletions(-) create mode 100644 tests/test_wp5_text.py diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 4c7ead4..ae7a9fe 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -41,6 +41,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | Automatic term recognition (§7.4, MAY) | ❌ Not supported | | Links (§8.7) | ✅ Every link is a hyperlink in HTML and shows `text ` in plain text, whatever its scheme (`ftp:`, `sip:`, `urn:`, ...), except one whose URL is empty or uses `javascript:`, `vbscript:`, `file:`, or `data:`, which shows its text alone. A link with no text shows its URL; the text writer omits `` for an autolink | | Images and file-reference paths (§8.7, §2.3) | ✅ A relative path (`../` included, as §2.3 defines no default document root) renders `` in HTML and its alt text in plain text. A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is never fetched or embedded: the alt text is shown and the Error `path-not-relative` is reported, which `--strict` refuses. An empty path shows the alt text with no diagnostic | +| Plain-text layout (§13.6, optional) | ✅ `attachments.separator` (§13.8): `none` prints no separator, `rule` and `page-break` both print a `---` line before each attachment (text has no pages). One quote stays one quote (blank lines keep the `>`), and the blocks of a list item are set apart by blank lines and indented under the item's label, whatever its condition label. A pipe in a table cell is backslash-escaped, and the separator row marks column alignment (`:---`, `:---:`, `---:`). A label a style blanks drops its `label: ` prefix in the title block and party details; HTML keeps an empty `
` so its grid layout and the `
` stay valid | | Extended tables through raw HTML (§9.2, MAY) | ❌ Not supported; the HTML table is dropped with the Warning | | Visual blank for a placeholder (§13.5 rule 6) | ✅ `placeholders.blank`; a blank that is empty or only whitespace renders `[TBD: id]` | | Question prompts next to blanks (§15.8, MAY) | ✅ `placeholders.show_prompt` | diff --git a/docs/architecture.md b/docs/architecture.md index a3e9d25..378944b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -176,7 +176,7 @@ clamped to 1–5, as the validator numbers them. | Writer | Notes | |---|---| -| `TextWriter` | Plain text (§13.6). A soft line break inside a paragraph is a space, a hard break a new line, and lists are indented with their labels. It is the **test oracle** for resolution | +| `TextWriter` | Plain text (§13.6). A soft line break inside a paragraph is a space, a hard break a new line, and lists are indented with their labels, one quote stays one quote, and tables keep escaped cells and alignment. It takes the style's `attachments.separator` (default `page-break`; `rule` and `page-break` both print `---` before each attachment). It is the **test oracle** for resolution | | `HtmlWriter` | A self-contained HTML5 page, or with `standalone=False` just the `
`. Semantic markup, with labels as real text rather than CSS counters, and a stylesheet generated from the style's presentation settings, including print rules | | DOCX, PDF | Planned ([ADR 0004](decisions/0004-html-first.md)) | diff --git a/docs/style-templates.md b/docs/style-templates.md index ac76ff4..9ce9074 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -110,7 +110,7 @@ placeholders: title_block: { parties: true, effective_date: true, version: false } contents: { enabled: false, depth: 2, attachments: true } # table of contents after the title block; depth 1-5 # counts number levels ("2.1" is 2); attachments listed last -attachments: { render: placeholder, separator: page-break } # placeholder | omit; page-break | rule | none +attachments: { render: placeholder, separator: page-break } # placeholder | omit; page-break | rule | none (plain text: `---` before each attachment unless none) signatures: { enabled: true } # the document's include_signatures: false wins template_view: { choice_separator: " / ", empty_choice: "" } diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index 508ce04..1ece903 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -194,7 +194,7 @@ def _resolve(document: Document, result: ValidationResult, style: Style) -> tupl def _writer(options: RenderOptions, style: Style) -> HtmlWriter | TextWriter: if options.format == "text": - return TextWriter() + return TextWriter(style.attachments.separator) from . import __version__ return HtmlWriter(style, standalone=options.standalone, generator=f"legaldown-render {__version__}") diff --git a/src/legaldown_render/writers/text.py b/src/legaldown_render/writers/text.py index e8fb00c..3e1a873 100644 --- a/src/legaldown_render/writers/text.py +++ b/src/legaldown_render/writers/text.py @@ -4,6 +4,12 @@ doubles as the test oracle for resolution (docs/architecture.md, Testing). Line breaks inside a paragraph are joined, so output does not depend on how the source was wrapped. + +Structure survives the plain layout: one quote stays one quote (every line of +it, blank ones included, carries the ``>`` marker), the blocks of a list item +are set apart by blank lines and indented under the item's label, and a table +is a pipe table whose cells escape ``|`` and whose separator row carries the +column alignment. """ from __future__ import annotations @@ -59,6 +65,12 @@ def _link_text(href: str, text: str) -> str: return text if url.startswith("#") or text in (url, address) else f"{text} <{url}>" +def _labelled(label: str, colon: str, value: str) -> str: + """A ``label: value`` row. A style that blanks the label (§13.5) leaves the + value alone rather than a bare ``: value``.""" + return f"{label}{colon}{value}" if label else value + + def inline_text(inlines: tuple[Inline, ...]) -> str: parts: list[str] = [] for inline in inlines: @@ -84,11 +96,20 @@ def inline_text(inlines: tuple[Inline, ...]) -> str: class TextWriter: - """Writes a resolved tree as plain text.""" + """Writes a resolved tree as plain text. + + *attachment_separator* is the style's ``attachments.separator`` (§13.8): + ``none`` prints nothing between attachments; ``rule`` and ``page-break`` + both print a ``---`` line before each attachment, as plain text has no + pages to break. The default is the default style's value. + """ extension = "txt" media_type = "text/plain" + def __init__(self, attachment_separator: str = "page-break") -> None: + self.attachment_separator = attachment_separator + def write(self, tree: RenderTree) -> str: chunks: list[str] = [] title = inline_text(tree.title).strip() @@ -98,12 +119,12 @@ def write(self, tree: RenderTree) -> str: if subtitle: chunks.append(subtitle) if tree.header: - chunks.append("\n".join(f"{label}{tree.colon}{inline_text(value)}" for label, value in tree.header)) + chunks.append("\n".join(_labelled(label, tree.colon, inline_text(value)) for label, value in tree.header)) for side in tree.sides: lines = [inline_text(side.label)] for party in side.parties: lines.append(_INDENT + inline_text(party.name)) - lines += [f"{_INDENT}{label}{tree.colon}{inline_text(value)}" for label, value in party.details] + lines += [_INDENT + _labelled(label, tree.colon, inline_text(value)) for label, value in party.details] chunks.append("\n".join(lines)) if tree.contents: lines = [tree.contents_label] if tree.contents_label else [] @@ -121,9 +142,10 @@ def write(self, tree: RenderTree) -> str: chunks.append(f"[{section.condition_label}]") chunks += [self.block(block) for block in section.blocks] if tree.attachments: - chunks.append("---") chunks.append(tree.attachments_label) for attachment in tree.attachments: + if self.attachment_separator != "none": + chunks.append("---") lines = [inline_text(attachment.title)] if attachment.condition_label: lines.append(f"[{attachment.condition_label}]") @@ -159,38 +181,46 @@ def block(self, block: Block) -> str: def list(self, block: List, indent: str) -> list[str]: """The lines of *block*, each item's label at *indent*. Everything - else in an item, a nested list included, lines up with its text.""" + else in an item, a nested list included, lines up with its text, which + starts one column after the item's own label (or bullet): a condition + label (§15.8) does not move it. Blocks of an item are set apart by a + blank line, except that a nested list stays tight under the paragraph + or list it follows.""" lines: list[str] = [] for item in block.items: - marker = item.label if item.label else "-" - if item.condition_label: - marker = f"{marker} [{item.condition_label}]" - content = indent + " " * (len(marker) + 1) - first = True + label = item.label if item.label else "-" + marker = f"{label} [{item.condition_label}]" if item.condition_label else label + content = indent + " " * (len(label) + 1) + previous: Block | None = None for child in item.blocks: + if previous is not None and not (isinstance(child, List) and isinstance(previous, Paragraph | List)): + lines.append("") if isinstance(child, List): - if first: + if previous is None: # The item opens with a nested list: its own label # still comes first, on a line of its own. lines.append(indent + marker) - first = False lines += self.list(child, content) - continue - text = self.block(child) - if first: - lines.append(_prefix(f"{marker} {text}", indent, hanging=" " * (len(marker) + 1))) - first = False + elif previous is None: + lines.append(_prefix(f"{marker} {self.block(child)}", indent, hanging=" " * (len(label) + 1))) else: - lines.append(_prefix(text, content)) - if first: + lines.append(_prefix(self.block(child), content)) + previous = child + if previous is None: lines.append(indent + marker) return lines def table(self, table: Table) -> str: - header = [inline_text(cell) for cell in table.header] - rows = [[inline_text(cell) for cell in row] for row in table.rows] - lines = ["| " + " | ".join(header) + " |", "|" + "|".join("---" for _ in header) + "|"] - lines += ["| " + " | ".join(row) + " |" for row in rows] + """A pipe table, unpadded. ``|`` in a cell is escaped so a cell stays + a cell; the separator row marks each column's alignment.""" + def cell(inlines: tuple[Inline, ...]) -> str: + return inline_text(inlines).replace("|", "\\|") + + rule = {"left": ":---", "center": ":---:", "right": "---:"} + align = table.align + ("",) * (len(table.header) - len(table.align)) + lines = ["| " + " | ".join(cell(c) for c in table.header) + " |", + "|" + "|".join(rule.get(a, "---") for a in align) + "|"] + lines += ["| " + " | ".join(cell(c) for c in row) + " |" for row in table.rows] return "\n".join(lines) def signature(self, signature: SignatureParty, labels: dict[str, str], colon: str) -> str: @@ -209,7 +239,9 @@ def signature(self, signature: SignatureParty, labels: dict[str, str], colon: st def _prefix(text: str, prefix: str, *, hanging: str | None = None) -> str: """Every line of *text* prefixed; with *hanging*, lines after the first - get *prefix* plus *hanging* instead.""" + get *prefix* plus *hanging* instead. A blank line gets the prefix with its + trailing space trimmed: ``>`` keeps a quote one quote, an indent leaves the + line empty.""" lines = text.split("\n") rest = prefix + hanging if hanging is not None else prefix - return "\n".join([prefix + lines[0]] + [(rest + line) if line else line for line in lines[1:]]) + return "\n".join([prefix + lines[0]] + [(rest + line) if line else rest.rstrip() for line in lines[1:]]) diff --git a/tests/golden/features.continental.txt b/tests/golden/features.continental.txt index 0299a7e..38d3d91 100644 --- a/tests/golden/features.continental.txt +++ b/tests/golden/features.continental.txt @@ -47,7 +47,7 @@ This Services Agreement (this Agreement) is made between Acme and Beta Industrie (1) The Client shall pay $10,000.00 per month, and €1,234.50 for setup, within 30 days of July 1, 2026. The Provider's VAT number is DE123456789. | Service | Fee | Notes | -|---|---|---| +|:---|---:|:---:| | Hosting | $500.00 | per month | | Support | $250.75 | see 3 | @@ -65,13 +65,15 @@ This Services Agreement (this Agreement) is made between Acme and Beta Industrie (2) Raw HTML is dropped. A safe link stays; an unsafe one does not. ---- - Attachments +--- + Schedule A: Service Description File: attachments/service-description.lgd +--- + Exhibit 1: Price List File: attachments/price-list.pdf diff --git a/tests/golden/features.default.txt b/tests/golden/features.default.txt index 4610d64..2f88b4a 100644 --- a/tests/golden/features.default.txt +++ b/tests/golden/features.default.txt @@ -47,7 +47,7 @@ A change under item 2.1(b)(i) or step 2.1(2) is made in writing under Section 2. The Client shall pay $10,000.00 per month, and €1,234.50 for setup, within 30 days of July 1, 2026. The Provider's VAT number is DE123456789. | Service | Fee | Notes | -|---|---|---| +|:---|---:|:---:| | Hosting | $500.00 | per month | | Support | $250.75 | see 3 | @@ -65,13 +65,15 @@ These are deliberately wrong: [BROKEN REF: nowhere], [UNDEFINED: nothing], [INVA Raw HTML is dropped. A safe link stays; an unsafe one does not. ---- - Attachments +--- + Schedule A: Service Description File: attachments/service-description.lgd +--- + Exhibit 1: Price List File: attachments/price-list.pdf diff --git a/tests/golden/features.outline.txt b/tests/golden/features.outline.txt index 52d599e..d28ca11 100644 --- a/tests/golden/features.outline.txt +++ b/tests/golden/features.outline.txt @@ -47,7 +47,7 @@ III. Fees The Client shall pay $10,000.00 per month, and €1,234.50 for setup, within 30 days of July 1, 2026. The Provider's VAT number is DE123456789. | Service | Fee | Notes | -|---|---|---| +|:---|---:|:---:| | Hosting | $500.00 | per month | | Support | $250.75 | see III | @@ -65,13 +65,15 @@ These are deliberately wrong: [BROKEN REF: nowhere], [UNDEFINED: nothing], [INVA Raw HTML is dropped. A safe link stays; an unsafe one does not. ---- - Attachments +--- + Schedule A: Service Description File: attachments/service-description.lgd +--- + Exhibit 1: Price List File: attachments/price-list.pdf diff --git a/tests/test_render.py b/tests/test_render.py index 8aac3c0..37a0676 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -766,7 +766,8 @@ def test_the_first_definition_in_document_order_keeps_the_anchor() -> None: def test_nested_lists_line_up_with_their_item_s_text() -> None: output = text("# A\n\n1. one\n\n more\n\n - child\n") - assert "1. one\n more\n (i) child" in output + # Blocks of an item are set apart (W5); a nested list stays tight under its paragraph. + assert "1. one\n\n more\n (i) child" in output def _nested_list(levels: int) -> str: @@ -978,7 +979,7 @@ def test_a_later_paragraph_in_a_list_item_is_the_item_s() -> None: # its marker is misplaced (§5.7) and a reference to it is broken, as the # validator reports. result = render(FRONT + "# A\n\n- item one\n\n second {#sec}\n\nSee {{ref: sec}}.\n", format="text") - assert "(a) item one\n second" in result.output + assert "(a) item one\n\n second" in result.output assert "[BROKEN REF: sec]" in result.output assert {"anchor-misplaced", "ref-broken"} <= {d.rule for d in result.diagnostics} @@ -1080,7 +1081,8 @@ def test_template_decision_is_the_validators() -> None: def test_a_heading_in_a_quote_or_item_is_a_bold_line_not_a_section() -> None: result = render(FRONT + "# A\n\n> # Inner\n> text\n\n- ## Item heading\n\n> title: x\n> ---\n> after\n", format="text") - assert "\n\n> Inner\n\n> text\n\n(a) Item heading\n\n> title: x\n\n> after\n" in result.output + # One quote stays one quote: its blank line keeps the `>` (W5). + assert "\n\n> Inner\n>\n> text\n\n(a) Item heading\n\n> title: x\n>\n> after\n" in result.output assert [s.designation for s in result.tree.sections] == ["1"] html = render(FRONT + "# A\n\n> # Inner\n", standalone=False).output assert "Inner" in html diff --git a/tests/test_wp5_text.py b/tests/test_wp5_text.py new file mode 100644 index 0000000..b6c37d9 --- /dev/null +++ b/tests/test_wp5_text.py @@ -0,0 +1,150 @@ +"""Text writer fidelity (WP5): attachment separators (W1), table cells and +alignment (W4), quotes and list items (W5), list indent (W6), blank labels (W7).""" +from __future__ import annotations + +import pytest + +from legaldown_render import render +from legaldown_render.tree import Table, Text +from legaldown_render.writers import TextWriter + +FRONT = """--- +title: T +sides: + - name: providers + label: Provider + parties: + - name: acme + label: Acme + type: legal_entity + legal_name: Acme Corporation + identification_number: DE-1 +attachments: + - id: sch-a + title: Schedule A + file: a.lgd + - id: sch-b + title: Schedule B + file: b.pdf +effective_date: 2026-06-01 +questions: + - id: forum + type: choice + options: [courts, arb] +--- +""" + + +def text(body: str, **overrides: object) -> str: + return render(FRONT + body, format="text", overrides=overrides).output + + +# -- W1: attachments.separator (§13.8) ------------------------------------------- + + +@pytest.mark.parametrize("separator", ["rule", "page-break"]) +def test_a_separator_precedes_each_attachment(separator: str) -> None: + output = text("# A\n\nBody.\n", **{"attachments.separator": separator}) + assert "Attachments\n\n---\n\nSchedule A\nFile: a.lgd\n\n---\n\nSchedule B\nFile: b.pdf\n" in output + + +def test_none_prints_no_separator_between_or_before_attachments() -> None: + output = text("# A\n\nBody.\n", **{"attachments.separator": "none"}) + assert "Body.\n\nAttachments\n\nSchedule A\nFile: a.lgd\n\nSchedule B\nFile: b.pdf\n" in output + # The one `---` left is the one before the signatures. + assert output.count("---") == 1 + assert output.index("---") > output.index("Schedule B") + + +def test_the_default_style_separates_attachments() -> None: + assert text("# A\n\nBody.\n").count("---") >= 2 + + +def test_the_writer_needs_no_style_and_defaults_to_the_default_styles_separator() -> None: + assert TextWriter().attachment_separator == "page-break" + result = render(FRONT + "# A\n\nBody.\n", format="text") + assert TextWriter().write(result.tree) == result.output + + +# -- W4: tables ------------------------------------------------------------------ + + +def test_a_pipe_in_a_cell_is_escaped() -> None: + output = text("# A\n\n| a | b |\n|---|---|\n| x \\| y | z |\n") + assert "| a | b |\n|---|---|\n| x \\| y | z |" in output + + +def test_alignment_survives_in_the_separator_row() -> None: + output = text("# A\n\n| a | b | c | d |\n|:--|:-:|--:|---|\n| 1 | 2 | 3 | 4 |\n") + assert "| a | b | c | d |\n|:---|:---:|---:|---|\n| 1 | 2 | 3 | 4 |" in output + + +def test_a_table_with_fewer_alignments_than_columns_still_prints() -> None: + cell = (Text("x"),) + table = Table((cell, cell), ((cell, cell),), ("right",)) + assert TextWriter().table(table) == "| x | x |\n|---:|---|\n| x | x |" + + +# -- W5: quotes and list items --------------------------------------------------- + + +def test_one_quote_stays_one_quote() -> None: + output = text("# A\n\n> first\n>\n> second\n") + assert "> first\n>\n> second\n" in output + + +def test_nested_quotes_keep_their_markers_on_blank_lines() -> None: + output = text("# A\n\n> outer\n>\n> > inner one\n> >\n> > inner two\n>\n> back\n") + assert "> outer\n>\n> > inner one\n> >\n> > inner two\n>\n> back\n" in output + + +def test_a_quote_in_a_list_item_stays_one_quote() -> None: + output = text("# A\n\n- item\n\n > one\n >\n > two\n") + assert "(a) item\n\n > one\n >\n > two\n" in output + + +def test_blocks_of_a_list_item_are_separated() -> None: + output = text("# A\n\n1. one\n\n more\n\n and more\n2. two\n") + assert "1. one\n\n more\n\n and more\n2. two\n" in output + assert " \n" not in output # blank lines carry no trailing space + + +def test_a_nested_list_stays_tight_under_its_paragraph() -> None: + output = text("# A\n\n1. one\n\n more\n\n - child\n - child two\n") + assert "1. one\n\n more\n (i) child\n (ii) child two\n" in output + + +def test_a_paragraph_after_a_nested_list_is_separated() -> None: + output = text("# A\n\n1. one\n - child\n\n after\n") + assert "1. one\n (i) child\n\n after\n" in output + + +# -- W6: list indent ------------------------------------------------------------- + + +def test_a_conditional_item_indents_by_its_label_not_the_condition_label() -> None: + body = ("# A\n\n- first\n- by courts {#i when=forum:courts}\n\n nested\n\n - deep\n" + "- by arbitration {#i when=forum:arb}\n- last\n") + output = text(body) + assert "(b) [Only if: forum:courts] by courts\n\n nested\n (i) deep\n" in output + + +# -- W7: blank labels ------------------------------------------------------------ + + +def test_a_blanked_label_drops_its_prefix_in_party_details() -> None: + output = text("# A\n\nBody.\n", **{"labels.identification_number": ""}) + assert "\n DE-1\n" in output + assert ": DE-1" not in output + + +def test_a_blanked_title_block_label_drops_its_prefix() -> None: + output = text("# A\n\nBody.\n", **{"labels.effective_date": ""}) + assert "\nJune 1, 2026\n" in output + assert ": June 1, 2026" not in output + + +def test_a_label_still_prefixes_its_value() -> None: + output = text("# A\n\nBody.\n") + assert "Registration No.: DE-1" in output + assert "Effective date: June 1, 2026" in output From 8579abdbb43efd1c12e8d79134fa869c64df6105 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:55:16 +0000 Subject: [PATCH 013/118] {{term:}} resolves the term as the definition does: a {{ref:}} or placeholder inside it is kept The term text of a {{term:}} was collected in Resolver._survey with a text-only pass, before sections and anchors were registered and before inconsistent placeholder types were known: a {{ref:}} in the term was dropped ("Fee per") and a placeholder used with conflicting types showed a blank. _survey now records only the first DefinitionSource per id; Resolver._term_text resolves its children during inline resolution (all targets known), takes plain text and caches it, under the existing _terms_open cycle guard. It restores used_anchors, so a definition nested in a term keeps its document-order anchor; before targets are known (a none-scheme heading) the text-only pass stands in, uncached. _warn and _not_relative deduplicate, so no repeated diagnostics. Docs updated. No existing test or golden changed. --- CONFORMANCE.md | 3 +- docs/architecture.md | 2 +- src/legaldown_render/resolve/resolver.py | 63 +++++++++++---- tests/test_wp9_review_fixes.py | 98 ++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 17 deletions(-) create mode 100644 tests/test_wp9_review_fixes.py diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 4c7ead4..ac3dc4e 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -68,7 +68,8 @@ It is checked against the specification's own examples and fixtures corpus (`tes no content at all keeps its place (CommonMark), and so does an empty drafting note, whose label is the template construct (§15.6). - **`{{term:}}` text.** A `{{term:}}` shows the term as rendered at its first definition in the - body, with its Markdown resolved (`"A *b* c" {{def: abc}}` is "A b c"), not the term's raw + body, with its Markdown, `{{ref:}}`s and placeholders resolved as at the definition + (`"A *b* c" {{def: abc}}` is "A b c", `"Fee per {{ref: h}}"` is "Fee per 1"), not the term's raw source; `label=` overrides it. A definition not in the body (a template's other alternative) falls back to the validator's term text. A term that refers to itself, directly or through other terms (`"A {{term: a}}" {{def: a}}`), is diff --git a/docs/architecture.md b/docs/architecture.md index a3e9d25..45fbd2e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -140,7 +140,7 @@ Guessing at lost structure was tried, and it traded each gap for new bugs. | Directive | Resolves to | Failure markers | |---|---|---| | `{{ref:}}` | `CrossRef(designation, anchor)` (§13.3). A ref to an unnumbered item or paragraph falls back to its section's designation, with `ref-not-enumerated` | `[BROKEN REF: id]` | -| `{{term:}}` | `TermRef(label or the definition's rendered term text, def anchor)` (§7.3, §13.4); the text is collected in `Resolver._survey` | `[UNDEFINED: id]` | +| `{{term:}}` | `TermRef(label or the definition's rendered term text, def anchor)` (§7.3, §13.4); the text is the first definition's term resolved as at the definition, worked out on first use by `Resolver._term_text` (`_survey` only records which definition is first) | `[UNDEFINED: id]` | | `"Term" {{def:}}` | `DefinedTerm` without quotation marks, anchored at `def:` (§7.2) | n/a | | `{{date:}}` `{{money:}}` `{{duration:}}` | `Value`, formatted for the locale by Babel (ADR 0005): amounts and durations keep every written digit, and a duration without a pattern in the locale falls back to another length, the language alone, then English with `render-locale-fallback` | `[INVALID DATE: …]`, `[INVALID AMOUNT: …]`, `[UNKNOWN CURRENCY: …]`, `[INVALID DURATION: …]`, `[INVALID DURATION UNIT: …]` | | `{{party:}}` `{{side:}}` | `Value(label or display name)` (§3.6) | `[INVALID PARTY: …]`, `[UNKNOWN PARTY: …]`, and the side forms | diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index d542d24..748e907 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -151,13 +151,18 @@ def __init__( questions = self.metadata.questions self.questions: dict[str, Any] = questions if isinstance(questions, dict) else {} self.inconsistent_placeholders: set[str] = set() - # A definition id -> the plain text of its term as rendered (Markdown - # resolved), from the first definition in the body (see _survey). + # A definition id -> the first {{def:}} of it in the body (see _survey), + # and -> the plain text of its term as rendered there, worked out the + # first time a {{term:}} needs it (see _term_text). + self.term_sources: dict[str, DefinitionSource] = {} self.term_texts: dict[str, str] = {} # The definitions whose term text is being worked out, so that a # {{term:}} inside a term, directly or through other terms, cannot # send the computation round in a circle. self._terms_open: list[str] = [] + # False until every target is registered and the placeholder types are + # known (run() sets it just before the inlines are resolved). + self._targets_known = False self.attachment_anchors: dict[str, str | None] = {} self.section_presences: list[Presence] = [] @@ -172,6 +177,7 @@ def run(self) -> RenderTree: replace(section, blocks=self._structure_section(section, presence)) for section, presence in zip(sections, self.section_presences, strict=True) ) + self._targets_known = True resolve = self._resolve_inlines # In document order, so that the first of several definitions of a # term is the one that keeps its anchor. @@ -249,22 +255,15 @@ def _settle_term_target(self, inline: Inline) -> Inline: def _survey(self) -> None: """One pass over the tree before resolving: which placeholder ids - are used with conflicting types (§10.7), and what each defined term - reads as, for ``{{term:}}`` (§7.3).""" + are used with conflicting types (§10.7), and which definition each + defined term reads as, for ``{{term:}}`` (§7.3): the term's own text is worked out + later, by :meth:`_term_text`, when its references can be resolved.""" types: dict[str, set[str]] = {} for inline in iter_tree_inlines(self.tree): if isinstance(inline, DefinitionSource): # The first definition of an id is the one the validator's - # lookup and the anchor keep. The term is the text the reader - # sees at the definition: its Markdown is resolved, not the - # validator's raw ``anchor.term``. - identifier = self._definition_id(inline) - if identifier not in self.term_texts: - self._terms_open.append(identifier) - try: - self.term_texts[identifier] = self._title_text(inline.children) - finally: - self._terms_open.pop() + # lookup and the anchor keep. + self.term_sources.setdefault(self._definition_id(inline), inline) elif isinstance(inline, DirectiveSource): directive = inline.directive if directive.name == "placeholder" and directive.positional and not directive.malformed: @@ -626,11 +625,45 @@ def _term(self, directive: Directive, definition_id: str) -> Inline: # The term as rendered at its definition; the validator's own # text (an empty term reads as its id, a definition not in the # body) is the fallback. - text = label or self.term_texts.get(definition_id) or self._plain_open(definition_id, term) + text = label or self._term_text(definition_id) or self._plain_open(definition_id, term) # Settled once everything is resolved (see _settle_term_target). target = DEFINITION_ANCHOR_PREFIX + definition_id return TermRef(text, target, self.style.definitions.term_style) + def _term_text(self, definition_id: str) -> str: + """The plain text of the term at the first definition of *definition_id* + in the body, as the reader sees it there: its Markdown, references, + placeholders and other terms resolved exactly as at the definition + (§7.3), not the validator's raw ``anchor.term``. Empty when the + definition is not in the body. + + Worked out the first time it is needed and kept, which is once every + target is registered and the placeholder types are known. Before that + (a heading read as text by the ``none`` scheme, §13.3) a text-only pass + stands in, which leaves references out and is not kept. Resolving the + term gives out no anchor: a definition inside a term keeps its place in + document order. The definition is open meanwhile, so a ``{{term:}}`` + inside it, directly or through others, shows its label or id and goes + no further.""" + source = self.term_sources.get(definition_id) + if source is None: + return "" + if definition_id in self.term_texts: + return self.term_texts[definition_id] + self._terms_open.append(definition_id) + try: + if not self._targets_known: + return self._title_text(source.children) + anchors = set(self.used_anchors) + try: + text = plain_text(self._resolve_inlines(source.children)) + finally: + self.used_anchors = anchors + finally: + self._terms_open.pop() + self.term_texts[definition_id] = text + return text + def _plain_open(self, definition_id: str, term: str) -> str: """The validator's *term* text of a definition as plain text, with the definition marked open while its directives resolve.""" diff --git a/tests/test_wp9_review_fixes.py b/tests/test_wp9_review_fixes.py new file mode 100644 index 0000000..b621028 --- /dev/null +++ b/tests/test_wp9_review_fixes.py @@ -0,0 +1,98 @@ +"""Review fixes after wave 1 (WP9): a ``{{ref:}}`` or placeholder inside a +defined term as ``{{term:}}`` shows it, a fraction beyond Python's int/str digit +limit renders, ``-o`` rejects only the extensions of formats the renderer does +not write, in-process ``main()`` leaves the streams alone, and output has LF +line endings.""" +from __future__ import annotations + +import io +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from conftest import DOCUMENTS + +from legaldown_render import render +from legaldown_render.cli import main + +SRC = Path(__file__).resolve().parents[1] / "src" +FEATURES = str(DOCUMENTS / "features.lgd") +FRONT = "---\ntitle: T\n---\n\n" + + +def text(body: str, **settings: object) -> str: + return render(FRONT + body, format="text", **settings).output + + +# -- 1: {{term:}} shows what the definition shows ------------------------------- + + +def test_a_term_reference_keeps_a_ref_inside_the_term() -> None: + output = text('# H {#h}\n\n"Fee per {{ref: h}}" {{def: a}}\n\n{{term: a}}\n') + assert output.count("Fee per 1") == 2 + + +def test_a_term_reference_before_the_heading_keeps_the_ref() -> None: + output = text('{{term: a}}\n\n# H {#h}\n\n"Fee per {{ref: h}}" {{def: a}}\n') + assert output.count("Fee per 1") == 2 + + +def test_a_term_reference_before_its_definition_and_its_target() -> None: + output = text('# First\n\nSee {{term: a}}.\n\n# H {#h}\n\n"Fee per {{ref: h}}" {{def: a}}\n') + assert "See Fee per 2." in output + + +def test_a_term_reference_keeps_a_ref_to_a_subsection() -> None: + body = '# A\n\n"Fee per {{ref: s}}" {{def: a}}\n\n{{term: a}}\n\n## Sub {#s}\n\nx\n' + output = text(body) + assert output.count("Fee per 1.1") == 2 + + +def test_a_term_reference_in_a_textual_heading_and_in_the_body() -> None: + # The none scheme reads a heading as text before every target is known; the + # term text must not be fixed from that early pass. + body = '# Fee {{term: a}}\n\n"Fee per {{ref: h}}" {{def: a}}\n\n# H {#h}\n\n{{term: a}}\n' + output = render(FRONT + body, format="text", overrides={"numbering.scheme": "none"}).output + assert output.rstrip().endswith("Fee per H") + assert "[BROKEN REF" not in output + + +def test_a_term_reference_shows_the_placeholder_as_the_definition_does() -> None: + body = '# A\n\n"Fee {{placeholder: p}}" {{def: a}} {{term: a}} {{placeholder: p, type=money}}\n' + result = render(FRONT + body, format="text") + line = next(row for row in result.output.splitlines() if "INVALID PLACEHOLDER" in row) + assert line.count("Fee [INVALID PLACEHOLDER: p]") == 2 + assert "[_____]" not in result.output + + +def test_a_definition_that_defines_a_term_twice_uses_the_first() -> None: + body = '# H {#h}\n\n"First {{ref: h}}" {{def: t}}\n\n"Second" {{def: t}}\n\n{{term: t}}\n' + assert text(body).rstrip().endswith("First 1") + + +def test_a_term_with_a_ref_adds_no_duplicate_diagnostics() -> None: + # The term holds an image with a remote path: one report however often it is resolved. + body = '# H {#h}\n\n"Fee ![x](http://e.com/i.png) {{ref: h}}" {{def: a}}\n\n{{term: a}} {{term: a}}\n' + diagnostics = render(FRONT + body, format="text").diagnostics + keys = [(d.rule, d.message) for d in diagnostics] + assert len(keys) == len(set(keys)) + assert sum(d.rule == "path-not-relative" for d in diagnostics) == 1 + + +def test_a_term_that_holds_a_definition_does_not_take_its_anchor() -> None: + body = '# A\n\nSee {{term: outer}}.\n\n"Outer "Inner" {{def: inner}}" {{def: outer}}\n\n{{term: inner}}\n' + html = render(FRONT + body, format="html").output + assert html.count('id="def:inner"') == 1 + assert html.count('id="def:outer"') == 1 + + +@pytest.mark.parametrize("output_format", ["text", "html"]) +def test_cycles_through_refs_still_terminate(output_format: str) -> None: + body = ('# H {#h}\n\n"A {{term: b}} {{ref: h}}" {{def: a}} x.\n\n"B {{term: a}}" {{def: b}} y.\n\n' + '"C {{term: c}}" {{def: c}}\n\nSee {{term: a}}, {{term: b}}, {{term: c}}.\n') + out = render(FRONT + body, format=output_format).output + assert "{{" not in out + # Which term shows what inside a cycle follows the order they are first needed in. + assert "A b 1" in out and "C c" in out From 2f0cceb581aef2290afb8960ad08de0bfda57b7f Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:55:21 +0000 Subject: [PATCH 014/118] Preamble labels and print page breaks (R5, W2) R5: a list item in the preamble has no section, so `{section}` was filled with "" and the continental label read ".1" while its designation was "1". `numbering.fill` now removes `{section}` and one following "." when the section is given as the empty string, so the label is "1". Items in a section keep "1.1". The text and HTML output of a preamble list change (".1 first pre" becomes "1 first pre"); no golden contains one. W2: the print stylesheet broke before each `.ld-attachment`, so the "Attachments" heading was stranded at the foot of the previous page. The break now goes before the `.ld-attachments` container (its heading goes with the first attachment), before each later attachment, and the heading has `break-after: avoid`. Goldens features.default.html and template.default.html changed only in those three CSS lines (`--update-golden`, hunks inspected). Tests: tests/test_wp7_small.py. Docs: CONFORMANCE.md (preamble label) and docs/style-templates.md (`{section}` in the preamble). --- CONFORMANCE.md | 2 +- docs/style-templates.md | 8 +- src/legaldown_render/resolve/numbering.py | 8 +- src/legaldown_render/writers/html.py | 4 +- tests/golden/features.default.html | 4 +- tests/golden/template.default.html | 4 +- tests/test_wp7_small.py | 106 ++++++++++++++++++++++ 7 files changed, 128 insertions(+), 8 deletions(-) create mode 100644 tests/test_wp7_small.py diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 4c7ead4..e9be48f 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -31,7 +31,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | Feature | Status | |---|---| | List enumeration (§13.2) | ✅ Configurable per depth; can be disabled. Ordered lists are always renumbered. Nested lists keep their structure, so a nested item is referenced as "2.1(b)(i)" | -| Paragraph numbering and section-qualified items (§13.2) | ✅ Style settings; the `continental` style uses both | +| Paragraph numbering and section-qualified items (§13.2) | ✅ Style settings; the `continental` style uses both. A preamble item has no section, so its label is "1", not ".1", as its designation is (`{section}` and the `.` after it are dropped) | | Style templates in a separate file (§13.7) | ✅ YAML, layered and validated; see [docs/style-templates.md](docs/style-templates.md) | | Signature blocks (§2.2) | ✅ One per party, with `legal_name` and each representative | | Labels following the document language (§13.7) | ✅ Built in for `en`, `cs`, `de`, `fr`, `pl`, and `sk`; any label can be set by a style | diff --git a/docs/style-templates.md b/docs/style-templates.md index ac76ff4..507e922 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -150,9 +150,11 @@ Headings, list items, and paragraphs are all numbered with the same building blo | `ref` | This level's piece of a **designation**, the text `{{ref:}}` renders | Both templates can use `{n}` (this level's counter), `{path}` (the full designation up to this -level), and `{section}` (the containing section's designation, for items and paragraphs). For -example, decimal headings use `label: "{path}"` and `ref: ".{n}"`. Legal enumeration uses -`label: "({n})"` and `ref: "({n})"`, so the second item of Section 2.3 is shown as "(b)" and +level), and `{section}` (the containing section's designation, for items and paragraphs). In the +preamble, which has no section, `{section}` is empty and it is removed together with the `.` after +it, so `label: "{section}.{n}"` shows "1", not ".1". For example, decimal headings use +`label: "{path}"` and `ref: ".{n}"`. Legal enumeration uses `label: "({n})"` and `ref: "({n})"`, so the +second item of Section 2.3 is shown as "(b)" and referenced as "2.3(b)". Under the `none` scheme, designations are heading text, and item pieces are set off by a space ("Termination (a)", §13.3). diff --git a/src/legaldown_render/resolve/numbering.py b/src/legaldown_render/resolve/numbering.py index 61925fa..de95a8e 100644 --- a/src/legaldown_render/resolve/numbering.py +++ b/src/legaldown_render/resolve/numbering.py @@ -71,11 +71,17 @@ def heading_levels(numbering: Numbering) -> tuple[LevelFormat, ...]: _FIELD_RE = re.compile(r"\{(n|path|section)\}") +_EMPTY_SECTION_RE = re.compile(r"\{section\}\.") def fill(template: str, **values: str) -> str: """*template* with ``{n}``, ``{path}``, and ``{section}`` replaced. Any - other braces are kept as written.""" + other braces are kept as written. When *section* is given as the empty + string (a preamble item has no section, §4.4), ``{section}`` and ONE + immediately following ``.`` are removed first, so "{section}.{n}" gives + "1", the same as the item's designation.""" + if values.get("section") == "": + template = _EMPTY_SECTION_RE.sub("", template) return _FIELD_RE.sub(lambda match: values.get(match.group(1), ""), template) diff --git a/src/legaldown_render/writers/html.py b/src/legaldown_render/writers/html.py index bf04ff2..01a440d 100644 --- a/src/legaldown_render/writers/html.py +++ b/src/legaldown_render/writers/html.py @@ -414,7 +414,9 @@ def inline(self, inline: Inline) -> str: .ld-signature-line { border-top: 1px solid currentColor; width: 80%; margin-bottom: 0.3em; } @media print { .ld-document { max-width: none; padding: 0; } - .ld-separator-page-break .ld-attachment { break-before: page; } + .ld-attachments.ld-separator-page-break { break-before: page; } + .ld-separator-page-break .ld-attachment + .ld-attachment { break-before: page; } + .ld-attachments-heading { break-after: avoid; } .ld-signatures { break-inside: avoid; } a { color: inherit; } } diff --git a/tests/golden/features.default.html b/tests/golden/features.default.html index a6610a2..a7a45a5 100644 --- a/tests/golden/features.default.html +++ b/tests/golden/features.default.html @@ -58,7 +58,9 @@ .ld-signature-line { border-top: 1px solid currentColor; width: 80%; margin-bottom: 0.3em; } @media print { .ld-document { max-width: none; padding: 0; } - .ld-separator-page-break .ld-attachment { break-before: page; } + .ld-attachments.ld-separator-page-break { break-before: page; } + .ld-separator-page-break .ld-attachment + .ld-attachment { break-before: page; } + .ld-attachments-heading { break-after: avoid; } .ld-signatures { break-inside: avoid; } a { color: inherit; } } diff --git a/tests/golden/template.default.html b/tests/golden/template.default.html index 75099b2..c789a6f 100644 --- a/tests/golden/template.default.html +++ b/tests/golden/template.default.html @@ -58,7 +58,9 @@ .ld-signature-line { border-top: 1px solid currentColor; width: 80%; margin-bottom: 0.3em; } @media print { .ld-document { max-width: none; padding: 0; } - .ld-separator-page-break .ld-attachment { break-before: page; } + .ld-attachments.ld-separator-page-break { break-before: page; } + .ld-separator-page-break .ld-attachment + .ld-attachment { break-before: page; } + .ld-attachments-heading { break-after: avoid; } .ld-signatures { break-inside: avoid; } a { color: inherit; } } diff --git a/tests/test_wp7_small.py b/tests/test_wp7_small.py new file mode 100644 index 0000000..1df7b50 --- /dev/null +++ b/tests/test_wp7_small.py @@ -0,0 +1,106 @@ +"""Small fixes (WP7): a label filled with an empty ``{section}`` drops the field +and its dot, so a preamble item reads "1", not ".1" (R5); the print stylesheet +breaks before the attachments container, not inside it (W2).""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +from legaldown_render import render +from legaldown_render.resolve.numbering import fill + +FRONT = "---\ntitle: T\n---\n" +PREAMBLE_AND_SECTION = ( + "\nPreamble.\n\n- first pre\n- second pre\n\n# A {#a}\n\n- in a\n" +) + + +def text(body: str, **settings: object) -> str: + return render(FRONT + body, format="text", **settings).output + + +def html(body: str, **settings: object) -> str: + return render(FRONT + body, format="html", **settings).output + + +# -- R5: an empty section drops its dot (numbering.fill) ----------------------------------------- + +def test_fill_drops_an_empty_section_with_one_following_dot() -> None: + assert fill("{section}.{n}", n="1", section="") == "1" + assert fill("{section}.{n}", n="1", section="2") == "2.1" + # Only ONE dot goes with the field. + assert fill("{section}..{n}", n="1", section="") == ".1" + + +def test_fill_keeps_other_text_of_an_empty_section_as_written() -> None: + # No dot right after the field: the field alone is removed, as before. + assert fill("({section}){n}", n="a", section="") == "()a" + assert fill("{section}", n="1", section="") == "" + + +def test_fill_without_a_section_value_keeps_the_dot() -> None: + # The rule applies only when the section value is given and empty. + assert fill("{section}.{n}", n="1") == ".1" + + +# -- R5: preamble list items agree with their designation ---------------------------------------- + +def test_a_preamble_list_is_numbered_without_a_leading_dot_in_text() -> None: + output = text(PREAMBLE_AND_SECTION, style="continental") + assert "1 first pre" in output + assert "2 second pre" in output + assert not any(line.startswith(".") for line in output.splitlines()) + + +def test_a_preamble_list_is_numbered_without_a_leading_dot_in_html() -> None: + output = html(PREAMBLE_AND_SECTION, style="continental") + assert '1

first pre

' in output + assert '2

second pre

' in output + assert 'class="ld-label">.1<' not in output + + +def test_a_list_in_a_section_keeps_its_section_prefix() -> None: + output = text(PREAMBLE_AND_SECTION, style="continental") + assert "1.1 in a" in output + html_output = html(PREAMBLE_AND_SECTION, style="continental") + assert '1.1

in a

' in html_output + + +# -- W2: print breaks before the attachments container -------------------------------------------- + +SRC = Path(__file__).resolve().parents[1] / "src" +ATTACHED = ( + "---\ntitle: T\nattachments:\n" + " - id: one\n title: First attachment\n file: one.pdf\n" + " - id: two\n title: Second attachment\n file: two.pdf\n" + "---\n\n# A {#a}\n\nBody.\n" +) + + +def _cli(*args: str, stdin: str) -> subprocess.CompletedProcess[str]: + env = dict(os.environ, PYTHONPATH=str(SRC), PYTHONUTF8="1") + return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *args], input=stdin, + capture_output=True, text=True, env=env, check=False) + + +def test_a_standalone_page_break_render_has_the_attachment_break_rules() -> None: + done = _cli("-", "--set", "attachments.separator=page-break", stdin=ATTACHED) + assert done.returncode == 0, done.stderr + assert "" in done.stdout + assert 'class="ld-attachments ld-separator-page-break"' in done.stdout + # The break goes before the container (heading included), then before each later attachment. + assert ".ld-attachments.ld-separator-page-break { break-before: page; }" in done.stdout + assert ".ld-separator-page-break .ld-attachment + .ld-attachment { break-before: page; }" in done.stdout + assert ".ld-attachments-heading { break-after: avoid; }" in done.stdout + # The old rule broke before every attachment, the first one included, inside the container. + assert ".ld-separator-page-break .ld-attachment { break-before: page; }" not in done.stdout + + +def test_without_a_page_break_separator_the_container_is_not_broken() -> None: + done = _cli("-", "--set", "attachments.separator=none", stdin=ATTACHED) + assert done.returncode == 0, done.stderr + # The rules are scoped by the separator class, which no element carries here. + assert 'class="ld-attachments ld-separator-none"' in done.stdout + assert 'ld-separator-page-break"' not in done.stdout From f9a42bb7454ef7eab9ff2aa024094ded2827323d Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:56:10 +0000 Subject: [PATCH 015/118] A fraction of more than 4300 digits renders: plural category falls back to 'other' (review: values) Babel's plural operands convert the fraction digits with int(), which Python limits to 4300 digits; {{duration:}} and name-mode {{money:}} with a longer fraction raised a raw ValueError. values._plural_category catches it and uses 'other' (the CLDR category of a non-integer in nearly every locale); name-mode money puts Babel's long-name pattern together with 'other' when its own lookup would fail. sys.set_int_max_str_digits is process-wide and is not touched. --- src/legaldown_render/resolve/values.py | 52 ++++++++++++++++++++++---- tests/test_wp9_review_fixes.py | 21 +++++++++++ 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/src/legaldown_render/resolve/values.py b/src/legaldown_render/resolve/values.py index 2e20bbc..ba94c13 100644 --- a/src/legaldown_render/resolve/values.py +++ b/src/legaldown_render/resolve/values.py @@ -20,7 +20,14 @@ from babel import Locale, UnknownLocaleError from babel.dates import format_date -from babel.numbers import format_currency, get_currency_precision, parse_pattern +from babel.numbers import ( + NumberPattern, + format_currency, + get_currency_name, + get_currency_precision, + get_currency_unit_pattern, + parse_pattern, +) from ..errors import InternalError from ..style.model import Values @@ -100,6 +107,22 @@ def _unit_patterns(locale: Locale, unit_id: str, length: str) -> dict[str, str]: return {category: pattern for category, pattern in patterns.items() if category != "per"} +def _plural_category(locale: Locale, value: Decimal) -> str: + """The CLDR plural category of *value* in *locale*. + + CLDR's plural operands of a value are its integer and fraction digits read + as integers, and Python refuses to convert a string of more than 4300 digits + to an int (``ValueError``). Such a value is a non-integer or a very long + integer whose category cannot be computed under that limit, and ``other`` is + the category CLDR gives non-integers in nearly every locale, so ``other`` it + is. The limit is not raised: ``sys.set_int_max_str_digits`` is process-wide. + """ + try: + return locale.plural_form(value) + except ValueError: + return "other" + + def _duration_pattern(locale: Locale, unit_id: str, length: str, value: Decimal) -> tuple[str, bool]: """The CLDR pattern for a duration of *value*, and whether it is English for a locale that is not. @@ -117,14 +140,14 @@ def _duration_pattern(locale: Locale, unit_id: str, length: str, value: Decimal) if language is not None and str(language) != str(locale): candidates.append(language) for candidate in candidates: - category = candidate.plural_form(value) + category = _plural_category(candidate, value) for each in lengths: patterns = _unit_patterns(candidate, unit_id, each) pattern = patterns.get(category) or patterns.get("other") if pattern: return pattern, False english = Locale("en") - category = english.plural_form(value) + category = _plural_category(english, value) for each in lengths: patterns = _unit_patterns(english, unit_id, each) pattern = patterns.get(category) or patterns.get("other") @@ -183,15 +206,30 @@ def _money(self, amount: str, currency: str) -> str: pattern = parse_pattern(source) digits = max(get_currency_precision(currency), len(amount.partition(".")[2])) pattern.frac_prec = (digits, digits) - text = format_currency(value, currency, format=pattern, locale=self.locale, currency_digits=False, - format_type="name" if mode == "name" else "standard", - decimal_quantization=False) if mode == "name": - return text + return self._money_name(value, currency, pattern) + text = format_currency(value, currency, format=pattern, locale=self.locale, currency_digits=False, + format_type="standard", decimal_quantization=False) # CLDR spaces an alphabetic symbol or code from the digits, on # whichever side it stands ("USD 10,000.00", "10 000,00 CZK"). return _LETTER_DIGIT.sub("\u00a0", text) + def _money_name(self, value: Decimal, currency: str, pattern: NumberPattern) -> str: + """*value* with the currency's name ("1,234.50 euros"), through Babel + unless the plural category of *value* cannot be computed (see + :func:`_plural_category`): Babel's own lookup of the unit pattern would + then fail, so the ``other`` pattern and name are put together here, the + way Babel does it (``_format_currency_long_name``).""" + try: + self.locale.plural_form(value) + except ValueError: + number = pattern.apply(value, self.locale, currency=currency, currency_digits=False, + decimal_quantization=False) + unit_pattern = get_currency_unit_pattern(currency, locale=self.locale) + return unit_pattern.format(number, get_currency_name(currency, count=value, locale=self.locale)) + return format_currency(value, currency, format=pattern, locale=self.locale, currency_digits=False, + format_type="name", decimal_quantization=False) + def duration(self, value: str, unit: str) -> str: """A duration (already validated) such as "12 months" or "12 měsíců".""" return self.format_duration(value, unit).text diff --git a/tests/test_wp9_review_fixes.py b/tests/test_wp9_review_fixes.py index b621028..0e1284a 100644 --- a/tests/test_wp9_review_fixes.py +++ b/tests/test_wp9_review_fixes.py @@ -96,3 +96,24 @@ def test_cycles_through_refs_still_terminate(output_format: str) -> None: assert "{{" not in out # Which term shows what inside a cycle follows the order they are first needed in. assert "A b 1" in out and "C c" in out + + +# -- 2: a fraction beyond Python's int/str digit limit --------------------------- + +LONG = "2" * 4400 + + +def test_a_duration_with_a_4400_digit_fraction_renders() -> None: + output = render(FRONT + f"# A\n\n{{{{duration: 1.{LONG}, unit=D}}}}", format="text").output + assert f"1.{LONG} days" in output + + +def test_a_money_amount_in_name_mode_with_a_4400_digit_fraction_renders() -> None: + output = render(FRONT + f"# A\n\n{{{{money: 1.{LONG}, currency=EUR}}}}", format="text", + overrides={"values.money": "name"}).output + assert f"1.{LONG} euros" in output + + +def test_a_4400_digit_fraction_uses_the_other_category_in_czech() -> None: + output = render(FRONT + f"# A\n\n{{{{duration: 1.{LONG}, unit=Y}}}}", format="text", locale="cs-CZ").output + assert output.rstrip().endswith(" let") # "other", not the "many" of "2,5 roku" From 61adf8ca6bf69ea257c718c307a5203b2909cd10 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:55:48 +0000 Subject: [PATCH 016/118] Text writer: attachment separator mirrors the HTML layout; blank signature labels (W1, W7 follow-up) W1: with rule/page-break the text writer prints "---" before the attachments heading (as it did originally) and between consecutive attachments, not between the heading and the first attachment, so the heading stays with what it heads. With none, no "---" at all. Updates the TextWriter docstring, CONFORMANCE row and docs. W7: the blank-label rule also covers the signature blocks (name, title, date, place): a blanked label prints the value alone, and a row with neither label nor value is omitted instead of a lone ":". Goldens (features.default/continental/outline.txt): the "---" moves back before the "Attachments" heading, relative to the previous commit; a "---" now also separates the two attachments. Tests in tests/test_wp5_text.py updated and extended for both changes. --- CONFORMANCE.md | 2 +- docs/architecture.md | 2 +- docs/style-templates.md | 2 +- src/legaldown_render/writers/text.py | 31 ++++++++++++++++----------- tests/golden/features.continental.txt | 4 ++-- tests/golden/features.default.txt | 4 ++-- tests/golden/features.outline.txt | 4 ++-- tests/test_wp5_text.py | 27 ++++++++++++++++++++--- 8 files changed, 52 insertions(+), 24 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index ae7a9fe..63175df 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -41,7 +41,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | Automatic term recognition (§7.4, MAY) | ❌ Not supported | | Links (§8.7) | ✅ Every link is a hyperlink in HTML and shows `text ` in plain text, whatever its scheme (`ftp:`, `sip:`, `urn:`, ...), except one whose URL is empty or uses `javascript:`, `vbscript:`, `file:`, or `data:`, which shows its text alone. A link with no text shows its URL; the text writer omits `` for an autolink | | Images and file-reference paths (§8.7, §2.3) | ✅ A relative path (`../` included, as §2.3 defines no default document root) renders `` in HTML and its alt text in plain text. A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is never fetched or embedded: the alt text is shown and the Error `path-not-relative` is reported, which `--strict` refuses. An empty path shows the alt text with no diagnostic | -| Plain-text layout (§13.6, optional) | ✅ `attachments.separator` (§13.8): `none` prints no separator, `rule` and `page-break` both print a `---` line before each attachment (text has no pages). One quote stays one quote (blank lines keep the `>`), and the blocks of a list item are set apart by blank lines and indented under the item's label, whatever its condition label. A pipe in a table cell is backslash-escaped, and the separator row marks column alignment (`:---`, `:---:`, `---:`). A label a style blanks drops its `label: ` prefix in the title block and party details; HTML keeps an empty `
` so its grid layout and the `
` stay valid | +| Plain-text layout (§13.6, optional) | ✅ `attachments.separator` (§13.8): `none` prints no `---`; `rule` and `page-break` both print one before the attachments heading and between consecutive attachments, so the heading stays with the first (text has no pages). One quote stays one quote (blank lines keep the `>`), and the blocks of a list item are set apart by blank lines and indented under the item's label, whatever its condition label. A pipe in a table cell is backslash-escaped, and the separator row marks column alignment (`:---`, `:---:`, `---:`). A label a style blanks drops its `label: ` prefix in the title block, party details and signature blocks (a signature row with neither label nor value is omitted); HTML keeps an empty `
` so its grid layout and the `
` stay valid | | Extended tables through raw HTML (§9.2, MAY) | ❌ Not supported; the HTML table is dropped with the Warning | | Visual blank for a placeholder (§13.5 rule 6) | ✅ `placeholders.blank`; a blank that is empty or only whitespace renders `[TBD: id]` | | Question prompts next to blanks (§15.8, MAY) | ✅ `placeholders.show_prompt` | diff --git a/docs/architecture.md b/docs/architecture.md index 378944b..cad1c5d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -176,7 +176,7 @@ clamped to 1–5, as the validator numbers them. | Writer | Notes | |---|---| -| `TextWriter` | Plain text (§13.6). A soft line break inside a paragraph is a space, a hard break a new line, and lists are indented with their labels, one quote stays one quote, and tables keep escaped cells and alignment. It takes the style's `attachments.separator` (default `page-break`; `rule` and `page-break` both print `---` before each attachment). It is the **test oracle** for resolution | +| `TextWriter` | Plain text (§13.6). A soft line break inside a paragraph is a space, a hard break a new line, and lists are indented with their labels, one quote stays one quote, and tables keep escaped cells and alignment. It takes the style's `attachments.separator` (default `page-break`; `rule` and `page-break` both print `---` before the attachments heading and between attachments). It is the **test oracle** for resolution | | `HtmlWriter` | A self-contained HTML5 page, or with `standalone=False` just the `
`. Semantic markup, with labels as real text rather than CSS counters, and a stylesheet generated from the style's presentation settings, including print rules | | DOCX, PDF | Planned ([ADR 0004](decisions/0004-html-first.md)) | diff --git a/docs/style-templates.md b/docs/style-templates.md index 9ce9074..d424d17 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -110,7 +110,7 @@ placeholders: title_block: { parties: true, effective_date: true, version: false } contents: { enabled: false, depth: 2, attachments: true } # table of contents after the title block; depth 1-5 # counts number levels ("2.1" is 2); attachments listed last -attachments: { render: placeholder, separator: page-break } # placeholder | omit; page-break | rule | none (plain text: `---` before each attachment unless none) +attachments: { render: placeholder, separator: page-break } # placeholder | omit; page-break | rule | none (plain text: `---` around attachments unless none) signatures: { enabled: true } # the document's include_signatures: false wins template_view: { choice_separator: " / ", empty_choice: "" } diff --git a/src/legaldown_render/writers/text.py b/src/legaldown_render/writers/text.py index 3e1a873..4ca0a81 100644 --- a/src/legaldown_render/writers/text.py +++ b/src/legaldown_render/writers/text.py @@ -99,9 +99,10 @@ class TextWriter: """Writes a resolved tree as plain text. *attachment_separator* is the style's ``attachments.separator`` (§13.8): - ``none`` prints nothing between attachments; ``rule`` and ``page-break`` - both print a ``---`` line before each attachment, as plain text has no - pages to break. The default is the default style's value. + ``none`` prints no ``---`` at all; ``rule`` and ``page-break`` both print + one before the attachments heading and between consecutive attachments, as + plain text has no pages to break (the HTML layout: the heading stays with + the first attachment). The default is the default style's value. """ extension = "txt" @@ -142,9 +143,11 @@ def write(self, tree: RenderTree) -> str: chunks.append(f"[{section.condition_label}]") chunks += [self.block(block) for block in section.blocks] if tree.attachments: + if self.attachment_separator != "none": + chunks.append("---") chunks.append(tree.attachments_label) - for attachment in tree.attachments: - if self.attachment_separator != "none": + for index, attachment in enumerate(tree.attachments): + if index and self.attachment_separator != "none": chunks.append("---") lines = [inline_text(attachment.title)] if attachment.condition_label: @@ -225,15 +228,19 @@ def cell(inlines: tuple[Inline, ...]) -> str: def signature(self, signature: SignatureParty, labels: dict[str, str], colon: str) -> str: lines = [inline_text(signature.side), inline_text(signature.name)] + + def row(key: str, value: str = "", *, strip: str | None = " ") -> list[str]: + # A blanked label leaves the value alone; a row with neither is + # omitted. Date and place keep a no-break space of the colon (their + # value is to be written in); name and title lose all trailing space. + line = _labelled(labels[key], colon, value).rstrip(strip) + return [line] if line.strip() else [] + # With no declared representative, one blank signing line. for name, title in signature.signatories or (((), ()),): - lines += [ - "", - "_" * 30, - f"{labels['name']}{colon}{inline_text(name)}".rstrip(), - f"{labels['title']}{colon}{inline_text(title)}".rstrip(), - ] - lines += [f"{labels['date']}{colon}".rstrip(" "), f"{labels['place']}{colon}".rstrip(" ")] + lines += ["", "_" * 30, *row("name", inline_text(name), strip=None), + *row("title", inline_text(title), strip=None)] + lines += [*row("date"), *row("place")] return "\n".join(lines) diff --git a/tests/golden/features.continental.txt b/tests/golden/features.continental.txt index 38d3d91..d44aedb 100644 --- a/tests/golden/features.continental.txt +++ b/tests/golden/features.continental.txt @@ -65,10 +65,10 @@ This Services Agreement (this Agreement) is made between Acme and Beta Industrie (2) Raw HTML is dropped. A safe link stays; an unsafe one does not. -Attachments - --- +Attachments + Schedule A: Service Description File: attachments/service-description.lgd diff --git a/tests/golden/features.default.txt b/tests/golden/features.default.txt index 2f88b4a..d7c8ab9 100644 --- a/tests/golden/features.default.txt +++ b/tests/golden/features.default.txt @@ -65,10 +65,10 @@ These are deliberately wrong: [BROKEN REF: nowhere], [UNDEFINED: nothing], [INVA Raw HTML is dropped. A safe link stays; an unsafe one does not. -Attachments - --- +Attachments + Schedule A: Service Description File: attachments/service-description.lgd diff --git a/tests/golden/features.outline.txt b/tests/golden/features.outline.txt index d28ca11..b8f4da2 100644 --- a/tests/golden/features.outline.txt +++ b/tests/golden/features.outline.txt @@ -65,10 +65,10 @@ These are deliberately wrong: [BROKEN REF: nowhere], [UNDEFINED: nothing], [INVA Raw HTML is dropped. A safe link stays; an unsafe one does not. -Attachments - --- +Attachments + Schedule A: Service Description File: attachments/service-description.lgd diff --git a/tests/test_wp5_text.py b/tests/test_wp5_text.py index b6c37d9..6402e4a 100644 --- a/tests/test_wp5_text.py +++ b/tests/test_wp5_text.py @@ -45,10 +45,11 @@ def text(body: str, **overrides: object) -> str: @pytest.mark.parametrize("separator", ["rule", "page-break"]) def test_a_separator_precedes_each_attachment(separator: str) -> None: output = text("# A\n\nBody.\n", **{"attachments.separator": separator}) - assert "Attachments\n\n---\n\nSchedule A\nFile: a.lgd\n\n---\n\nSchedule B\nFile: b.pdf\n" in output + # The heading stays with the first attachment; a rule falls before it and between attachments. + assert "Body.\n\n---\n\nAttachments\n\nSchedule A\nFile: a.lgd\n\n---\n\nSchedule B\nFile: b.pdf\n" in output -def test_none_prints_no_separator_between_or_before_attachments() -> None: +def test_none_prints_no_separator_around_attachments() -> None: output = text("# A\n\nBody.\n", **{"attachments.separator": "none"}) assert "Body.\n\nAttachments\n\nSchedule A\nFile: a.lgd\n\nSchedule B\nFile: b.pdf\n" in output # The one `---` left is the one before the signatures. @@ -57,7 +58,7 @@ def test_none_prints_no_separator_between_or_before_attachments() -> None: def test_the_default_style_separates_attachments() -> None: - assert text("# A\n\nBody.\n").count("---") >= 2 + assert text("# A\n\nBody.\n").count("---") == 3 # heading, between attachments, signatures def test_the_writer_needs_no_style_and_defaults_to_the_default_styles_separator() -> None: @@ -148,3 +149,23 @@ def test_a_label_still_prefixes_its_value() -> None: output = text("# A\n\nBody.\n") assert "Registration No.: DE-1" in output assert "Effective date: June 1, 2026" in output + + +def test_a_blanked_signature_label_prints_the_value_alone() -> None: + body = "# A\n\nBody.\n" + output = text(body, **{"labels.signature_name": "", "labels.signature_title": ""}) + assert "Name:" not in output and "Title:" not in output + assert "Date:\nPlace:" in output + + +def test_a_signature_row_with_no_label_and_no_value_is_omitted() -> None: + output = text("# A\n\nBody.\n", **{"labels.signature_date": "", "labels.signature_place": ""}) + assert output.endswith("______________________________\nName:\nTitle:\n") + assert "\n:" not in output + + +def test_a_blanked_signature_label_with_a_representative_shows_the_name() -> None: + source = FRONT.replace("identification_number: DE-1", "representatives:\n - name: John Smith\n" + " title: CEO") + "# A\n\nBody.\n" + output = render(source, format="text", overrides={"labels.signature_name": "", "labels.signature_title": ""}).output + assert "______________________________\nJohn Smith\nCEO\nDate:\nPlace:\n" in output From 3c92e56d59ea6b2e7c1219e7da8847761a6240a2 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:57:00 +0000 Subject: [PATCH 017/118] -o: only the extension of a format the renderer does not write is a usage error (review: cli) contract.v2, report.2025-10-27 and .out failed with exit 2 since the S6 fix. Now only .docx .doc .odt .rtf .pdf .md .markdown .epub .tex (any case) need --format; any other extension, or none, keeps the html default. -o and -f --help and the README state the exact rule. tests/test_wp2_cli.py: the S6 parametrization grows to the full list, and tests for the kept default, known extensions and help. --- README.md | 7 +++++-- src/legaldown_render/cli.py | 18 +++++++++++++----- tests/test_wp2_cli.py | 29 +++++++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1eeed6a..396d5cb 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,11 @@ read (a document that is not UTF-8 included) or the output file cannot be writte style, setting, or usage, and 70 for an internal error (a bug to report). Documents are read as UTF-8, and standard output is written as UTF-8 whatever the console's encoding. -`-o` takes its format from the file extension (`.html`, `.htm`, `.txt`, `.text`); a name with no -extension is HTML, and any other extension (`.docx`, `.pdf`, `.md`) needs an explicit `--format`. +Without `--format`, `-o` takes the format from the file extension: `.html` and `.htm` write HTML, +`.txt` and `.text` plain text. The extension of a document format this renderer does not write +(`.docx`, `.doc`, `.odt`, `.rtf`, `.pdf`, `.md`, `.markdown`, `.epub`, `.tex`, in any case) is a usage +error (exit 2) unless `--format` says what to write. Any other name, with any other extension or +none (`contract`, `contract.v2`, `report.2025-10-27`, `.out`), is just a name and writes HTML. ## Python diff --git a/src/legaldown_render/cli.py b/src/legaldown_render/cli.py index 8509c08..881b6ae 100644 --- a/src/legaldown_render/cli.py +++ b/src/legaldown_render/cli.py @@ -25,6 +25,12 @@ from .style import builtin_styles, dump_style, load_style, parse_override from .writers import FORMATS, format_for_path +#: Extensions of document formats this renderer does not write. An output file +#: named so (``x.docx``) and no ``--format`` is a mistake to refuse: HTML under +#: that name would mislead. Any other extension (``contract.v2``, +#: ``report.2025-10-27``), or none, is just a name and keeps the HTML default. +_UNWRITTEN_EXTENSIONS = frozenset({".docx", ".doc", ".odt", ".rtf", ".pdf", ".md", ".markdown", ".epub", ".tex"}) + def _setting(text: str) -> tuple[str, Any]: key, separator, value = text.partition("=") @@ -48,8 +54,12 @@ def _parser() -> argparse.ArgumentParser: "See every setting with --print-style.", ) parser.add_argument("input", nargs="?", help="the .lgd file to render, or - for standard input") - parser.add_argument("-o", "--output", help="output file (default: standard output); its extension sets the format") - parser.add_argument("-f", "--format", choices=sorted(FORMATS), help="output format (default: from --output, else html)") + parser.add_argument("-o", "--output", help="output file (default: standard output). Without --format, " + ".html/.htm write HTML and .txt/.text plain text; any other name, with or without an " + "extension, writes HTML, except the extension of a format not written (.docx, .doc, " + ".odt, .rtf, .pdf, .md, .markdown, .epub, .tex), which needs --format") + parser.add_argument("-f", "--format", choices=sorted(FORMATS), + help="output format (default: html, or text for --output *.txt/*.text)") parser.add_argument("-s", "--style", help="style template: a built-in name or a path to a .yaml file") parser.add_argument("--set", dest="overrides", action="append", default=[], type=_setting, metavar="KEY=VALUE", help="override one style setting; repeatable. Text settings take the value as " @@ -123,9 +133,7 @@ def main(argv: list[str] | None = None) -> int: if not output_format and args.output: file_name = Path(args.output).name output_format = format_for_path(file_name) - if not output_format and "." in file_name.rstrip("."): - # A name with an extension we do not write (x.docx, x.pdf, x.md): HTML under that - # name would mislead. A name without one keeps the html default. + if not output_format and Path(file_name).suffix.lower() in _UNWRITTEN_EXTENSIONS: parser.error(f"cannot tell the output format from '{args.output}'; use --format html or text") output_format = output_format or "html" name = "" if args.input == "-" else args.input diff --git a/tests/test_wp2_cli.py b/tests/test_wp2_cli.py index 949fcff..7c0c43d 100644 --- a/tests/test_wp2_cli.py +++ b/tests/test_wp2_cli.py @@ -149,9 +149,10 @@ def test_output_that_is_a_directory(tmp_path: Path, capsys: pytest.CaptureFixtur assert "cannot write" in capsys.readouterr().err -# --- S6: unknown extension ------------------------------------------------------------------- +# --- S6: an extension of a format not written ----------------------------------------------- -@pytest.mark.parametrize("name", ["x.docx", "x.pdf", "x.md", "x.DOCX"]) +@pytest.mark.parametrize("name", ["x.docx", "x.pdf", "x.md", "x.DOCX", "x.doc", "x.odt", "x.rtf", "x.markdown", + "x.epub", "x.tex", "x.v2.Pdf"]) def test_unknown_output_extension_without_format_is_a_usage_error( tmp_path: Path, capsys: pytest.CaptureFixture[str], name: str) -> None: target = tmp_path / name @@ -177,6 +178,30 @@ def test_output_without_an_extension_keeps_the_html_default(tmp_path: Path) -> N assert target.read_text(encoding="utf-8").startswith("") +@pytest.mark.parametrize("name", ["contract.v2", "report.2025-10-27", ".out", "out.", "notes.final", "x.json"]) +def test_any_other_extension_keeps_the_html_default(tmp_path: Path, name: str) -> None: + target = tmp_path / name + assert main([FEATURES, "-q", "-o", str(target)]) == 0 + assert target.read_text(encoding="utf-8").startswith("") + + +def test_a_known_extension_still_sets_the_format(tmp_path: Path) -> None: + for name, start in (("a.TXT", "Services Agreement"), ("a.text", "Services Agreement"), + ("a.htm", ""), ("a.html", "")): + target = tmp_path / name + assert main([FEATURES, "-q", "-o", str(target)]) == 0 + assert target.read_text(encoding="utf-8").startswith(start) + + +def test_the_help_states_the_output_rule(capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + main(["--help"]) + help_text = " ".join(capsys.readouterr().out.split()) + assert "its extension sets the format" not in help_text + assert "any other name, with or without an extension, writes HTML" in help_text + assert ".docx, .doc, .odt, .rtf, .pdf, .md, .markdown, .epub, .tex" in help_text + + # --- S2: error hierarchy --------------------------------------------------------------------- def test_style_error_is_a_render_error_and_still_a_value_error() -> None: From 73a845c601bf5c51533ca015f9057dbbb70a2c65 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:58:09 +0000 Subject: [PATCH 018/118] CLI: only the program reconfigures the streams; output is UTF-8 with LF everywhere (review: cli) main() reconfigured sys.stdout/sys.stderr on every call, so an in-process main([...]) changed the caller's streams for good. It now does so only when argv is None (the console script and python -m call it so). Standard output is reconfigured with newline="\n" and -o files are written with newline="\n", so no platform translates LF to CRLF. README states both. tests/test_wp2_cli.py: comments on the exact-byte comparisons (LF on every platform). New tests in tests/test_wp9_review_fixes.py. No golden changed. --- README.md | 5 ++- src/legaldown_render/cli.py | 21 ++++++--- tests/test_wp2_cli.py | 3 ++ tests/test_wp9_review_fixes.py | 79 ++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 396d5cb..d8197a1 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,10 @@ or a template that cannot be assembled with its answers) or the document or answ read (a document that is not UTF-8 included) or the output file cannot be written, 2 for an invalid style, setting, or usage, and 70 for an internal error (a bug to report). -Documents are read as UTF-8, and standard output is written as UTF-8 whatever the console's encoding. +Documents are read as UTF-8, and output is UTF-8 with LF line endings, on standard output and in +`-o` files, whatever the console's encoding and the platform's newline. The command reconfigures +the standard streams only when it runs as the program; calling `legaldown_render.cli.main([...])` +from Python leaves `sys.stdout` and `sys.stderr` as they are. Without `--format`, `-o` takes the format from the file extension: `.html` and `.htm` write HTML, `.txt` and `.text` plain text. The extension of a document format this renderer does not write (`.docx`, `.doc`, `.odt`, `.rtf`, `.pdf`, `.md`, `.markdown`, `.epub`, `.tex`, in any case) is a usage diff --git a/src/legaldown_render/cli.py b/src/legaldown_render/cli.py index 881b6ae..a7be21f 100644 --- a/src/legaldown_render/cli.py +++ b/src/legaldown_render/cli.py @@ -7,7 +7,8 @@ report). Documents and output are always UTF-8 (the HTML says ````), -never the console's or the locale's encoding. +never the console's or the locale's encoding, and output has LF line endings on +every platform. """ from __future__ import annotations @@ -85,13 +86,15 @@ def _parser() -> argparse.ArgumentParser: def _configure_streams() -> None: """Make the standard streams safe for any content, whatever the console's encoding: the document and the style are UTF-8, so what is written to - standard output is UTF-8 as well, and a diagnostic that quotes characters + standard output is UTF-8 as well, with LF line endings (no translation to + CRLF on Windows), and a diagnostic that quotes characters the console cannot show escapes them (``\\u2192``) instead of failing with a UnicodeEncodeError. Replaced streams without ``reconfigure`` (a test - capture, a StringIO) are left alone.""" + capture, a StringIO) are left alone. :func:`main` calls this only when it + runs as the program (see there).""" if hasattr(sys.stdout, "reconfigure"): with contextlib.suppress(ValueError, OSError): - sys.stdout.reconfigure(encoding="utf-8") + sys.stdout.reconfigure(encoding="utf-8", newline="\n") if hasattr(sys.stderr, "reconfigure"): with contextlib.suppress(ValueError, OSError): sys.stderr.reconfigure(errors="backslashreplace") @@ -109,7 +112,13 @@ def _read_source(path: str) -> str: def main(argv: list[str] | None = None) -> int: - _configure_streams() + """Run the command: *argv* is the argument list, or None for the process's + own (``sys.argv``). Only then is ``main`` the program, as the console script + and ``python -m legaldown_render.cli`` call it, and only then are the + standard streams reconfigured: a caller that passes *argv* is in-process + code whose ``sys.stdout`` and ``sys.stderr`` are not ours to change.""" + if argv is None: + _configure_streams() parser = _parser() args = parser.parse_args(argv) overrides = dict(args.overrides) @@ -181,7 +190,7 @@ def main(argv: list[str] | None = None) -> int: _report(name, result.diagnostics, quiet=args.quiet) if args.output: try: - Path(args.output).write_text(result.output, encoding="utf-8") + Path(args.output).write_text(result.output, encoding="utf-8", newline="\n") except OSError as error: print(f"legaldown-render: cannot write {args.output}: {error.strerror or error}", file=sys.stderr) return 1 diff --git a/tests/test_wp2_cli.py b/tests/test_wp2_cli.py index 7c0c43d..1d1e199 100644 --- a/tests/test_wp2_cli.py +++ b/tests/test_wp2_cli.py @@ -55,6 +55,8 @@ def test_stdin_to_stdout_is_utf8_whatever_the_console_encoding(fmt: str) -> None expected = render(CZECH, format=fmt).output.encode("utf-8") done = _run(["-", "-f", fmt, "-q"], CZECH.encode("utf-8")) assert done.returncode == 0, done.stderr + # Exact bytes: the output is UTF-8 with LF line endings on every platform, so + # the same comparison holds on Windows (no CRLF translation of stdout). assert done.stdout == expected assert "Příloha".encode() in done.stdout @@ -64,6 +66,7 @@ def test_file_to_stdout_is_utf8_whatever_the_console_encoding(tmp_path: Path) -> path.write_bytes(CZECH.encode("utf-8")) done = _run([str(path), "-f", "text", "-q"]) assert done.returncode == 0, done.stderr + # Exact bytes, as above: UTF-8 with LF line endings on every platform. assert done.stdout == render(CZECH, format="text").output.encode("utf-8") diff --git a/tests/test_wp9_review_fixes.py b/tests/test_wp9_review_fixes.py index 0e1284a..3d9b5a3 100644 --- a/tests/test_wp9_review_fixes.py +++ b/tests/test_wp9_review_fixes.py @@ -117,3 +117,82 @@ def test_a_money_amount_in_name_mode_with_a_4400_digit_fraction_renders() -> Non def test_a_4400_digit_fraction_uses_the_other_category_in_czech() -> None: output = render(FRONT + f"# A\n\n{{{{duration: 1.{LONG}, unit=Y}}}}", format="text", locale="cs-CZ").output assert output.rstrip().endswith(" let") # "other", not the "many" of "2,5 roku" + + +# -- 4: only the program reconfigures the streams ------------------------------- + + +def _wrapper(encoding: str, **options: object) -> io.TextIOWrapper: + return io.TextIOWrapper(io.BytesIO(), encoding=encoding, **options) # type: ignore[arg-type] + + +def test_in_process_main_leaves_the_streams_alone(monkeypatch: pytest.MonkeyPatch) -> None: + out, err = _wrapper("cp1252", newline="\r\n"), _wrapper("cp1252") + monkeypatch.setattr(sys, "stdout", out) + monkeypatch.setattr(sys, "stderr", err) + assert main([FEATURES, "-f", "text", "-q"]) == 0 + assert main(["--list-styles"]) == 0 + assert (sys.stdout.encoding, sys.stdout.errors) == ("cp1252", "strict") + assert (sys.stderr.encoding, sys.stderr.errors) == ("cp1252", "strict") + out.flush() + # Not even the newline translation of the caller's stream changed. + assert b"\r\n" in out.buffer.getvalue() # type: ignore[attr-defined] + + +def test_main_as_the_program_reconfigures_the_streams(monkeypatch: pytest.MonkeyPatch) -> None: + out, err = _wrapper("cp1252", newline="\r\n"), _wrapper("cp1252") + monkeypatch.setattr(sys, "stdout", out) + monkeypatch.setattr(sys, "stderr", err) + monkeypatch.setattr(sys, "argv", ["legaldown-render", "--list-styles"]) + assert main() == 0 + out.flush() + assert (out.encoding, err.errors) == ("utf-8", "backslashreplace") + data = out.buffer.getvalue() # type: ignore[attr-defined] + assert data and b"\r" not in data and data.endswith(b"\n") + + +def test_the_console_script_calls_main_without_argv() -> None: + # The script entry point calls main() with no argument, so it is "the program". + pyproject = (SRC.parent / "pyproject.toml").read_text(encoding="utf-8") + assert 'legaldown-render = "legaldown_render.cli:main"' in pyproject + + +# -- 5: LF line endings everywhere ------------------------------------------------ + + +def _run(args: list[str]) -> subprocess.CompletedProcess[bytes]: + env = dict(os.environ, PYTHONPATH=str(SRC), PYTHONIOENCODING="cp1252", PYTHONUTF8="0") + return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *args], capture_output=True, env=env, + check=False) + + +@pytest.mark.parametrize("fmt", ["html", "text"]) +def test_standard_output_has_lf_line_endings(fmt: str) -> None: + done = _run([FEATURES, "-f", fmt, "-q"]) + assert done.returncode == 0, done.stderr + assert b"\n" in done.stdout and b"\r" not in done.stdout + + +@pytest.mark.parametrize("fmt", ["html", "text"]) +def test_an_output_file_has_lf_line_endings(tmp_path: Path, fmt: str) -> None: + target = tmp_path / "out.bin" + assert main([FEATURES, "-q", "-f", fmt, "-o", str(target)]) == 0 + data = target.read_bytes() + assert b"\n" in data and b"\r" not in data + assert data == render(Path(FEATURES).read_text(encoding="utf-8"), format=fmt).output.encode("utf-8") + + +def test_an_output_file_is_written_without_newline_translation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # Windows' text mode would turn "\n" into CRLF unless the file is opened + # with newline="\n": that is what the CLI asks for. + calls: list[dict[str, object]] = [] + real = Path.write_text + + def spy(self: Path, data: str, *args: object, **kwargs: object) -> int: + calls.append(kwargs) + return real(self, data, *args, **kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(Path, "write_text", spy) + assert main([FEATURES, "-q", "-o", str(tmp_path / "out.txt")]) == 0 + assert calls == [{"encoding": "utf-8", "newline": "\n"}] From 50742867630ca435c3b0652b9ed6bfef653751e7 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:04:38 +0000 Subject: [PATCH 019/118] Docs and labels: neutral party-id labels, packaging, documented limitations (S13, S14, S15, B1, B5, B7, B10, R7/B11, W9/S12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit S13 labels (style/labels.py): the party identifier label is shown for §3.4 identification_number, which natural persons may have too. cs "IČO" -> "Identifikační číslo", sk "IČO" -> "Identifikačné číslo", fr "Représentée par" -> "Représenté(e) par"; pl "Reprezentacja" kept (no gendered form). Existing test changed: tests/test_render.py French label assertion. No golden shows these labels. New tests/test_wp8_labels.py. S14: README and __init__ docstring examples open files with encoding="utf-8" (and write with newline="\n"); the --locale example says it formats values and README states labels follow the document language; README links are absolute GitHub URLs for PyPI; CONFORMANCE.md added to the sdist (checked with python -m build --sdist). docs: locale vs language wording in concepts.md and style-templates.md. S15: dropped the deprecated License classifier (license = "MIT" is SPDX); .github/PUBLISHING.md recommends gating the publish workflow on the tests (workflow file not changed). DOC rows, each verified by running an example: CONFORMANCE.md Known limitations for text after a comment (B1), directives in link/image destinations (B5), the exact nesting-limit rule (B7), table cells beyond the header (B10), parties block vs {{party:}} (R7/B11); architecture.md Security rewritten for trusted style input (W9/S12), link-gate sentences de-duplicated. --- .github/PUBLISHING.md | 7 +++++ CONFORMANCE.md | 28 +++++++++++++++++-- README.md | 33 ++++++++++++---------- docs/architecture.md | 32 +++++++++++++++------ docs/concepts.md | 4 +-- docs/style-templates.md | 19 ++++++++++--- pyproject.toml | 3 +- src/legaldown_render/__init__.py | 6 ++-- src/legaldown_render/style/labels.py | 6 ++-- tests/test_render.py | 2 +- tests/test_wp8_labels.py | 42 ++++++++++++++++++++++++++++ 11 files changed, 142 insertions(+), 40 deletions(-) create mode 100644 tests/test_wp8_labels.py diff --git a/.github/PUBLISHING.md b/.github/PUBLISHING.md index 21ad195..e3fafa2 100644 --- a/.github/PUBLISHING.md +++ b/.github/PUBLISHING.md @@ -89,4 +89,11 @@ markdown-it-py) come from PyPI. overrides the installed validator with `main`, so it tests main regardless of the range). The tests fail on any `legaldown.LegaldownDeprecationWarning`, so a release never calls what the validator's next minor version removes. +- **Recommendation: gate the upload on the tests.** The `publish` workflow builds and uploads + without running the test suite, and `ci.yml` runs separately, so a release can reach PyPI from + a commit whose tests fail. Add the lint, test, and conformance jobs (or one called workflow) + as `needs:` of the `build` job in `publish.yml`, or require a passing CI run on the tagged + commit before the Release is published. This repository's workflows are not changed here. +- The license is declared once, as the SPDX expression `license = "MIT"` in `pyproject.toml` + (PEP 639), so the deprecated `License :: OSI Approved :: MIT License` classifier is not listed. - The package ships a `py.typed` marker, so type checkers use its annotations directly. diff --git a/CONFORMANCE.md b/CONFORMANCE.md index d0c3021..aa52e49 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -37,7 +37,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | Labels following the document language (§13.7) | ✅ Built in for `en`, `cs`, `de`, `fr`, `pl`, and `sk`; any label can be set by a style | | A conditional unit's condition is shown (§15.8, MUST) | ✅ Also for a custom style: `labels.condition` must contain `{condition}`, or the style is refused | | `ref-not-enumerated` Warning (§6.3, §16.3) | ✅ | -| `raw-html` Warning; raw HTML never emitted (§8.7) | ✅ Nothing is emitted: an HTML block is dropped whole, and an inline tag is dropped with its text kept. The Warning is the validator's, once per block or text that holds raw HTML other than comments | +| `raw-html` Warning; raw HTML never emitted (§8.7) | ✅ Nothing is emitted: an HTML block is dropped whole, and an inline tag is dropped with its text kept. The Warning is the validator's, once per block or text that holds raw HTML other than comments (see "Text after a comment", below) | | Automatic term recognition (§7.4, MAY) | ❌ Not supported | | Links (§8.7) | ✅ Every link is a hyperlink in HTML and shows `text ` in plain text, whatever its scheme (`ftp:`, `sip:`, `urn:`, ...), except one whose URL is empty or uses `javascript:`, `vbscript:`, `file:`, or `data:`, which shows its text alone. A link with no text shows its URL; the text writer omits `` for an autolink | | Images and file-reference paths (§8.7, §2.3) | ✅ A relative path (`../` included, as §2.3 defines no default document root) renders `` in HTML and its alt text in plain text. A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is never fetched or embedded: the alt text is shown and the Error `path-not-relative` is reported, which `--strict` refuses. An empty path shows the alt text with no diagnostic | @@ -61,13 +61,35 @@ It is checked against the specification's own examples and fixtures corpus (`tes The containment rule (`path-outside-root`) is not checked: no document root is configured. - **Nesting limit.** The validator reads lists 64 levels deep and quotes 16; deeper, an item's or quote's content is its text. The renderer follows it, and refuses (`DocumentError`) a document - whose lists, quotes, and inline formatting (emphasis, links) together nest more than 100 levels, - while building it and before resolving or writing it, so no document can exhaust the stack. + in which the enclosing lists and quotes plus one text's inline formatting (emphasis, links) nest + more than 100 levels, while building it and before resolving or writing it, so no document can + exhaust the stack. The count is per text: a defined term's own text (`"A *b*" {{def: x}}`) is + built separately, so the emphasis inside the term does not add to the emphasis around it. Each + is checked against the limit with the enclosing lists and quotes, and their sum can exceed 100. - **Comment-only items and quotes.** A list item whose blocks are all comments or raw HTML is left out (§8.6: the author removed it), and so is a list left with no items and a quote that holds only comments. Numbers and designations are those of the items that remain. An item with no content at all keeps its place (CommonMark), and so does an empty drafting note, whose label is the template construct (§15.6). +- **Text after a comment.** A comment that starts a line is a CommonMark HTML block, which ends at + the line holding `-->`. Anything after the `-->` on that line belongs to the block and is dropped + with it, with the validator's `raw-html` Warning: ` text` renders nothing (the lines + after it are ordinary paragraphs again). A line holding only the comment is dropped with no + Warning, and a comment inside a line of text (`before after`) is dropped alone, + keeping the text around it (§8.6). +- **Directives in a link or image destination.** A destination is not text: `[x](#{{ref: a}})`, + `` and `![i](p/{{field: x}}.png)` are not resolved. The `href` or `src` + holds the directive's source, percent-encoded (`#%7B%7Bref:%20a%7D%7D`). The validator still reads + the directive, so a broken `{{ref:}}` there is reported (`ref-broken`), and the visible text of + an autolink shows it resolved, unlike its target. Directives in link text and alt text are + resolved. +- **Table cells.** A row with more cells than the header has loses the extra cells, as GFM + specifies, with no diagnostic from the validator or the renderer; a shorter row is padded with + empty cells. +- **Parties and `{{party:}}`.** The title block lists each party as an identification block, by + `legal_name` (then `label`, then `name`), and the signature blocks use `legal_name` (§3.6). The + text uses `label`, falling back to `legal_name` (§3.6): "the Seller" in a sentence, "Acme + Corporation" in the title block. - **`{{term:}}` text.** A `{{term:}}` shows the term as rendered at its first definition in the body, with its Markdown, `{{ref:}}`s and placeholders resolved as at the definition (`"A *b* c" {{def: abc}}` is "A b c", `"Fee per {{ref: h}}"` is "Fee per 1"), not the term's raw diff --git a/README.md b/README.md index d8197a1..ef9fdc4 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Python 3.11 or newer. The dependencies are all pure Python: `legaldown-validator ```bash legaldown-render contract.lgd -o contract.html # HTML page legaldown-render contract.lgd -o contract.txt # plain text (format from extension) -legaldown-render contract.lgd --style continental --locale cs-CZ -o smlouva.html +legaldown-render contract.lgd --style continental --locale cs-CZ -o smlouva.html # Czech dates, money, durations legaldown-render contract.lgd --set numbering.scheme=legal-outline --set enumeration.enabled=false legaldown-render contract.lgd --set contents.enabled=true --set contents.depth=3 # table of contents legaldown-render contract.lgd --strict # refuse if the document has errors @@ -66,6 +66,9 @@ legaldown-render template.lgd --answers answers.yaml -o nda.html # fill in a tem legaldown-render --print-style --style continental # every setting, as YAML ``` +`--locale` sets how dates, money, and durations are formatted. The words the renderer generates +(such as "Represented by" or "Signatures") follow the document's `language` (§13.7), whatever the locale. + Diagnostics go to stderr, one per line with its rule id. The exit code is 0 when the document rendered, even if it has errors (they show as markers), 1 when rendering was refused (strict mode, or a template that cannot be assembled with its answers) or the document or answers file cannot be @@ -87,10 +90,12 @@ none (`contract`, `contract.v2`, `report.2025-10-27`, `.out`), is just a name an ```python from legaldown_render import render -result = render(open("contract.lgd").read(), format="html", style="continental", locale="cs-CZ") +with open("contract.lgd", encoding="utf-8") as source: + result = render(source.read(), format="html", style="continental", locale="cs-CZ") for diagnostic in result.diagnostics: print(diagnostic.level, diagnostic.rule, diagnostic.message) -open("contract.html", "w").write(result.output) +with open("contract.html", "w", encoding="utf-8", newline="\n") as output: + output.write(result.output) ``` `render()` also accepts a `RenderOptions`. Pass `answers={...}` to render a template filled in. @@ -117,9 +122,9 @@ typography: { font_family: "Georgia, serif", justify: true } ``` **Render options** say what one job does: the output format, strict mode, the final check, the -answers to a template, a full page or an HTML fragment. Values layer in this order: built-in defaults, then the `extends` chain, then your -style, then per-job `--set` overrides. Every value is validated before rendering starts. See -[Styles and preferences](docs/style-templates.md). +answers to a template, a full page or an HTML fragment. Values layer in this order: built-in +defaults, then the `extends` chain, then your style, then per-job `--set` overrides. Every value is validated before rendering starts. See +[Styles and preferences](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/style-templates.md). Built-in styles are `default` (1. / 1.1, with (a) / (i) list items), `continental` (numbered articles, 2.1-style items, numbered paragraphs), and `outline` (I. / A. / 1. / a.). @@ -128,12 +133,12 @@ articles, 2.1-style items, numbered paragraphs), and `outline` (I. / A. / 1. / a | Document | What it covers | |---|---| -| [Concepts](docs/concepts.md) | What a renderer is responsible for, principles, and vocabulary | -| [Architecture](docs/architecture.md) | The pipeline, the render tree, the package layout, and testing | -| [Styles and preferences](docs/style-templates.md) | The style format, layering, and every setting | -| [Conformance](CONFORMANCE.md) | What the Rendering level requires, and what is covered | -| [Roadmap](docs/roadmap.md) | Milestones, and the changes needed upstream | -| [Decision records](docs/decisions/) | Why the main choices were made | +| [Concepts](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/concepts.md) | What a renderer is responsible for, principles, and vocabulary | +| [Architecture](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/architecture.md) | The pipeline, the render tree, the package layout, and testing | +| [Styles and preferences](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/style-templates.md) | The style format, layering, and every setting | +| [Conformance](https://github.com/ForLegalAI/legaldown-render/blob/main/CONFORMANCE.md) | What the Rendering level requires, and what is covered | +| [Roadmap](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/roadmap.md) | Milestones, and the changes needed upstream | +| [Decision records](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/decisions/) | Why the main choices were made | ## Development @@ -145,8 +150,8 @@ LEGALDOWN_SPEC_DIR=../LegalDown pytest -m conformance # the specification's ex ruff check . ``` -Releases go to PyPI from a GitHub Release; see [PUBLISHING.md](.github/PUBLISHING.md). +Releases go to PyPI from a GitHub Release; see [PUBLISHING.md](https://github.com/ForLegalAI/legaldown-render/blob/main/.github/PUBLISHING.md). ## License -[MIT](LICENSE) +[MIT](https://github.com/ForLegalAI/legaldown-render/blob/main/LICENSE) diff --git a/docs/architecture.md b/docs/architecture.md index 835f4a1..492a95f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -102,7 +102,9 @@ replaces are punctuation, so that CommonMark's emphasis flanking rules see the s in the source: `_text_{{ref: a}}` closes its emphasis. A defined term in a link title or alt text shows its term but is not the definition's anchor. -An HTML block in the model (`kind="html"`), a comment block included, renders nothing (§8.6). +An HTML block in the model (`kind="html"`), a comment block included, renders nothing (§8.6). A +comment that starts a line is such a block, and it ends at the line holding `-->`, so text after +the `-->` on that line is dropped with it (CONFORMANCE.md, "Text after a comment"). Within a block's text, inline comments and tags are dropped; where one stood between two spaces, one space stays, and before a line break none does. A list item whose blocks are all comments, and a quote that holds only comments, are dropped (with a list left empty), after the whole list @@ -154,7 +156,9 @@ Guessing at lost structure was tried, and it traded each gap for new bugs. A `label` never hides a failure marker (§11.5). Frontmatter strings go through the same resolver, so placeholders in frontmatter render as blanks (§3.10). -The resolver also builds the **title block** (effective date, sides and parties), **attachment +The resolver also builds the **title block** (effective date, sides and parties; a party is named +there by `legal_name`, as an identification block, where `{{party:}}` in the text shows its +`label`, §3.6), **attachment placeholders** (§13.8; files are not read at this level), and **signature blocks** (§2.2, always with `legal_name`, per §3.6). @@ -248,11 +252,10 @@ line public (`Document.line_of`, U2). `file`, or `data` (a deny-list, `urls.DENIED_SCHEMES`, judged as a browser reads the URL: tabs, newlines, controls, and case are ignored). Every other scheme (`ftp:`, `sip:`, `urn:`, ...) is linked. A link that is not made renders as its text alone, in HTML and in text alike. The - builder keeps such links as nodes (it turns off markdown-it-py's own `validateLink`, which - would leave them as literal Markdown), so the writers' check is the one that applies; - that deny-list is the gate for every `href` and `src` the HTML writer emits (links, images, - value links), and markdown-it-py's check is off so that the resolver can report §2.3 paths. - A link with no text shows its URL; the text writer leaves out ` ` when the URL, less a + builder turns off markdown-it-py's own `validateLink` (which would leave such links as literal + Markdown) and keeps them as nodes, so that the resolver can report §2.3 paths and the writers' + deny-list is the one gate for every `href` and `src` the HTML writer emits (links, images, + value links). A link with no text shows its URL; the text writer leaves out ` ` when the URL, less a `mailto:` or `tel:` prefix, is the text. - An image path must be relative with no URI scheme (§2.3), so a remote resource is never fetched or embedded. Resolution, not a writer, enforces it: an image whose path has a scheme (`https:`, @@ -264,8 +267,19 @@ line public (`Document.line_of`, U2). - At Rendering level, the only files read are the document, the style template, and the answers file the CLI reads with `--answers` (the Python API takes answers as a mapping). Attachment files and includes are never opened. -- Style values that are placed into CSS are stripped of `;{}<>`. `html.extra_css` is trusted - configuration, so a hosted service must not accept styles from untrusted users. +- **A style template is trusted input.** Nothing in it is a security boundary: + - A value placed into the stylesheet (fonts, sizes, colours, margins, `page.size`) is stripped + of `;{}<>` only (`writers/html.py:_css`). That keeps it from ending its declaration or the + `` included. + - `extends` names any path the process can read, absolute or with `../`; only cycles are + rejected. (A style file is read with `yaml.safe_load`, and as UTF-8.) + A service that renders for several parties must choose the style itself, and never accept a + style file, `--set` values, or `extends` paths from a user it does not trust. Document source + has no such power: it is never emitted as HTML or CSS (§8.7). ## Testing diff --git a/docs/concepts.md b/docs/concepts.md index 1509192..e5d895a 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -75,8 +75,8 @@ errors, because a draft with visible markers is useful. Under `--strict` it refu **4. Content and presentation stay separate.** Everything visual comes from the style template, including numbering schemes, locale, and the labels used in the template view. The same source -must render correctly with any compatible template (§13.7). Locale is a render-time setting and -is never read from the document; `language` is only a hint (§10.1). +must render correctly with any compatible template (§13.7). Locale is a render-time setting +of the style: the document's `language` is only the hint used when the style sets none (§10.1). **5. Output is deterministic.** Identical inputs produce byte-identical output: no timestamps, no random ids, stable anchor names, stable ordering. This makes golden-file tests possible and diff --git a/docs/style-templates.md b/docs/style-templates.md index dc3f186..54177b7 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -25,8 +25,9 @@ Values are layered. Each layer overrides the one before it, **field by field**: 1. **Built-in defaults.** Every setting has one, defined in `legaldown_render/style/model.py`. The `default` style is exactly these defaults. 2. **The `extends` chain.** A style names the style it builds on, which can be a built-in name or - a path relative to the file. Chains are resolved base first, and cycles are rejected, however - the files are spelled (`../dir/a.yaml`, a symbolic link). + a path relative to the file (any path the process can read, `../` and absolute paths + included). Chains are resolved base first, and cycles are rejected, however the files are + spelled (`../dir/a.yaml`, a symbolic link). 3. **The chosen style**: `--style NAME|PATH`, or `RenderOptions(style=...)`. 4. **Per-job overrides**: `--set numbering.scheme=mixed` or `overrides={"numbering.scheme": "mixed"}`, plus `--locale`. On the command line, text settings take the value as written @@ -136,7 +137,7 @@ headings: # per level 1–5 page: { size: A4, margin: 25mm } # print stylesheet now; the PDF writer later html: max_width: 46rem - extra_css: "" # appended to the stylesheet; trusted configuration only + extra_css: "" # raw CSS, appended to the stylesheet as written (see "Trusted input") ``` ### Level formats @@ -185,9 +186,19 @@ legaldown-render: Invalid style 'default': A `ref` is one piece of a designation, so it cannot show the whole `{path}`. A label left unset takes the built-in one for the document language, which always passes. +### Trusted input + +A style template, a `--set` value, and an `extends` path are the operator's, not the document +author's. Values placed into the stylesheet (`typography`, `headings`, `page`, `html.max_width`) +are stripped of `;{}<>` and are otherwise not checked as CSS, so a value such as `12pt (` is +emitted as written and breaks the rest of the stylesheet. `html.extra_css` is raw CSS. Do not +render with styles from users you do not trust +([Security](architecture.md#security)). + ### Built-in labels -Labels follow the document's `language`. English (`en`), Czech (`cs`), German (`de`), French +Labels follow the document's `language`, not `locale` (which formats dates, money, and durations). +English (`en`), Czech (`cs`), German (`de`), French (`fr`), Polish (`pl`), and Slovak (`sk`) are built in, and other languages fall back to English. A style sets any label explicitly to override it, which is also how a firm adds a language. diff --git a/pyproject.toml b/pyproject.toml index 8ea3cbe..63a65a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ classifiers = [ "Environment :: Console", "Intended Audience :: Developers", "Intended Audience :: Legal Industry", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", @@ -59,7 +58,7 @@ path = "src/legaldown_render/__init__.py" packages = ["src/legaldown_render"] [tool.hatch.build.targets.sdist] -include = ["src/legaldown_render", "tests", "docs", "README.md", "LICENSE", "pyproject.toml"] +include = ["src/legaldown_render", "tests", "docs", "README.md", "CONFORMANCE.md", "LICENSE", "pyproject.toml"] [tool.ruff] line-length = 120 diff --git a/src/legaldown_render/__init__.py b/src/legaldown_render/__init__.py index 8845c22..366b307 100644 --- a/src/legaldown_render/__init__.py +++ b/src/legaldown_render/__init__.py @@ -6,10 +6,12 @@ from legaldown_render import render - result = render(open("contract.lgd").read(), format="html", style="default") + with open("contract.lgd", encoding="utf-8") as source: + result = render(source.read(), format="html", style="default") for diagnostic in result.diagnostics: print(diagnostic.level, diagnostic.rule, diagnostic.message) - open("contract.html", "w").write(result.output) + with open("contract.html", "w", encoding="utf-8", newline="\n") as output: + output.write(result.output) Parsing and Core validation come from legaldown-validator; this package claims the Rendering conformance level (§17.3). Exact coverage: diff --git a/src/legaldown_render/style/labels.py b/src/legaldown_render/style/labels.py index 19a432a..62b7f5e 100644 --- a/src/legaldown_render/style/labels.py +++ b/src/legaldown_render/style/labels.py @@ -29,7 +29,7 @@ "cs": Labels( drafting_note="Poznámka pro zpracovatele", condition="Pouze pokud: {condition}", - identification_number="IČO", + identification_number="Identifikační číslo", date_of_birth="Datum narození", address="Adresa", represented_by="Zastoupení", @@ -71,7 +71,7 @@ identification_number="N° d'immatriculation", date_of_birth="Date de naissance", address="Adresse", - represented_by="Représentée par", + represented_by="Représenté(e) par", effective_date="Date d'effet", version="Version", attachments="Annexes", @@ -106,7 +106,7 @@ "sk": Labels( drafting_note="Poznámka pre spracovateľa", condition="Iba ak: {condition}", - identification_number="IČO", + identification_number="Identifikačné číslo", date_of_birth="Dátum narodenia", address="Adresa", represented_by="Zastúpenie", diff --git a/tests/test_render.py b/tests/test_render.py index 37a0676..d432e3a 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -331,7 +331,7 @@ def test_representatives_are_labelled_and_french_labels_space_their_colons() -> relabelled = render(source, format="text", overrides={"labels.represented_by": "Acting for"}).output assert "Acting for: John Smith" in relabelled french = render(source.replace("language: en", "language: fr"), format="text").output - assert "Représentée par\xa0: John Smith" in french + assert "Représenté(e) par\xa0: John Smith" in french assert "Nom\xa0: " in french and "\nLieu\xa0:\n" in french assert "Date d'effet\xa0: " in french diff --git a/tests/test_wp8_labels.py b/tests/test_wp8_labels.py new file mode 100644 index 0000000..cb2a3fd --- /dev/null +++ b/tests/test_wp8_labels.py @@ -0,0 +1,42 @@ +"""WP8 (S13): the party identifier label is neutral, since §3.4's +``identification_number`` also covers a natural person's national ID.""" +from __future__ import annotations + +import pytest + +from legaldown_render import render +from legaldown_render.style.labels import BUILTIN_LABELS + +DOCUMENT = """--- +legaldown: "0.2" +title: T +language: {language} +sides: + - name: s1 + parties: + - name: a + type: natural_person + legal_name: Jan Novak + identification_number: "800101/1234" + representatives: + - {{name: Eva, title: Jednatel}} +--- +# A +""" + + +@pytest.mark.parametrize(("language", "identifier", "represented_by"), [ + ("cs", "Identifikační číslo", "Zastoupení"), + ("sk", "Identifikačné číslo", "Zastúpenie"), + ("fr", "N° d'immatriculation", "Représenté(e) par"), + ("pl", "Nr rejestrowy", "Reprezentacja"), +]) +def test_party_labels_in_each_language(language: str, identifier: str, represented_by: str) -> None: + colon = BUILTIN_LABELS[language].colon + output = render(DOCUMENT.format(language=language), format="text").output + assert f"{identifier}{colon}800101/1234" in output + assert f"{represented_by}{colon}Eva, Jednatel" in output + + +def test_cs_and_sk_do_not_call_every_party_identifier_an_ico() -> None: + assert "IČO" not in {BUILTIN_LABELS["cs"].identification_number, BUILTIN_LABELS["sk"].identification_number} From f5ad89f8c78ad133867ca1438f87f67bb2a49f52 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:08:44 +0000 Subject: [PATCH 020/118] Links: allow-list of schemes, refused link reported, diagnostics escape controls (review wave 2) - urls.LINK_SCHEMES (http, https, mailto, tel, ftp, ftps, sftp, sip, sips, sms, urn, geo, news, plus scheme-less URLs) replaces the DENIED_SCHEMES deny-list: protocol handlers (ms-msdt:, vscode:, ...) and pseudo-schemes (C:\x, localhost:8080) render as their text. One predicate, is_link_target. - resolver: a refused non-empty link is the Warning render-link-refused. - urls.printable escapes C0/C1, DEL, Cf, U+2028/9 in quoted paths/URLs; used in path-not-relative and the new warning (no ESC/BEL/newline forged on stderr). - path-not-relative dedupe matches the quoted ''; message is generic. - A link with no text shows the URL as written (unquoted), in both writers. - tests/test_wp3_links.py: magnet: is no longer a link; 'a:b' is no longer a safe href (allow-list). --- src/legaldown_render/build.py | 2 +- src/legaldown_render/resolve/resolver.py | 31 ++++- src/legaldown_render/urls.py | 63 +++++++-- src/legaldown_render/writers/html.py | 11 +- src/legaldown_render/writers/text.py | 4 +- tests/test_wp10_review_fixes.py | 155 +++++++++++++++++++++++ tests/test_wp3_links.py | 8 +- 7 files changed, 243 insertions(+), 31 deletions(-) create mode 100644 tests/test_wp10_review_fixes.py diff --git a/src/legaldown_render/build.py b/src/legaldown_render/build.py index c9aa8e5..2b9ecfc 100644 --- a/src/legaldown_render/build.py +++ b/src/legaldown_render/build.py @@ -185,7 +185,7 @@ def __init__(self, document: Document, result: ValidationResult) -> None: # prose. The builder keeps the node instead: the resolver reports an # image path that is not relative (§2.3) and replaces the image with # its alt text, and the writers show a refused link as its text - # (`urls.DENIED_SCHEMES`), which is the safety markdown-it was for. + # (`urls.LINK_SCHEMES`), which is the safety markdown-it was for. self.md.validateLink = lambda url: True self.env: dict = {} self.payloads: list[_Payload] = [] diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 748e907..87fa466 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -17,7 +17,8 @@ failures shown here, so the resolver only adds the diagnostics that need a style template or a renderer: ``ref-not-enumerated``, ``render-ref-ambiguous``, and ``render-not-processed`` for constructs beyond the Rendering level (§17.5), -and ``path-not-relative`` for an image whose path is not relative (§2.3), which +``render-link-refused`` for a link whose URL scheme is not one links may use +(§8.7), and ``path-not-relative`` for an image whose path is not relative (§2.3), which the validator does not check yet. """ from __future__ import annotations @@ -84,7 +85,7 @@ plain_text, unlinked, ) -from ..urls import is_empty_path, is_relative_path +from ..urls import is_empty_path, is_link_target, is_relative_path, printable from .numbering import RENUMBERED, extend, fill, format_counter, heading_levels from .values import DURATION_UNITS, Formatter @@ -532,7 +533,9 @@ def _resolve_inline(self, inline: Inline) -> Inline | tuple[Inline, ...] | None: return Emphasis(self._resolve_inlines(children)) case Strong(children=children): return Strong(self._resolve_inlines(children)) - case Link(children=children, title=title): + case Link(href=href, children=children, title=title): + if href.strip() and not is_link_target(href): + self._link_refused(href) return replace(inline, children=self._resolve_inlines(children), title=self._resolve_inlines(_attribute_text(title))) case Image(src=src, children=children, title=title): @@ -890,6 +893,18 @@ def _warn(self, rule: str, message: str) -> None: if diagnostic not in self.diagnostics: self.diagnostics.append(diagnostic) + def _link_refused(self, href: str) -> None: + """Report a link whose URL does not use a scheme links may use + (:data:`~legaldown_render.urls.LINK_SCHEMES`) as ``render-link-refused``: + the writers show its text and say nothing (ADR 0003), so the reader of + the document would not otherwise know a link was lost.""" + shown = printable(unquote(href.strip())) + self._warn( + "render-link-refused", + f"Link to '{shown}' is not rendered as a link: only http, https, mailto, tel, ftp, ftps, sftp, sip, " + f"sips, sms, urn, geo, news and relative links are; its text is shown.", + ) + def _not_relative(self, path: str) -> None: """Report an image *path* that is not relative as the Core Error ``path-not-relative`` (§2.3, §16.2), unless the validator has @@ -898,13 +913,17 @@ def _not_relative(self, path: str) -> None: percent-encoded the path ("C:\\x" reads "C:%5Cx"), so it is decoded to name the path as written.""" path = unquote(path) - if any(d.rule == "path-not-relative" and path in d.message for d in self.result.diagnostics): + # Assumption: the validator quotes the path in its message as '', + # as written. Matching the quoted form keeps a path that is a prefix of + # another ("/a" and "/ab") from being counted as already reported. + if any(d.rule == "path-not-relative" and f"'{path}'" in d.message for d in self.result.diagnostics): return + path = printable(path) diagnostic = Diagnostic( rule="path-not-relative", level="error", - message=f"Image path '{path}' is not relative (§2.3): an image path is a relative path with no URI " - f"scheme, and a remote resource is never fetched. Its alt text is shown instead.", + message=f"Image path '{path}' is not relative (§2.3): file references are relative paths with no " + f"URI scheme. Its alt text is shown instead.", ) if diagnostic not in self.diagnostics: self.diagnostics.append(diagnostic) diff --git a/src/legaldown_render/urls.py b/src/legaldown_render/urls.py index a76ef7c..6b89a8f 100644 --- a/src/legaldown_render/urls.py +++ b/src/legaldown_render/urls.py @@ -1,5 +1,6 @@ """Reading URLs the way the renderer needs to: which schemes a link may use -(§8.7), and whether an image path is a relative one (§2.3). +(§8.7), whether an image path is a relative one (§2.3), and how to quote one in +a message. The checks are syntactic, like the validator's (§2.3). They are shared by the resolver, which decides what an image is, and by both writers, which decide @@ -8,16 +9,25 @@ from __future__ import annotations import re +import unicodedata from urllib.parse import unquote -#: Schemes whose links are never made: script (``javascript:``, ``vbscript:``), -#: local files (``file:``), and ``data:``. This list is the gate: markdown-it's -#: own ``validateLink`` is turned off in the builder, so that the resolver can -#: see and report an image path that is not relative (§2.3), and the HTML -#: writer applies this list to every ``href`` and ``src`` it emits (a link, an -#: image, a value's link). Every other scheme (``ftp:``, ``sip:``, ``urn:``, -#: ...) is a hyperlink, since §8.7 requires links to be rendered as hyperlinks. -DENIED_SCHEMES = frozenset({"javascript", "vbscript", "file", "data"}) +#: The schemes a link may use: the web, mail and phone, file transfer, and a +#: few that name a thing rather than start a program. Nothing else is a link: +#: a protocol handler (``ms-msdt:``, ``search-ms:``, ``vscode:``, ``intent:``), +#: a pseudo-scheme (``C:\x``, ``localhost:8080``), script, ``file:``, and +#: ``data:`` render as their text, and the resolver reports it. §8.7 requires +#: links to be hyperlinks, but a renderer reads documents it did not write +#: (docs/concepts.md, principle 6), so it is an allow-list rather than a list +#: of what is known to be dangerous. A URL with no scheme (a relative path, a +#: ``#fragment``) is a link too. markdown-it's own ``validateLink`` is turned +#: off in the builder, so that the resolver can see and report an image path +#: that is not relative (§2.3); this list is the gate, and the HTML writer +#: applies it to every ``href`` and ``src`` it emits (a link, an image, a +#: value's link). +LINK_SCHEMES = frozenset({ + "http", "https", "mailto", "tel", "ftp", "ftps", "sftp", "sip", "sips", "sms", "urn", "geo", "news", +}) _SCHEME_RE = re.compile(r"^([a-zA-Z][a-zA-Z0-9+.-]*):") # What the WHATWG URL parser ignores before it reads a scheme: tabs and @@ -39,17 +49,44 @@ def url_scheme(url: str) -> str | None: def is_safe_href(href: str) -> bool: - """True unless *href* uses a scheme in :data:`DENIED_SCHEMES`, judged as a + """True if *href* has no scheme, or one in :data:`LINK_SCHEMES`, judged as a browser parses the URL.""" - return url_scheme(href) not in DENIED_SCHEMES + scheme = url_scheme(href) + return scheme is None or scheme in LINK_SCHEMES def is_link_target(href: str) -> bool: - """True if a link to *href* is made: it names something, and is safe. - An empty destination would be a link to the page itself.""" + """True if a link to *href* is made: it names something, and its scheme is + one links may use. An empty destination would be a link to the page itself. + Both writers and the resolver's report use this one predicate.""" return bool(_as_read(href)) and is_safe_href(href) +def displayed_url(href: str) -> str: + """*href* as the document wrote it, to show where a link has no text: the + percent-encoding markdown-it added is undone (``a%20b`` reads ``a b``), + unless that would show a control or format character.""" + url = href.strip() + shown = unquote(url) + return url if any(unicodedata.category(c) in ("Cc", "Cf") for c in shown) else shown + + +def printable(text: str) -> str: + """*text*, a path or URL from a document, made safe to quote in a message: + every control character (C0, C1, DEL), Unicode format character (bidi + controls, zero-width, ...), and line or paragraph separator is written as + ``\\xNN`` or ``\\uNNNN``, so that it cannot move the cursor, set a title, + or start a line of its own on a terminal.""" + return "".join(_escaped(c) for c in text) + + +def _escaped(c: str) -> str: + if unicodedata.category(c) not in ("Cc", "Cf") and c not in "\u2028\u2029": + return c + code = ord(c) + return f"\\x{code:02x}" if code < 0x100 else f"\\u{code:04x}" if code < 0x10000 else f"\\U{code:08x}" + + def is_empty_path(path: str) -> bool: """True if *path* names nothing: empty, or only what a browser ignores. As an image's source it would request the page itself.""" diff --git a/src/legaldown_render/writers/html.py b/src/legaldown_render/writers/html.py index 01a440d..82ada1d 100644 --- a/src/legaldown_render/writers/html.py +++ b/src/legaldown_render/writers/html.py @@ -9,9 +9,10 @@ Safety (docs/architecture.md): every text node and attribute is escaped, raw HTML from the source never reaches this writer (§8.7), and a link whose URL -is empty or uses a script, ``file:``, or ``data:`` scheme -(:data:`~legaldown_render.urls.DENIED_SCHEMES`, judged as a browser reads the -URL) renders as its text alone. Every other scheme is a hyperlink (§8.7). +is empty or uses a scheme outside the allow-list +(:data:`~legaldown_render.urls.LINK_SCHEMES`, judged as a browser reads the +URL: script, ``file:``, ``data:``, protocol handlers) renders as its text +alone, and the resolver reports it. Every other link is a hyperlink (§8.7). This check is the gate for every ``href`` and ``src`` the writer emits (a link, an image, a value's link): the builder turns markdown-it's own check off. An image path that is not relative (§2.3) never gets here: resolution has @@ -56,7 +57,7 @@ has_visible_content, plain_text, ) -from ..urls import is_link_target, is_safe_href +from ..urls import displayed_url, is_link_target, is_safe_href __all__ = ["HtmlWriter", "is_safe_href", "stylesheet"] @@ -314,7 +315,7 @@ def inline(self, inline: Inline) -> str: return f"{_text(text)}" case Link(href=href, children=children, title=title): # A link with no text shows its URL (§8.7: it is still a hyperlink). - shown = children if has_visible_content(children) else (Text(href.strip()),) + shown = children if has_visible_content(children) else (Text(displayed_url(href)),) if not is_link_target(href): return self.inlines(children) title_attr = f' title="{_attr(plain_text(title))}"' if title else "" diff --git a/src/legaldown_render/writers/text.py b/src/legaldown_render/writers/text.py index 4ca0a81..aa6f328 100644 --- a/src/legaldown_render/writers/text.py +++ b/src/legaldown_render/writers/text.py @@ -41,7 +41,7 @@ Text, Value, ) -from ..urls import is_link_target +from ..urls import displayed_url, is_link_target _INDENT = " " _ADDRESS_SCHEMES = ("mailto:", "tel:") @@ -54,7 +54,7 @@ def _link_text(href: str, text: str) -> str: text alone, and one with no text shows its URL.""" if not is_link_target(href): return text - url = href.strip() + url = displayed_url(href) if not text.strip(): return url address = url diff --git a/tests/test_wp10_review_fixes.py b/tests/test_wp10_review_fixes.py new file mode 100644 index 0000000..e0aca2c --- /dev/null +++ b/tests/test_wp10_review_fixes.py @@ -0,0 +1,155 @@ +"""Review fixes after wave 2 (WP10): links are an allow-list and a refused one is +reported, control characters never reach a diagnostic, style templates are +validated only where a setting is used, and small wording fixes.""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from legaldown_render import render +from legaldown_render.urls import displayed_url, is_link_target, printable + +SRC = Path(__file__).resolve().parents[1] / "src" +FRONT = "---\ntitle: T\n---\n\n" + + +def html(body: str, **settings: object) -> str: + return render(FRONT + body, format="html", **settings).output + + +def text(body: str, **settings: object) -> str: + return render(FRONT + body, format="text", **settings).output + + +# -- 1: links are an allow-list --------------------------------------------------- + +HANDLERS = [ + "ms-msdt:/id PCWDiagnostic", "ms-word:ofe|u|https://x/a.docx", "ms-officecmd:{}", "search-ms:query=x", + "ms-appinstaller:?source=https://x/a.appx", "vscode://file/x", "intent://x#Intent;end", "blob:https://x/1", + "about:blank", "jar:https://x/a.jar!/", "view-source:https://x", "mhtml:file://x", "ms-its:x.chm::/a.htm", + "C:\\x", "C:/x", "localhost:8080", "magnet:?xt=urn:btih:x", "javascript:alert(1)", "file:///etc/passwd", + "data:text/html,x", " MS-MSDT:x", +] +ALLOWED = [ + "http://e/x", "https://e/x", "mailto:a@e", "tel:+1", "ftp://e/x", "ftps://e/x", "sftp://e/x", "sip:a@e", + "sips:a@e", "sms:+1", "urn:isbn:1", "geo:1,2", "news:a.b", "a/b.pdf", "../a.pdf", "#top", "?q=1", +] + + +@pytest.mark.parametrize("href", HANDLERS) +def test_a_protocol_handler_or_pseudo_scheme_is_not_a_link(href: str) -> None: + assert not is_link_target(href) + body = f"# A\n\nSee [click](<{href}>) now.\n" + result = render(FRONT + body, format="html") + assert "See click now." in result.output + assert "")[1] + assert "See click now." in text(body) + assert "<" not in text(body).split("See click now.")[0].split("\n")[-1] + + +def test_the_scheme_is_read_as_a_browser_reads_it() -> None: + # markdown-it percent-encodes a tab in a URL it parses ("ms%09-msdt:x" is a + # relative path to a browser); the check still holds for a URL it is given + # directly, as a browser would read it. + for href in ("ms\t-msdt:x", "jav\nascript:x", "\x01javascript:x", "\u0000 vscode://x"): + assert not is_link_target(href), repr(href) + + +@pytest.mark.parametrize("href", ALLOWED) +def test_an_allowed_scheme_or_a_relative_url_is_a_link(href: str) -> None: + assert is_link_target(href) + assert f'click' in html(f"# A\n\nSee [click](<{href}>).\n") + + +def test_a_refused_link_is_reported_once_as_a_warning() -> None: + body = "# A\n\n[a](ms-msdt:x) and [b](ms-msdt:x) and [c](vscode://x) and [d](http://ok) and [e]().\n" + result = render(FRONT + body, format="html") + refused = [d for d in result.diagnostics if d.rule == "render-link-refused"] + assert [d.level for d in refused] == ["warning", "warning"] + assert "Link to 'ms-msdt:x' is not rendered as a link" in refused[0].message + assert "its text is shown" in refused[0].message + assert "vscode://x" in refused[1].message + assert result.ok # a Warning does not refuse, even under --strict's Errors + assert render(FRONT + body, format="text").diagnostics == result.diagnostics + + +def test_an_empty_or_allowed_link_is_not_reported() -> None: + result = render(FRONT + "# A\n\n[a]() [b](<>) [c](https://e) [d](#x) [e](rel/path)\n") + assert not [d for d in result.diagnostics if d.rule == "render-link-refused"] + + +# -- 3, 4, 5: what a diagnostic shows of a document ------------------------------ + + +def test_printable_escapes_controls_and_format_characters() -> None: + assert printable("a\x1b[2J\x07\n\x7f\x85") == "a\\x1b[2J\\x07\\x0a\\x7f\\x85" + assert printable("\u202e\u200b\ufeff\u2028\u2029") == "\\u202e\\u200b\\ufeff\\u2028\\u2029" + assert printable("ž é €") == "ž é €" + + +EVIL = "https://e/%1b[2J%1b]0;pwned%07%0aerror: [x] forged%e2%80%ae" + + +def test_a_control_character_in_an_image_path_is_escaped_in_the_message() -> None: + result = render(FRONT + f"# A\n\n![x](<{EVIL}>)\n") + messages = [d.message for d in result.diagnostics if d.rule == "path-not-relative"] + assert messages + for message in messages: + assert "\x1b" not in message and "\x07" not in message and "\n" not in message + assert any("\\x1b[2J\\x1b]0;pwned\\x07\\x0aerror: [x] forged\\u202e" in m for m in messages) + + +def test_a_control_character_in_a_link_is_escaped_in_the_message() -> None: + result = render(FRONT + f"# A\n\n[x]()\n") + message = next(d.message for d in result.diagnostics if d.rule == "render-link-refused") + assert "\x1b" not in message and "\x07" not in message and "\n" not in message and "\u202e" not in message + assert "\\x1b" in message + + +def test_the_cli_writes_no_control_character_from_a_document_to_standard_error(tmp_path: Path) -> None: + source = tmp_path / "evil.lgd" + source.write_text(FRONT + f"# A\n\n![x](<{EVIL}>) [y]()\n", encoding="utf-8") + env = dict(os.environ, PYTHONPATH=str(SRC)) + done = subprocess.run([sys.executable, "-m", "legaldown_render.cli", str(source), "-f", "text"], + capture_output=True, env=env, check=False) + err = done.stderr.decode("utf-8") + assert "path-not-relative" in err and "render-link-refused" in err + assert "\x1b" not in err and "\x07" not in err and "\u202e" not in err + assert "\r" not in err + assert not any(line.startswith("error: [x]") for line in err.splitlines()) + + +def test_a_path_that_starts_another_is_not_covered_by_the_validators_report() -> None: + # Whatever the validator reports for one path, a different path whose text + # it contains is still the renderer's to report. + result = render(FRONT + "# A\n\n![a](/x.png) ![b](/x.png2)\n") + messages = [d.message for d in result.diagnostics if d.rule == "path-not-relative"] + assert any("'/x.png2'" in m for m in messages) + assert any("'/x.png'" in m for m in messages) + + +@pytest.mark.parametrize("path", ["javascript:alert(1)", "data:text/html,x", "https://e/p.png"]) +def test_the_not_relative_message_is_generic(path: str) -> None: + result = render(FRONT + f"# A\n\n![x](<{path}>)\n") + message = next(d.message for d in result.diagnostics if d.rule == "path-not-relative") + assert "never fetched" not in message + assert "file references are relative paths with no URI scheme. Its alt text is shown instead." in message + + +# -- 8: a link with no text shows the URL as written ---------------------------- + + +def test_displayed_url_undoes_the_encoding_unless_it_shows_a_control() -> None: + assert displayed_url(" https://e/a%20b ") == "https://e/a b" + assert displayed_url("https://e/a%0ab") == "https://e/a%0ab" + assert displayed_url("https://e/%E2%80%AE") == "https://e/%E2%80%AE" + + +def test_a_link_with_no_text_shows_the_written_url() -> None: + body = "# A\n\nSee []().\n" + assert 'href="https://e/a%20b/%C3%A9">https://e/a b/é' in html(body) + assert "See https://e/a b/é." in text(body) diff --git a/tests/test_wp3_links.py b/tests/test_wp3_links.py index e745292..c212fbe 100644 --- a/tests/test_wp3_links.py +++ b/tests/test_wp3_links.py @@ -127,9 +127,9 @@ def validate(document, **kwargs): # type: ignore[no-untyped-def] @pytest.mark.parametrize("href", [ "ftp://example.com/f", "sip:alice@example.com", "urn:isbn:0451450523", "https://example.com", - "mailto:a@example.com", "tel:+420123456789", "magnet:?xt=urn:btih:x", "attachments/a.pdf", "../a.pdf", "#top", + "mailto:a@example.com", "tel:+420123456789", "attachments/a.pdf", "../a.pdf", "#top", ]) -def test_every_link_but_the_dangerous_ones_is_a_hyperlink(href: str) -> None: +def test_every_link_of_an_allowed_scheme_is_a_hyperlink(href: str) -> None: output = html(f"# A\n\nSee [the link](<{href}>).\n") assert f'the link' in output @@ -152,12 +152,12 @@ def test_a_refused_link_is_its_text_in_both_writers(href: str) -> None: assert not is_safe_href(href) -def test_the_deny_list_is_judged_as_a_browser_reads_the_url() -> None: +def test_the_allow_list_is_judged_as_a_browser_reads_the_url() -> None: for href in ("java\tscript:alert(1)", "jav\nascript:x", "\x01javascript:x", " JAVASCRIPT:x", "data:text/html,x", "VBScript:x", "FILE:///x"): assert not is_safe_href(href), repr(href) for href in ("https://example.com", "#section", "attachments/a.pdf", "mailto:a@example.com", "ftp://x/y", - "sip:a@b", "urn:x:y", "a:b", "x/data:y", "javascript"): + "sip:a@b", "urn:x:y", "x/data:y", "javascript", "HTTPS://X", "\thttps://x"): assert is_safe_href(href), href From b19901d9c5431d85bdf1a78b0e9525c203f3ca9e Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:10:18 +0000 Subject: [PATCH 021/118] Style: validate templates only where a setting is used (review wave 2) - template_settings: a ref is required to contain {n} only where it is read (numbering unless scheme none; enumeration when enabled or ordered: enumerate; paragraphs when numbered). Unknown fields are still flagged. - Plain labels (labels.* but condition and attachment_file) substitute nothing, so braces in them are text and they are no longer checked. - tests/test_wp6_style.py: labels.drafting_note/colon with fields are valid; paragraph format cases set paragraphs.numbered. --- src/legaldown_render/style/loader.py | 4 ++- src/legaldown_render/style/model.py | 40 ++++++++++++++++------ tests/test_wp10_review_fixes.py | 51 +++++++++++++++++++++++++++- tests/test_wp6_style.py | 7 ++-- 4 files changed, 86 insertions(+), 16 deletions(-) diff --git a/src/legaldown_render/style/loader.py b/src/legaldown_render/style/loader.py index d53f58a..7e472da 100644 --- a/src/legaldown_render/style/loader.py +++ b/src/legaldown_render/style/loader.py @@ -312,7 +312,9 @@ def _template_problems(style: Style) -> list[str]: hide the condition, which §15.8 requires to be shown). An empty template is exempt from the required fields where the resolver falls back to the bare field for it (``labels.condition``): it already shows the condition. - Where there is no such fallback, an empty template lacks its fields.""" + Where there is no such fallback, an empty template lacks its fields. + :func:`~.model.template_settings` lists only the settings that substitute + fields, and drops the required fields of one the style does not use.""" problems = [] for key, template, fields in template_settings(style): found = set(_FIELD_RE.findall(template)) diff --git a/src/legaldown_render/style/model.py b/src/legaldown_render/style/model.py index 8032faf..971834c 100644 --- a/src/legaldown_render/style/model.py +++ b/src/legaldown_render/style/model.py @@ -16,7 +16,7 @@ from __future__ import annotations from collections.abc import Iterator -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field, fields, replace from typing import Literal Counter = Literal["decimal", "lower-alpha", "upper-alpha", "lower-roman", "upper-roman"] @@ -332,19 +332,39 @@ class TemplateFields: } +def _unused(fields: TemplateFields) -> TemplateFields: + """*fields* for a setting the style does not use: a typo is still a typo, + but nothing is missing from a setting nothing reads.""" + return replace(fields, required=frozenset(), why="") + + def template_settings(style: Style) -> Iterator[tuple[str, str, TemplateFields]]: - """Every text template setting of *style* that has been set, as - ``(key path, template, fields)``. A label left unset (``None``) takes a - built-in one, which is correct by construction.""" - for name, kind_label, kind_ref in (("numbering", HEADING_LABEL, HEADING_REF), - ("enumeration", ITEM_LABEL, ITEM_REF)): + """Every text template setting of *style* that has been set and substitutes + fields, as ``(key path, template, fields)``. A label left unset (``None``) + takes a built-in one, which is correct by construction. A plain label + (``labels.*`` but ``condition`` and ``attachment_file``) substitutes + nothing, so braces in it are text and it is not listed. + + A ``ref`` that is not in use does not have to tell its units apart: the + heading levels' under ``numbering.scheme: none`` (a reference shows the + heading text, §13.3), the item levels' when lists are not enumerated + (``enumeration.enabled: false`` and ``ordered: renumber``), the + paragraph's unless ``paragraphs.numbered``. Their fields are still checked + for typos.""" + numbered = style.numbering.scheme != "none" + enumerated = style.enumeration.enabled or style.enumeration.ordered == "enumerate" + for name, kind_label, kind_ref, in_use in ( + ("numbering", HEADING_LABEL, HEADING_REF, numbered), + ("enumeration", ITEM_LABEL, ITEM_REF, enumerated), + ): for index, level in enumerate(getattr(style, name).levels): yield f"{name}.levels[{index}].label", level.label, kind_label - yield f"{name}.levels[{index}].ref", level.ref, kind_ref + yield f"{name}.levels[{index}].ref", level.ref, kind_ref if in_use else _unused(kind_ref) yield "paragraphs.format.label", style.paragraphs.format.label, ITEM_LABEL - yield "paragraphs.format.ref", style.paragraphs.format.ref, ITEM_REF + yield ("paragraphs.format.ref", style.paragraphs.format.ref, + ITEM_REF if style.paragraphs.numbered else _unused(ITEM_REF)) yield "references.format", style.references.format, REFERENCE_FORMAT for item in fields(Labels): template = getattr(style.labels, item.name) - if template is not None: - yield f"labels.{item.name}", template, LABEL_FIELDS.get(item.name, TemplateFields()) + if template is not None and item.name in LABEL_FIELDS: + yield f"labels.{item.name}", template, LABEL_FIELDS[item.name] diff --git a/tests/test_wp10_review_fixes.py b/tests/test_wp10_review_fixes.py index e0aca2c..324144d 100644 --- a/tests/test_wp10_review_fixes.py +++ b/tests/test_wp10_review_fixes.py @@ -10,7 +10,8 @@ import pytest -from legaldown_render import render +from legaldown_render import StyleError, render +from legaldown_render.style import load_style from legaldown_render.urls import displayed_url, is_link_target, printable SRC = Path(__file__).resolve().parents[1] / "src" @@ -153,3 +154,51 @@ def test_a_link_with_no_text_shows_the_written_url() -> None: body = "# A\n\nSee []().\n" assert 'href="https://e/a%20b/%C3%A9">https://e/a b/é' in html(body) assert "See https://e/a b/é." in text(body) + + +# -- 6: a style is validated where its settings are used ------------------------ + +def _problems(overrides: dict[str, object]) -> list[str]: + with pytest.raises(StyleError) as caught: + load_style("default", overrides=overrides) + return caught.value.problems + + +NO_N = [{"label": "x", "ref": "x"}] + + +@pytest.mark.parametrize(("overrides", "key"), [ + ({"enumeration.levels": NO_N}, "enumeration.levels[0].ref"), + ({"enumeration.levels": NO_N, "enumeration.enabled": False, "enumeration.ordered": "enumerate"}, + "enumeration.levels[0].ref"), + ({"numbering.levels": NO_N}, "numbering.levels[0].ref"), + ({"paragraphs.numbered": True, "paragraphs.format": {"label": "x", "ref": "x"}}, "paragraphs.format.ref"), +]) +def test_a_ref_that_is_in_use_must_have_its_counter(overrides: dict[str, object], key: str) -> None: + assert any(p.startswith(key) and "{n}" in p for p in _problems(overrides)) + + +@pytest.mark.parametrize("overrides", [ + {"enumeration.levels": NO_N, "enumeration.enabled": False}, + {"numbering.levels": NO_N, "numbering.scheme": "none"}, + {"paragraphs.format": {"label": "x", "ref": "x"}}, + {"paragraphs.format": {"label": "x", "ref": ""}}, +]) +def test_a_ref_that_is_not_in_use_may_leave_out_its_counter(overrides: dict[str, object]) -> None: + load_style("default", overrides=overrides) + + +def test_a_typo_is_flagged_even_where_the_setting_is_not_used() -> None: + problems = _problems({"numbering.scheme": "none", "numbering.levels": [{"label": "{n}", "ref": "{m}"}]}) + assert any(p.startswith("numbering.levels[0].ref") and "{m}" in p and "unknown field" in p for p in problems) + + +def test_a_plain_label_may_contain_braced_words() -> None: + style = load_style("default", overrides={"labels.drafting_note": "Note {see below}", "labels.colon": "{x}:"}) + assert style.labels.drafting_note == "Note {see below}" + assert style.labels.colon == "{x}:" + + +def test_the_labels_that_substitute_still_check_their_fields() -> None: + assert _problems({"labels.condition": "{condtion}"}) + assert _problems({"labels.attachment_file": "{fil}"}) diff --git a/tests/test_wp6_style.py b/tests/test_wp6_style.py index f0cdfd5..c033687 100644 --- a/tests/test_wp6_style.py +++ b/tests/test_wp6_style.py @@ -215,8 +215,6 @@ def test_a_typo_in_a_field_is_named() -> None: @pytest.mark.parametrize(("setting", "value"), [ ("labels.condition", "Only if {condition} {n}"), ("labels.attachment_file", "{condition}"), - ("labels.drafting_note", "{note}"), - ("labels.colon", "{n}"), ("references.format", "{n}{designation}"), ("numbering.levels", [{"label": "{n}", "ref": "{path}"}]), ("numbering.levels", [{"label": "{section}", "ref": "{n}"}]), @@ -227,7 +225,8 @@ def test_a_typo_in_a_field_is_named() -> None: ("paragraphs.format", {"label": "{n}", "ref": "{section}"}), ]) def test_invalid_templates_are_refused(setting: str, value: object) -> None: - problems = _problems({setting: value}) + # A paragraph's ref is read only where paragraphs are numbered. + problems = _problems({setting: value, "paragraphs.numbered": True}) assert any(p.startswith(setting) for p in problems), problems @@ -270,7 +269,7 @@ def test_an_empty_template_without_a_fallback_still_lacks_its_field() -> None: for setting in ("references.format", "numbering.levels", "paragraphs.format"): value: object = {"label": "{n}", "ref": ""} if setting == "paragraphs.format" else ( [{"ref": ""}] if setting == "numbering.levels" else "") - assert any(p.startswith(setting) for p in _problems({setting: value})) + assert any(p.startswith(setting) for p in _problems({setting: value, "paragraphs.numbered": True})) def test_unset_labels_are_not_checked() -> None: From 4e11aba35db8b3b5cffa6c1def78d6c417d6bc24 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:10:43 +0000 Subject: [PATCH 022/118] Placeholders: a blank of whitespace or format characters is no visual blank (review wave 2) --- CONFORMANCE.md | 2 +- docs/style-templates.md | 2 +- src/legaldown_render/resolve/resolver.py | 8 ++++++-- tests/test_wp10_review_fixes.py | 16 ++++++++++++++++ 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index aa52e49..66aa38c 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -43,7 +43,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | Images and file-reference paths (§8.7, §2.3) | ✅ A relative path (`../` included, as §2.3 defines no default document root) renders `` in HTML and its alt text in plain text. A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is never fetched or embedded: the alt text is shown and the Error `path-not-relative` is reported, which `--strict` refuses. An empty path shows the alt text with no diagnostic | | Plain-text layout (§13.6, optional) | ✅ `attachments.separator` (§13.8): `none` prints no `---`; `rule` and `page-break` both print one before the attachments heading and between consecutive attachments, so the heading stays with the first (text has no pages). One quote stays one quote (blank lines keep the `>`), and the blocks of a list item are set apart by blank lines and indented under the item's label, whatever its condition label. A pipe in a table cell is backslash-escaped, and the separator row marks column alignment (`:---`, `:---:`, `---:`). A label a style blanks drops its `label: ` prefix in the title block, party details and signature blocks (a signature row with neither label nor value is omitted); HTML keeps an empty `
` so its grid layout and the `
` stay valid | | Extended tables through raw HTML (§9.2, MAY) | ❌ Not supported; the HTML table is dropped with the Warning | -| Visual blank for a placeholder (§13.5 rule 6) | ✅ `placeholders.blank`; a blank that is empty or only whitespace renders `[TBD: id]` | +| Visual blank for a placeholder (§13.5 rule 6) | ✅ `placeholders.blank`; a blank that is empty, or only whitespace or invisible format characters (U+200B, U+FEFF), renders `[TBD: id]` | | Question prompts next to blanks (§15.8, MAY) | ✅ `placeholders.show_prompt` | | Final check (§15.9, SHOULD) | ✅ `--final` / `final=True`: the validator's `placeholder-unfilled` and `template-construct-present` Errors; with `--strict`, the document is refused | diff --git a/docs/style-templates.md b/docs/style-templates.md index 54177b7..cb1f28b 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -105,7 +105,7 @@ values: duration: long # long (12 months) | short | narrow placeholders: - blank: "[_____]" # empty or whitespace only: renders [TBD: id] (§13.5 rule 6) + blank: "[_____]" # empty, or only whitespace/invisible characters: renders [TBD: id] (§13.5 rule 6) show_prompt: false # show the question's prompt next to the blank (§15.8) title_block: { parties: true, effective_date: true, version: false } diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 87fa466..189e414 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -23,6 +23,7 @@ """ from __future__ import annotations +import unicodedata from collections.abc import Callable from dataclasses import dataclass, replace from typing import Any, NamedTuple @@ -752,9 +753,12 @@ def _placeholder(self, directive: Directive, placeholder_id: str) -> Inline: prompt = "" if self.style.placeholders.show_prompt and isinstance(declared, dict): prompt = str(declared.get("prompt") or "") - # §13.5 rule 6: a style whose blank is empty offers no visual blank. + # §13.5 rule 6: a style whose blank is empty offers no visual blank, + # and neither does one of whitespace or format characters only + # (U+200B zero-width space, U+FEFF, bidi controls): it shows nothing. blank = self.style.placeholders.blank - return Blank(placeholder_id, blank if blank.strip() else f"[TBD: {placeholder_id}]", prompt) + shows = any(not c.isspace() and unicodedata.category(c) != "Cf" for c in blank) + return Blank(placeholder_id, blank if shows else f"[TBD: {placeholder_id}]", prompt) def _attach(self, directive: Directive, attachment_id: str) -> Inline: title = self.result.index.attachment_lookup.get(attachment_id) diff --git a/tests/test_wp10_review_fixes.py b/tests/test_wp10_review_fixes.py index 324144d..2c6ca00 100644 --- a/tests/test_wp10_review_fixes.py +++ b/tests/test_wp10_review_fixes.py @@ -202,3 +202,19 @@ def test_a_plain_label_may_contain_braced_words() -> None: def test_the_labels_that_substitute_still_check_their_fields() -> None: assert _problems({"labels.condition": "{condtion}"}) assert _problems({"labels.attachment_file": "{fil}"}) + + +# -- 7: a blank of invisible characters is no visual blank ------------------------ + + +@pytest.mark.parametrize("blank", ["​", "", " ​‍ ", " ", "‮⁠"]) +@pytest.mark.parametrize("fmt", ["html", "text"]) +def test_a_blank_of_invisible_characters_falls_back_to_tbd(blank: str, fmt: str) -> None: + source = FRONT.replace("title: T", "title: T\nplaceholders:\n - id: fee\n type: text") + "# A\n\nFee {{placeholder: fee}}.\n" + result = render(source, format=fmt, overrides={"placeholders.blank": blank}) + assert "[TBD: fee]" in result.output + + +def test_a_blank_with_a_visible_character_is_kept() -> None: + source = FRONT.replace("title: T", "title: T\nplaceholders:\n - id: fee\n type: text") + "# A\n\nFee {{placeholder: fee}}.\n" + assert "Fee ​__​." in render(source, format="text", overrides={"placeholders.blank": "​__​"}).output From 12879c5c0cc19f86ac878a95472ea0bff6508269 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:11:34 +0000 Subject: [PATCH 023/118] =?UTF-8?q?Labels:=20neutral=20identifier=20labels?= =?UTF-8?q?,=20since=20identification=5Fnumber=20covers=20natural=20person?= =?UTF-8?q?s=20(=C2=A73.4)=20(review=20wave=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit en Identification No., de Identifikationsnummer, fr Numéro d'identification, pl Numer identyfikacyjny. Goldens features.* change only that line; test_wp5_text and test_wp8_labels follow (de added). --- src/legaldown_render/style/labels.py | 8 ++++---- tests/golden/features.continental.txt | 2 +- tests/golden/features.default.html | 2 +- tests/golden/features.default.txt | 2 +- tests/golden/features.outline.txt | 2 +- tests/test_wp5_text.py | 2 +- tests/test_wp8_labels.py | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/legaldown_render/style/labels.py b/src/legaldown_render/style/labels.py index 62b7f5e..015d5c3 100644 --- a/src/legaldown_render/style/labels.py +++ b/src/legaldown_render/style/labels.py @@ -10,7 +10,7 @@ "en": Labels( drafting_note="Drafting note", condition="Only if: {condition}", - identification_number="Registration No.", + identification_number="Identification No.", date_of_birth="Date of birth", address="Address", represented_by="Represented by", @@ -48,7 +48,7 @@ "de": Labels( drafting_note="Bearbeitungshinweis", condition="Nur wenn: {condition}", - identification_number="Registernummer", + identification_number="Identifikationsnummer", date_of_birth="Geburtsdatum", address="Anschrift", represented_by="Vertreten durch", @@ -68,7 +68,7 @@ "fr": Labels( drafting_note="Note de rédaction", condition="Uniquement si\u00a0: {condition}", - identification_number="N° d'immatriculation", + identification_number="Numéro d'identification", date_of_birth="Date de naissance", address="Adresse", represented_by="Représenté(e) par", @@ -87,7 +87,7 @@ "pl": Labels( drafting_note="Uwaga redakcyjna", condition="Tylko jeżeli: {condition}", - identification_number="Nr rejestrowy", + identification_number="Numer identyfikacyjny", date_of_birth="Data urodzenia", address="Adres", represented_by="Reprezentacja", diff --git a/tests/golden/features.continental.txt b/tests/golden/features.continental.txt index d44aedb..3454d11 100644 --- a/tests/golden/features.continental.txt +++ b/tests/golden/features.continental.txt @@ -6,7 +6,7 @@ Effective date: June 1, 2026 Provider Acme Corporation - Registration No.: DE-12345678 + Identification No.: DE-12345678 Address: 123 Main Street, Dover, DE 19901 Represented by: John Smith, Chief Executive Officer diff --git a/tests/golden/features.default.html b/tests/golden/features.default.html index a7a45a5..09c78d3 100644 --- a/tests/golden/features.default.html +++ b/tests/golden/features.default.html @@ -89,7 +89,7 @@

Services Agreement

Acme Corporation

-
Registration No.
DE-12345678
+
Identification No.
DE-12345678
Address
123 Main Street, Dover, DE 19901
Represented by
John Smith, Chief Executive Officer
diff --git a/tests/golden/features.default.txt b/tests/golden/features.default.txt index d7c8ab9..5ea8aa9 100644 --- a/tests/golden/features.default.txt +++ b/tests/golden/features.default.txt @@ -6,7 +6,7 @@ Effective date: June 1, 2026 Provider Acme Corporation - Registration No.: DE-12345678 + Identification No.: DE-12345678 Address: 123 Main Street, Dover, DE 19901 Represented by: John Smith, Chief Executive Officer diff --git a/tests/golden/features.outline.txt b/tests/golden/features.outline.txt index b8f4da2..09536b2 100644 --- a/tests/golden/features.outline.txt +++ b/tests/golden/features.outline.txt @@ -6,7 +6,7 @@ Effective date: June 1, 2026 Provider Acme Corporation - Registration No.: DE-12345678 + Identification No.: DE-12345678 Address: 123 Main Street, Dover, DE 19901 Represented by: John Smith, Chief Executive Officer diff --git a/tests/test_wp5_text.py b/tests/test_wp5_text.py index 6402e4a..a896a6d 100644 --- a/tests/test_wp5_text.py +++ b/tests/test_wp5_text.py @@ -147,7 +147,7 @@ def test_a_blanked_title_block_label_drops_its_prefix() -> None: def test_a_label_still_prefixes_its_value() -> None: output = text("# A\n\nBody.\n") - assert "Registration No.: DE-1" in output + assert "Identification No.: DE-1" in output assert "Effective date: June 1, 2026" in output diff --git a/tests/test_wp8_labels.py b/tests/test_wp8_labels.py index cb2a3fd..5cfbce2 100644 --- a/tests/test_wp8_labels.py +++ b/tests/test_wp8_labels.py @@ -28,8 +28,8 @@ @pytest.mark.parametrize(("language", "identifier", "represented_by"), [ ("cs", "Identifikační číslo", "Zastoupení"), ("sk", "Identifikačné číslo", "Zastúpenie"), - ("fr", "N° d'immatriculation", "Représenté(e) par"), - ("pl", "Nr rejestrowy", "Reprezentacja"), + ("fr", "Numéro d'identification", "Représenté(e) par"), + ("pl", "Numer identyfikacyjny", "Reprezentacja"), ]) def test_party_labels_in_each_language(language: str, identifier: str, represented_by: str) -> None: colon = BUILTIN_LABELS[language].colon From cc4d7b7d4613a3d937fde2c5fca41f6a3e26a202 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:12:39 +0000 Subject: [PATCH 024/118] Docs: link allow-list, render-link-refused, escaped diagnostics, style validation scope, overrides note (review wave 2) --- CONFORMANCE.md | 5 +++-- README.md | 3 ++- docs/architecture.md | 30 +++++++++++++++++++++--------- docs/concepts.md | 6 ++++-- docs/style-templates.md | 20 +++++++++++++------- 5 files changed, 43 insertions(+), 21 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 66aa38c..3f95325 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -39,7 +39,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `ref-not-enumerated` Warning (§6.3, §16.3) | ✅ | | `raw-html` Warning; raw HTML never emitted (§8.7) | ✅ Nothing is emitted: an HTML block is dropped whole, and an inline tag is dropped with its text kept. The Warning is the validator's, once per block or text that holds raw HTML other than comments (see "Text after a comment", below) | | Automatic term recognition (§7.4, MAY) | ❌ Not supported | -| Links (§8.7) | ✅ Every link is a hyperlink in HTML and shows `text ` in plain text, whatever its scheme (`ftp:`, `sip:`, `urn:`, ...), except one whose URL is empty or uses `javascript:`, `vbscript:`, `file:`, or `data:`, which shows its text alone. A link with no text shows its URL; the text writer omits `` for an autolink | +| Links (§8.7) | ✅ A link is a hyperlink in HTML and shows `text ` in plain text when its URL is relative, a fragment, or has one of the schemes `http`, `https`, `mailto`, `tel`, `ftp`, `ftps`, `sftp`, `sip`, `sips`, `sms`, `urn`, `geo`, `news` (an allow-list, judged as a browser reads the URL). Any other scheme (`javascript:`, `file:`, `data:`, protocol handlers such as `ms-msdt:`, `vscode:`, `search-ms:`, and pseudo-schemes such as `C:\x` or `localhost:8080`), or an empty URL, is not a link: its text is shown alone, and the Warning `render-link-refused` is reported for a non-empty URL. A link with no text shows its URL as written; the text writer omits `` for an autolink | | Images and file-reference paths (§8.7, §2.3) | ✅ A relative path (`../` included, as §2.3 defines no default document root) renders `` in HTML and its alt text in plain text. A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is never fetched or embedded: the alt text is shown and the Error `path-not-relative` is reported, which `--strict` refuses. An empty path shows the alt text with no diagnostic | | Plain-text layout (§13.6, optional) | ✅ `attachments.separator` (§13.8): `none` prints no `---`; `rule` and `page-break` both print one before the attachments heading and between consecutive attachments, so the heading stays with the first (text has no pages). One quote stays one quote (blank lines keep the `>`), and the blocks of a list item are set apart by blank lines and indented under the item's label, whatever its condition label. A pipe in a table cell is backslash-escaped, and the separator row marks column alignment (`:---`, `:---:`, `---:`). A label a style blanks drops its `label: ` prefix in the title block, party details and signature blocks (a signature row with neither label nor value is omitted); HTML keeps an empty `
` so its grid layout and the `
` stay valid | | Extended tables through raw HTML (§9.2, MAY) | ❌ Not supported; the HTML table is dropped with the Warning | @@ -53,7 +53,8 @@ It is checked against the specification's own examples and fixtures corpus (`tes files, or `translations` is refused when rendered with answers, as §17.6 requires below Full. A template with Errors is refused too, since §15.7.2 defines no output for one. - **Line numbers.** The validator's diagnostics name their line, and the CLI prints it. The - renderer's own (`render-ref-ambiguous`, `ref-not-enumerated`, `render-not-processed`) do not + renderer's own (`render-ref-ambiguous`, `ref-not-enumerated`, `render-not-processed`, + `render-link-refused`) do not yet ([validator#93](https://github.com/ForLegalAI/legaldown-validator/issues/93)). - **`path-not-relative` for images.** The renderer reports the Core rule `path-not-relative` (§2.3, §16.2) for an image path, because the validator (0.4) does not check image paths; it diff --git a/README.md b/README.md index ef9fdc4..3e72d7a 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,8 @@ Problems in the document never raise. Only an invalid style (`StyleError`), an u format (`OptionsError`), an unreadable document (`DocumentError`), strict mode, or a template that cannot be assembled with its answers (`RenderRefused`) do. All of them are `RenderError`s, and `StyleError` and `OptionsError` are also `ValueError`s. They can be pickled, so they cross process -boundaries (`multiprocessing`, `concurrent.futures`) intact. +boundaries (`multiprocessing`, `concurrent.futures`) intact. `overrides=` takes dotted settings and +may not contain `version` or `extends`, so `overrides=style_to_dict(style)` must drop those two keys. ## Styles and preferences diff --git a/docs/architecture.md b/docs/architecture.md index 492a95f..4799b8c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -239,7 +239,7 @@ turns into an error. Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can evaluate: `ref-not-enumerated` and `path-not-relative` (an Error for an image path that is not relative, §2.3, reported here because validator 0.4 does not check image paths), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), and `render-ref-ambiguous`. The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), and `render-ref-ambiguous`. The validator's diagnostics carry their line (§16.9), and the CLI prints it as `file:line:`. The renderer's own have none yet: they are about a directive within a block, and the validator makes only a block's line public (`Document.line_of`, U2). @@ -248,15 +248,27 @@ line public (`Document.line_of`, U2). - Raw HTML in the source is never emitted (§8.7). - Every text node and attribute is escaped. -- A link is a hyperlink (§8.7) unless its URL is empty or its scheme is `javascript`, `vbscript`, - `file`, or `data` (a deny-list, `urls.DENIED_SCHEMES`, judged as a browser reads the URL: tabs, - newlines, controls, and case are ignored). Every other scheme (`ftp:`, `sip:`, `urn:`, ...) is - linked. A link that is not made renders as its text alone, in HTML and in text alike. The - builder turns off markdown-it-py's own `validateLink` (which would leave such links as literal - Markdown) and keeps them as nodes, so that the resolver can report §2.3 paths and the writers' - deny-list is the one gate for every `href` and `src` the HTML writer emits (links, images, - value links). A link with no text shows its URL; the text writer leaves out ` ` when the URL, less a +- A link is a hyperlink (§8.7) only if its URL is relative (no scheme, a fragment) or its scheme + is one of `urls.LINK_SCHEMES`: `http`, `https`, `mailto`, `tel`, `ftp`, `ftps`, `sftp`, `sip`, + `sips`, `sms`, `urn`, `geo`, `news`. It is an allow-list, because a renderer reads documents it + did not write (principle 6 in concepts.md): protocol handlers (`ms-msdt:`, `ms-word:`, + `search-ms:`, `vscode:`, `intent:`, `blob:`, `jar:`, ...) and pseudo-schemes (`C:\x`, + `localhost:8080`) are not links, nor are `javascript:`, `file:`, `data:`. The scheme is read as + a browser reads the URL (tabs, newlines, controls, and case are ignored). One predicate, + `urls.is_link_target`, serves both writers and the resolver. A link that is not made renders as + its text alone, in HTML and in text alike, and the resolver reports a non-empty one as the + Warning `render-link-refused` (the writers emit no diagnostics, ADR 0003); an empty URL is + text with no diagnostic. The builder turns off markdown-it-py's own `validateLink` (which would + leave such links as literal Markdown) and keeps them as nodes, so that the resolver can report + §2.3 paths and the allow-list is the one gate for every `href` and `src` the HTML writer emits + (links, images, value links). A link with no text shows its URL as written (percent-encoding + undone; the `href` stays encoded); the text writer leaves out ` ` when the URL, less a `mailto:` or `tel:` prefix, is the text. +- A diagnostic never carries a raw control character from the document: a path or URL quoted in a + message (`path-not-relative`, `render-link-refused`) goes through `urls.printable`, which writes + every C0/C1 control, DEL, Unicode format character (bidi controls, zero-width), and U+2028/9 as + `\xNN` or `\uNNNN`, so a crafted URL cannot move the cursor, set a terminal title, or forge a + line on stderr. - An image path must be relative with no URI scheme (§2.3), so a remote resource is never fetched or embedded. Resolution, not a writer, enforces it: an image whose path has a scheme (`https:`, `file:`, `data:`, a Windows drive such as `C:\x`) or starts with `/` or `\` (`//host`) is diff --git a/docs/concepts.md b/docs/concepts.md index e5d895a..2d66b40 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -83,8 +83,10 @@ no random ids, stable anchor names, stable ordering. This makes golden-file test lets rendered output be diffed in Git just like the source. **6. It is safe with documents you did not write.** Raw HTML in the source is never emitted -(§8.7). All text is escaped for the target format. A link whose URL is empty or uses a script, -`file:`, or `data:` scheme is shown as its text, and an image whose path is not relative is shown as +(§8.7). All text is escaped for the target format. A link is made only for a relative URL or +one of a short list of schemes (`http`, `https`, `mailto`, `tel`, `ftp`, ...); any other +(`javascript:`, `file:`, `data:`, protocol handlers such as `ms-msdt:`) is shown as its text and +reported, and an image whose path is not relative is shown as its alt text and reported (§2.3). At Rendering level the renderer reads no files other than the document, the chosen style template, and the answers file the CLI is given with `--answers`. diff --git a/docs/style-templates.md b/docs/style-templates.md index cb1f28b..780cda3 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -161,21 +161,27 @@ are set off by a space ("Termination (a)", §13.3). ### Text templates and their fields -A text setting that holds `{field}` names is checked when the style is loaded. A field is a name -in braces that is an identifier (`{n}`, `{designation}`); any other braces (`{}`, `{1}`, `{a b}`) -are literal text. An identifier the setting does not accept is an error naming the setting (a -typo would otherwise be shown literally), and so is a missing required field. +A text setting that substitutes `{field}` names is checked when the style is loaded. A field is a +name in braces that is an identifier (`{n}`, `{designation}`); any other braces (`{}`, `{1}`, +`{a b}`) are literal text. An identifier the setting does not accept is an error naming the +setting (a typo would otherwise be shown literally), and so is a missing required field, but only +where the setting is in use: the `ref` of `numbering.levels` is required to hold `{n}` unless +`numbering.scheme` is `none` (a reference then shows the heading text); the `ref` of +`enumeration.levels` only when lists are enumerated (`enumeration.enabled: true`, or `ordered: +enumerate`); `paragraphs.format.ref` only when `paragraphs.numbered` is true. A typo in a field is +flagged either way. Plain labels substitute nothing, so they are not checked: `{word}` in one is +just text. | Setting | Accepted fields | Required | |---|---|---| | `numbering.levels[].label` | `{n}`, `{path}` | none | -| `numbering.levels[].ref` | `{n}` | `{n}` | +| `numbering.levels[].ref` | `{n}` | `{n}`, unless `numbering.scheme: none` | | `enumeration.levels[].label`, `paragraphs.format.label` | `{n}`, `{path}`, `{section}` | none | -| `enumeration.levels[].ref`, `paragraphs.format.ref` | `{n}`, `{section}` | `{n}` | +| `enumeration.levels[].ref`, `paragraphs.format.ref` | `{n}`, `{section}` | `{n}`, when lists are enumerated / paragraphs are numbered | | `references.format` | `{designation}` | `{designation}`, or every reference reads the same | | `labels.condition` | `{condition}` | `{condition}`, as §15.8 requires the condition to be shown; an empty value is allowed and shows the bare condition | | `labels.attachment_file` | `{file}` | none | -| every other label | none | none | +| every other label | not checked: braces are text | none | ``` $ legaldown-render contract.lgd --set references.format=nope From 6652ab6afb7f61ca560bea66413d67312cc90b42 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:19:08 +0000 Subject: [PATCH 025/118] Wave 3 review fixes: plural of huge fractions, text list items, -o extension, term cache in cycles 1. Plural fallback for huge fractions (values.py _plural_category, Formatter._money_name): when Babel's plural operands fail (4300-digit limit), the category is computed from the value with its fraction cut to the first 30 digits (integer part exact, v != 0); only if that fails is it "other". The old comment ("other is the non-integer category in nearly every locale") was false for cs, sk, lt (many). Name-mode money does the same. 2. Text writer (TextWriter.list): an item whose first block is not a paragraph (table, code block, quote, as for a nested list) gets its marker on its own line and the block indented under the item text, instead of being flattened onto the label line. Paragraph-first items and all goldens are unchanged. 3. -o extension rule (cli.py, writers/__init__.py): new file_extension() strips trailing dots and spaces and takes the text after the last dot even in a dotfile name, so ".pdf", ".md", "x.pdf." and "x.docx " are refused without -f; format_for_path uses it too (".txt", "x.txt." write text). No extension / unknown extensions still write HTML; -f still wins. 4. Term text caching (resolver.py _term_text): a text worked out while another term's text is open depends on which term was open, so only the outermost text is cached. Existing tests changed because they pinned the old behaviour: - tests/test_wp9_review_fixes.py: a 4400-digit fraction in Czech years is now "roku" (many), not " let". - tests/test_render.py test_an_item_opening_with_a_drafting_note_or_code: the label now stands alone and the quote/code is indented under the item text (was "(b) code" and "(a) > [Drafting note]"). Docs: README (-o rule), CONFORMANCE.md (term cycles, text list layout), docs/architecture.md (plural). --- CONFORMANCE.md | 6 +- README.md | 6 +- docs/architecture.md | 2 +- src/legaldown_render/cli.py | 15 +- src/legaldown_render/resolve/resolver.py | 10 +- src/legaldown_render/resolve/values.py | 58 ++++++-- src/legaldown_render/writers/__init__.py | 16 ++- src/legaldown_render/writers/text.py | 15 +- tests/test_render.py | 6 +- tests/test_wp11_review_fixes.py | 166 +++++++++++++++++++++++ tests/test_wp9_review_fixes.py | 6 +- 11 files changed, 266 insertions(+), 40 deletions(-) create mode 100644 tests/test_wp11_review_fixes.py diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 3f95325..fc158d7 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -41,7 +41,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | Automatic term recognition (§7.4, MAY) | ❌ Not supported | | Links (§8.7) | ✅ A link is a hyperlink in HTML and shows `text ` in plain text when its URL is relative, a fragment, or has one of the schemes `http`, `https`, `mailto`, `tel`, `ftp`, `ftps`, `sftp`, `sip`, `sips`, `sms`, `urn`, `geo`, `news` (an allow-list, judged as a browser reads the URL). Any other scheme (`javascript:`, `file:`, `data:`, protocol handlers such as `ms-msdt:`, `vscode:`, `search-ms:`, and pseudo-schemes such as `C:\x` or `localhost:8080`), or an empty URL, is not a link: its text is shown alone, and the Warning `render-link-refused` is reported for a non-empty URL. A link with no text shows its URL as written; the text writer omits `` for an autolink | | Images and file-reference paths (§8.7, §2.3) | ✅ A relative path (`../` included, as §2.3 defines no default document root) renders `` in HTML and its alt text in plain text. A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is never fetched or embedded: the alt text is shown and the Error `path-not-relative` is reported, which `--strict` refuses. An empty path shows the alt text with no diagnostic | -| Plain-text layout (§13.6, optional) | ✅ `attachments.separator` (§13.8): `none` prints no `---`; `rule` and `page-break` both print one before the attachments heading and between consecutive attachments, so the heading stays with the first (text has no pages). One quote stays one quote (blank lines keep the `>`), and the blocks of a list item are set apart by blank lines and indented under the item's label, whatever its condition label. A pipe in a table cell is backslash-escaped, and the separator row marks column alignment (`:---`, `:---:`, `---:`). A label a style blanks drops its `label: ` prefix in the title block, party details and signature blocks (a signature row with neither label nor value is omitted); HTML keeps an empty `
` so its grid layout and the `
` stay valid | +| Plain-text layout (§13.6, optional) | ✅ `attachments.separator` (§13.8): `none` prints no `---`; `rule` and `page-break` both print one before the attachments heading and between consecutive attachments, so the heading stays with the first (text has no pages). One quote stays one quote (blank lines keep the `>`), and the blocks of a list item are set apart by blank lines and indented under the item's label, whatever its condition label; an item that opens with a nested list, table, code block or quote has its label on a line of its own, so that block stays valid. A pipe in a table cell is backslash-escaped, and the separator row marks column alignment (`:---`, `:---:`, `---:`). A label a style blanks drops its `label: ` prefix in the title block, party details and signature blocks (a signature row with neither label nor value is omitted); HTML keeps an empty `
` so its grid layout and the `
` stay valid | | Extended tables through raw HTML (§9.2, MAY) | ❌ Not supported; the HTML table is dropped with the Warning | | Visual blank for a placeholder (§13.5 rule 6) | ✅ `placeholders.blank`; a blank that is empty, or only whitespace or invisible format characters (U+200B, U+FEFF), renders `[TBD: id]` | | Question prompts next to blanks (§15.8, MAY) | ✅ `placeholders.show_prompt` | @@ -97,7 +97,9 @@ It is checked against the specification's own examples and fixtures corpus (`tes source; `label=` overrides it. A definition not in the body (a template's other alternative) falls back to the validator's term text. A term that refers to itself, directly or through other terms (`"A {{term: a}}" {{def: a}}`), is - rendered without looping: the inner reference shows its `label=`, else the definition id. The + rendered without looping: the inner reference shows its `label=`, else the definition id. A term text + worked out while another term's text is open depends on which one is open, so only a text worked out + at the outermost level is kept and reused. The validator (0.4.0) does not report these cycles; the specification's `definition-circular` Warning (§16) is not emitted yet. - **Ambiguous references.** Each list starts again at its first number, as contracts are usually diff --git a/README.md b/README.md index 3e72d7a..3fa4b17 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,10 @@ from Python leaves `sys.stdout` and `sys.stderr` as they are. Without `--format`, `-o` takes the format from the file extension: `.html` and `.htm` write HTML, `.txt` and `.text` plain text. The extension of a document format this renderer does not write (`.docx`, `.doc`, `.odt`, `.rtf`, `.pdf`, `.md`, `.markdown`, `.epub`, `.tex`, in any case) is a usage -error (exit 2) unless `--format` says what to write. Any other name, with any other extension or -none (`contract`, `contract.v2`, `report.2025-10-27`, `.out`), is just a name and writes HTML. +error (exit 2) unless `--format` says what to write. The extension is the text after the last dot, +also in a name that starts with one (`.pdf`), and trailing dots and spaces are ignored (`x.pdf.`, +`"x.docx "`). Any other name, with any other extension or none (`contract`, `contract.v2`, +`report.2025-10-27`, `.out`), is just a name and writes HTML. ## Python diff --git a/docs/architecture.md b/docs/architecture.md index 4799b8c..4dd47df 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -144,7 +144,7 @@ Guessing at lost structure was tried, and it traded each gap for new bugs. | `{{ref:}}` | `CrossRef(designation, anchor)` (§13.3). A ref to an unnumbered item or paragraph falls back to its section's designation, with `ref-not-enumerated` | `[BROKEN REF: id]` | | `{{term:}}` | `TermRef(label or the definition's rendered term text, def anchor)` (§7.3, §13.4); the text is the first definition's term resolved as at the definition, worked out on first use by `Resolver._term_text` (`_survey` only records which definition is first) | `[UNDEFINED: id]` | | `"Term" {{def:}}` | `DefinedTerm` without quotation marks, anchored at `def:` (§7.2) | n/a | -| `{{date:}}` `{{money:}}` `{{duration:}}` | `Value`, formatted for the locale by Babel (ADR 0005): amounts and durations keep every written digit, and a duration without a pattern in the locale falls back to another length, the language alone, then English with `render-locale-fallback` | `[INVALID DATE: …]`, `[INVALID AMOUNT: …]`, `[UNKNOWN CURRENCY: …]`, `[INVALID DURATION: …]`, `[INVALID DURATION UNIT: …]` | +| `{{date:}}` `{{money:}}` `{{duration:}}` | `Value`, formatted for the locale by Babel (ADR 0005): amounts and durations keep every written digit (a fraction beyond Python's 4300-digit limit takes its plural form from its first 30 digits), and a duration without a pattern in the locale falls back to another length, the language alone, then English with `render-locale-fallback` | `[INVALID DATE: …]`, `[INVALID AMOUNT: …]`, `[UNKNOWN CURRENCY: …]`, `[INVALID DURATION: …]`, `[INVALID DURATION UNIT: …]` | | `{{party:}}` `{{side:}}` | `Value(label or display name)` (§3.6) | `[INVALID PARTY: …]`, `[UNKNOWN PARTY: …]`, and the side forms | | `{{field:}}` | `Value(raw value)` | `[INVALID FIELD: …]` | | `{{placeholder:}}` | `Blank`, with the question's prompt if the style asks for it | `[INVALID PLACEHOLDER: …]`, `[UNKNOWN CURRENCY: …]`, `[INVALID DURATION UNIT: …]` | diff --git a/src/legaldown_render/cli.py b/src/legaldown_render/cli.py index a7be21f..e0cdea8 100644 --- a/src/legaldown_render/cli.py +++ b/src/legaldown_render/cli.py @@ -24,13 +24,14 @@ from .api import RenderOptions, render from .errors import DocumentError, InternalError, RenderRefused, StyleError from .style import builtin_styles, dump_style, load_style, parse_override -from .writers import FORMATS, format_for_path +from .writers import FORMATS, file_extension, format_for_path -#: Extensions of document formats this renderer does not write. An output file -#: named so (``x.docx``) and no ``--format`` is a mistake to refuse: HTML under -#: that name would mislead. Any other extension (``contract.v2``, -#: ``report.2025-10-27``), or none, is just a name and keeps the HTML default. -_UNWRITTEN_EXTENSIONS = frozenset({".docx", ".doc", ".odt", ".rtf", ".pdf", ".md", ".markdown", ".epub", ".tex"}) +#: Extensions (:func:`file_extension`, so no dot) of document formats this +#: renderer does not write. An output file named so (``x.docx``, ``.pdf``, +#: ``x.pdf.``) and no ``--format`` is a mistake to refuse: HTML under that name +#: would mislead. Any other extension (``contract.v2``, ``report.2025-10-27``), +#: or none, is just a name and keeps the HTML default. +_UNWRITTEN_EXTENSIONS = frozenset({"docx", "doc", "odt", "rtf", "pdf", "md", "markdown", "epub", "tex"}) def _setting(text: str) -> tuple[str, Any]: @@ -142,7 +143,7 @@ def main(argv: list[str] | None = None) -> int: if not output_format and args.output: file_name = Path(args.output).name output_format = format_for_path(file_name) - if not output_format and Path(file_name).suffix.lower() in _UNWRITTEN_EXTENSIONS: + if not output_format and file_extension(file_name) in _UNWRITTEN_EXTENSIONS: parser.error(f"cannot tell the output format from '{args.output}'; use --format html or text") output_format = output_format or "html" name = "" if args.input == "-" else args.input diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 189e414..83af663 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -642,7 +642,11 @@ def _term_text(self, definition_id: str) -> str: definition is not in the body. Worked out the first time it is needed and kept, which is once every - target is registered and the placeholder types are known. Before that + target is registered and the placeholder types are known. Only a text + worked out at the outermost level is kept: one worked out while another + term's text is being worked out has that term open, so a reference back + to it shows its label or id there, and the same term resolved alone + would read differently (a cycle). Before that (a heading read as text by the ``none`` scheme, §13.3) a text-only pass stands in, which leaves references out and is not kept. Resolving the term gives out no anchor: a definition inside a term keeps its place in @@ -655,6 +659,7 @@ def _term_text(self, definition_id: str) -> str: if definition_id in self.term_texts: return self.term_texts[definition_id] self._terms_open.append(definition_id) + outermost = len(self._terms_open) == 1 try: if not self._targets_known: return self._title_text(source.children) @@ -665,7 +670,8 @@ def _term_text(self, definition_id: str) -> str: self.used_anchors = anchors finally: self._terms_open.pop() - self.term_texts[definition_id] = text + if outermost: + self.term_texts[definition_id] = text return text def _plain_open(self, definition_id: str, term: str) -> str: diff --git a/src/legaldown_render/resolve/values.py b/src/legaldown_render/resolve/values.py index ba94c13..52be250 100644 --- a/src/legaldown_render/resolve/values.py +++ b/src/legaldown_render/resolve/values.py @@ -107,20 +107,44 @@ def _unit_patterns(locale: Locale, unit_id: str, length: str) -> dict[str, str]: return {category: pattern for category, pattern in patterns.items() if category != "per"} +_PLURAL_FRACTION_DIGITS = 30 +"""Fraction digits kept when the plural category of a value with a huge +fraction is computed from a shortened copy (see :func:`_plural_category`).""" + + +def _truncated_fraction(value: Decimal) -> Decimal: + """*value* with its fraction cut to its first :data:`_PLURAL_FRACTION_DIGITS` + digits (cut, not rounded), so the integer part stays exact and a value that + had a fraction still has one (CLDR's operand ``v`` stays non-zero). The + digits are rearranged, not computed with, so no precision limit applies.""" + sign, digits, exponent = value.as_tuple() + if not isinstance(exponent, int) or exponent >= -_PLURAL_FRACTION_DIGITS: + return value + keep = len(digits) + exponent + _PLURAL_FRACTION_DIGITS + digits = digits[:keep] if keep > 0 else (0,) + return Decimal((sign, digits, -_PLURAL_FRACTION_DIGITS)) + + def _plural_category(locale: Locale, value: Decimal) -> str: """The CLDR plural category of *value* in *locale*. CLDR's plural operands of a value are its integer and fraction digits read as integers, and Python refuses to convert a string of more than 4300 digits - to an int (``ValueError``). Such a value is a non-integer or a very long - integer whose category cannot be computed under that limit, and ``other`` is - the category CLDR gives non-integers in nearly every locale, so ``other`` it - is. The limit is not raised: ``sys.set_int_max_str_digits`` is process-wide. + to an int (``ValueError``). For a value whose fraction is that long the + category is computed from the same value with its fraction cut to its first + 30 digits: that keeps the integer part and the fact that there is a + fraction, which is what the category depends on in practice ("1,333..." is + ``many`` in Czech, Slovak and Lithuanian, not ``other``). A value whose + integer part is itself too long has no category that can be computed, and + ``other`` it is. The limit is not raised: ``sys.set_int_max_str_digits`` is + process-wide. """ - try: - return locale.plural_form(value) - except ValueError: - return "other" + for candidate in (value, _truncated_fraction(value)): + try: + return locale.plural_form(candidate) + except ValueError: + continue + return "other" def _duration_pattern(locale: Locale, unit_id: str, length: str, value: Decimal) -> tuple[str, bool]: @@ -216,17 +240,23 @@ def _money(self, amount: str, currency: str) -> str: def _money_name(self, value: Decimal, currency: str, pattern: NumberPattern) -> str: """*value* with the currency's name ("1,234.50 euros"), through Babel - unless the plural category of *value* cannot be computed (see - :func:`_plural_category`): Babel's own lookup of the unit pattern would - then fail, so the ``other`` pattern and name are put together here, the - way Babel does it (``_format_currency_long_name``).""" + unless the plural category of *value* cannot be computed from its own + digits (see :func:`_plural_category`): Babel's own lookup of the unit + pattern would then fail, so the pattern and name are put together here, + the way Babel does it (``_format_currency_long_name``), for the + category of the value with its fraction shortened (``other`` when even + that fails).""" try: self.locale.plural_form(value) except ValueError: number = pattern.apply(value, self.locale, currency=currency, currency_digits=False, decimal_quantization=False) - unit_pattern = get_currency_unit_pattern(currency, locale=self.locale) - return unit_pattern.format(number, get_currency_name(currency, count=value, locale=self.locale)) + shortened = _truncated_fraction(value) + try: + unit_pattern = get_currency_unit_pattern(currency, count=shortened, locale=self.locale) + except ValueError: + unit_pattern = get_currency_unit_pattern(currency, locale=self.locale) + return unit_pattern.format(number, get_currency_name(currency, count=shortened, locale=self.locale)) return format_currency(value, currency, format=pattern, locale=self.locale, currency_digits=False, format_type="name", decimal_quantization=False) diff --git a/src/legaldown_render/writers/__init__.py b/src/legaldown_render/writers/__init__.py index a3d25b0..cf77336 100644 --- a/src/legaldown_render/writers/__init__.py +++ b/src/legaldown_render/writers/__init__.py @@ -12,10 +12,22 @@ } +def file_extension(name: str) -> str: + """The lower-cased extension of a file name, without the dot, or ``""``. + + It is the text after the last dot, also in a name that starts with one + (``.pdf`` is a PDF name here, not a name without extension), and trailing + dots and spaces are not part of the name (Windows drops them, so + ``x.pdf.`` and ``x.docx `` name the same files as ``x.pdf`` and ``x.docx``). + """ + name = name.rstrip(". ") + return name.rsplit(".", 1)[-1].lower() if "." in name else "" + + def format_for_path(path: str) -> str | None: """The output format a file name implies, or None.""" - extension = path.rsplit(".", 1)[-1].lower() if "." in path else "" + extension = file_extension(path) return next((name for name, extensions in FORMATS.items() if extension in extensions), None) -__all__ = ["FORMATS", "HtmlWriter", "TextWriter", "format_for_path", "stylesheet"] +__all__ = ["FORMATS", "HtmlWriter", "TextWriter", "file_extension", "format_for_path", "stylesheet"] diff --git a/src/legaldown_render/writers/text.py b/src/legaldown_render/writers/text.py index aa6f328..449ba0a 100644 --- a/src/legaldown_render/writers/text.py +++ b/src/legaldown_render/writers/text.py @@ -188,7 +188,9 @@ def list(self, block: List, indent: str) -> list[str]: starts one column after the item's own label (or bullet): a condition label (§15.8) does not move it. Blocks of an item are set apart by a blank line, except that a nested list stays tight under the paragraph - or list it follows.""" + or list it follows. An item whose first block is not a paragraph (a + nested list, table, code block or quote) has its label on a line of its + own, so the block starts at a line start and stays what it is.""" lines: list[str] = [] for item in block.items: label = item.label if item.label else "-" @@ -198,13 +200,14 @@ def list(self, block: List, indent: str) -> list[str]: for child in item.blocks: if previous is not None and not (isinstance(child, List) and isinstance(previous, Paragraph | List)): lines.append("") + if previous is None and not isinstance(child, Paragraph): + # The item opens with a block that must start a line + # (a nested list, a table, code, a quote): its own label + # still comes first, on a line of its own. + lines.append(indent + marker) if isinstance(child, List): - if previous is None: - # The item opens with a nested list: its own label - # still comes first, on a line of its own. - lines.append(indent + marker) lines += self.list(child, content) - elif previous is None: + elif previous is None and isinstance(child, Paragraph): lines.append(_prefix(f"{marker} {self.block(child)}", indent, hanging=" " * (len(label) + 1))) else: lines.append(_prefix(self.block(child), content)) diff --git a/tests/test_render.py b/tests/test_render.py index d432e3a..93c507d 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -1113,8 +1113,10 @@ def test_a_marker_copy_inside_a_comment_is_not_the_marker() -> None: def test_an_item_opening_with_a_drafting_note_or_code() -> None: output = text("# A\n\n- > [!DRAFTING]\n > note here\n- ```\n code\n ```\n") - assert "(a) > [Drafting note]\n > note here" in output - assert "(b) code" in output + # The label stands alone and the block is indented under the item text (WP11): flattened + # onto the label line, a quote is no quote and code is mis-indented. + assert "(a)\n > [Drafting note]\n > note here" in output + assert "(b)\n code" in output assert "```" not in output diff --git a/tests/test_wp11_review_fixes.py b/tests/test_wp11_review_fixes.py new file mode 100644 index 0000000..e402ca5 --- /dev/null +++ b/tests/test_wp11_review_fixes.py @@ -0,0 +1,166 @@ +"""Review fixes after wave 3 (WP11): the plural of a huge fraction is computed +from its first digits, a list item opening with a table, code or quote keeps +that block intact in plain text, the ``-o`` extension rule has no holes, and a +term text worked out inside a cycle is not kept.""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from babel import Locale + +from legaldown_render import render +from legaldown_render.resolve.values import Formatter +from legaldown_render.style.model import Values +from legaldown_render.writers import file_extension, format_for_path + +SRC = Path(__file__).resolve().parents[1] / "src" +FRONT = "---\ntitle: T\n---\n\n" +FEATURES = str(Path(__file__).resolve().parent / "documents" / "features.lgd") + + +def text(body: str, **settings: object) -> str: + return render(FRONT + body, format="text", **settings).output + + +# -- 1: the plural category of a huge fraction ----------------------------------- + + +def _formatter(locale: str, **values: str) -> Formatter: + return Formatter(Locale.parse(locale), Values(**values)) + + +@pytest.mark.parametrize("unit", ["D", "Y", "W"]) +@pytest.mark.parametrize("locale", ["cs", "sk", "lt", "pl", "ru", "en"]) +def test_a_huge_fraction_has_the_unit_form_of_its_first_digits(locale: str, unit: str) -> None: + formatter = _formatter(locale) + huge = formatter.format_duration("1." + "3" * 4400, unit).text + short = formatter.format_duration("1." + "3" * 40, unit).text + assert huge.replace("3" * 4400, "") == short.replace("3" * 40, "") + + +def test_a_huge_fraction_is_many_in_czech() -> None: + formatter = _formatter("cs") + assert formatter.format_duration("1." + "3" * 4400, "D").text.endswith(" dne") + assert formatter.format_duration("1." + "3" * 40, "D").text.endswith(" dne") + + +def test_a_huge_fraction_of_zeros_ending_in_a_digit_keeps_its_fraction() -> None: + # Cut to 30 digits it is "1.000...0": still a fraction (v != 0), so "many" in Czech. + value = "1." + "0" * 4399 + "1" + assert _formatter("cs").format_duration(value, "D").text.rsplit(" ", 1)[-1] == "dne" + + +def test_a_name_mode_money_amount_with_a_huge_fraction_uses_the_form_of_its_first_digits() -> None: + formatter = _formatter("cs", money="name") + huge = formatter.money("1." + "3" * 4400, "EUR") + short = formatter.money("1." + "3" * 40, "EUR") + assert huge.replace("3" * 4400, "") == short.replace("3" * 40, "") + assert huge.endswith("eura") # "many", not the "other" form "eur" + + +def test_an_integer_part_beyond_the_limit_falls_back_to_other() -> None: + # No category can be computed (not even from the first digits): "other" ("dnů" in Czech). + other = _formatter("cs").format_duration("1.5", "D").text.rsplit(" ", 1)[-1] + assert other == "dne" + assert _formatter("cs").format_duration("1" * 4400, "D").text.endswith(" dnů") + + +# -- 2: an item that opens with a table, code or a quote ------------------------- + + +def test_an_item_that_opens_with_a_table_keeps_a_valid_table() -> None: + out = text("# A\n\n1. | x | y |\n |---|---|\n | 1 | 2 |\n") + assert "\n1.\n | x | y |\n |---|---|\n | 1 | 2 |\n" in out + assert "1. |" not in out + + +def test_an_item_that_opens_with_code_indents_it_under_the_item_text() -> None: + out = text("# A\n\n1. ```\n code\n more\n ```\n2. next\n") + assert "\n1.\n code\n more\n2. next\n" in out + + +def test_an_item_that_opens_with_a_quote_puts_the_quote_on_its_own_lines() -> None: + out = text("# A\n\n1. > one\n >\n > two\n2. next\n") + assert "\n1.\n > one\n >\n > two\n2. next\n" in out + assert "1. >" not in out + + +def test_blocks_after_a_non_paragraph_first_block_are_set_apart() -> None: + out = text("# A\n\n1. | x |\n |---|\n | 1 |\n\n then text\n") + assert "\n1.\n | x |\n |---|\n | 1 |\n\n then text\n" in out + + +def test_an_item_that_opens_with_a_paragraph_is_unchanged() -> None: + out = text("# A\n\n1. first\n > quote\n2. second\n | x |\n |---|\n | 1 |\n") + assert "\n1. first\n\n > quote\n2. second\n\n | x |\n |---|\n | 1 |\n" in out + + +# -- 3: the extension rule of -o -------------------------------------------------- + + +@pytest.mark.parametrize(("name", "extension"), [ + ("x.pdf", "pdf"), (".pdf", "pdf"), (".md", "md"), ("x.pdf.", "pdf"), ("x.docx ", "docx"), + ("x.pdf. .", "pdf"), ("X.PDF", "pdf"), ("a.b.txt", "txt"), ("contract", ""), ("", ""), (".", ""), + ("..", ""), ("x.", ""), +]) +def test_the_extension_of_a_file_name(name: str, extension: str) -> None: + assert file_extension(name) == extension + + +@pytest.mark.parametrize(("name", "expected"), [ + (".txt", "text"), ("x.txt.", "text"), ("x.html ", "html"), (".htm", "html"), ("x.unknown", None), +]) +def test_the_format_a_file_name_implies(name: str, expected: str | None) -> None: + assert format_for_path(name) == expected + + +def _run(*arguments: str, cwd: Path) -> subprocess.CompletedProcess[str]: + env = {**os.environ, "PYTHONPATH": str(SRC)} + return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *arguments], + capture_output=True, text=True, cwd=cwd, env=env, check=False) + + +@pytest.mark.parametrize("name", [".pdf", ".md", "x.pdf.", "x.docx ", "x.docx. .", "dir.pdf", "x.MD"]) +def test_an_unwritten_extension_is_refused_however_it_is_spelled(tmp_path: Path, name: str) -> None: + result = _run(FEATURES, "-o", name, "-q", cwd=tmp_path) + assert result.returncode == 2, result.stderr + assert "cannot tell the output format" in result.stderr + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("name", ["x.pdf.", ".md", "x.docx "]) +def test_format_overrides_an_unwritten_extension(tmp_path: Path, name: str) -> None: + result = _run(FEATURES, "-o", name, "-f", "text", "-q", cwd=tmp_path) + assert result.returncode == 0, result.stderr + assert not (tmp_path / name).read_text(encoding="utf-8").lstrip().startswith("<") + + +@pytest.mark.parametrize("name", ["contract", "contract.v2", "report.2025-10-27", ".out", "x.", "x.."]) +def test_a_name_without_a_known_extension_still_writes_html(tmp_path: Path, name: str) -> None: + result = _run(FEATURES, "-o", name, "-q", cwd=tmp_path) + assert result.returncode == 0, result.stderr + assert " None: + result = _run(FEATURES, "-o", name, "-q", cwd=tmp_path) + assert result.returncode == 0, result.stderr + assert marker not in (tmp_path / name).read_text(encoding="utf-8").lower() + + +# -- 4: a term text worked out inside a cycle is not kept ------------------------ + +CYCLE = ('# A\n\nUsed first: {{term: ca}} and {{term: cb}}.\n\n' + '"Alpha {{term: cb}}" {{def: ca}}\n\n"Beta {{term: ca}}" {{def: cb}}\n') + + +def test_a_term_reference_reads_as_its_definition_in_a_cycle() -> None: + lines = text(CYCLE).splitlines() + used = next(line for line in lines if line.startswith("Used first:")) + site = next(line for line in lines if line.startswith("Beta ")) + assert used.endswith(f"and {site}.") diff --git a/tests/test_wp9_review_fixes.py b/tests/test_wp9_review_fixes.py index 3d9b5a3..1a01418 100644 --- a/tests/test_wp9_review_fixes.py +++ b/tests/test_wp9_review_fixes.py @@ -114,9 +114,11 @@ def test_a_money_amount_in_name_mode_with_a_4400_digit_fraction_renders() -> Non assert f"1.{LONG} euros" in output -def test_a_4400_digit_fraction_uses_the_other_category_in_czech() -> None: +def test_a_4400_digit_fraction_uses_the_many_category_in_czech() -> None: + # Wave 3 (WP11): the category comes from the first 30 fraction digits, so it is the + # "many" of "2,5 roku", not "other" (" let"). output = render(FRONT + f"# A\n\n{{{{duration: 1.{LONG}, unit=Y}}}}", format="text", locale="cs-CZ").output - assert output.rstrip().endswith(" let") # "other", not the "many" of "2,5 roku" + assert output.rstrip().endswith(" roku") # -- 4: only the program reconfigures the streams ------------------------------- From a89a0b12dc2531f004f63cd0026bd1dd3e43327e Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:32:05 +0000 Subject: [PATCH 026/118] Term texts: memoize per term in every pass, so branching chains are linear (review wave 3) The previous commit kept a term text only when worked out at the outermost level, which made a chain of terms that each use the next one twice exponential (a DoS from a short document); the text-only pass before the targets are known and the validator-text fallback were never memoized at all (exponential in every version). Resolver._while_open now keeps texts for the three passes (term_texts, term_texts_early, term_texts_plain). A text worked out with no cycle cut at a term opened before it is the term's own text and is reused everywhere (so a reference in a cycle still reads as the definition site: the earlier cycle test passes). A text cut short at an enclosing open term records the terms it named and the ones it was cut at, and is reused only where exactly those named terms are open (at most 32 per term). Chains without a cycle are worked out once per term; a cycle costs one more per way round. Tests (tests/test_wp11_review_fixes.py): branching chain under the decimal and none schemes, used in body text and in a heading, with and without a cycle back to t0, each under 2 s. The chain is n=20 (n=9 with a cycle), not 40: the text of t0 is itself 2**n words (4**n with a cycle), so longer chains are bound by output size, not by the term cache. --- CONFORMANCE.md | 5 +- src/legaldown_render/resolve/resolver.py | 102 +++++++++++++++++------ tests/test_wp11_review_fixes.py | 31 +++++++ 3 files changed, 112 insertions(+), 26 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index fc158d7..af6a33c 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -98,8 +98,9 @@ It is checked against the specification's own examples and fixtures corpus (`tes falls back to the validator's term text. A term that refers to itself, directly or through other terms (`"A {{term: a}}" {{def: a}}`), is rendered without looping: the inner reference shows its `label=`, else the definition id. A term text - worked out while another term's text is open depends on which one is open, so only a text worked out - at the outermost level is kept and reused. The + worked out inside a cycle depends on which terms are open, so a text with + a cut at a term open outside it is reused only where the same of its terms are open; every other text + is worked out once per term, also in a heading read before the targets are known (§13.3). The validator (0.4.0) does not report these cycles; the specification's `definition-circular` Warning (§16) is not emitted yet. - **Ambiguous references.** Each list starts again at its first number, as contracts are usually diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 83af663..524d9d4 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -114,6 +114,20 @@ } +#: How many texts (one per different set of open terms around it) are kept for +#: one term, so that a pathological tangle of cycles cannot fill memory. +_TERM_TEXTS_PER_TERM = 32 + + +class _TermText(NamedTuple): + """A term's text, the terms it named, and those it was cut short at + because an enclosing term had them open (see ``Resolver._while_open``).""" + + text: str + seen: frozenset[str] + cut: frozenset[str] + + @dataclass(frozen=True, slots=True) class _Target: """What a ``{{ref:}}`` to an anchor resolves to.""" @@ -157,11 +171,20 @@ def __init__( # and -> the plain text of its term as rendered there, worked out the # first time a {{term:}} needs it (see _term_text). self.term_sources: dict[str, DefinitionSource] = {} - self.term_texts: dict[str, str] = {} + self.term_texts: dict[str, list[_TermText]] = {} + # The same for the text-only pass before the targets are known, and + # for the validator's term text where a definition is not in the body. + self.term_texts_early: dict[str, list[_TermText]] = {} + self.term_texts_plain: dict[str, list[_TermText]] = {} # The definitions whose term text is being worked out, so that a # {{term:}} inside a term, directly or through other terms, cannot # send the computation round in a circle. self._terms_open: list[str] = [] + # Since the text now being worked out began (see _while_open): every + # term a {{term:}} named, and those of them it was cut short at + # because they were open. + self._seen: set[str] = set() + self._cut: set[str] = set() # False until every target is registered and the placeholder types are # known (run() sets it just before the inlines are resolved). self._targets_known = False @@ -625,7 +648,10 @@ def _term(self, directive: Directive, definition_id: str) -> Inline: # A term that refers to itself, directly or through others: the # reference shows its label, else the id, and goes no further. text = label or definition_id + self._seen.add(definition_id) + self._cut.add(definition_id) else: + self._seen.add(definition_id) # The term as rendered at its definition; the validator's own # text (an empty term reads as its id, a definition not in the # body) is the fallback. @@ -642,46 +668,74 @@ def _term_text(self, definition_id: str) -> str: definition is not in the body. Worked out the first time it is needed and kept, which is once every - target is registered and the placeholder types are known. Only a text - worked out at the outermost level is kept: one worked out while another - term's text is being worked out has that term open, so a reference back - to it shows its label or id there, and the same term resolved alone - would read differently (a cycle). Before that + target is registered and the placeholder types are known. Before that (a heading read as text by the ``none`` scheme, §13.3) a text-only pass - stands in, which leaves references out and is not kept. Resolving the - term gives out no anchor: a definition inside a term keeps its place in - document order. The definition is open meanwhile, so a ``{{term:}}`` - inside it, directly or through others, shows its label or id and goes - no further.""" + stands in, which leaves references out; it is kept apart from the + text of the later pass. Resolving the term gives out no anchor: a + definition inside a term keeps its place in document order. The + definition is open meanwhile, so a ``{{term:}}`` inside it, directly or + through others, shows its label or id and goes no further (see + :meth:`_while_open`).""" source = self.term_sources.get(definition_id) if source is None: return "" - if definition_id in self.term_texts: - return self.term_texts[definition_id] - self._terms_open.append(definition_id) - outermost = len(self._terms_open) == 1 - try: - if not self._targets_known: + known = self._targets_known + + def work() -> str: + if not known: return self._title_text(source.children) anchors = set(self.used_anchors) try: - text = plain_text(self._resolve_inlines(source.children)) + return plain_text(self._resolve_inlines(source.children)) finally: self.used_anchors = anchors - finally: - self._terms_open.pop() - if outermost: - self.term_texts[definition_id] = text - return text + + return self._while_open(self.term_texts if known else self.term_texts_early, definition_id, work) def _plain_open(self, definition_id: str, term: str) -> str: """The validator's *term* text of a definition as plain text, with the definition marked open while its directives resolve.""" + key = f"{definition_id}\0{int(self._targets_known)}" + return self._while_open(self.term_texts_plain, definition_id, lambda: self._plain_value(term), key=key) + + def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, work: Callable[[], str], + *, key: str | None = None) -> str: + """The text *work* gives with *definition_id* open, kept in *cache* + (under *key*, default the id), so a chain of terms that each use the + next one twice is worked out once per term, not once per use. + + What a text reads depends on which terms are open around it only + where a reference was cut short at an open term (it shows that term's + label or id). A text with no cut at a term opened *before* this one is + the term's own text, as the reader sees it at its definition, and is + reused wherever the term is needed. A text with such a cut (a cycle + back through an enclosing term) records the terms it named and the + ones it was cut short at, and is reused only where exactly those of + the terms it named are open. A cut at this term itself, or at one + opened inside it, is its own business and does not count. Without a + cycle every term is worked out once; with one, once for each way round + it is reached. + """ + key = definition_id if key is None else key + around = set(self._terms_open) + for kept in cache.get(key, ()): + if not kept.cut or kept.seen & around == kept.cut: + self._seen |= kept.seen + self._cut |= kept.cut + return kept.text self._terms_open.append(definition_id) + outer = self._seen, self._cut + self._seen, self._cut = set(), set() try: - return self._plain_value(term) + text = work() + seen, cut = frozenset(self._seen), frozenset(self._cut & around) + if len(cache.setdefault(key, [])) < _TERM_TEXTS_PER_TERM: + cache[key].append(_TermText(text, seen, cut)) finally: self._terms_open.pop() + self._seen = outer[0] | self._seen + self._cut = outer[1] | (self._cut & around) + return text def _date(self, directive: Directive, value: str) -> Inline: if not is_valid_iso_date(value): diff --git a/tests/test_wp11_review_fixes.py b/tests/test_wp11_review_fixes.py index e402ca5..ab17940 100644 --- a/tests/test_wp11_review_fixes.py +++ b/tests/test_wp11_review_fixes.py @@ -7,6 +7,7 @@ import os import subprocess import sys +import time from pathlib import Path import pytest @@ -164,3 +165,33 @@ def test_a_term_reference_reads_as_its_definition_in_a_cycle() -> None: used = next(line for line in lines if line.startswith("Used first:")) site = next(line for line in lines if line.startswith("Beta ")) assert used.endswith(f"and {site}.") + + +def _chain(n: int, *, cycle: bool, heading: bool = False) -> str: + """Terms t0..tn, each reading the next one twice; the last one reads t0 when *cycle*.""" + defs = [f'"T{i} {{{{term: t{i + 1}}}}} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + defs.append(f'"T{n}{" {{term: t0}}" if cycle else ""}" {{{{def: t{n}}}}}') + title = "About {{term: t0}}" if heading else "A" + return f"# {title}\n\nUsed: {{{{term: t0}}}}\n\n" + "\n\n".join(defs) + "\n" + + +@pytest.mark.parametrize("heading", [False, True]) +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +@pytest.mark.parametrize("cycle", [False, True]) +def test_a_branching_chain_of_terms_is_worked_out_once_per_term(cycle: bool, scheme: str, heading: bool) -> None: + # Each term reads the next twice: worked out afresh at each use, 20 terms is 2**20 resolutions. + # The text of t0 is itself 2**n words long (4**n with a cycle, which reads the chain round + # twice), so the chains cannot be longer. Under the none scheme a heading is read as text + # before the targets are known (§13.3). + started = time.perf_counter() + out = text(_chain(9 if cycle else 20, cycle=cycle, heading=heading), overrides={"numbering.scheme": scheme}) + assert time.perf_counter() - started < 2 + assert "{{" not in out + + +def test_a_chain_without_a_cycle_reads_as_each_term_does() -> None: + lines = text(_chain(3, cycle=False)).splitlines() + used = next(line for line in lines if line.startswith("Used: ")) + # t1 as used inside t0 is the same text as at t1's own definition. + site = next(line for line in lines if line.startswith("T1 ")) + assert used == f"Used: T0 {site} {site}" From fdbca3da21b8d8f5bbf3b448cbf277a7dee45930 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:34:16 +0000 Subject: [PATCH 027/118] Cap a term text at 1,000 characters, render-term-too-long (review wave 3) A chain of terms that each read the next twice asks for a text of 2**n words (4**n through a cycle), so a short document could exhaust memory however well the texts were cached. _term_text now checks the length as each term text is built (both the text-only pass before the targets are known and the later pass); a longer one becomes its definition id, is cached like any text, and the Warning render-term-too-long names the id once. A label= still wins. The definition site renders nested {{term:}} through the same path, so nothing is built past the source plus the nested references times the cap. Tests (tests/test_wp11_review_fixes.py): the n=40 branching chain and the n=14 cyclic chain (OOM-killed before) render in under 2 s, small, with the Warning, under both numbering schemes; over the cap shows the id with one Warning; label wins; exactly 1,000 characters is kept; a normal nested term is unaffected. Docs: CONFORMANCE.md (the cap), docs/architecture.md (the rule id). --- CONFORMANCE.md | 6 ++- docs/architecture.md | 2 +- src/legaldown_render/resolve/resolver.py | 37 +++++++++++++---- tests/test_wp11_review_fixes.py | 53 ++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 10 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index af6a33c..c98d2a8 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -100,7 +100,11 @@ It is checked against the specification's own examples and fixtures corpus (`tes rendered without looping: the inner reference shows its `label=`, else the definition id. A term text worked out inside a cycle depends on which terms are open, so a text with a cut at a term open outside it is reused only where the same of its terms are open; every other text - is worked out once per term, also in a heading read before the targets are known (§13.3). The + is worked out once per term, also in a heading read before the targets are known (§13.3). A term text is capped at 1,000 characters + of plain text, checked as it is built (no real defined term comes near it; a chain of terms that each + read the next twice would otherwise ask for a text of 2^n words): a longer one is shown by its + definition id in `{{term:}}` (a `label=` still wins), and the Warning `render-term-too-long` names the + id, once. The validator (0.4.0) does not report these cycles; the specification's `definition-circular` Warning (§16) is not emitted yet. - **Ambiguous references.** Each list starts again at its first number, as contracts are usually diff --git a/docs/architecture.md b/docs/architecture.md index 4dd47df..d20372a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -239,7 +239,7 @@ turns into an error. Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can evaluate: `ref-not-enumerated` and `path-not-relative` (an Error for an image path that is not relative, §2.3, reported here because validator 0.4 does not check image paths), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), and `render-ref-ambiguous`. The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's diagnostics carry their line (§16.9), and the CLI prints it as `file:line:`. The renderer's own have none yet: they are about a directive within a block, and the validator makes only a block's line public (`Document.line_of`, U2). diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 524d9d4..90312bc 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -18,7 +18,8 @@ style template or a renderer: ``ref-not-enumerated``, ``render-ref-ambiguous``, and ``render-not-processed`` for constructs beyond the Rendering level (§17.5), ``render-link-refused`` for a link whose URL scheme is not one links may use -(§8.7), and ``path-not-relative`` for an image whose path is not relative (§2.3), which +(§8.7), ``render-term-too-long`` for a term whose text is longer than a +``{{term:}}`` may show, and ``path-not-relative`` for an image whose path is not relative (§2.3), which the validator does not check yet. """ from __future__ import annotations @@ -114,6 +115,15 @@ } +#: The longest plain text a ``{{term:}}`` may show, in characters. No real +#: defined term comes near it; one longer is a term that reads other terms +#: that read other terms (each reference repeating the text of the next, so a +#: short document can ask for a text of 2**n words), and its reference shows +#: the definition id instead (``render-term-too-long``). Checked as each term +#: text is built, so no text grows past the source plus the nested references +#: times this. +_MAX_TERM_TEXT = 1000 + #: How many texts (one per different set of open terms around it) are kept for #: one term, so that a pathological tangle of cycles cannot fill memory. _TERM_TEXTS_PER_TERM = 32 @@ -675,7 +685,9 @@ def _term_text(self, definition_id: str) -> str: definition inside a term keeps its place in document order. The definition is open meanwhile, so a ``{{term:}}`` inside it, directly or through others, shows its label or id and goes no further (see - :meth:`_while_open`).""" + :meth:`_while_open`). A text longer than :data:`_MAX_TERM_TEXT` is the + definition id instead, and the Warning ``render-term-too-long`` says + so, once per id.""" source = self.term_sources.get(definition_id) if source is None: return "" @@ -683,12 +695,21 @@ def _term_text(self, definition_id: str) -> str: def work() -> str: if not known: - return self._title_text(source.children) - anchors = set(self.used_anchors) - try: - return plain_text(self._resolve_inlines(source.children)) - finally: - self.used_anchors = anchors + text = self._title_text(source.children) + else: + anchors = set(self.used_anchors) + try: + text = plain_text(self._resolve_inlines(source.children)) + finally: + self.used_anchors = anchors + if len(text) > _MAX_TERM_TEXT: + self._warn( + "render-term-too-long", + f"The term of '{printable(definition_id)}' is longer than {_MAX_TERM_TEXT} characters once its " + f"references are resolved; '{{{{term: {printable(definition_id)}}}}}' shows the definition id.", + ) + return definition_id + return text return self._while_open(self.term_texts if known else self.term_texts_early, definition_id, work) diff --git a/tests/test_wp11_review_fixes.py b/tests/test_wp11_review_fixes.py index ab17940..bd2df16 100644 --- a/tests/test_wp11_review_fixes.py +++ b/tests/test_wp11_review_fixes.py @@ -195,3 +195,56 @@ def test_a_chain_without_a_cycle_reads_as_each_term_does() -> None: # t1 as used inside t0 is the same text as at t1's own definition. site = next(line for line in lines if line.startswith("T1 ")) assert used == f"Used: T0 {site} {site}" + + +# -- 4b: a term text is capped ---------------------------------------------------- + + +def _warnings(body: str, **settings: object) -> list[str]: + return [d.message for d in render(FRONT + body, format="text", **settings).diagnostics + if d.rule == "render-term-too-long"] + + +@pytest.mark.parametrize(("n", "cycle"), [(40, False), (14, True)]) +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +def test_a_branching_chain_of_terms_is_capped_and_reported(n: int, cycle: bool, scheme: str) -> None: + started = time.perf_counter() + result = render(FRONT + _chain(n, cycle=cycle, heading=True), format="text", + overrides={"numbering.scheme": scheme}) + assert time.perf_counter() - started < 2 + assert len(result.output) < 100_000 + messages = [d.message for d in result.diagnostics if d.rule == "render-term-too-long"] + assert messages + assert len(messages) == len(set(messages)) # one per id + assert all("longer than 1000 characters" in message and "shows the definition id" in message + for message in messages) + assert "{{" not in result.output + + +def test_a_term_over_the_cap_shows_its_id_and_one_warning_names_it() -> None: + long_term = "word " * 250 + body = f'# A\n\n"{long_term}" {{{{def: big}}}}\n\nSee {{{{term: big}}}} and {{{{term: big}}}}.\n' + out = text(body) + assert "See big and big." in out + assert _warnings(body) == [ + "The term of 'big' is longer than 1000 characters once its references are resolved; " + "'{{term: big}}' shows the definition id."] + + +def test_a_label_still_wins_over_the_cap() -> None: + body = '# A\n\n"' + "word " * 250 + '" {{def: big}}\n\nSee {{term: big, label=Big}}.\n' + assert "See Big." in text(body) + + +def test_a_term_at_the_cap_is_shown_whole() -> None: + term = "x" * 1000 + body = f'# A\n\n"{term}" {{{{def: edge}}}}\n\nSee {{{{term: edge}}}}.\n' + assert f"See {term}." in text(body) + assert _warnings(body) == [] + + +def test_a_normal_nested_term_is_unaffected() -> None: + body = ('# A\n\n"Company" {{def: company}}\n\n"Affiliate of the {{term: company}}" {{def: affiliate}}\n\n' + 'See {{term: affiliate}}.\n') + assert "See Affiliate of the Company." in text(body) + assert _warnings(body) == [] From bceecee42ba926343cd39f71de7da2231d1546d8 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:53:35 +0000 Subject: [PATCH 028/118] Final review fixes: term budget and depth limit, exact cache reuse, printable CLI diagnostics, docs 1. render-term-limit: k terms that each name all k took time exponential in k (k=10 over two minutes) whatever the cache. _while_open now counts real term-text computations per render (limit 2,000) and the terms open inside one another (limit 32); past either, a {{term:}} shows its label or definition id, nothing is kept, and one Warning is reported. Also fixes a RecursionError for 150+ chained terms. 2. A cached term text is reused only where the terms it named are open exactly as when it was worked out (the "no cut" shortcut reused a text although a term it named was open). The result equals the uncached open-set recursion (differential fuzz test). Changed test: test_wp11_review_fixes.test_a_term_reference_reads_as_its_definition_in_a_cycle asserted that a reference reads as the definition site, which only held through the old cache reusing a text across contexts; a reference is the term's text with that term open (the site renders with none open), and it no longer depends on the order of use. Replaced by an order independence test. 3. cli: the file name and every diagnostic message (validator's and renderer's), and the DocumentError and RenderRefused messages, go through urls.printable, so control characters from the document never reach stderr. 4. Docs: control-character claim narrowed to the CLI's stderr; allow-list wording; style_to_dict import path; CONFORMANCE Links row marked partial (documented deviation); term-cycle wording matches the real bound; render-term-limit documented. --- CONFORMANCE.md | 11 +- README.md | 2 +- docs/architecture.md | 17 +-- src/legaldown_render/cli.py | 11 +- src/legaldown_render/resolve/resolver.py | 69 ++++++++--- tests/test_wp11_review_fixes.py | 13 +- tests/test_wp12_final_fixes.py | 150 +++++++++++++++++++++++ 7 files changed, 234 insertions(+), 39 deletions(-) create mode 100644 tests/test_wp12_final_fixes.py diff --git a/CONFORMANCE.md b/CONFORMANCE.md index c98d2a8..76a2046 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -39,7 +39,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `ref-not-enumerated` Warning (§6.3, §16.3) | ✅ | | `raw-html` Warning; raw HTML never emitted (§8.7) | ✅ Nothing is emitted: an HTML block is dropped whole, and an inline tag is dropped with its text kept. The Warning is the validator's, once per block or text that holds raw HTML other than comments (see "Text after a comment", below) | | Automatic term recognition (§7.4, MAY) | ❌ Not supported | -| Links (§8.7) | ✅ A link is a hyperlink in HTML and shows `text ` in plain text when its URL is relative, a fragment, or has one of the schemes `http`, `https`, `mailto`, `tel`, `ftp`, `ftps`, `sftp`, `sip`, `sips`, `sms`, `urn`, `geo`, `news` (an allow-list, judged as a browser reads the URL). Any other scheme (`javascript:`, `file:`, `data:`, protocol handlers such as `ms-msdt:`, `vscode:`, `search-ms:`, and pseudo-schemes such as `C:\x` or `localhost:8080`), or an empty URL, is not a link: its text is shown alone, and the Warning `render-link-refused` is reported for a non-empty URL. A link with no text shows its URL as written; the text writer omits `` for an autolink | +| Links (§8.7) | ⚠️ Partial, a documented deviation for safety with untrusted documents: links with schemes outside the allow-list render as text with a `render-link-refused` Warning. A link is a hyperlink in HTML and shows `text ` in plain text when its URL is relative, a fragment, or has one of the schemes `http`, `https`, `mailto`, `tel`, `ftp`, `ftps`, `sftp`, `sip`, `sips`, `sms`, `urn`, `geo`, `news` (an allow-list, judged as a browser reads the URL). Any other scheme (`javascript:`, `file:`, `data:`, protocol handlers such as `ms-msdt:`, `vscode:`, `search-ms:`, and pseudo-schemes such as `C:\x` or `localhost:8080`), or an empty URL, is not a link: its text is shown alone, and the Warning `render-link-refused` is reported for a non-empty URL. A link with no text shows its URL as written; the text writer omits `` for an autolink | | Images and file-reference paths (§8.7, §2.3) | ✅ A relative path (`../` included, as §2.3 defines no default document root) renders `` in HTML and its alt text in plain text. A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is never fetched or embedded: the alt text is shown and the Error `path-not-relative` is reported, which `--strict` refuses. An empty path shows the alt text with no diagnostic | | Plain-text layout (§13.6, optional) | ✅ `attachments.separator` (§13.8): `none` prints no `---`; `rule` and `page-break` both print one before the attachments heading and between consecutive attachments, so the heading stays with the first (text has no pages). One quote stays one quote (blank lines keep the `>`), and the blocks of a list item are set apart by blank lines and indented under the item's label, whatever its condition label; an item that opens with a nested list, table, code block or quote has its label on a line of its own, so that block stays valid. A pipe in a table cell is backslash-escaped, and the separator row marks column alignment (`:---`, `:---:`, `---:`). A label a style blanks drops its `label: ` prefix in the title block, party details and signature blocks (a signature row with neither label nor value is omitted); HTML keeps an empty `
` so its grid layout and the `
` stay valid | | Extended tables through raw HTML (§9.2, MAY) | ❌ Not supported; the HTML table is dropped with the Warning | @@ -98,9 +98,12 @@ It is checked against the specification's own examples and fixtures corpus (`tes falls back to the validator's term text. A term that refers to itself, directly or through other terms (`"A {{term: a}}" {{def: a}}`), is rendered without looping: the inner reference shows its `label=`, else the definition id. A term text - worked out inside a cycle depends on which terms are open, so a text with - a cut at a term open outside it is reused only where the same of its terms are open; every other text - is worked out once per term, also in a heading read before the targets are known (§13.3). A term text is capped at 1,000 characters + worked out inside a cycle depends on which terms are open, so it is kept with the open terms it + depends on and reused only where those are open; a text with no cycle is worked out once per term, + also in a heading read before the targets are known (§13.3). Terms that name each other can still + ask for more different texts than any cache holds, so the work is bounded: past 2,000 texts worked + out in one document, or 32 terms open inside one another, a `{{term:}}` shows its `label=` or + definition id and the Warning `render-term-limit` is reported, once. A term text is capped at 1,000 characters of plain text, checked as it is built (no real defined term comes near it; a chain of terms that each read the next twice would otherwise ask for a text of 2^n words): a longer one is shown by its definition id in `{{term:}}` (a `label=` still wins), and the Warning `render-term-too-long` names the diff --git a/README.md b/README.md index 3fa4b17..d4e9262 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ format (`OptionsError`), an unreadable document (`DocumentError`), strict mode, cannot be assembled with its answers (`RenderRefused`) do. All of them are `RenderError`s, and `StyleError` and `OptionsError` are also `ValueError`s. They can be pickled, so they cross process boundaries (`multiprocessing`, `concurrent.futures`) intact. `overrides=` takes dotted settings and -may not contain `version` or `extends`, so `overrides=style_to_dict(style)` must drop those two keys. +may not contain `version` or `extends`, so `overrides=legaldown_render.style.style_to_dict(style)` must drop those two keys. ## Styles and preferences diff --git a/docs/architecture.md b/docs/architecture.md index d20372a..f7dfc39 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -239,7 +239,7 @@ turns into an error. Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can evaluate: `ref-not-enumerated` and `path-not-relative` (an Error for an image path that is not relative, §2.3, reported here because validator 0.4 does not check image paths), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested or 2,000 computed term texts a `{{term:}}` shows its id), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's diagnostics carry their line (§16.9), and the CLI prints it as `file:line:`. The renderer's own have none yet: they are about a directive within a block, and the validator makes only a block's line public (`Document.line_of`, U2). @@ -264,11 +264,14 @@ line public (`Document.line_of`, U2). (links, images, value links). A link with no text shows its URL as written (percent-encoding undone; the `href` stays encoded); the text writer leaves out ` ` when the URL, less a `mailto:` or `tel:` prefix, is the text. -- A diagnostic never carries a raw control character from the document: a path or URL quoted in a - message (`path-not-relative`, `render-link-refused`) goes through `urls.printable`, which writes - every C0/C1 control, DEL, Unicode format character (bidi controls, zero-width), and U+2028/9 as - `\xNN` or `\uNNNN`, so a crafted URL cannot move the cursor, set a terminal title, or forge a - line on stderr. +- What the CLI prints on stderr never carries a raw control character from the document: `cli._report` + passes the file name and every diagnostic message (the renderer's and the validator's, which quote + ids, paths and terms as written) through `urls.printable`, which writes every C0/C1 control, DEL, + Unicode format character (bidi controls, zero-width), and U+2028/9 as `\xNN` or `\uNNNN`, so a + crafted document cannot move the cursor, set a terminal title, or forge a line. The messages the + API returns are not all escaped: only a path or URL quoted by the renderer's own rules + (`path-not-relative`, `render-link-refused`) is, at its source; a caller that shows other + messages must do the same. - An image path must be relative with no URI scheme (§2.3), so a remote resource is never fetched or embedded. Resolution, not a writer, enforces it: an image whose path has a scheme (`https:`, `file:`, `data:`, a Windows drive such as `C:\x`) or starts with `/` or `\` (`//host`) is @@ -299,7 +302,7 @@ line public (`Document.line_of`, U2). |---|---| | `tests/test_golden.py` | The test documents under each built-in style, compared byte for byte with `tests/golden/`. Regenerate with `pytest --update-golden` and review the diff | | `tests/test_render.py` | Numbering schemes, designations, fallbacks, every failure marker, template view, labels by language, HTML safety, unique anchors, determinism, strict mode, locales | -| `tests/test_wp3_links.py` | Which images are shown and which links are links: non-relative image paths, the link scheme deny-list, empty URLs and empty link text | +| `tests/test_wp3_links.py` | Which images are shown and which links are links: non-relative image paths, the link scheme allow-list, empty URLs and empty link text | | `tests/test_style.py`, `tests/test_values.py` | Style layering and validation; counters and locale formatting | | `tests/test_cli.py` | The command | | `tests/conformance/` | Every example and fixture in the specification repository renders, every internal link has a target, and each invalid fixture renders its rule's failure marker. Needs `$LEGALDOWN_SPEC_DIR` or a sibling `../LegalDown` checkout | diff --git a/src/legaldown_render/cli.py b/src/legaldown_render/cli.py index e0cdea8..b97b438 100644 --- a/src/legaldown_render/cli.py +++ b/src/legaldown_render/cli.py @@ -24,6 +24,7 @@ from .api import RenderOptions, render from .errors import DocumentError, InternalError, RenderRefused, StyleError from .style import builtin_styles, dump_style, load_style, parse_override +from .urls import printable from .writers import FORMATS, file_extension, format_for_path #: Extensions (:func:`file_extension`, so no dot) of document formats this @@ -179,10 +180,10 @@ def main(argv: list[str] | None = None) -> int: return 2 except RenderRefused as error: _report(name, error.diagnostics, quiet=args.quiet) - print(f"legaldown-render: {error}", file=sys.stderr) + print(f"legaldown-render: {printable(str(error))}", file=sys.stderr) return 1 except DocumentError as error: - print(f"{name}: error: {error}", file=sys.stderr) + print(f"{printable(name)}: error: {printable(str(error))}", file=sys.stderr) return 1 except InternalError as error: print(f"legaldown-render: internal error: {error}", file=sys.stderr) @@ -205,8 +206,10 @@ def _report(name: str, diagnostics: list, *, quiet: bool) -> None: return for diagnostic in diagnostics: # As the validator's CLI prints them: the line when the finding has one. - where = f"{name}:{diagnostic.line}" if diagnostic.line else name - print(f"{where}: {diagnostic.level}: [{diagnostic.rule}] {diagnostic.message}", file=sys.stderr) + # The message quotes the document (an id, a path, a term), and a terminal + # obeys the control characters in it, so both parts are made printable. + where = f"{printable(name)}:{diagnostic.line}" if diagnostic.line else printable(name) + print(f"{where}: {diagnostic.level}: [{diagnostic.rule}] {printable(diagnostic.message)}", file=sys.stderr) if __name__ == "__main__": diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 90312bc..4d626b6 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -19,7 +19,8 @@ and ``render-not-processed`` for constructs beyond the Rendering level (§17.5), ``render-link-refused`` for a link whose URL scheme is not one links may use (§8.7), ``render-term-too-long`` for a term whose text is longer than a -``{{term:}}`` may show, and ``path-not-relative`` for an image whose path is not relative (§2.3), which +``{{term:}}`` may show, ``render-term-limit`` for terms that refer to each other too deeply or too often, +and ``path-not-relative`` for an image whose path is not relative (§2.3), which the validator does not check yet. """ from __future__ import annotations @@ -124,6 +125,16 @@ #: times this. _MAX_TERM_TEXT = 1000 +#: The most term texts worked out in one document (each real computation, not +#: a text taken from the cache), and the most terms open inside one another. +#: Past either, a ``{{term:}}`` shows its label or definition id and the +#: Warning ``render-term-limit`` is reported. A document of terms that name +#: each other (k terms each naming all k) would otherwise take time +#: exponential in k, and a long chain of terms would exhaust the stack. No +#: real document comes near them. +_MAX_TERM_RUNS = 2000 +_MAX_TERM_DEPTH = 32 + #: How many texts (one per different set of open terms around it) are kept for #: one term, so that a pathological tangle of cycles cannot fill memory. _TERM_TEXTS_PER_TERM = 32 @@ -195,6 +206,7 @@ def __init__( # because they were open. self._seen: set[str] = set() self._cut: set[str] = set() + self._term_runs = 0 # False until every target is registered and the placeholder types are # known (run() sets it just before the inlines are resolved). self._targets_known = False @@ -665,12 +677,16 @@ def _term(self, directive: Directive, definition_id: str) -> Inline: # The term as rendered at its definition; the validator's own # text (an empty term reads as its id, a definition not in the # body) is the fallback. - text = label or self._term_text(definition_id) or self._plain_open(definition_id, term) + text = label or self._term_text(definition_id) + if text == "": + text = self._plain_open(definition_id, term) + if text is None: + text = definition_id # past the limits (see _while_open) # Settled once everything is resolved (see _settle_term_target). target = DEFINITION_ANCHOR_PREFIX + definition_id return TermRef(text, target, self.style.definitions.term_style) - def _term_text(self, definition_id: str) -> str: + def _term_text(self, definition_id: str) -> str | None: """The plain text of the term at the first definition of *definition_id* in the body, as the reader sees it there: its Markdown, references, placeholders and other terms resolved exactly as at the definition @@ -687,7 +703,8 @@ def _term_text(self, definition_id: str) -> str: through others, shows its label or id and goes no further (see :meth:`_while_open`). A text longer than :data:`_MAX_TERM_TEXT` is the definition id instead, and the Warning ``render-term-too-long`` says - so, once per id.""" + so, once per id. ``None`` is a term past the limits of + :meth:`_while_open`.""" source = self.term_sources.get(definition_id) if source is None: return "" @@ -713,37 +730,53 @@ def work() -> str: return self._while_open(self.term_texts if known else self.term_texts_early, definition_id, work) - def _plain_open(self, definition_id: str, term: str) -> str: + def _plain_open(self, definition_id: str, term: str) -> str | None: """The validator's *term* text of a definition as plain text, with the definition marked open while its directives resolve.""" key = f"{definition_id}\0{int(self._targets_known)}" return self._while_open(self.term_texts_plain, definition_id, lambda: self._plain_value(term), key=key) def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, work: Callable[[], str], - *, key: str | None = None) -> str: + *, key: str | None = None) -> str | None: """The text *work* gives with *definition_id* open, kept in *cache* (under *key*, default the id), so a chain of terms that each use the next one twice is worked out once per term, not once per use. - What a text reads depends on which terms are open around it only - where a reference was cut short at an open term (it shows that term's - label or id). A text with no cut at a term opened *before* this one is - the term's own text, as the reader sees it at its definition, and is - reused wherever the term is needed. A text with such a cut (a cycle - back through an enclosing term) records the terms it named and the - ones it was cut short at, and is reused only where exactly those of - the terms it named are open. A cut at this term itself, or at one - opened inside it, is its own business and does not count. Without a - cycle every term is worked out once; with one, once for each way round - it is reached. + What a text reads depends on which terms are open around it: a + reference to an open term is cut short and shows its label or id. A + kept text records the terms it named (``seen``) and those it was cut + short at that were open *before* this one (``cut``: a cycle back + through an enclosing term). It is reused only where exactly those of + the terms it named are open, so it reads as it would if worked out + afresh there. A cut at this term itself, or at one opened inside it, is + its own business and does not count. Without a cycle every term is + worked out once. + + A cycle can still ask for a great many different texts (k terms that + each name all k have more ways round than any cache holds), so the work + is limited: past :data:`_MAX_TERM_DEPTH` terms open at once, or + :data:`_MAX_TERM_RUNS` texts worked out in the document, the result is + ``None`` (the reference shows its label or id) and the Warning + ``render-term-limit`` says so, once. """ key = definition_id if key is None else key around = set(self._terms_open) for kept in cache.get(key, ()): - if not kept.cut or kept.seen & around == kept.cut: + if kept.seen & around == kept.cut: self._seen |= kept.seen self._cut |= kept.cut return kept.text + if len(self._terms_open) >= _MAX_TERM_DEPTH or self._term_runs >= _MAX_TERM_RUNS: + # Terms that refer to each other this deeply or this often are + # not resolved further: like a cut, the reference shows its label + # or id, and nothing is kept. + self._warn( + "render-term-limit", + "Defined terms refer to each other too deeply or too often to resolve fully; " + "some {{term:}} references show their definition id.", + ) + return None + self._term_runs += 1 self._terms_open.append(definition_id) outer = self._seen, self._cut self._seen, self._cut = set(), set() diff --git a/tests/test_wp11_review_fixes.py b/tests/test_wp11_review_fixes.py index bd2df16..9cf45af 100644 --- a/tests/test_wp11_review_fixes.py +++ b/tests/test_wp11_review_fixes.py @@ -160,11 +160,14 @@ def test_a_text_name_with_a_dot_quirk_writes_text(tmp_path: Path, name: str, mar '"Alpha {{term: cb}}" {{def: ca}}\n\n"Beta {{term: ca}}" {{def: cb}}\n') -def test_a_term_reference_reads_as_its_definition_in_a_cycle() -> None: - lines = text(CYCLE).splitlines() - used = next(line for line in lines if line.startswith("Used first:")) - site = next(line for line in lines if line.startswith("Beta ")) - assert used.endswith(f"and {site}.") +def test_a_term_reference_in_a_cycle_does_not_depend_on_the_order_of_use() -> None: + # WP12: a term shown by {{term:}} is its text with that term open (a reference back to it + # shows its id), whichever term is used first. (The definition site renders with no term + # open, so it can read longer: "Beta Alpha Beta ca" there, "Beta Alpha cb" in a reference.) + ca_first = text(CYCLE) + cb_first = text(CYCLE.replace("{{term: ca}} and {{term: cb}}", "{{term: cb}} and {{term: ca}}")) + assert "Used first: Alpha Beta ca and Beta Alpha cb." in ca_first + assert "Used first: Beta Alpha cb and Alpha Beta ca." in cb_first def _chain(n: int, *, cycle: bool, heading: bool = False) -> str: diff --git a/tests/test_wp12_final_fixes.py b/tests/test_wp12_final_fixes.py new file mode 100644 index 0000000..704c927 --- /dev/null +++ b/tests/test_wp12_final_fixes.py @@ -0,0 +1,150 @@ +"""Final review fixes (WP12): terms that name each other are resolved within a +budget (``render-term-limit``), a cached term text is reused only where it would +read the same, long chains of terms do not exhaust the stack, and nothing from a +document reaches stderr as a control character.""" +from __future__ import annotations + +import os +import random +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from legaldown_render import render +from legaldown_render.resolve.resolver import Resolver + +SRC = Path(__file__).resolve().parents[1] / "src" +FRONT = "---\ntitle: T\n---\n\n" +LIMIT = ("Defined terms refer to each other too deeply or too often to resolve fully; " + "some {{term:}} references show their definition id.") + + +def messages(result, rule: str) -> list[str]: # noqa: ANN001 + return [d.message for d in result.diagnostics if d.rule == rule] + + +# -- 1: terms that all name each other --------------------------------------------- + + +def _tangle(k: int) -> str: + defs = [f'"T{i} {" ".join(f"{{{{term: t{j}}}}}" for j in range(k))}" {{{{def: t{i}}}}}' for i in range(k)] + return "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n" + + +@pytest.mark.parametrize("k", [3, 9, 12, 20, 40]) +def test_k_terms_that_each_name_all_k_render_quickly(k: int) -> None: + started = time.perf_counter() + result = render(FRONT + _tangle(k), format="text") + assert time.perf_counter() - started < 2 + assert "{{" not in result.output + if k >= 9: + assert messages(result, "render-term-limit") == [LIMIT] + + +def test_a_small_cycle_is_resolved_in_full_without_the_warning() -> None: + assert messages(render(FRONT + _tangle(3), format="text"), "render-term-limit") == [] + + +# -- 3: a long chain of terms ------------------------------------------------------ + + +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +@pytest.mark.parametrize("heading", [False, True]) +def test_a_chain_of_500_terms_does_not_exhaust_the_stack(scheme: str, heading: bool) -> None: + n = 500 + defs = [f'"T{i} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + [f'"End" {{{{def: t{n}}}}}'] + title = "About {{term: t0}}" if heading else "A" + body = f"# {title}\n\nUsed: {{{{term: t0}}}}\n\n" + "\n\n".join(defs) + "\n" + started = time.perf_counter() + result = render(FRONT + body, format="text", overrides={"numbering.scheme": scheme}) + assert time.perf_counter() - started < 2 + assert messages(result, "render-term-limit") == [LIMIT] + used = next(line for line in result.output.splitlines() if line.startswith("Used: ")) + assert used.startswith("Used: T0 T1 T2 ") + assert "{{" not in result.output + + +def test_a_chain_inside_the_depth_limit_is_resolved_in_full() -> None: + n = 20 + defs = [f'"T{i} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + [f'"End" {{{{def: t{n}}}}}'] + result = render(FRONT + "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n", format="text") + assert f"Used: {' '.join(f'T{i}' for i in range(n))} End" in result.output + assert messages(result, "render-term-limit") == [] + + +# -- 2: a cached text is reused only where it would read the same ------------------ + + +def _oracle(self: Resolver, cache: object, definition_id: str, work, *, key: str | None = None) -> str: # noqa: ANN001 + """The term text with the open terms alone deciding, nothing kept.""" + self._terms_open.append(definition_id) + try: + return work() + finally: + self._terms_open.pop() + + +def _graph(rng: random.Random) -> str: + ids = [f"t{i}" for i in range(rng.randint(2, 6))] + defs = [] + for identifier in ids: + parts = [identifier.upper()] + for _ in range(rng.randint(0, 3)): + target = rng.choice(ids) + parts.append(f"{{{{term: {target}, label=L{target}}}}}" if rng.random() < 0.15 + else f"{{{{term: {target}}}}}") + defs.append(f'"{" ".join(parts)}" {{{{def: {identifier}}}}}') + rng.shuffle(defs) + heading = rng.choice(["A", f"About {{{{term: {rng.choice(ids)}}}}}"]) + use = " ".join(f"{{{{term: {rng.choice(ids)}}}}}" for _ in range(rng.randint(1, 4))) + return f"# {heading}\n\nUsed: {use}\n\n" + "\n\n".join(defs) + "\n" + + +def test_cached_term_texts_equal_the_texts_worked_out_afresh(monkeypatch: pytest.MonkeyPatch) -> None: + rng = random.Random(20261027) + cached_original = Resolver._while_open + for _ in range(150): + source = FRONT + _graph(rng) + for scheme in ("decimal", "none"): + overrides = {"numbering.scheme": scheme} + monkeypatch.setattr(Resolver, "_while_open", cached_original) + cached = render(source, format="text", overrides=overrides) + monkeypatch.setattr(Resolver, "_while_open", _oracle) + fresh = render(source, format="text", overrides=overrides) + assert cached.output == fresh.output, source + assert [(d.rule, d.message) for d in cached.diagnostics] == \ + [(d.rule, d.message) for d in fresh.diagnostics], source + + +def test_a_text_that_named_a_term_is_not_reused_where_that_term_is_open() -> None: + # ca's text names cb. Worked out with nothing open it is "Alpha Beta ca"; with cb open it is + # "Alpha cb", and that is what a reference inside cb must read. + body = ('# A\n\nFirst {{term: ca}}. Then {{term: cb}}.\n\n"Alpha {{term: cb}}" {{def: ca}}\n\n' + '"Beta {{term: ca}}" {{def: cb}}\n') + out = render(FRONT + body, format="text").output + assert "First Alpha Beta ca. Then Beta Alpha cb." in out + + +# -- 4: control characters never reach stderr ---------------------------------------- + + +def _run(source: str, tmp_path: Path, *arguments: str) -> subprocess.CompletedProcess[bytes]: + document = tmp_path / "d.lgd" + document.write_text(source, encoding="utf-8") + env = {**os.environ, "PYTHONPATH": str(SRC)} + return subprocess.run([sys.executable, "-m", "legaldown_render.cli", str(document), "-f", "text", *arguments], + capture_output=True, cwd=tmp_path, env=env, check=False) + + +def test_no_control_character_from_the_document_reaches_stderr(tmp_path: Path) -> None: + body = ("# A {#a}\n\n{{include: a\x1b]0;pwn\x07.lgd}}\n\n{{ref: x\x1b[2Jy}}\n\n{{term: q\x1b[31m}}\n\n" + "See {{ref: a}}.\n") + result = _run(FRONT + body, tmp_path) + errors = result.stderr.decode("utf-8") + assert errors.count("\n") >= 3 # the three findings, one line each + assert not any(ord(c) < 0x20 and c != "\n" or 0x7F <= ord(c) < 0xA0 for c in errors) + assert "\\x1b" in errors # written out, not dropped + assert "pwn" in errors From 2091550beb97ffc20d7c02acba82e6c64cf0c0fb Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:09:45 +0000 Subject: [PATCH 029/118] Term budget charges only repeat work; CLI escapes every echoed command-line value 1. The budget no longer charges acyclic terms: a term's first text is free (bounded by the number of terms per pass); only a repeat computation (the cache already holds a text of the term, for other open terms) counts. 3000 independent terms, with and without headings, under both schemes, show no render-term-limit and every real text. 2. The budget is now a work budget: the inline nodes resolved inside repeat computations (nested ones included) are counted, limit _MAX_TERM_WORK = 200,000, so the extra work cycles can cause is a small multiple of the document, whatever the number of references in a term. The depth limit (32) stays. Past either: label or id, nothing kept, one render-term-limit Warning. 3. cli: every error line goes through _safe (urls.printable per line, keeping the line breaks of multi-line messages), and argparse errors through a parser whose error() does the same, so an input, answers, output or style name, a locale or a --set key echoed there is escaped. Docs: CONFORMANCE.md (work budget), docs/architecture.md (limit, stderr). --- CONFORMANCE.md | 8 +- docs/architecture.md | 6 +- src/legaldown_render/cli.py | 50 ++++--- src/legaldown_render/resolve/resolver.py | 44 +++--- tests/test_wp13_budget.py | 162 +++++++++++++++++++++++ 5 files changed, 235 insertions(+), 35 deletions(-) create mode 100644 tests/test_wp13_budget.py diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 76a2046..cc22c1b 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -101,9 +101,11 @@ It is checked against the specification's own examples and fixtures corpus (`tes worked out inside a cycle depends on which terms are open, so it is kept with the open terms it depends on and reused only where those are open; a text with no cycle is worked out once per term, also in a heading read before the targets are known (§13.3). Terms that name each other can still - ask for more different texts than any cache holds, so the work is bounded: past 2,000 texts worked - out in one document, or 32 terms open inside one another, a `{{term:}}` shows its `label=` or - definition id and the Warning `render-term-limit` is reported, once. A term text is capped at 1,000 characters + ask for more different texts than any cache holds, so the extra work is bounded: a term's first text + is free, and past 200,000 inline nodes resolved inside later texts of the same term (nested ones + included), or 32 terms open inside one another, a `{{term:}}` shows its `label=` or definition id + and the Warning `render-term-limit` is reported, once. Any number of terms that do not name each + other resolve in full and never reach the limit. A term text is capped at 1,000 characters of plain text, checked as it is built (no real defined term comes near it; a chain of terms that each read the next twice would otherwise ask for a text of 2^n words): a longer one is shown by its definition id in `{{term:}}` (a `label=` still wins), and the Warning `render-term-too-long` names the diff --git a/docs/architecture.md b/docs/architecture.md index f7dfc39..f73c800 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -239,7 +239,7 @@ turns into an error. Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can evaluate: `ref-not-enumerated` and `path-not-relative` (an Error for an image path that is not relative, §2.3, reported here because validator 0.4 does not check image paths), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested or 2,000 computed term texts a `{{term:}}` shows its id), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, or 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), a `{{term:}}` shows its id), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's diagnostics carry their line (§16.9), and the CLI prints it as `file:line:`. The renderer's own have none yet: they are about a directive within a block, and the validator makes only a block's line public (`Document.line_of`, U2). @@ -264,7 +264,9 @@ line public (`Document.line_of`, U2). (links, images, value links). A link with no text shows its URL as written (percent-encoding undone; the `href` stays encoded); the text writer leaves out ` ` when the URL, less a `mailto:` or `tel:` prefix, is the text. -- What the CLI prints on stderr never carries a raw control character from the document: `cli._report` +- What the CLI prints on stderr never carries a raw control character from the document or the command + line: every error line and every argparse error goes through `cli._safe` (`urls.printable` per line), so + an input, answers, output or style name, a locale or a `--set` key echoed in one is escaped, and `cli._report` passes the file name and every diagnostic message (the renderer's and the validator's, which quote ids, paths and terms as written) through `urls.printable`, which writes every C0/C1 control, DEL, Unicode format character (bidi controls, zero-width), and U+2028/9 as `\xNN` or `\uNNNN`, so a diff --git a/src/legaldown_render/cli.py b/src/legaldown_render/cli.py index b97b438..993e704 100644 --- a/src/legaldown_render/cli.py +++ b/src/legaldown_render/cli.py @@ -16,7 +16,7 @@ import contextlib import sys from pathlib import Path -from typing import Any +from typing import Any, NoReturn from legaldown import AnswersError, load_answers @@ -35,20 +35,40 @@ _UNWRITTEN_EXTENSIONS = frozenset({"docx", "doc", "odt", "rtf", "pdf", "md", "markdown", "epub", "tex"}) +def _safe(text: str) -> str: + """*text* for stderr, with every control character written out (see + :func:`~legaldown_render.urls.printable`) except the line breaks that + structure a multi-line message. Command-line values (file names, a style + name, a locale, a setting key) and the library's messages that quote them + reach a terminal only through this.""" + return "\n".join(printable(line) for line in text.split("\n")) + + +def _complain(message: str) -> None: + print(_safe(message), file=sys.stderr) + + +class _Parser(argparse.ArgumentParser): + def error(self, message: str) -> NoReturn: + # argparse echoes the offending value (an invalid choice, an + # unrecognized argument): written out, not obeyed by the terminal. + super().error(_safe(message)) + + def _setting(text: str) -> tuple[str, Any]: key, separator, value = text.partition("=") if not separator or not key.strip(): - raise argparse.ArgumentTypeError(f"expected KEY=VALUE, got '{text}'") + raise argparse.ArgumentTypeError(f"expected KEY=VALUE, got '{printable(text)}'") # Typed by the setting: text stays as written, anything else is YAML. try: return key.strip(), parse_override(key.strip(), value) except StyleError as error: # argparse shows only ArgumentTypeError (and TypeError/ValueError, with a generic message). - raise argparse.ArgumentTypeError(str(error)) from None + raise argparse.ArgumentTypeError(_safe(str(error))) from None def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( + parser = _Parser( prog="legaldown-render", description="Render a LegalDown document to HTML or plain text.", epilog="Style settings: --style picks a template (built-in: " @@ -134,7 +154,7 @@ def main(argv: list[str] | None = None) -> int: try: print(dump_style(load_style(args.style, overrides=overrides)), end="") except StyleError as error: - print(f"legaldown-render: {error}", file=sys.stderr) + _complain(f"legaldown-render: {error}") return 2 return 0 if not args.input: @@ -145,17 +165,17 @@ def main(argv: list[str] | None = None) -> int: file_name = Path(args.output).name output_format = format_for_path(file_name) if not output_format and file_extension(file_name) in _UNWRITTEN_EXTENSIONS: - parser.error(f"cannot tell the output format from '{args.output}'; use --format html or text") + parser.error(f"cannot tell the output format from '{printable(args.output)}'; use --format html or text") output_format = output_format or "html" name = "" if args.input == "-" else args.input try: source = _read_source(args.input) except OSError as error: - print(f"legaldown-render: cannot read {name}: {error.strerror or error}", file=sys.stderr) + _complain(f"legaldown-render: cannot read {name}: {error.strerror or error}") return 1 except UnicodeDecodeError as error: - print(f"legaldown-render: cannot read {name}: not valid UTF-8 " - f"(byte 0x{error.object[error.start]:02x} at offset {error.start})", file=sys.stderr) + _complain(f"legaldown-render: cannot read {name}: not valid UTF-8 " + f"(byte 0x{error.object[error.start]:02x} at offset {error.start})") return 1 answers = None @@ -165,10 +185,10 @@ def main(argv: list[str] | None = None) -> int: try: answers = load_answers(args.answers) except OSError as error: - print(f"legaldown-render: cannot read {Path(args.answers)}: {error}", file=sys.stderr) + _complain(f"legaldown-render: cannot read {Path(args.answers)}: {error}") return 1 except AnswersError as error: - print(f"legaldown-render: {error}", file=sys.stderr) + _complain(f"legaldown-render: {error}") return 1 options = RenderOptions(format=output_format, style=args.style, overrides=overrides, @@ -176,17 +196,17 @@ def main(argv: list[str] | None = None) -> int: try: result = render(source, options) except StyleError as error: - print(f"legaldown-render: {error}", file=sys.stderr) + _complain(f"legaldown-render: {error}") return 2 except RenderRefused as error: _report(name, error.diagnostics, quiet=args.quiet) - print(f"legaldown-render: {printable(str(error))}", file=sys.stderr) + _complain(f"legaldown-render: {printable(str(error))}") return 1 except DocumentError as error: print(f"{printable(name)}: error: {printable(str(error))}", file=sys.stderr) return 1 except InternalError as error: - print(f"legaldown-render: internal error: {error}", file=sys.stderr) + _complain(f"legaldown-render: internal error: {error}") return 70 _report(name, result.diagnostics, quiet=args.quiet) @@ -194,7 +214,7 @@ def main(argv: list[str] | None = None) -> int: try: Path(args.output).write_text(result.output, encoding="utf-8", newline="\n") except OSError as error: - print(f"legaldown-render: cannot write {args.output}: {error.strerror or error}", file=sys.stderr) + _complain(f"legaldown-render: cannot write {args.output}: {error.strerror or error}") return 1 else: sys.stdout.write(result.output) diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 4d626b6..6954803 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -125,14 +125,17 @@ #: times this. _MAX_TERM_TEXT = 1000 -#: The most term texts worked out in one document (each real computation, not -#: a text taken from the cache), and the most terms open inside one another. -#: Past either, a ``{{term:}}`` shows its label or definition id and the -#: Warning ``render-term-limit`` is reported. A document of terms that name -#: each other (k terms each naming all k) would otherwise take time -#: exponential in k, and a long chain of terms would exhaust the stack. No -#: real document comes near them. -_MAX_TERM_RUNS = 2000 +#: The most inline nodes resolved, in one document, inside a term text that is +#: worked out again (a second or later text of a term, for another set of open +#: terms: only terms that name each other cause these), nested ones included; +#: and the most terms open inside one another. Past either, a ``{{term:}}`` +#: shows its label or definition id and the Warning ``render-term-limit`` is +#: reported. A term's first text costs nothing (it is bounded by the number of +#: terms), so this bounds the extra work cycles can cause to a small multiple of +#: the document, where k terms each naming all k would otherwise take time +#: exponential in k, and a long chain of terms would exhaust the stack. No real +#: document comes near them. +_MAX_TERM_WORK = 200_000 _MAX_TERM_DEPTH = 32 #: How many texts (one per different set of open terms around it) are kept for @@ -206,7 +209,9 @@ def __init__( # because they were open. self._seen: set[str] = set() self._cut: set[str] = set() - self._term_runs = 0 + # Inline nodes resolved inside repeat computations (see _while_open). + self._term_work = 0 + self._repeating = 0 # False until every target is registered and the placeholder types are # known (run() sets it just before the inlines are resolved). self._targets_known = False @@ -401,6 +406,8 @@ def _title_text(self, title: tuple[Inline, ...]) -> str: def text(inlines: tuple[Inline, ...]) -> str: parts: list[str] = [] for inline in inlines: + if self._repeating: + self._term_work += 1 match inline: case DirectiveSource(directive=directive): if directive.name != "ref": @@ -567,6 +574,8 @@ def _resolve_inlines(self, inlines: tuple[Inline, ...]) -> tuple[Inline, ...]: return tuple(out) def _resolve_inline(self, inline: Inline) -> Inline | tuple[Inline, ...] | None: + if self._repeating: + self._term_work += 1 match inline: case DirectiveSource(directive=directive): return self._directive(directive) @@ -754,10 +763,13 @@ def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, wor A cycle can still ask for a great many different texts (k terms that each name all k have more ways round than any cache holds), so the work - is limited: past :data:`_MAX_TERM_DEPTH` terms open at once, or - :data:`_MAX_TERM_RUNS` texts worked out in the document, the result is - ``None`` (the reference shows its label or id) and the Warning - ``render-term-limit`` says so, once. + is limited. A text worked out when the cache already holds one for + this term (for other open terms) is a repeat, and the inline nodes + resolved inside a repeat, nested ones included, are counted; first + texts are free. Past :data:`_MAX_TERM_WORK` such nodes, or with + :data:`_MAX_TERM_DEPTH` terms open at once, the result is ``None`` (the + reference shows its label or id), and the Warning ``render-term-limit`` + says so, once. """ key = definition_id if key is None else key around = set(self._terms_open) @@ -766,7 +778,8 @@ def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, wor self._seen |= kept.seen self._cut |= kept.cut return kept.text - if len(self._terms_open) >= _MAX_TERM_DEPTH or self._term_runs >= _MAX_TERM_RUNS: + repeat = bool(cache.get(key)) + if len(self._terms_open) >= _MAX_TERM_DEPTH or (repeat and self._term_work >= _MAX_TERM_WORK): # Terms that refer to each other this deeply or this often are # not resolved further: like a cut, the reference shows its label # or id, and nothing is kept. @@ -776,7 +789,7 @@ def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, wor "some {{term:}} references show their definition id.", ) return None - self._term_runs += 1 + self._repeating += repeat self._terms_open.append(definition_id) outer = self._seen, self._cut self._seen, self._cut = set(), set() @@ -786,6 +799,7 @@ def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, wor if len(cache.setdefault(key, [])) < _TERM_TEXTS_PER_TERM: cache[key].append(_TermText(text, seen, cut)) finally: + self._repeating -= repeat self._terms_open.pop() self._seen = outer[0] | self._seen self._cut = outer[1] | (self._cut & around) diff --git a/tests/test_wp13_budget.py b/tests/test_wp13_budget.py new file mode 100644 index 0000000..f9e9aed --- /dev/null +++ b/tests/test_wp13_budget.py @@ -0,0 +1,162 @@ +"""Review fixes (WP13): the term budget charges only repeat work, and every +command-line value echoed to stderr is escaped.""" +from __future__ import annotations + +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from legaldown_render import render + +SRC = Path(__file__).resolve().parents[1] / "src" +FRONT = "---\ntitle: T\n---\n\n" + + +def rules(result, rule: str) -> list[str]: # noqa: ANN001 + return [d.message for d in result.diagnostics if d.rule == rule] + + +# -- 1: acyclic terms are free ------------------------------------------------------ + + +def _independent(n: int, *, heading: bool) -> str: + parts = [] + for i in range(n): + if heading: + parts.append(f"## H{i} {{{{term: t{i}}}}}\n\n") + parts.append(f'"Term {i}" {{{{def: t{i}}}}} used {{{{term: t{i}}}}}\n\n') + return "# A\n\n" + "".join(parts) + + +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +@pytest.mark.parametrize("heading", [False, True]) +def test_3000_independent_terms_show_their_text_and_no_warning(scheme: str, heading: bool) -> None: + result = render(FRONT + _independent(3000, heading=heading), format="text", overrides={"numbering.scheme": scheme}) + assert rules(result, "render-term-limit") == [] + lines = result.output.splitlines() + assert "Term 2999 used Term 2999" in lines + assert "Term 0 used Term 0" in lines + assert not any(line.endswith(" used t2999") for line in lines) + if heading and scheme == "none": + assert any(line.endswith("H2999 Term 2999") for line in lines) + + +# -- 2: a work budget on repeat computations ------------------------------------------ + + +def _ring(k: int, n: int) -> str: + """k terms in a cycle, each holding n references to the next.""" + defs = [f'"T{i} {" ".join(f"{{{{term: t{(i + 1) % k}}}}}" for _ in range(n))}" {{{{def: t{i}}}}}' for i in range(k)] + return "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n" + + +def test_a_ring_of_terms_with_many_references_costs_a_small_multiple_of_the_document() -> None: + document = FRONT + _ring(12, 2000) + started = time.perf_counter() + result = render(document, format="text", overrides={"numbering.scheme": "none"}) + cyclic = time.perf_counter() - started + # The same document with the references pointing at a term that names nothing. + flat = document.replace("{{term: t0}}", "{{term: leaf}}") + for i in range(12): + flat = flat.replace(f"{{{{term: t{i}}}}}", "{{term: leaf}}") + flat += '\n"Leaf" {{def: leaf}}\n' + started = time.perf_counter() + render(flat, format="text", overrides={"numbering.scheme": "none"}) + acyclic = time.perf_counter() - started + assert rules(result, "render-term-limit") + assert cyclic < 3 * acyclic + 1 + + +@pytest.mark.parametrize("k", [7, 12, 40]) +def test_all_to_all_terms_stay_fast(k: int) -> None: + defs = [f'"T{i} {" ".join(f"{{{{term: t{j}}}}}" for j in range(k))}" {{{{def: t{i}}}}}' for i in range(k)] + started = time.perf_counter() + result = render(FRONT + "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n", format="text") + assert time.perf_counter() - started < 2 + assert "{{" not in result.output + + +def test_a_chain_of_500_terms_stays_fast_and_reports_the_limit() -> None: + defs = [f'"T{i} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(500)] + ['"End" {{def: t500}}'] + started = time.perf_counter() + result = render(FRONT + "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n", format="text") + assert time.perf_counter() - started < 2 + assert len(rules(result, "render-term-limit")) == 1 + + +# -- 3: command-line values are escaped on stderr ------------------------------------- + + +def _cli(*arguments: str, cwd: Path) -> subprocess.CompletedProcess[bytes]: + env = {**os.environ, "PYTHONPATH": str(SRC)} + return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *arguments], + capture_output=True, cwd=cwd, env=env, check=False) + + +def _clean(errors: bytes) -> str: + text = errors.decode("utf-8") + assert not any((ord(c) < 0x20 and c != "\n") or 0x7F <= ord(c) < 0xA0 for c in text), text + return text + + +ESC = "x\x1b]0;pwn\x07y" + + +def test_an_input_file_name_is_escaped(tmp_path: Path) -> None: + result = _cli(ESC + ".lgd", cwd=tmp_path) + assert result.returncode == 1 + assert "cannot read" in _clean(result.stderr) and "\\x1b" in _clean(result.stderr) + + +def test_an_answers_path_is_escaped(tmp_path: Path) -> None: + document = tmp_path / "d.lgd" + document.write_text(FRONT + "# A\n", encoding="utf-8") + result = _cli(str(document), "--answers", ESC + ".yaml", cwd=tmp_path) + assert result.returncode == 1 + assert "\\x1b" in _clean(result.stderr) + + +def test_an_output_path_is_escaped(tmp_path: Path) -> None: + document = tmp_path / "d.lgd" + document.write_text(FRONT + "# A\n", encoding="utf-8") + result = _cli(str(document), "-o", "no-such-dir\x1b[2J/out.html", cwd=tmp_path) + assert result.returncode == 1 + assert "\\x1b" in _clean(result.stderr) + refused = _cli(str(document), "-o", "x\x1b[2J.pdf", cwd=tmp_path) + assert refused.returncode == 2 + assert "\\x1b" in _clean(refused.stderr) + + +def test_a_style_name_is_escaped(tmp_path: Path) -> None: + document = tmp_path / "d.lgd" + document.write_text(FRONT + "# A\n", encoding="utf-8") + for arguments in (["-s", ESC], ["-s", ESC, "--print-style"]): + result = _cli(str(document), *arguments, cwd=tmp_path) + assert result.returncode == 2 + assert "\\x1b" in _clean(result.stderr) + + +def test_a_locale_is_escaped(tmp_path: Path) -> None: + document = tmp_path / "d.lgd" + document.write_text(FRONT + "# A\n", encoding="utf-8") + result = _cli(str(document), "--locale", "zz\x1b[31m", cwd=tmp_path) + _clean(result.stderr) + + +def test_a_set_key_is_escaped_in_argparse_errors(tmp_path: Path) -> None: + document = tmp_path / "d.lgd" + document.write_text(FRONT + "# A\n", encoding="utf-8") + for setting in ("bad\x1b[31m", "no\x1b[31m.such=1"): + result = _cli(str(document), "--set", setting, cwd=tmp_path) + assert result.returncode == 2 + assert "\\x1b" in _clean(result.stderr) + + +def test_an_unrecognized_argument_is_escaped(tmp_path: Path) -> None: + result = _cli("--bogus\x1b[31m", cwd=tmp_path) + assert result.returncode == 2 + assert "\\x1b" in _clean(result.stderr) From 1336bbb9ab3230bd7acedbbfb9736cdae535ab06 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:33:26 +0000 Subject: [PATCH 030/118] Reorganise the tests by topic The tests that grew in modules named after the order they were written in are now grouped by what they check: values, numbering, links, building the tree, defined terms (text and budget), the plain-text writer, the HTML print stylesheet, built-in labels, style validation, the command line (options and byte-level input and output), errors and escaping of diagnostics. Shared helpers (the minimal document header, the text and html renderers, and running the command as a program) are in conftest.py. Docstrings and comments no longer refer to internal work items. Tests that asserted the same thing twice are merged or dropped: the extension and link-scheme cases that appeared in two modules, the all-to-all and 500-term chain cases of the term budget, and a few single assertions already covered by an existing test. --- tests/conftest.py | 39 +++ tests/{test_wp4_build.py => test_build.py} | 148 ++-------- tests/test_cli.py | 24 +- tests/{test_wp2_cli.py => test_cli_io.py} | 250 ++++++++-------- tests/test_errors.py | 78 +++++ tests/test_escaping.py | 130 +++++++++ tests/test_html.py | 34 +++ tests/{test_wp8_labels.py => test_labels.py} | 5 +- tests/{test_wp3_links.py => test_links.py} | 133 ++++++--- tests/test_numbering.py | 71 +++++ tests/test_render.py | 31 +- tests/test_style.py | 6 +- ..._wp6_style.py => test_style_validation.py} | 104 +++++-- tests/test_terms.py | 211 ++++++++++++++ tests/test_terms_budget.py | 219 ++++++++++++++ .../{test_wp5_text.py => test_text_writer.py} | 45 ++- tests/test_values.py | 275 +++++++++++++++++- tests/test_wp10_review_fixes.py | 220 -------------- tests/test_wp11_review_fixes.py | 253 ---------------- tests/test_wp12_final_fixes.py | 150 ---------- tests/test_wp13_budget.py | 162 ----------- tests/test_wp1_values.py | 206 ------------- tests/test_wp7_small.py | 106 ------- tests/test_wp9_review_fixes.py | 200 ------------- 24 files changed, 1429 insertions(+), 1671 deletions(-) rename tests/{test_wp4_build.py => test_build.py} (60%) rename tests/{test_wp2_cli.py => test_cli_io.py} (51%) create mode 100644 tests/test_errors.py create mode 100644 tests/test_escaping.py create mode 100644 tests/test_html.py rename tests/{test_wp8_labels.py => test_labels.py} (90%) rename tests/{test_wp3_links.py => test_links.py} (64%) create mode 100644 tests/test_numbering.py rename tests/{test_wp6_style.py => test_style_validation.py} (74%) create mode 100644 tests/test_terms.py create mode 100644 tests/test_terms_budget.py rename tests/{test_wp5_text.py => test_text_writer.py} (78%) delete mode 100644 tests/test_wp10_review_fixes.py delete mode 100644 tests/test_wp11_review_fixes.py delete mode 100644 tests/test_wp12_final_fixes.py delete mode 100644 tests/test_wp13_budget.py delete mode 100644 tests/test_wp1_values.py delete mode 100644 tests/test_wp7_small.py delete mode 100644 tests/test_wp9_review_fixes.py diff --git a/tests/conftest.py b/tests/conftest.py index 61a55d5..a77cca9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,52 @@ from __future__ import annotations import os +import subprocess +import sys from pathlib import Path import pytest +from legaldown_render import render + ROOT = Path(__file__).parent +SRC = Path(__file__).resolve().parents[1] / "src" DOCUMENTS = ROOT / "documents" GOLDEN = ROOT / "golden" +FEATURES = str(DOCUMENTS / "features.lgd") + +#: The smallest valid header, for tests whose body is all that matters. +FRONT = "---\ntitle: T\n---\n" + + +def text(body: str, **settings: object) -> str: + """The plain-text rendering of *body* under :data:`FRONT`.""" + return render(FRONT + body, format="text", **settings).output + + +def html(body: str, **settings: object) -> str: + """The HTML rendering of *body* under :data:`FRONT`.""" + return render(FRONT + body, format="html", **settings).output + + +def run_cli(*args: str, stdin: bytes | str | None = None, cwd: Path | None = None, + narrow: bool = False) -> subprocess.CompletedProcess[bytes]: + """Run ``legaldown-render`` as a program on this checkout's sources. + + *narrow* gives it a Windows-like console, where neither stream is UTF-8.""" + env = dict(os.environ, PYTHONPATH=str(SRC)) + if narrow: + env.update(PYTHONIOENCODING="cp1252", PYTHONUTF8="0") + data = stdin.encode("utf-8") if isinstance(stdin, str) else stdin + return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *args], input=data, + capture_output=True, cwd=cwd, env=env, check=False) + + +def run_cli_text(*args: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]: + """Like :func:`run_cli`, with UTF-8 text for the streams.""" + env = dict(os.environ, PYTHONPATH=str(SRC), PYTHONUTF8="1") + return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *args], input=stdin, + capture_output=True, text=True, env=env, check=False) def pytest_addoption(parser: pytest.Parser) -> None: diff --git a/tests/test_wp4_build.py b/tests/test_build.py similarity index 60% rename from tests/test_wp4_build.py rename to tests/test_build.py index 6862f29..38020f9 100644 --- a/tests/test_wp4_build.py +++ b/tests/test_build.py @@ -1,17 +1,17 @@ -"""Build fidelity (WP4): what the builder keeps of a comment-only item or -quote (B2), how emphasis next to a directive parses (B3), the spacing where -an inline comment was dropped (B4), and what ``{{term:}}`` shows (B8).""" +"""Building the tree from the validator's document: what is kept of a +comment-only list item or quote (§8.6), how emphasis next to a directive +parses, and the spacing where an inline comment was dropped.""" from __future__ import annotations import html as html_escape import unicodedata import pytest +from conftest import FRONT, html, text from legaldown import parse, validate from markdown_it import MarkdownIt from markdown_it.common.utils import isPunctChar -from legaldown_render import render from legaldown_render.build import _WRAPPERS, build_tree from legaldown_render.tree import ( DirectiveSource, @@ -23,16 +23,6 @@ Text, ) -FRONT = "---\ntitle: T\n---\n" - - -def text(body: str, **settings: object) -> str: - return render(FRONT + body, format="text", **settings).output - - -def html(body: str, **settings: object) -> str: - return render(FRONT + body, format="html", **settings).output - def built(body: str): document = parse(FRONT + body) @@ -43,8 +33,7 @@ def blocks_of(body: str): return built(body).sections[0].blocks -# -- B2: comment-only list items and quotes ---------------------------------- - +# -- Comment-only list items and quotes ------------------------------------------ def test_a_list_item_holding_only_a_comment_is_dropped() -> None: output = text("# A\n\n- a\n- \n- c\n") @@ -69,13 +58,6 @@ def test_a_list_left_with_no_items_is_dropped() -> None: assert "\n") -def test_an_empty_item_keeps_its_place_but_a_comment_only_one_does_not() -> None: - # The validator keeps an empty item (CommonMark); a comment is the - # author's way to remove one. - output = text("# A\n\n- \n- \n- b\n") - assert "(a)\n(b) b" in output - - def test_an_item_with_a_comment_and_text_stays() -> None: assert "(a) kept" in text("# A\n\n- kept \n") @@ -111,8 +93,25 @@ def test_a_comment_only_item_does_not_leave_an_empty_list_in_an_item() -> None: assert "(a) a\n(b) b" in output -# -- B3: emphasis flanking next to a directive ------------------------------- +def test_a_comment_only_item_with_a_marker_is_kept() -> None: + # Defensive: an item a marker belongs to never vanishes. The + # validator does not place a marker on a comment-only item, so a marker + # is made up here to drive the builder's guard. + from legaldown import PlacedMarker + + from legaldown_render.build import _Builder + + document = parse(FRONT + "# A\n\n- \n- b\n") + result = validate(document) + marker = PlacedMarker(section=0, block=0, fragment=0, offset=0, source="", + identifier="x", condition="", item=0) + out = _Builder(document, result).list(document.sections[0].blocks[0], [marker]) + assert out is not None and [item.anchor_id for item in out.items] == ["x", ""] + plain = _Builder(document, result).list(document.sections[0].blocks[0], []) + assert plain is not None and len(plain.items) == 1 + +# -- Emphasis flanking next to a directive --------------------------------------- def _signature(inlines) -> str: out = [] @@ -173,8 +172,7 @@ def test_wrappers_do_not_leak_into_code_urls_or_alt_text() -> None: assert char not in output -# -- B4: spacing where an inline comment was dropped ------------------------- - +# -- Spacing where an inline comment was dropped --------------------------------- @pytest.mark.parametrize(("body", "expected"), [ ("one \ntwo\n", "one two"), @@ -201,101 +199,3 @@ def test_a_dropped_comment_between_text_and_a_hard_break_leaves_no_space() -> No paragraph = blocks_of("# A\n\none \\\ntwo\n")[0] first = paragraph.inlines[0] assert isinstance(first, Text) and first.text == "one" - - -# -- B8: what {{term:}} shows ------------------------------------------------ - - -def test_a_term_reference_shows_the_rendered_text_of_the_term() -> None: - output = text('# A\n\n"A *b* c" {{def: abc}} means it.\n\nSee {{term: abc}}.\n') - assert "See A b c." in output - assert "*" not in output.split("See", 1)[1] - - -def test_a_term_reference_label_still_wins() -> None: - output = text('# A\n\n"A *b* c" {{def: abc}} means it.\n\nSee {{term: abc, label=the thing}}.\n') - assert "See the thing." in output - - -def test_a_term_reference_resolves_directives_inside_the_term() -> None: - body = '# A {#a}\n\n"Fee **under** {{ref: a}}" {{def: fee}} means it.\n\nSee {{term: fee}}.\n' - output = text(body) - assert "See Fee under" in output - - -def test_a_term_reference_uses_the_first_definition() -> None: - body = '# A\n\n"First *one*" {{def: t}} here.\n\n"Second" {{def: t}} there.\n\nSee {{term: t}}.\n' - assert "See First one." in text(body) - - -def test_a_term_reference_before_its_definition() -> None: - body = '# A\n\nSee {{term: abc}} first.\n\n"A *b* c" {{def: abc}} means it.\n' - assert "See A b c first." in text(body) - - -def test_a_plain_term_reference_is_unchanged() -> None: - assert "See Fee." in text('# A\n\n"Fee" {{def: fee}} means it.\n\nSee {{term: fee}}.\n') - - -def test_an_empty_term_falls_back_to_the_validator() -> None: - output = text('# A\n\n"" {{def: late-fee}} means it.\n\nSee {{term: late-fee}}.\n') - assert "See Late Fee." in output - - -def test_an_undefined_term_is_still_reported() -> None: - assert "[UNDEFINED: nope]" in text("# A\n\nSee {{term: nope}}.\n") - - -# -- B8: a term that refers to itself ---------------------------------------- - -SELF = '# A\n\n"Self {{term: s}}" {{def: s}} means it.\n\nSee {{term: s}} and {{term: s, label=the label}}.\n' -INDIRECT = ('# A\n\n"A {{term: b}}" {{def: a}} x.\n\n"B {{term: a}}" {{def: b}} y.\n\n' - "See {{term: a}}, {{term: b}} and {{term: a, label=the label}}.\n") - - -@pytest.mark.parametrize("output_format", ["text", "html"]) -@pytest.mark.parametrize("body", [SELF, INDIRECT], ids=["direct", "indirect"]) -def test_a_cyclic_term_renders_in_both_writers(body: str, output_format: str) -> None: - result = render(FRONT + body, format=output_format) - assert "the label" in result.output - assert "[UNDEFINED" not in result.output - - -def test_a_term_inside_its_own_definition_shows_its_id() -> None: - output = text(SELF) - # At the definition and at a reference alike: the inner reference shows - # the id (its label when it has one), and goes no further. - assert "Self Self s means it." in output - assert "See Self s and the label." in output - - -def test_a_term_reference_in_a_cycle_shows_its_label_else_its_id() -> None: - output = text(INDIRECT) - assert output.rstrip().endswith("and the label.") - # No term shows its own reference as raw directive source. - assert "{{" not in output - - -def test_a_cycle_through_a_reference_site_inside_another_terms_definition() -> None: - body = ('# A\n\n"A" {{def: a}} x.\n\n"B {{term: a}} and {{term: c}}" {{def: b}} y.\n\n' - '"C {{term: b}}" {{def: c}} z.\n\nSee {{term: b}}.\n') - assert "See B A and" in text(body) - assert "B A and" in html(body) - - -def test_a_comment_only_item_with_a_marker_is_kept() -> None: - # Defensive (B2): an item a marker belongs to never vanishes. The - # validator does not place a marker on a comment-only item, so a marker - # is made up here to drive the builder's guard. - from legaldown import PlacedMarker - - from legaldown_render.build import _Builder - - document = parse(FRONT + "# A\n\n- \n- b\n") - result = validate(document) - marker = PlacedMarker(section=0, block=0, fragment=0, offset=0, source="", - identifier="x", condition="", item=0) - out = _Builder(document, result).list(document.sections[0].blocks[0], [marker]) - assert out is not None and [item.anchor_id for item in out.items] == ["x", ""] - plain = _Builder(document, result).list(document.sections[0].blocks[0], []) - assert plain is not None and len(plain.items) == 1 diff --git a/tests/test_cli.py b/tests/test_cli.py index aad5ff3..c3565d7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,14 +1,18 @@ +"""The command line: the format from ``-o``, ``--style``, ``--set``, +``--locale``, ``--strict``, ``--final``, ``--answers``, ``--print-style`` and +``--list-styles``, their exit codes, and the diagnostics on standard error. How +output is encoded and written is in test_cli_io.py.""" from __future__ import annotations from pathlib import Path import pytest -from conftest import DOCUMENTS +from conftest import DOCUMENTS, FEATURES +import legaldown_render.cli as cli +from legaldown_render import StyleError from legaldown_render.cli import main -FEATURES = str(DOCUMENTS / "features.lgd") - def test_render_to_file_infers_the_format(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: out = tmp_path / "out.txt" @@ -90,3 +94,17 @@ def test_set_keeps_text_values_as_written(capsys: pytest.CaptureFixture[str]) -> def test_quote_bounded_literal_setting_is_kept(capsys: pytest.CaptureFixture[str]) -> None: assert main([FEATURES, "-f", "text", "-q", "--set", 'references.format="§" {designation} "x"']) == 0 assert 'under Section "§" 2.1 "x".' in capsys.readouterr().out + + +def test_set_style_error_is_shown_by_argparse(monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str]) -> None: + def refuse(key: str, text: str) -> object: + raise StyleError("overrides", [f"{key}: the value is no good"]) + + monkeypatch.setattr(cli, "parse_override", refuse) + with pytest.raises(SystemExit) as exit_info: + main([FEATURES, "--set", "numbering.scheme=oops"]) + assert exit_info.value.code == 2 + err = capsys.readouterr().err + assert "numbering.scheme: the value is no good" in err + assert "--set" in err diff --git a/tests/test_wp2_cli.py b/tests/test_cli_io.py similarity index 51% rename from tests/test_wp2_cli.py rename to tests/test_cli_io.py index 1d1e199..58da13c 100644 --- a/tests/test_wp2_cli.py +++ b/tests/test_cli_io.py @@ -1,59 +1,39 @@ -"""CLI and API robustness: stream encodings (S1), non-UTF-8 input (S4), an -unwritable output (S5), an unknown output extension (S6), the error -hierarchy (S2), picklable errors (S3), and the ``--set`` error path (S10, CLI -side).""" +"""How the command line reads and writes bytes: UTF-8 whatever the console's +encoding, a clean error for input that is not UTF-8 or output that cannot be +written, LF line endings, in-process ``main()`` leaving the streams alone, and +the rule that ``-o`` names a format by its extension (§13.8).""" from __future__ import annotations import io -import os -import pickle -import subprocess import sys from pathlib import Path import pytest -from conftest import DOCUMENTS -from legaldown import Diagnostic +from conftest import FEATURES, SRC, run_cli import legaldown_render -import legaldown_render.cli as cli -from legaldown_render import ( - DocumentError, - OptionsError, - RenderError, - RenderOptions, - RenderRefused, - StyleError, - load_style, - render, - render_file, -) +from legaldown_render import DocumentError, StyleError, load_style, render, render_file from legaldown_render.cli import main +from legaldown_render.writers import file_extension, format_for_path -SRC = Path(__file__).resolve().parents[1] / "src" -FEATURES = str(DOCUMENTS / "features.lgd") CZECH = "---\nlegaldown: '0.2'\ntitle: Smlouva o dílo\nlanguage: cs\n---\n\n# Příloha\n\nŽluťoučký kůň úpěl ďábelské ódy – “citát” → konec, ōsaka.\n" -def _run(args: list[str], stdin: bytes | None = None, *, narrow: bool = True) -> subprocess.CompletedProcess[bytes]: - env = dict(os.environ, PYTHONPATH=str(SRC)) - if narrow: - # A Windows-like console: neither stream is UTF-8. - env.update(PYTHONIOENCODING="cp1252", PYTHONUTF8="0") - return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *args], input=stdin, - capture_output=True, env=env, check=False) +def _narrow(args: list[str], stdin: bytes | None = None): # type: ignore[no-untyped-def] + """The program on a Windows-like console: neither stream is UTF-8.""" + return run_cli(*args, stdin=stdin, narrow=True) def test_the_package_under_test_is_this_worktree() -> None: assert Path(legaldown_render.__file__).resolve().is_relative_to(SRC) -# --- S1: encodings --------------------------------------------------------------------------- +# -- Encodings ------------------------------------------------------------------- @pytest.mark.parametrize("fmt", ["html", "text"]) def test_stdin_to_stdout_is_utf8_whatever_the_console_encoding(fmt: str) -> None: expected = render(CZECH, format=fmt).output.encode("utf-8") - done = _run(["-", "-f", fmt, "-q"], CZECH.encode("utf-8")) + done = _narrow(["-", "-f", fmt, "-q"], CZECH.encode("utf-8")) assert done.returncode == 0, done.stderr # Exact bytes: the output is UTF-8 with LF line endings on every platform, so # the same comparison holds on Windows (no CRLF translation of stdout). @@ -64,27 +44,27 @@ def test_stdin_to_stdout_is_utf8_whatever_the_console_encoding(fmt: str) -> None def test_file_to_stdout_is_utf8_whatever_the_console_encoding(tmp_path: Path) -> None: path = tmp_path / "smlouva.lgd" path.write_bytes(CZECH.encode("utf-8")) - done = _run([str(path), "-f", "text", "-q"]) + done = _narrow([str(path), "-f", "text", "-q"]) assert done.returncode == 0, done.stderr # Exact bytes, as above: UTF-8 with LF line endings on every platform. assert done.stdout == render(CZECH, format="text").output.encode("utf-8") def test_print_style_survives_a_narrow_console() -> None: - done = _run(["--print-style", "--style", "continental", "--set", "placeholders.blank=…"]) + done = _narrow(["--print-style", "--style", "continental", "--set", "placeholders.blank=…"]) assert done.returncode == 0, done.stderr assert "…".encode() in done.stdout def test_stderr_never_crashes_on_a_narrow_console() -> None: - done = _run(["žlutý→.lgd"]) + done = _narrow(["žlutý→.lgd"]) assert done.returncode == 1 assert b"Traceback" not in done.stderr assert b"cannot read" in done.stderr assert b"\\u2192" in done.stderr # not encodable in cp1252: backslash-escaped, not an error -# --- S4: not UTF-8 --------------------------------------------------------------------------- +# -- Input that is not UTF-8 ----------------------------------------------------- def test_latin1_input_file_is_a_clean_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: path = tmp_path / "latin1.lgd" @@ -137,7 +117,7 @@ def test_render_file_raises_document_error_for_non_utf8(tmp_path: Path) -> None: assert "Příloha" in render_file(good, format="text").output -# --- S5: unwritable output ------------------------------------------------------------------- +# -- Output that cannot be written ----------------------------------------------- def test_unwritable_output_is_a_clean_error(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: target = tmp_path / "missing-folder" / "out.html" @@ -152,10 +132,96 @@ def test_output_that_is_a_directory(tmp_path: Path, capsys: pytest.CaptureFixtur assert "cannot write" in capsys.readouterr().err -# --- S6: an extension of a format not written ----------------------------------------------- +# -- Streams and line endings ---------------------------------------------------- + +def _wrapper(encoding: str, **options: object) -> io.TextIOWrapper: + return io.TextIOWrapper(io.BytesIO(), encoding=encoding, **options) # type: ignore[arg-type] + + +def test_in_process_main_leaves_the_streams_alone(monkeypatch: pytest.MonkeyPatch) -> None: + out, err = _wrapper("cp1252", newline="\r\n"), _wrapper("cp1252") + monkeypatch.setattr(sys, "stdout", out) + monkeypatch.setattr(sys, "stderr", err) + assert main([FEATURES, "-f", "text", "-q"]) == 0 + assert main(["--list-styles"]) == 0 + assert (sys.stdout.encoding, sys.stdout.errors) == ("cp1252", "strict") + assert (sys.stderr.encoding, sys.stderr.errors) == ("cp1252", "strict") + out.flush() + # Not even the newline translation of the caller's stream changed. + assert b"\r\n" in out.buffer.getvalue() # type: ignore[attr-defined] + + +def test_main_as_the_program_reconfigures_the_streams(monkeypatch: pytest.MonkeyPatch) -> None: + out, err = _wrapper("cp1252", newline="\r\n"), _wrapper("cp1252") + monkeypatch.setattr(sys, "stdout", out) + monkeypatch.setattr(sys, "stderr", err) + monkeypatch.setattr(sys, "argv", ["legaldown-render", "--list-styles"]) + assert main() == 0 + out.flush() + assert (out.encoding, err.errors) == ("utf-8", "backslashreplace") + data = out.buffer.getvalue() # type: ignore[attr-defined] + assert data and b"\r" not in data and data.endswith(b"\n") + + +def test_the_console_script_calls_main_without_argv() -> None: + # The script entry point calls main() with no argument, so it is "the program". + pyproject = (SRC.parent / "pyproject.toml").read_text(encoding="utf-8") + assert 'legaldown-render = "legaldown_render.cli:main"' in pyproject + + +@pytest.mark.parametrize("fmt", ["html", "text"]) +def test_standard_output_has_lf_line_endings(fmt: str) -> None: + done = _narrow([FEATURES, "-f", fmt, "-q"]) + assert done.returncode == 0, done.stderr + assert b"\n" in done.stdout and b"\r" not in done.stdout + + +@pytest.mark.parametrize("fmt", ["html", "text"]) +def test_an_output_file_has_lf_line_endings(tmp_path: Path, fmt: str) -> None: + target = tmp_path / "out.bin" + assert main([FEATURES, "-q", "-f", fmt, "-o", str(target)]) == 0 + data = target.read_bytes() + assert b"\n" in data and b"\r" not in data + assert data == render(Path(FEATURES).read_text(encoding="utf-8"), format=fmt).output.encode("utf-8") + + +def test_an_output_file_is_written_without_newline_translation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # Windows' text mode would turn "\n" into CRLF unless the file is opened + # with newline="\n": that is what the CLI asks for. + calls: list[dict[str, object]] = [] + real = Path.write_text + + def spy(self: Path, data: str, *args: object, **kwargs: object) -> int: + calls.append(kwargs) + return real(self, data, *args, **kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(Path, "write_text", spy) + assert main([FEATURES, "-q", "-o", str(tmp_path / "out.txt")]) == 0 + assert calls == [{"encoding": "utf-8", "newline": "\n"}] + + +# -- The extension of -o sets the format, but only for a format that is written ---- + +@pytest.mark.parametrize(("name", "extension"), [ + ("x.pdf", "pdf"), (".pdf", "pdf"), (".md", "md"), ("x.pdf.", "pdf"), ("x.docx ", "docx"), + ("x.pdf. .", "pdf"), ("X.PDF", "pdf"), ("a.b.txt", "txt"), ("contract", ""), ("", ""), (".", ""), + ("..", ""), ("x.", ""), +]) +def test_the_extension_of_a_file_name(name: str, extension: str) -> None: + assert file_extension(name) == extension + + +@pytest.mark.parametrize(("name", "expected"), [ + (".txt", "text"), ("x.txt.", "text"), ("x.html ", "html"), (".htm", "html"), ("x.unknown", None), +]) +def test_the_format_a_file_name_implies(name: str, expected: str | None) -> None: + assert format_for_path(name) == expected + @pytest.mark.parametrize("name", ["x.docx", "x.pdf", "x.md", "x.DOCX", "x.doc", "x.odt", "x.rtf", "x.markdown", - "x.epub", "x.tex", "x.v2.Pdf"]) + "x.epub", "x.tex", "x.v2.Pdf", ".pdf", ".md", "x.pdf.", "x.docx ", "x.docx. .", + "dir.pdf", "x.MD"]) def test_unknown_output_extension_without_format_is_a_usage_error( tmp_path: Path, capsys: pytest.CaptureFixture[str], name: str) -> None: target = tmp_path / name @@ -166,10 +232,12 @@ def test_unknown_output_extension_without_format_is_a_usage_error( assert f"cannot tell the output format from '{target}'" in err assert "--format html or text" in err assert not target.exists() + assert list(tmp_path.iterdir()) == [] -def test_explicit_format_is_honoured_whatever_the_extension(tmp_path: Path) -> None: - target = tmp_path / "x.docx" +@pytest.mark.parametrize("name", ["x.docx", "x.pdf.", ".md", "x.docx "]) +def test_explicit_format_is_honoured_whatever_the_extension(tmp_path: Path, name: str) -> None: + target = tmp_path / name assert main([FEATURES, "-q", "-f", "text", "-o", str(target)]) == 0 assert target.read_text(encoding="utf-8").startswith("Services Agreement") @@ -181,19 +249,22 @@ def test_output_without_an_extension_keeps_the_html_default(tmp_path: Path) -> N assert target.read_text(encoding="utf-8").startswith("") -@pytest.mark.parametrize("name", ["contract.v2", "report.2025-10-27", ".out", "out.", "notes.final", "x.json"]) -def test_any_other_extension_keeps_the_html_default(tmp_path: Path, name: str) -> None: +@pytest.mark.parametrize("name", ["contract", "contract.v2", "report.2025-10-27", ".out", "out.", "notes.final", + "x.json", "x.."]) +def test_any_other_name_keeps_the_html_default(tmp_path: Path, name: str) -> None: target = tmp_path / name assert main([FEATURES, "-q", "-o", str(target)]) == 0 assert target.read_text(encoding="utf-8").startswith("") -def test_a_known_extension_still_sets_the_format(tmp_path: Path) -> None: - for name, start in (("a.TXT", "Services Agreement"), ("a.text", "Services Agreement"), - ("a.htm", ""), ("a.html", "")): - target = tmp_path / name - assert main([FEATURES, "-q", "-o", str(target)]) == 0 - assert target.read_text(encoding="utf-8").startswith(start) +@pytest.mark.parametrize(("name", "start"), [ + ("a.TXT", "Services Agreement"), ("a.text", "Services Agreement"), ("a.htm", ""), + ("a.html", ""), (".txt", "Services Agreement"), ("x.txt.", "Services Agreement"), +]) +def test_a_known_extension_still_sets_the_format(tmp_path: Path, name: str, start: str) -> None: + target = tmp_path / name + assert main([FEATURES, "-q", "-o", str(target)]) == 0 + assert target.read_text(encoding="utf-8").startswith(start) def test_the_help_states_the_output_rule(capsys: pytest.CaptureFixture[str]) -> None: @@ -203,84 +274,3 @@ def test_the_help_states_the_output_rule(capsys: pytest.CaptureFixture[str]) -> assert "its extension sets the format" not in help_text assert "any other name, with or without an extension, writes HTML" in help_text assert ".docx, .doc, .odt, .rtf, .pdf, .md, .markdown, .epub, .tex" in help_text - - -# --- S2: error hierarchy --------------------------------------------------------------------- - -def test_style_error_is_a_render_error_and_still_a_value_error() -> None: - assert issubclass(StyleError, RenderError) - assert issubclass(StyleError, ValueError) - - -def test_style_error_is_importable_from_where_it_always_was() -> None: - from legaldown_render import errors - from legaldown_render.style import StyleError as from_style - from legaldown_render.style.loader import StyleError as from_loader - - assert errors.StyleError is StyleError is from_style is from_loader - - -def test_unknown_format_raises_options_error() -> None: - assert issubclass(OptionsError, RenderError) - assert issubclass(OptionsError, ValueError) - with pytest.raises(OptionsError, match="unknown output format 'pdf'"): - render("# x\n", RenderOptions(format="pdf")) - with pytest.raises(OptionsError): - render("# x\n", format="docx") - with pytest.raises(RenderError): - render("# x\n", format="docx") - - -def test_invalid_style_through_the_api_is_a_render_error() -> None: - with pytest.raises(RenderError): - render("# x\n", style="no-such-style") - - -# --- S3: pickling ---------------------------------------------------------------------------- - -def test_render_refused_round_trips_through_pickle() -> None: - diagnostics = [Diagnostic(rule="ref-broken", level="error", message="No such target", line=3)] - for reason in (None, "Assembly refused: 1 error(s)."): - error = RenderRefused(diagnostics, reason) - copy = pickle.loads(pickle.dumps(error)) - assert type(copy) is RenderRefused - assert copy.diagnostics == diagnostics - assert str(copy) == str(error) - assert copy.reason == error.reason - - -def test_a_real_refusal_round_trips_through_pickle() -> None: - with pytest.raises(RenderRefused) as caught: - render(Path(FEATURES).read_text(encoding="utf-8"), strict=True) - copy = pickle.loads(pickle.dumps(caught.value)) - assert copy.diagnostics == caught.value.diagnostics - assert str(copy) == str(caught.value) - - -def test_style_error_round_trips_through_pickle() -> None: - error = StyleError("overrides", ["a.b: wrong", "c: also wrong"]) - copy = pickle.loads(pickle.dumps(error)) - assert type(copy) is StyleError - assert (copy.source, copy.problems) == ("overrides", ["a.b: wrong", "c: also wrong"]) - assert str(copy) == str(error) - - -def test_options_error_round_trips_through_pickle() -> None: - error = OptionsError("unknown output format 'pdf'") - assert str(pickle.loads(pickle.dumps(error))) == str(error) - - -# --- S10 (CLI side): a StyleError from --set reaches the user --------------------------------- - -def test_set_style_error_is_shown_by_argparse(monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str]) -> None: - def refuse(key: str, text: str) -> object: - raise StyleError("overrides", [f"{key}: the value is no good"]) - - monkeypatch.setattr(cli, "parse_override", refuse) - with pytest.raises(SystemExit) as exit_info: - main([FEATURES, "--set", "numbering.scheme=oops"]) - assert exit_info.value.code == 2 - err = capsys.readouterr().err - assert "numbering.scheme: the value is no good" in err - assert "--set" in err diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..1968390 --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,78 @@ +"""The exception hierarchy: every error the renderer raises is a +``RenderError``, ``StyleError`` and ``OptionsError`` are still ``ValueError``, +``StyleError`` is importable from where it always was, and the errors survive +pickling (for a process pool).""" +from __future__ import annotations + +import pickle +from pathlib import Path + +import pytest +from conftest import FEATURES +from legaldown import Diagnostic + +from legaldown_render import OptionsError, RenderError, RenderOptions, RenderRefused, StyleError, render + +# -- Hierarchy ------------------------------------------------------------------- + +def test_style_error_is_a_render_error_and_still_a_value_error() -> None: + assert issubclass(StyleError, RenderError) + assert issubclass(StyleError, ValueError) + + +def test_style_error_is_importable_from_where_it_always_was() -> None: + from legaldown_render import errors + from legaldown_render.style import StyleError as from_style + from legaldown_render.style.loader import StyleError as from_loader + + assert errors.StyleError is StyleError is from_style is from_loader + + +def test_unknown_format_raises_options_error() -> None: + assert issubclass(OptionsError, RenderError) + assert issubclass(OptionsError, ValueError) + with pytest.raises(OptionsError, match="unknown output format 'pdf'"): + render("# x\n", RenderOptions(format="pdf")) + with pytest.raises(OptionsError): + render("# x\n", format="docx") + with pytest.raises(RenderError): + render("# x\n", format="docx") + + +def test_invalid_style_through_the_api_is_a_render_error() -> None: + with pytest.raises(RenderError): + render("# x\n", style="no-such-style") + + +# -- Pickling -------------------------------------------------------------------- + +def test_render_refused_round_trips_through_pickle() -> None: + diagnostics = [Diagnostic(rule="ref-broken", level="error", message="No such target", line=3)] + for reason in (None, "Assembly refused: 1 error(s)."): + error = RenderRefused(diagnostics, reason) + copy = pickle.loads(pickle.dumps(error)) + assert type(copy) is RenderRefused + assert copy.diagnostics == diagnostics + assert str(copy) == str(error) + assert copy.reason == error.reason + + +def test_a_real_refusal_round_trips_through_pickle() -> None: + with pytest.raises(RenderRefused) as caught: + render(Path(FEATURES).read_text(encoding="utf-8"), strict=True) + copy = pickle.loads(pickle.dumps(caught.value)) + assert copy.diagnostics == caught.value.diagnostics + assert str(copy) == str(caught.value) + + +def test_style_error_round_trips_through_pickle() -> None: + error = StyleError("overrides", ["a.b: wrong", "c: also wrong"]) + copy = pickle.loads(pickle.dumps(error)) + assert type(copy) is StyleError + assert (copy.source, copy.problems) == ("overrides", ["a.b: wrong", "c: also wrong"]) + assert str(copy) == str(error) + + +def test_options_error_round_trips_through_pickle() -> None: + error = OptionsError("unknown output format 'pdf'") + assert str(pickle.loads(pickle.dumps(error))) == str(error) diff --git a/tests/test_escaping.py b/tests/test_escaping.py new file mode 100644 index 0000000..696bdc0 --- /dev/null +++ b/tests/test_escaping.py @@ -0,0 +1,130 @@ +"""Nothing a document or the command line supplies reaches a diagnostic or +standard error as a control character: controls, format characters (bidi +overrides, zero-width) and line separators are written out as ``\\xNN`` or +``\\uNNNN``, so a hostile path cannot move the cursor, set a title or forge a +line of its own.""" +from __future__ import annotations + +from pathlib import Path + +from conftest import FRONT, run_cli + +from legaldown_render import render +from legaldown_render.urls import printable + +ESC = "x\x1b]0;pwn\x07y" + + +# -- What a diagnostic shows of a document --------------------------------------- + +def test_printable_escapes_controls_and_format_characters() -> None: + assert printable("a\x1b[2J\x07\n\x7f\x85") == "a\\x1b[2J\\x07\\x0a\\x7f\\x85" + assert printable("\u202e\u200b\ufeff\u2028\u2029") == "\\u202e\\u200b\\ufeff\\u2028\\u2029" + assert printable("ž é €") == "ž é €" + + +EVIL = "https://e/%1b[2J%1b]0;pwned%07%0aerror: [x] forged%e2%80%ae" + + +def test_a_control_character_in_an_image_path_is_escaped_in_the_message() -> None: + result = render(FRONT + f"# A\n\n![x](<{EVIL}>)\n") + messages = [d.message for d in result.diagnostics if d.rule == "path-not-relative"] + assert messages + for message in messages: + assert "\x1b" not in message and "\x07" not in message and "\n" not in message + assert any("\\x1b[2J\\x1b]0;pwned\\x07\\x0aerror: [x] forged\\u202e" in m for m in messages) + + +def test_a_control_character_in_a_link_is_escaped_in_the_message() -> None: + result = render(FRONT + f"# A\n\n[x]()\n") + message = next(d.message for d in result.diagnostics if d.rule == "render-link-refused") + assert "\x1b" not in message and "\x07" not in message and "\n" not in message and "\u202e" not in message + assert "\\x1b" in message + + +# -- Standard error of the command ----------------------------------------------- + +def _clean(errors: bytes) -> str: + text = errors.decode("utf-8") + assert not any((ord(c) < 0x20 and c != "\n") or 0x7F <= ord(c) < 0xA0 for c in text), text + return text + + +def test_the_cli_writes_no_control_character_from_a_document_to_standard_error(tmp_path: Path) -> None: + source = tmp_path / "evil.lgd" + source.write_text(FRONT + f"# A\n\n![x](<{EVIL}>) [y]()\n", encoding="utf-8") + done = run_cli(str(source), "-f", "text") + err = done.stderr.decode("utf-8") + assert "path-not-relative" in err and "render-link-refused" in err + assert "\x1b" not in err and "\x07" not in err and "\u202e" not in err + assert "\r" not in err + assert not any(line.startswith("error: [x]") for line in err.splitlines()) + + +def test_no_control_character_from_the_document_reaches_stderr(tmp_path: Path) -> None: + body = ("# A {#a}\n\n{{include: a\x1b]0;pwn\x07.lgd}}\n\n{{ref: x\x1b[2Jy}}\n\n{{term: q\x1b[31m}}\n\n" + "See {{ref: a}}.\n") + document = tmp_path / "d.lgd" + document.write_text(FRONT + body, encoding="utf-8") + errors = _clean(run_cli(str(document), "-f", "text", cwd=tmp_path).stderr) + assert errors.count("\n") >= 3 # the three findings, one line each + assert "\\x1b" in errors # written out, not dropped + assert "pwn" in errors + + +# -- Values from the command line ------------------------------------------------ + +def test_an_input_file_name_is_escaped(tmp_path: Path) -> None: + result = run_cli(ESC + ".lgd", cwd=tmp_path) + assert result.returncode == 1 + assert "cannot read" in _clean(result.stderr) and "\\x1b" in _clean(result.stderr) + + +def test_an_answers_path_is_escaped(tmp_path: Path) -> None: + document = tmp_path / "d.lgd" + document.write_text(FRONT + "# A\n", encoding="utf-8") + result = run_cli(str(document), "--answers", ESC + ".yaml", cwd=tmp_path) + assert result.returncode == 1 + assert "\\x1b" in _clean(result.stderr) + + +def test_an_output_path_is_escaped(tmp_path: Path) -> None: + document = tmp_path / "d.lgd" + document.write_text(FRONT + "# A\n", encoding="utf-8") + result = run_cli(str(document), "-o", "no-such-dir\x1b[2J/out.html", cwd=tmp_path) + assert result.returncode == 1 + assert "\\x1b" in _clean(result.stderr) + refused = run_cli(str(document), "-o", "x\x1b[2J.pdf", cwd=tmp_path) + assert refused.returncode == 2 + assert "\\x1b" in _clean(refused.stderr) + + +def test_a_style_name_is_escaped(tmp_path: Path) -> None: + document = tmp_path / "d.lgd" + document.write_text(FRONT + "# A\n", encoding="utf-8") + for arguments in (["-s", ESC], ["-s", ESC, "--print-style"]): + result = run_cli(str(document), *arguments, cwd=tmp_path) + assert result.returncode == 2 + assert "\\x1b" in _clean(result.stderr) + + +def test_a_locale_is_escaped(tmp_path: Path) -> None: + document = tmp_path / "d.lgd" + document.write_text(FRONT + "# A\n", encoding="utf-8") + result = run_cli(str(document), "--locale", "zz\x1b[31m", cwd=tmp_path) + _clean(result.stderr) + + +def test_a_set_key_is_escaped_in_argparse_errors(tmp_path: Path) -> None: + document = tmp_path / "d.lgd" + document.write_text(FRONT + "# A\n", encoding="utf-8") + for setting in ("bad\x1b[31m", "no\x1b[31m.such=1"): + result = run_cli(str(document), "--set", setting, cwd=tmp_path) + assert result.returncode == 2 + assert "\\x1b" in _clean(result.stderr) + + +def test_an_unrecognized_argument_is_escaped(tmp_path: Path) -> None: + result = run_cli("--bogus\x1b[31m", cwd=tmp_path) + assert result.returncode == 2 + assert "\\x1b" in _clean(result.stderr) diff --git a/tests/test_html.py b/tests/test_html.py new file mode 100644 index 0000000..0d4cd2c --- /dev/null +++ b/tests/test_html.py @@ -0,0 +1,34 @@ +"""The HTML writer's stylesheet for print (§13.8): a page break goes before the +attachments container, not inside it, and only where the separator asks for +one.""" +from __future__ import annotations + +from conftest import run_cli_text + +ATTACHED = ( + "---\ntitle: T\nattachments:\n" + " - id: one\n title: First attachment\n file: one.pdf\n" + " - id: two\n title: Second attachment\n file: two.pdf\n" + "---\n\n# A {#a}\n\nBody.\n" +) + + +def test_a_standalone_page_break_render_has_the_attachment_break_rules() -> None: + done = run_cli_text("-", "--set", "attachments.separator=page-break", stdin=ATTACHED) + assert done.returncode == 0, done.stderr + assert "" in done.stdout + assert 'class="ld-attachments ld-separator-page-break"' in done.stdout + # The break goes before the container (heading included), then before each later attachment. + assert ".ld-attachments.ld-separator-page-break { break-before: page; }" in done.stdout + assert ".ld-separator-page-break .ld-attachment + .ld-attachment { break-before: page; }" in done.stdout + assert ".ld-attachments-heading { break-after: avoid; }" in done.stdout + # The old rule broke before every attachment, the first one included, inside the container. + assert ".ld-separator-page-break .ld-attachment { break-before: page; }" not in done.stdout + + +def test_without_a_page_break_separator_the_container_is_not_broken() -> None: + done = run_cli_text("-", "--set", "attachments.separator=none", stdin=ATTACHED) + assert done.returncode == 0, done.stderr + # The rules are scoped by the separator class, which no element carries here. + assert 'class="ld-attachments ld-separator-none"' in done.stdout + assert 'ld-separator-page-break"' not in done.stdout diff --git a/tests/test_wp8_labels.py b/tests/test_labels.py similarity index 90% rename from tests/test_wp8_labels.py rename to tests/test_labels.py index 5cfbce2..ff876d1 100644 --- a/tests/test_wp8_labels.py +++ b/tests/test_labels.py @@ -1,5 +1,6 @@ -"""WP8 (S13): the party identifier label is neutral, since §3.4's -``identification_number`` also covers a natural person's national ID.""" +"""Built-in labels: the party identifier label is neutral, since §3.4's +``identification_number`` also covers a natural person's national ID, and each +language words the party labels in its own way.""" from __future__ import annotations import pytest diff --git a/tests/test_wp3_links.py b/tests/test_links.py similarity index 64% rename from tests/test_wp3_links.py rename to tests/test_links.py index c212fbe..d79107a 100644 --- a/tests/test_wp3_links.py +++ b/tests/test_links.py @@ -1,24 +1,15 @@ -"""Links and images (WP3): which image paths are shown (B6, §2.3), which links -are links (W3, §8.7), and links with no URL or no text (B9/W8).""" +"""Links and images: which image paths are shown (§2.3), which links are links +(§8.7), a link with no URL or no text, and how a refused link or image is +reported.""" from __future__ import annotations import pytest +from conftest import FRONT, html, text from legaldown_render import RenderRefused, render -from legaldown_render.writers.html import is_safe_href +from legaldown_render.urls import displayed_url, is_link_target, is_safe_href -FRONT = "---\ntitle: T\n---\n" - - -def text(body: str, **settings: object) -> str: - return render(FRONT + body, format="text", **settings).output - - -def html(body: str, **settings: object) -> str: - return render(FRONT + body, format="html", **settings).output - - -# -- B6: image paths (§2.3) --------------------------------------------------- +# -- Image paths (§2.3) ---------------------------------------------------------- NON_RELATIVE = [ "https://example.com/p.png", @@ -122,16 +113,66 @@ def validate(document, **kwargs): # type: ignore[no-untyped-def] assert any("/other.png" in d.message for d in errors) -# -- W3: which links are links (§8.7) ----------------------------------------- +def test_a_path_that_starts_another_is_not_covered_by_the_validators_report() -> None: + # Whatever the validator reports for one path, a different path whose text + # it contains is still the renderer's to report. + result = render(FRONT + "# A\n\n![a](/x.png) ![b](/x.png2)\n") + messages = [d.message for d in result.diagnostics if d.rule == "path-not-relative"] + assert any("'/x.png2'" in m for m in messages) + assert any("'/x.png'" in m for m in messages) -@pytest.mark.parametrize("href", [ +@pytest.mark.parametrize("path", ["javascript:alert(1)", "data:text/html,x", "https://e/p.png"]) +def test_the_not_relative_message_is_generic(path: str) -> None: + result = render(FRONT + f"# A\n\n![x](<{path}>)\n") + message = next(d.message for d in result.diagnostics if d.rule == "path-not-relative") + assert "never fetched" not in message + assert "file references are relative paths with no URI scheme. Its alt text is shown instead." in message + + +@pytest.mark.parametrize("empty", ["", " ", "%20"]) +def test_an_image_with_no_path_is_its_alt_text(empty: str) -> None: + for fmt in ("html", "text"): + result = render(FRONT + f"# A\n\nBefore ![the logo](<{empty}>) after.\n", format=fmt, strict=True) + assert " None: - output = html(f"# A\n\nSee [the link](<{href}>).\n") - assert f'the link' in output + "mailto:a@example.com", "tel:+420123456789", "attachments/a.pdf", +] + + +@pytest.mark.parametrize("href", ALLOWED) +def test_an_allowed_scheme_or_a_relative_url_is_a_link(href: str) -> None: + assert is_link_target(href) + assert f'click' in html(f"# A\n\nSee [click](<{href}>).\n") + + +@pytest.mark.parametrize("href", HANDLERS) +def test_a_protocol_handler_or_pseudo_scheme_is_not_a_link(href: str) -> None: + assert not is_link_target(href) + body = f"# A\n\nSee [click](<{href}>) now.\n" + result = render(FRONT + body, format="html") + assert "See click now." in result.output + assert "")[1] + assert "See click now." in text(body) + assert "<" not in text(body).split("See click now.")[0].split("\n")[-1] @pytest.mark.parametrize("href", [ @@ -152,10 +193,14 @@ def test_a_refused_link_is_its_text_in_both_writers(href: str) -> None: assert not is_safe_href(href) -def test_the_allow_list_is_judged_as_a_browser_reads_the_url() -> None: +def test_the_scheme_is_judged_as_a_browser_reads_the_url() -> None: + # markdown-it percent-encodes a tab in a URL it parses ("ms%09-msdt:x" is a + # relative path to a browser); the check still holds for a URL it is given + # directly, as a browser would read it. for href in ("java\tscript:alert(1)", "jav\nascript:x", "\x01javascript:x", " JAVASCRIPT:x", "data:text/html,x", - "VBScript:x", "FILE:///x"): + "VBScript:x", "FILE:///x", "ms\t-msdt:x", "\u0000 vscode://x"): assert not is_safe_href(href), repr(href) + assert not is_link_target(href), repr(href) for href in ("https://example.com", "#section", "attachments/a.pdf", "mailto:a@example.com", "ftp://x/y", "sip:a@b", "urn:x:y", "x/data:y", "javascript", "HTTPS://X", "\thttps://x"): assert is_safe_href(href), href @@ -176,8 +221,24 @@ def test_a_link_with_an_unusual_scheme_shows_its_url_in_text() -> None: assert "the file " in text("# A\n\nGet [the file](ftp://example.com/f).\n") -# -- B9/W8: links without a URL or without text ----------------------------- +def test_a_refused_link_is_reported_once_as_a_warning() -> None: + body = "# A\n\n[a](ms-msdt:x) and [b](ms-msdt:x) and [c](vscode://x) and [d](http://ok) and [e]().\n" + result = render(FRONT + body, format="html") + refused = [d for d in result.diagnostics if d.rule == "render-link-refused"] + assert [d.level for d in refused] == ["warning", "warning"] + assert "Link to 'ms-msdt:x' is not rendered as a link" in refused[0].message + assert "its text is shown" in refused[0].message + assert "vscode://x" in refused[1].message + assert result.ok # a Warning does not refuse, even under --strict's Errors + assert render(FRONT + body, format="text").diagnostics == result.diagnostics + +def test_an_empty_or_allowed_link_is_not_reported() -> None: + result = render(FRONT + "# A\n\n[a]() [b](<>) [c](https://e) [d](#x) [e](rel/path)\n") + assert not [d for d in result.diagnostics if d.rule == "render-link-refused"] + + +# -- Links without a URL or without text ----------------------------------------- def test_a_link_with_no_url_is_its_text() -> None: assert "")[1] @@ -198,6 +259,18 @@ def test_a_link_with_no_text_shows_its_url() -> None: assert "See http://z.y." in text("# A\n\nSee [](http://z.y).\n") +def test_displayed_url_undoes_the_encoding_unless_it_shows_a_control() -> None: + assert displayed_url(" https://e/a%20b ") == "https://e/a b" + assert displayed_url("https://e/a%0ab") == "https://e/a%0ab" + assert displayed_url("https://e/%E2%80%AE") == "https://e/%E2%80%AE" + + +def test_a_link_with_no_text_shows_the_written_url() -> None: + body = "# A\n\nSee []().\n" + assert 'href="https://e/a%20b/%C3%A9">https://e/a b/é' in html(body) + assert "See https://e/a b/é." in text(body) + + def test_an_autolink_is_its_address_alone_in_text() -> None: assert "Mail a@b.c now" in text("# A\n\nMail now\n") assert "Call tel:+1 now" in text("# A\n\nCall [tel:+1](tel:+1) now\n") @@ -214,13 +287,3 @@ def test_a_link_text_that_differs_from_its_url_shows_both_in_text() -> None: def test_a_link_around_a_picture_keeps_the_picture_not_its_url() -> None: output = html("# A\n\n[![](img/p.png)](https://example.com)\n") assert '' in output - - -@pytest.mark.parametrize("empty", ["", " ", "%20"]) -def test_an_image_with_no_path_is_its_alt_text(empty: str) -> None: - for fmt in ("html", "text"): - result = render(FRONT + f"# A\n\nBefore ![the logo](<{empty}>) after.\n", format=fmt, strict=True) - assert " None: + assert format_counter(value, counter) == expected + + +def test_fill_and_extend() -> None: + assert fill("{section}.{n} {other}", n="2", section="5") == "5.2 {other}" + assert extend("", ".1", textual=False) == "1" + assert extend("2.1", "(b)", textual=False) == "2.1(b)" + assert extend("Termination", "(a)", textual=True) == "Termination (a)" + + +def test_fill_drops_an_empty_section_with_one_following_dot() -> None: + assert fill("{section}.{n}", n="1", section="") == "1" + assert fill("{section}.{n}", n="1", section="2") == "2.1" + # Only ONE dot goes with the field. + assert fill("{section}..{n}", n="1", section="") == ".1" + + +def test_fill_keeps_other_text_of_an_empty_section_as_written() -> None: + # No dot right after the field: the field alone is removed, as before. + assert fill("({section}){n}", n="a", section="") == "()a" + assert fill("{section}", n="1", section="") == "" + + +def test_fill_without_a_section_value_keeps_the_dot() -> None: + # The rule applies only when the section value is given and empty. + assert fill("{section}.{n}", n="1") == ".1" + + +# -- Preamble list items agree with their designation ---------------------------- + +PREAMBLE_AND_SECTION = ( + "\nPreamble.\n\n- first pre\n- second pre\n\n# A {#a}\n\n- in a\n" +) + + +def test_a_preamble_list_is_numbered_without_a_leading_dot_in_text() -> None: + output = text(PREAMBLE_AND_SECTION, style="continental") + assert "1 first pre" in output + assert "2 second pre" in output + assert not any(line.startswith(".") for line in output.splitlines()) + + +def test_a_preamble_list_is_numbered_without_a_leading_dot_in_html() -> None: + output = html(PREAMBLE_AND_SECTION, style="continental") + assert '1

first pre

' in output + assert '2

second pre

' in output + assert 'class="ld-label">.1<' not in output + + +def test_a_list_in_a_section_keeps_its_section_prefix() -> None: + output = text(PREAMBLE_AND_SECTION, style="continental") + assert "1.1 in a" in output + html_output = html(PREAMBLE_AND_SECTION, style="continental") + assert '1.1

in a

' in html_output diff --git a/tests/test_render.py b/tests/test_render.py index 93c507d..42e5746 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -473,16 +473,11 @@ def test_options_object_and_keywords_are_exclusive() -> None: render(FRONT, RenderOptions(), format="text") -# -- regressions from code review ------------------------------------------------- +# -- alternatives, sentinels and emphasis around terms ---------------------------- -def test_disguised_javascript_urls_are_not_links() -> None: - from legaldown_render.writers.html import is_safe_href - - for href in ("java\tscript:alert(1)", "jav\nascript:x", "\x01javascript:x", " JAVASCRIPT:x", "data:text/html,x"): - assert not is_safe_href(href), repr(href) - for href in ("https://example.com", "#section", "attachments/a.pdf", "mailto:a@example.com"): - assert is_safe_href(href), href +def test_a_disguised_javascript_attachment_path_is_not_a_link() -> None: + # The scheme judgement itself is tested in test_links.py; this is the attachment's own link. source = FRONT.replace("title: T", 'title: T\nattachments:\n - id: annex\n title: Annex\n file: "java\\tscript:alert(1)"') output = render(source + "# A\n\nSee {{attach: annex}}.\n", overrides={"attachments.render": "omit"}).output assert "script:alert" not in "".join(re.findall(r'href="([^"]*)"', output)) @@ -564,7 +559,7 @@ def test_an_unresolved_node_is_an_internal_error() -> None: arb: Arbitration""" -# -- regressions from the second code review --------------------------------------- +# -- sentinels, link titles and which alternatives share a number ------------------ def test_sentinels_written_as_entities_are_plain_text() -> None: @@ -610,7 +605,7 @@ def test_units_that_can_appear_together_do_not_share_a_number() -> None: assert "2. Two" in output -# -- regressions from the third code review ------------------------------------------- +# -- alternatives and definitions in titles ------------------------------------------ THREE_WAY = """title: T questions: @@ -675,7 +670,7 @@ def test_section_numbers_agree_with_the_validator() -> None: assert [section.designation for section in result.tree.sections] == expected == ["1", "2", "3", "4", "4"] -# -- regressions from the fourth code review ------------------------------------------- +# -- the preamble, level-six headings and definitions in headings ---------------------- def test_preamble_condition_is_literal_outside_a_template() -> None: @@ -738,7 +733,7 @@ def test_definition_in_a_link_inside_alt_text_takes_no_anchor() -> None: assert 'alt="pic the Fee"' in output -# -- regressions from the fifth code review -------------------------------------------- +# -- includes, ambiguous references and the structure of lists ------------------------- def test_a_malformed_include_is_not_an_include_paragraph() -> None: @@ -764,12 +759,6 @@ def test_the_first_definition_in_document_order_keeps_the_anchor() -> None: assert 'id="def:fee"' not in sections -def test_nested_lists_line_up_with_their_item_s_text() -> None: - output = text("# A\n\n1. one\n\n more\n\n - child\n") - # Blocks of an item are set apart (W5); a nested list stays tight under its paragraph. - assert "1. one\n\n more\n (i) child" in output - - def _nested_list(levels: int) -> str: return "# A\n\n" + "".join(" " * depth + f"- l{depth}\n" for depth in range(levels)) @@ -1075,13 +1064,13 @@ def test_template_decision_is_the_validators() -> None: assert "[Only if: flag] Preamble para" in result.output -# -- regressions from the seventh code review ------------------------------------------ +# -- quotes, markers, comments and breaks ---------------------------------------------- def test_a_heading_in_a_quote_or_item_is_a_bold_line_not_a_section() -> None: result = render(FRONT + "# A\n\n> # Inner\n> text\n\n- ## Item heading\n\n> title: x\n> ---\n> after\n", format="text") - # One quote stays one quote: its blank line keeps the `>` (W5). + # One quote stays one quote: its blank line keeps the `>`. assert "\n\n> Inner\n>\n> text\n\n(a) Item heading\n\n> title: x\n>\n> after\n" in result.output assert [s.designation for s in result.tree.sections] == ["1"] html = render(FRONT + "# A\n\n> # Inner\n", standalone=False).output @@ -1113,7 +1102,7 @@ def test_a_marker_copy_inside_a_comment_is_not_the_marker() -> None: def test_an_item_opening_with_a_drafting_note_or_code() -> None: output = text("# A\n\n- > [!DRAFTING]\n > note here\n- ```\n code\n ```\n") - # The label stands alone and the block is indented under the item text (WP11): flattened + # The label stands alone and the block is indented under the item text: flattened # onto the label line, a quote is no quote and code is mis-indented. assert "(a)\n > [Drafting note]\n > note here" in output assert "(b)\n code" in output diff --git a/tests/test_style.py b/tests/test_style.py index c0e5495..b3f0a09 100644 --- a/tests/test_style.py +++ b/tests/test_style.py @@ -1,3 +1,7 @@ +"""Styles: the built-in ones, layering with ``extends``, dotted overrides merged +field by field, how every problem is reported with its path, dumping a style to +YAML, and the typing of command-line values. Validation of templates and the +loader's edge cases are in test_style_validation.py.""" from __future__ import annotations from dataclasses import replace @@ -69,7 +73,7 @@ def test_dump_round_trips(tmp_path: Path) -> None: style = load_style("continental", overrides={"locale": "de-DE"}) path = tmp_path / "dumped.yaml" path.write_text(dump_style(style)) - # The dump is flattened (S8): the same style, without the record of what it extended. + # The dump is flattened: the same style, without the record of what it extended. assert load_style(path) == replace(style, extends=None) diff --git a/tests/test_wp6_style.py b/tests/test_style_validation.py similarity index 74% rename from tests/test_wp6_style.py rename to tests/test_style_validation.py index c033687..73069da 100644 --- a/tests/test_wp6_style.py +++ b/tests/test_style_validation.py @@ -1,33 +1,26 @@ -"""Style validation: an empty blank falls back to ``[TBD: id]`` (R6), ``extends`` -cycles are found on resolved paths (S7), ``--print-style`` is self-contained -(S8), ``version`` and ``extends`` cannot be overridden (S9), a YAML error in a -``--set`` value is reported (S10, loader side), and the ``{field}`` -placeholders of every text template are validated (S11).""" +"""Style validation and the style loader's edge cases: an empty placeholder +blank falls back to ``[TBD: id]`` (§13.5 rule 6), ``extends`` cycles are found +on resolved paths, ``--print-style`` is self-contained, ``version`` and +``extends`` cannot be overridden, a YAML error in a ``--set`` value is +reported, and the ``{field}`` placeholders of every text template are checked +where the setting is used (and for typos everywhere). Loading, layering and +dotted overrides are in test_style.py.""" from __future__ import annotations import dataclasses -import os -import subprocess -import sys from dataclasses import replace from pathlib import Path import pytest +from conftest import FRONT, run_cli_text from legaldown_render import RenderOptions, StyleError, builtin_styles, dump_style, load_style, render from legaldown_render.style import parse_override -SRC = Path(__file__).resolve().parents[1] / "src" DOC = "---\nlegaldown: '0.2'\ntitle: T\n---\n\n# One {#one}\n\nAmount {{placeholder: fee}} and {{ref: one}}.\n" -def _cli(*args: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]: - env = dict(os.environ, PYTHONPATH=str(SRC), PYTHONUTF8="1") - return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *args], input=stdin, - capture_output=True, text=True, env=env, check=False) - - -# --- R6: an empty blank -------------------------------------------------------------------------- +# -- An empty or invisible blank (§13.5) ----------------------------------------- @pytest.mark.parametrize("blank", ["", " ", "\t\n"]) @pytest.mark.parametrize("fmt", ["html", "text"]) @@ -43,7 +36,20 @@ def test_a_visible_blank_is_kept() -> None: assert "TBD" not in result.output -# --- S7: extends cycles ---------------------------------------------------------------------------- +@pytest.mark.parametrize("blank", ["​", "", " ​‍ ", " ", "‮⁠"]) +@pytest.mark.parametrize("fmt", ["html", "text"]) +def test_a_blank_of_invisible_characters_falls_back_to_tbd(blank: str, fmt: str) -> None: + source = FRONT.replace("title: T", "title: T\nplaceholders:\n - id: fee\n type: text") + "# A\n\nFee {{placeholder: fee}}.\n" + result = render(source, format=fmt, overrides={"placeholders.blank": blank}) + assert "[TBD: fee]" in result.output + + +def test_a_blank_with_a_visible_character_is_kept() -> None: + source = FRONT.replace("title: T", "title: T\nplaceholders:\n - id: fee\n type: text") + "# A\n\nFee {{placeholder: fee}}.\n" + assert "Fee ​__​." in render(source, format="text", overrides={"placeholders.blank": "​__​"}).output + + +# -- extends cycles -------------------------------------------------------------- def test_a_cycle_through_a_parent_directory_is_found(tmp_path: Path) -> None: (tmp_path / "dir").mkdir() @@ -77,7 +83,7 @@ def test_the_same_base_used_twice_is_not_a_cycle(tmp_path: Path) -> None: assert load_style(tmp_path / "top.yaml").numbering.scheme == "none" -# --- S8: a dump is self-contained ---------------------------------------------------------------------- +# -- A dump is self-contained ---------------------------------------------------- def test_a_dump_does_not_extend_anything(tmp_path: Path) -> None: (tmp_path / "base.yaml").write_text("version: 1\nnumbering: {scheme: none}\n") @@ -108,7 +114,7 @@ def test_a_dumped_builtin_reloads_equal(name: str, tmp_path: Path) -> None: def test_print_style_of_a_relative_extends_reloads_from_anywhere(tmp_path: Path) -> None: (tmp_path / "base.yaml").write_text("version: 1\nplaceholders: {blank: '[__]'}\n") (tmp_path / "child.yaml").write_text("version: 1\nextends: base.yaml\n") - done = _cli("--print-style", "--style", str(tmp_path / "child.yaml")) + done = run_cli_text("--print-style", "--style", str(tmp_path / "child.yaml")) assert done.returncode == 0, done.stderr assert "extends" not in done.stdout elsewhere = tmp_path / "other" @@ -117,7 +123,7 @@ def test_print_style_of_a_relative_extends_reloads_from_anywhere(tmp_path: Path) assert load_style(elsewhere / "dump.yaml").placeholders.blank == "[__]" -# --- S9: version and extends are not overridable ------------------------------------------------------------ +# -- version and extends are not overridable ------------------------------------- @pytest.mark.parametrize("key", ["version", "extends"]) def test_version_and_extends_cannot_be_overridden(key: str) -> None: @@ -134,14 +140,14 @@ def test_a_nested_override_of_version_is_refused_too() -> None: @pytest.mark.parametrize("print_style", [True, False]) def test_the_cli_refuses_them(setting: str, print_style: bool) -> None: args = ["--print-style"] if print_style else ["-", "-q"] - done = _cli(*args, "--set", setting, stdin=DOC) + done = run_cli_text(*args, "--set", setting, stdin=DOC) assert done.returncode == 2, done.stdout assert setting.split("=")[0] in done.stderr assert "cannot be overridden" in done.stderr assert "Traceback" not in done.stderr -# --- S10: a YAML error in a --set value ------------------------------------------------------------------- +# -- A YAML error in a --set value ----------------------------------------------- def test_a_yaml_error_in_an_override_names_the_key() -> None: with pytest.raises(StyleError) as caught: @@ -153,7 +159,7 @@ def test_a_yaml_error_in_an_override_names_the_key() -> None: def test_a_yaml_error_in_an_override_reaches_stderr() -> None: - done = _cli("--print-style", "--set", "numbering.levels=[{counter: decimal") + done = run_cli_text("--print-style", "--set", "numbering.levels=[{counter: decimal") assert done.returncode == 2 assert "numbering.levels" in done.stderr assert "YAML" in done.stderr @@ -162,7 +168,7 @@ def test_a_yaml_error_in_an_override_reaches_stderr() -> None: def test_a_yaml_error_in_a_render_run_reaches_stderr() -> None: - done = _cli("-", "-q", "--set", "enumeration.levels=[{a", stdin=DOC) + done = run_cli_text("-", "-q", "--set", "enumeration.levels=[{a", stdin=DOC) assert done.returncode != 0 assert "enumeration.levels" in done.stderr assert "YAML" in done.stderr @@ -172,15 +178,9 @@ def test_a_yaml_error_in_a_render_run_reaches_stderr() -> None: def test_valid_yaml_overrides_still_parse() -> None: assert parse_override("title_block.version", "true") is True assert parse_override("contents.depth", "3") == 3 - assert parse_override("numbering.levels", "[{counter: decimal}]") == [{"counter": "decimal"}] - - -# --- S11: template fields ------------------------------------------------------------------------------------ -@pytest.mark.parametrize("name", builtin_styles()) -def test_every_builtin_style_passes(name: str) -> None: - load_style(name) +# -- Template fields ------------------------------------------------------------- @pytest.mark.parametrize("language", ["en", "cs", "de", "fr", "pl", "sk"]) def test_every_builtin_label_passes(language: str) -> None: @@ -295,3 +295,45 @@ def test_all_template_problems_are_reported_together() -> None: assert any(p.startswith("references.format") for p in problems) assert any(p.startswith("labels.condition") for p in problems) assert any(p.startswith("placeholders.blank") for p in problems) + + +# -- Templates are validated where their setting is used ------------------------- + +NO_N = [{"label": "x", "ref": "x"}] + + +@pytest.mark.parametrize(("overrides", "key"), [ + ({"enumeration.levels": NO_N}, "enumeration.levels[0].ref"), + ({"enumeration.levels": NO_N, "enumeration.enabled": False, "enumeration.ordered": "enumerate"}, + "enumeration.levels[0].ref"), + ({"numbering.levels": NO_N}, "numbering.levels[0].ref"), + ({"paragraphs.numbered": True, "paragraphs.format": {"label": "x", "ref": "x"}}, "paragraphs.format.ref"), +]) +def test_a_ref_that_is_in_use_must_have_its_counter(overrides: dict[str, object], key: str) -> None: + assert any(p.startswith(key) and "{n}" in p for p in _problems(overrides)) + + +@pytest.mark.parametrize("overrides", [ + {"enumeration.levels": NO_N, "enumeration.enabled": False}, + {"numbering.levels": NO_N, "numbering.scheme": "none"}, + {"paragraphs.format": {"label": "x", "ref": "x"}}, + {"paragraphs.format": {"label": "x", "ref": ""}}, +]) +def test_a_ref_that_is_not_in_use_may_leave_out_its_counter(overrides: dict[str, object]) -> None: + load_style("default", overrides=overrides) + + +def test_a_typo_is_flagged_even_where_the_setting_is_not_used() -> None: + problems = _problems({"numbering.scheme": "none", "numbering.levels": [{"label": "{n}", "ref": "{m}"}]}) + assert any(p.startswith("numbering.levels[0].ref") and "{m}" in p and "unknown field" in p for p in problems) + + +def test_a_plain_label_may_contain_braced_words() -> None: + style = load_style("default", overrides={"labels.drafting_note": "Note {see below}", "labels.colon": "{x}:"}) + assert style.labels.drafting_note == "Note {see below}" + assert style.labels.colon == "{x}:" + + +def test_the_labels_that_substitute_still_check_their_fields() -> None: + assert _problems({"labels.condition": "{condtion}"}) + assert _problems({"labels.attachment_file": "{fil}"}) diff --git a/tests/test_terms.py b/tests/test_terms.py new file mode 100644 index 0000000..3d97fbe --- /dev/null +++ b/tests/test_terms.py @@ -0,0 +1,211 @@ +"""Defined terms: what ``{{term:}}`` shows (§7.3), which is the rendered text of +the term where it is defined, with its own Markdown, references and +placeholders resolved; terms that refer to themselves or to each other; and +the cap on the length of a term text. How much work nested terms may cost is in +test_terms_budget.py.""" +from __future__ import annotations + +import pytest +from conftest import FRONT, html, text + +from legaldown_render import render + +# -- The text of a term ---------------------------------------------------------- + +def test_a_term_reference_shows_the_rendered_text_of_the_term() -> None: + output = text('# A\n\n"A *b* c" {{def: abc}} means it.\n\nSee {{term: abc}}.\n') + assert "See A b c." in output + assert "*" not in output.split("See", 1)[1] + + +def test_a_term_reference_label_still_wins() -> None: + output = text('# A\n\n"A *b* c" {{def: abc}} means it.\n\nSee {{term: abc, label=the thing}}.\n') + assert "See the thing." in output + + +def test_a_term_reference_resolves_directives_inside_the_term() -> None: + body = '# A {#a}\n\n"Fee **under** {{ref: a}}" {{def: fee}} means it.\n\nSee {{term: fee}}.\n' + output = text(body) + assert "See Fee under" in output + + +def test_a_term_reference_uses_the_first_definition() -> None: + body = '# A\n\n"First *one*" {{def: t}} here.\n\n"Second" {{def: t}} there.\n\nSee {{term: t}}.\n' + assert "See First one." in text(body) + + +def test_a_term_reference_before_its_definition() -> None: + body = '# A\n\nSee {{term: abc}} first.\n\n"A *b* c" {{def: abc}} means it.\n' + assert "See A b c first." in text(body) + + +def test_a_plain_term_reference_is_unchanged() -> None: + assert "See Fee." in text('# A\n\n"Fee" {{def: fee}} means it.\n\nSee {{term: fee}}.\n') + + +def test_an_empty_term_falls_back_to_the_validator() -> None: + output = text('# A\n\n"" {{def: late-fee}} means it.\n\nSee {{term: late-fee}}.\n') + assert "See Late Fee." in output + + +def test_an_undefined_term_is_still_reported() -> None: + assert "[UNDEFINED: nope]" in text("# A\n\nSee {{term: nope}}.\n") + + +# -- A reference, placeholder or definition inside a term ------------------------ + +def test_a_term_reference_keeps_a_ref_inside_the_term() -> None: + output = text('# H {#h}\n\n"Fee per {{ref: h}}" {{def: a}}\n\n{{term: a}}\n') + assert output.count("Fee per 1") == 2 + + +def test_a_term_reference_before_the_heading_keeps_the_ref() -> None: + output = text('{{term: a}}\n\n# H {#h}\n\n"Fee per {{ref: h}}" {{def: a}}\n') + assert output.count("Fee per 1") == 2 + + +def test_a_term_reference_before_its_definition_and_its_target() -> None: + output = text('# First\n\nSee {{term: a}}.\n\n# H {#h}\n\n"Fee per {{ref: h}}" {{def: a}}\n') + assert "See Fee per 2." in output + + +def test_a_term_reference_keeps_a_ref_to_a_subsection() -> None: + body = '# A\n\n"Fee per {{ref: s}}" {{def: a}}\n\n{{term: a}}\n\n## Sub {#s}\n\nx\n' + output = text(body) + assert output.count("Fee per 1.1") == 2 + + +def test_a_term_reference_in_a_textual_heading_and_in_the_body() -> None: + # The none scheme reads a heading as text before every target is known; the + # term text must not be fixed from that early pass. + body = '# Fee {{term: a}}\n\n"Fee per {{ref: h}}" {{def: a}}\n\n# H {#h}\n\n{{term: a}}\n' + output = render(FRONT + body, format="text", overrides={"numbering.scheme": "none"}).output + assert output.rstrip().endswith("Fee per H") + assert "[BROKEN REF" not in output + + +def test_a_term_reference_shows_the_placeholder_as_the_definition_does() -> None: + body = '# A\n\n"Fee {{placeholder: p}}" {{def: a}} {{term: a}} {{placeholder: p, type=money}}\n' + result = render(FRONT + body, format="text") + line = next(row for row in result.output.splitlines() if "INVALID PLACEHOLDER" in row) + assert line.count("Fee [INVALID PLACEHOLDER: p]") == 2 + assert "[_____]" not in result.output + + +def test_a_definition_that_defines_a_term_twice_uses_the_first() -> None: + body = '# H {#h}\n\n"First {{ref: h}}" {{def: t}}\n\n"Second" {{def: t}}\n\n{{term: t}}\n' + assert text(body).rstrip().endswith("First 1") + + +def test_a_term_with_a_ref_adds_no_duplicate_diagnostics() -> None: + # The term holds an image with a remote path: one report however often it is resolved. + body = '# H {#h}\n\n"Fee ![x](http://e.com/i.png) {{ref: h}}" {{def: a}}\n\n{{term: a}} {{term: a}}\n' + diagnostics = render(FRONT + body, format="text").diagnostics + keys = [(d.rule, d.message) for d in diagnostics] + assert len(keys) == len(set(keys)) + assert sum(d.rule == "path-not-relative" for d in diagnostics) == 1 + + +def test_a_term_that_holds_a_definition_does_not_take_its_anchor() -> None: + body = '# A\n\nSee {{term: outer}}.\n\n"Outer "Inner" {{def: inner}}" {{def: outer}}\n\n{{term: inner}}\n' + html = render(FRONT + body, format="html").output + assert html.count('id="def:inner"') == 1 + assert html.count('id="def:outer"') == 1 + + +# -- Terms that refer to themselves or to each other ----------------------------- + +SELF = '# A\n\n"Self {{term: s}}" {{def: s}} means it.\n\nSee {{term: s}} and {{term: s, label=the label}}.\n' + +INDIRECT = ('# A\n\n"A {{term: b}}" {{def: a}} x.\n\n"B {{term: a}}" {{def: b}} y.\n\n' + "See {{term: a}}, {{term: b}} and {{term: a, label=the label}}.\n") + + +@pytest.mark.parametrize("output_format", ["text", "html"]) +@pytest.mark.parametrize("body", [SELF, INDIRECT], ids=["direct", "indirect"]) +def test_a_cyclic_term_renders_in_both_writers(body: str, output_format: str) -> None: + result = render(FRONT + body, format=output_format) + assert "the label" in result.output + assert "[UNDEFINED" not in result.output + + +def test_a_term_inside_its_own_definition_shows_its_id() -> None: + output = text(SELF) + # At the definition and at a reference alike: the inner reference shows + # the id (its label when it has one), and goes no further. + assert "Self Self s means it." in output + assert "See Self s and the label." in output + + +def test_a_term_reference_in_a_cycle_shows_its_label_else_its_id() -> None: + output = text(INDIRECT) + assert output.rstrip().endswith("and the label.") + # No term shows its own reference as raw directive source. + assert "{{" not in output + + +def test_a_cycle_through_a_reference_site_inside_another_terms_definition() -> None: + body = ('# A\n\n"A" {{def: a}} x.\n\n"B {{term: a}} and {{term: c}}" {{def: b}} y.\n\n' + '"C {{term: b}}" {{def: c}} z.\n\nSee {{term: b}}.\n') + assert "See B A and" in text(body) + assert "B A and" in html(body) + + +@pytest.mark.parametrize("output_format", ["text", "html"]) +def test_cycles_through_refs_still_terminate(output_format: str) -> None: + body = ('# H {#h}\n\n"A {{term: b}} {{ref: h}}" {{def: a}} x.\n\n"B {{term: a}}" {{def: b}} y.\n\n' + '"C {{term: c}}" {{def: c}}\n\nSee {{term: a}}, {{term: b}}, {{term: c}}.\n') + out = render(FRONT + body, format=output_format).output + assert "{{" not in out + # Which term shows what inside a cycle follows the order they are first needed in. + assert "A b 1" in out and "C c" in out + + +CYCLE = ('# A\n\nUsed first: {{term: ca}} and {{term: cb}}.\n\n' + '"Alpha {{term: cb}}" {{def: ca}}\n\n"Beta {{term: ca}}" {{def: cb}}\n') + + +def test_a_term_reference_in_a_cycle_does_not_depend_on_the_order_of_use() -> None: + # A term shown by {{term:}} is its text with that term open (a reference back to it + # shows its id), whichever term is used first. (The definition site renders with no term + # open, so it can read longer: "Beta Alpha Beta ca" there, "Beta Alpha cb" in a reference.) + ca_first = text(CYCLE) + cb_first = text(CYCLE.replace("{{term: ca}} and {{term: cb}}", "{{term: cb}} and {{term: ca}}")) + assert "Used first: Alpha Beta ca and Beta Alpha cb." in ca_first + assert "Used first: Beta Alpha cb and Alpha Beta ca." in cb_first + + +# -- The cap on a term text ------------------------------------------------------ + +def _warnings(body: str, **settings: object) -> list[str]: + return [d.message for d in render(FRONT + body, format="text", **settings).diagnostics + if d.rule == "render-term-too-long"] + + +def test_a_term_over_the_cap_shows_its_id_and_one_warning_names_it() -> None: + long_term = "word " * 250 + body = f'# A\n\n"{long_term}" {{{{def: big}}}}\n\nSee {{{{term: big}}}} and {{{{term: big}}}}.\n' + out = text(body) + assert "See big and big." in out + assert _warnings(body) == [ + "The term of 'big' is longer than 1000 characters once its references are resolved; " + "'{{term: big}}' shows the definition id."] + + +def test_a_label_still_wins_over_the_cap() -> None: + body = '# A\n\n"' + "word " * 250 + '" {{def: big}}\n\nSee {{term: big, label=Big}}.\n' + assert "See Big." in text(body) + + +def test_a_term_at_the_cap_is_shown_whole() -> None: + term = "x" * 1000 + body = f'# A\n\n"{term}" {{{{def: edge}}}}\n\nSee {{{{term: edge}}}}.\n' + assert f"See {term}." in text(body) + assert _warnings(body) == [] + + +def test_a_normal_nested_term_is_unaffected() -> None: + body = ('# A\n\n"Company" {{def: company}}\n\n"Affiliate of the {{term: company}}" {{def: affiliate}}\n\n' + 'See {{term: affiliate}}.\n') + assert "See Affiliate of the Company." in text(body) + assert _warnings(body) == [] diff --git a/tests/test_terms_budget.py b/tests/test_terms_budget.py new file mode 100644 index 0000000..13d34e9 --- /dev/null +++ b/tests/test_terms_budget.py @@ -0,0 +1,219 @@ +"""Terms that name each other, and how much is worked out for them (§7.3). + +A text worked out for a term is kept and reused only where it would read the +same (what it shows depends on which terms are open around it), so a chain of +terms is worked out once per term, not once per use. Terms that name each +other in a tangle have more texts than any cache holds, so the work is +limited: a text worked out again is charged to a budget, and past it, or past a +depth of nested terms, a reference shows its definition id and the Warning +``render-term-limit`` says so. A term text is also capped in length +(``render-term-too-long``). +""" +from __future__ import annotations + +import random +import time + +import pytest +from conftest import FRONT, text + +from legaldown_render import RenderResult, render +from legaldown_render.resolve.resolver import Resolver + +LIMIT = ("Defined terms refer to each other too deeply or too often to resolve fully; " + "some {{term:}} references show their definition id.") + + +def messages(result: RenderResult, rule: str) -> list[str]: + return [d.message for d in result.diagnostics if d.rule == rule] + + +def _chain(n: int, *, cycle: bool, heading: bool = False) -> str: + """Terms t0..tn, each reading the next one twice; the last one reads t0 when *cycle*.""" + defs = [f'"T{i} {{{{term: t{i + 1}}}}} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + defs.append(f'"T{n}{" {{term: t0}}" if cycle else ""}" {{{{def: t{n}}}}}') + title = "About {{term: t0}}" if heading else "A" + return f"# {title}\n\nUsed: {{{{term: t0}}}}\n\n" + "\n\n".join(defs) + "\n" + + +def _line(n: int, *, heading: bool) -> str: + """Terms t0..tn, each reading the next one once.""" + defs = [f'"T{i} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + [f'"End" {{{{def: t{n}}}}}'] + title = "About {{term: t0}}" if heading else "A" + return f"# {title}\n\nUsed: {{{{term: t0}}}}\n\n" + "\n\n".join(defs) + "\n" + + +def _tangle(k: int) -> str: + defs = [f'"T{i} {" ".join(f"{{{{term: t{j}}}}}" for j in range(k))}" {{{{def: t{i}}}}}' for i in range(k)] + return "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n" + + +def _ring(k: int, n: int) -> str: + """k terms in a cycle, each holding n references to the next.""" + defs = [f'"T{i} {" ".join(f"{{{{term: t{(i + 1) % k}}}}}" for _ in range(n))}" {{{{def: t{i}}}}}' for i in range(k)] + return "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n" + + +def _independent(n: int, *, heading: bool) -> str: + parts = [] + for i in range(n): + if heading: + parts.append(f"## H{i} {{{{term: t{i}}}}}\n\n") + parts.append(f'"Term {i}" {{{{def: t{i}}}}} used {{{{term: t{i}}}}}\n\n') + return "# A\n\n" + "".join(parts) + + +# -- A text is worked out once per term --------------------------------------- + + +@pytest.mark.parametrize("heading", [False, True]) +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +@pytest.mark.parametrize("cycle", [False, True]) +def test_a_branching_chain_of_terms_is_worked_out_once_per_term(cycle: bool, scheme: str, heading: bool) -> None: + # Each term reads the next twice: worked out afresh at each use, 20 terms is 2**20 resolutions. + # The text of t0 is itself 2**n words long (4**n with a cycle, which reads the chain round + # twice), so the chains cannot be longer. Under the none scheme a heading is read as text + # before the targets are known (§13.3). + started = time.perf_counter() + out = text(_chain(9 if cycle else 20, cycle=cycle, heading=heading), overrides={"numbering.scheme": scheme}) + assert time.perf_counter() - started < 2 + assert "{{" not in out + + +def test_a_chain_without_a_cycle_reads_as_each_term_does() -> None: + lines = text(_chain(3, cycle=False)).splitlines() + used = next(line for line in lines if line.startswith("Used: ")) + # t1 as used inside t0 is the same text as at t1's own definition. + site = next(line for line in lines if line.startswith("T1 ")) + assert used == f"Used: T0 {site} {site}" + + +@pytest.mark.parametrize(("n", "cycle"), [(40, False), (14, True)]) +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +def test_a_branching_chain_of_terms_is_capped_and_reported(n: int, cycle: bool, scheme: str) -> None: + started = time.perf_counter() + result = render(FRONT + _chain(n, cycle=cycle, heading=True), format="text", + overrides={"numbering.scheme": scheme}) + assert time.perf_counter() - started < 2 + assert len(result.output) < 100_000 + messages = [d.message for d in result.diagnostics if d.rule == "render-term-too-long"] + assert messages + assert len(messages) == len(set(messages)) # one per id + assert all("longer than 1000 characters" in message and "shows the definition id" in message + for message in messages) + assert "{{" not in result.output + + +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +@pytest.mark.parametrize("heading", [False, True]) +def test_3000_independent_terms_show_their_text_and_no_warning(scheme: str, heading: bool) -> None: + result = render(FRONT + _independent(3000, heading=heading), format="text", overrides={"numbering.scheme": scheme}) + assert messages(result, "render-term-limit") == [] + lines = result.output.splitlines() + assert "Term 2999 used Term 2999" in lines + assert "Term 0 used Term 0" in lines + assert not any(line.endswith(" used t2999") for line in lines) + if heading and scheme == "none": + assert any(line.endswith("H2999 Term 2999") for line in lines) + +# -- A cached text is reused only where it would read the same ----------------- + +def _oracle(self: Resolver, cache: object, definition_id: str, work, *, key: str | None = None) -> str: # noqa: ANN001 + """The term text with the open terms alone deciding, nothing kept.""" + self._terms_open.append(definition_id) + try: + return work() + finally: + self._terms_open.pop() + + +def _graph(rng: random.Random) -> str: + ids = [f"t{i}" for i in range(rng.randint(2, 6))] + defs = [] + for identifier in ids: + parts = [identifier.upper()] + for _ in range(rng.randint(0, 3)): + target = rng.choice(ids) + parts.append(f"{{{{term: {target}, label=L{target}}}}}" if rng.random() < 0.15 + else f"{{{{term: {target}}}}}") + defs.append(f'"{" ".join(parts)}" {{{{def: {identifier}}}}}') + rng.shuffle(defs) + heading = rng.choice(["A", f"About {{{{term: {rng.choice(ids)}}}}}"]) + use = " ".join(f"{{{{term: {rng.choice(ids)}}}}}" for _ in range(rng.randint(1, 4))) + return f"# {heading}\n\nUsed: {use}\n\n" + "\n\n".join(defs) + "\n" + + +def test_cached_term_texts_equal_the_texts_worked_out_afresh(monkeypatch: pytest.MonkeyPatch) -> None: + rng = random.Random(20261027) + cached_original = Resolver._while_open + for _ in range(150): + source = FRONT + _graph(rng) + for scheme in ("decimal", "none"): + overrides = {"numbering.scheme": scheme} + monkeypatch.setattr(Resolver, "_while_open", cached_original) + cached = render(source, format="text", overrides=overrides) + monkeypatch.setattr(Resolver, "_while_open", _oracle) + fresh = render(source, format="text", overrides=overrides) + assert cached.output == fresh.output, source + assert [(d.rule, d.message) for d in cached.diagnostics] == \ + [(d.rule, d.message) for d in fresh.diagnostics], source + + +# -- Terms that all name each other: the work budget ---------------------------- + + +@pytest.mark.parametrize("k", [3, 7, 9, 12, 20, 40]) +def test_k_terms_that_each_name_all_k_render_quickly(k: int) -> None: + started = time.perf_counter() + result = render(FRONT + _tangle(k), format="text") + assert time.perf_counter() - started < 2 + assert "{{" not in result.output + if k >= 9: + assert messages(result, "render-term-limit") == [LIMIT] + + +def test_a_small_cycle_is_resolved_in_full_without_the_warning() -> None: + assert messages(render(FRONT + _tangle(3), format="text"), "render-term-limit") == [] + + +def test_a_ring_of_terms_with_many_references_costs_a_small_multiple_of_the_document() -> None: + document = FRONT + _ring(12, 2000) + started = time.perf_counter() + result = render(document, format="text", overrides={"numbering.scheme": "none"}) + cyclic = time.perf_counter() - started + # The same document with the references pointing at a term that names nothing. + flat = document.replace("{{term: t0}}", "{{term: leaf}}") + for i in range(12): + flat = flat.replace(f"{{{{term: t{i}}}}}", "{{term: leaf}}") + flat += '\n"Leaf" {{def: leaf}}\n' + started = time.perf_counter() + render(flat, format="text", overrides={"numbering.scheme": "none"}) + acyclic = time.perf_counter() - started + assert messages(result, "render-term-limit") + assert cyclic < 3 * acyclic + 1 + + +# -- A long chain of terms: the depth limit ------------------------------------ + + +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +@pytest.mark.parametrize("heading", [False, True]) +def test_a_chain_of_500_terms_does_not_exhaust_the_stack(scheme: str, heading: bool) -> None: + n = 500 + defs = [f'"T{i} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + [f'"End" {{{{def: t{n}}}}}'] + title = "About {{term: t0}}" if heading else "A" + body = f"# {title}\n\nUsed: {{{{term: t0}}}}\n\n" + "\n\n".join(defs) + "\n" + started = time.perf_counter() + result = render(FRONT + body, format="text", overrides={"numbering.scheme": scheme}) + assert time.perf_counter() - started < 2 + assert messages(result, "render-term-limit") == [LIMIT] + used = next(line for line in result.output.splitlines() if line.startswith("Used: ")) + assert used.startswith("Used: T0 T1 T2 ") + assert "{{" not in result.output + + +def test_a_chain_inside_the_depth_limit_is_resolved_in_full() -> None: + n = 20 + result = render(FRONT + _line(n, heading=False), format="text") + assert f"Used: {' '.join(f'T{i}' for i in range(n))} End" in result.output + assert messages(result, "render-term-limit") == [] diff --git a/tests/test_wp5_text.py b/tests/test_text_writer.py similarity index 78% rename from tests/test_wp5_text.py rename to tests/test_text_writer.py index a896a6d..9ffef26 100644 --- a/tests/test_wp5_text.py +++ b/tests/test_text_writer.py @@ -1,5 +1,6 @@ -"""Text writer fidelity (WP5): attachment separators (W1), table cells and -alignment (W4), quotes and list items (W5), list indent (W6), blank labels (W7).""" +"""The plain-text writer's layout (§13.8): attachment separators, table cells +and alignment, quotes and list items (also those that open with a table, code +or a quote), list indent, and the labels a style may blank.""" from __future__ import annotations import pytest @@ -39,8 +40,7 @@ def text(body: str, **overrides: object) -> str: return render(FRONT + body, format="text", overrides=overrides).output -# -- W1: attachments.separator (§13.8) ------------------------------------------- - +# -- Attachment separators (§13.8) ----------------------------------------------- @pytest.mark.parametrize("separator", ["rule", "page-break"]) def test_a_separator_precedes_each_attachment(separator: str) -> None: @@ -67,8 +67,7 @@ def test_the_writer_needs_no_style_and_defaults_to_the_default_styles_separator( assert TextWriter().write(result.tree) == result.output -# -- W4: tables ------------------------------------------------------------------ - +# -- Tables ---------------------------------------------------------------------- def test_a_pipe_in_a_cell_is_escaped() -> None: output = text("# A\n\n| a | b |\n|---|---|\n| x \\| y | z |\n") @@ -86,8 +85,7 @@ def test_a_table_with_fewer_alignments_than_columns_still_prints() -> None: assert TextWriter().table(table) == "| x | x |\n|---:|---|\n| x | x |" -# -- W5: quotes and list items --------------------------------------------------- - +# -- Quotes and list items ------------------------------------------------------- def test_one_quote_stays_one_quote() -> None: output = text("# A\n\n> first\n>\n> second\n") @@ -120,9 +118,35 @@ def test_a_paragraph_after_a_nested_list_is_separated() -> None: assert "1. one\n (i) child\n\n after\n" in output -# -- W6: list indent ------------------------------------------------------------- +def test_an_item_that_opens_with_a_table_keeps_a_valid_table() -> None: + out = text("# A\n\n1. | x | y |\n |---|---|\n | 1 | 2 |\n") + assert "\n1.\n | x | y |\n |---|---|\n | 1 | 2 |\n" in out + assert "1. |" not in out + + +def test_an_item_that_opens_with_code_indents_it_under_the_item_text() -> None: + out = text("# A\n\n1. ```\n code\n more\n ```\n2. next\n") + assert "\n1.\n code\n more\n2. next\n" in out + + +def test_an_item_that_opens_with_a_quote_puts_the_quote_on_its_own_lines() -> None: + out = text("# A\n\n1. > one\n >\n > two\n2. next\n") + assert "\n1.\n > one\n >\n > two\n2. next\n" in out + assert "1. >" not in out +def test_blocks_after_a_non_paragraph_first_block_are_set_apart() -> None: + out = text("# A\n\n1. | x |\n |---|\n | 1 |\n\n then text\n") + assert "\n1.\n | x |\n |---|\n | 1 |\n\n then text\n" in out + + +def test_an_item_that_opens_with_a_paragraph_is_unchanged() -> None: + out = text("# A\n\n1. first\n > quote\n2. second\n | x |\n |---|\n | 1 |\n") + assert "\n1. first\n\n > quote\n2. second\n\n | x |\n |---|\n | 1 |\n" in out + + +# -- List indent ----------------------------------------------------------------- + def test_a_conditional_item_indents_by_its_label_not_the_condition_label() -> None: body = ("# A\n\n- first\n- by courts {#i when=forum:courts}\n\n nested\n\n - deep\n" "- by arbitration {#i when=forum:arb}\n- last\n") @@ -130,8 +154,7 @@ def test_a_conditional_item_indents_by_its_label_not_the_condition_label() -> No assert "(b) [Only if: forum:courts] by courts\n\n nested\n (i) deep\n" in output -# -- W7: blank labels ------------------------------------------------------------ - +# -- Blank labels ---------------------------------------------------------------- def test_a_blanked_label_drops_its_prefix_in_party_details() -> None: output = text("# A\n\nBody.\n", **{"labels.identification_number": ""}) diff --git a/tests/test_values.py b/tests/test_values.py index 55bea26..2bcd94a 100644 --- a/tests/test_values.py +++ b/tests/test_values.py @@ -1,9 +1,20 @@ +"""Value formatting (§10, §13.5): money, dates and durations in a locale. + +Money keeps every digit it was written with and is padded to the currency's +minor units, never rounded. A duration keeps its digits and takes the plural +of the value shown, and no raw CLDR unit id is ever printed: a locale without +a pattern falls back within the locale first and to English last, and says so. +""" from __future__ import annotations +import re + import pytest +from conftest import FRONT -from legaldown_render.resolve import Formatter, parse_locale -from legaldown_render.resolve.numbering import extend, fill, format_counter +from legaldown_render import render +from legaldown_render.resolve import DURATION_UNITS, Formatter, parse_locale +from legaldown_render.resolve.values import DurationText from legaldown_render.style.model import Values @@ -13,20 +24,7 @@ def formatter(tag: str, **values: str) -> Formatter: return Formatter(locale, Values(**values)) -@pytest.mark.parametrize(("value", "counter", "expected"), [ - (1, "lower-alpha", "a"), (26, "lower-alpha", "z"), (27, "lower-alpha", "aa"), (3, "upper-alpha", "C"), - (4, "lower-roman", "iv"), (14, "upper-roman", "XIV"), (7, "decimal", "7"), -]) -def test_counters(value: int, counter: str, expected: str) -> None: - assert format_counter(value, counter) == expected - - -def test_fill_and_extend() -> None: - assert fill("{section}.{n} {other}", n="2", section="5") == "5.2 {other}" - assert extend("", ".1", textual=False) == "1" - assert extend("2.1", "(b)", textual=False) == "2.1(b)" - assert extend("Termination", "(a)", textual=True) == "Termination (a)" - +# -- Money: exact amounts (§10.3) ------------------------------------------------ def test_money_is_padded_never_rounded() -> None: en = formatter("en-US") @@ -36,11 +34,256 @@ def test_money_is_padded_never_rounded() -> None: assert en.money("1234.50", None) == "1,234.50" +@pytest.mark.parametrize("digits", [29, 40, 60]) +def test_long_integers_format_exactly(digits: int) -> None: + value = "1234567890" * 6 + value = value[:digits] + grouped = f"{int(value):,}" + en = formatter("en-US") + assert en.number(value) == grouped + assert en.money(value, "USD") == f"${grouped}.00" + assert formatter("en-US", money="code").money(value, "USD") == f"USD\xa0{grouped}.00" + assert formatter("en-US", money="name").money(value, "USD") == f"{grouped}.00 US dollars" + + +def test_long_fraction_keeps_every_digit() -> None: + fraction = "1234567891" * 4 + en = formatter("en-US") + assert en.number("0." + fraction) == "0." + fraction + assert en.money("1." + fraction, "EUR") == "€1." + fraction + assert en.number("1" * 29 + ".5") == f"{int('1' * 29):,}.5" + + +def test_long_amount_renders_end_to_end() -> None: + source = "---\ntitle: T\n---\nPay {{money: " + "9" * 29 + "}} {{money: 12345678901234567890123456789.1234567890123456789}}.\n" + output = render(source, format="text", locale="en-US").output + assert f"{int('9' * 29):,}" in output + assert "12,345,678,901,234,567,890,123,456,789.1234567890123456789" in output + + +@pytest.mark.parametrize(("amount", "currency", "expected"), [ + ("1.1230", "EUR", "€1.1230"), + ("1.500", "EUR", "€1.500"), + ("1.50", "EUR", "€1.50"), + ("5", "EUR", "€5.00"), + ("5.5", "EUR", "€5.50"), + ("5", "JPY", "¥5"), + ("0.125", "JPY", "¥0.125"), + ("5.0", "JPY", "¥5.0"), + ("1234.5670", "USD", "$1,234.5670"), +]) +def test_money_fraction_digits_symbol(amount: str, currency: str, expected: str) -> None: + assert formatter("en-US").money(amount, currency) == expected + + +def test_money_fraction_digits_code_and_name() -> None: + code = formatter("en-US", money="code") + name = formatter("en-US", money="name") + assert code.money("1.500", "EUR") == "EUR\xa01.500" + assert code.money("5", "EUR") == "EUR\xa05.00" + assert code.money("0.125", "JPY") == "JPY\xa00.125" + assert code.money("5", "JPY") == "JPY\xa05" + assert name.money("1.500", "EUR") == "1.500 euros" + assert name.money("5", "EUR") == "5.00 euros" + assert name.money("0.125", "JPY") == "0.125 Japanese yen" + assert name.money("5", "JPY") == "5 Japanese yen" + + +def test_money_trailing_zeros_in_other_locales_and_long_amounts() -> None: + assert formatter("cs-CZ").money("1.500", "CZK") == "1,500\xa0Kč" + assert formatter("de", money="code").money("1.1230", "EUR") == "1,1230\xa0EUR" + assert formatter("en-US").money("1" * 40 + ".1230", "EUR") == f"€{int('1' * 40):,}.1230" + + +# -- Money: how the currency is shown -------------------------------------------- + def test_money_display_styles() -> None: assert formatter("en-US", money="code").money("10", "USD") == "USD\xa010.00" assert formatter("en-US", money="name").money("10", "USD") == "10.00 US dollars" +@pytest.mark.parametrize(("tag", "currency", "expected"), [ + ("en", "CZK", "CZK\xa05.00"), + ("en", "CHF", "CHF\xa05.00"), + ("en", "USD", "$5.00"), + ("en", "EUR", "€5.00"), + ("pt-BR", "BRL", "R$\xa05,00"), + ("en", "CAD", "CA$5.00"), + ("cs", "CZK", "5,00\xa0Kč"), +]) +def test_symbol_mode_spacing(tag: str, currency: str, expected: str) -> None: + assert formatter(tag).money("5", currency) == expected + + +def test_symbol_after_the_digits_is_spaced_too() -> None: + # Whatever the locale puts after the amount, a letter never touches a digit. + for tag in ("en", "de", "fr", "cs", "pl", "sv", "ja", "ru", "pt-PT", "es", "nl", "it"): + for currency in ("CZK", "CHF", "USD", "EUR", "SEK", "PLN", "XAF"): + text = formatter(tag).money("5", currency) + assert not re.search(r"[^\W\d_]\d|\d[^\W\d_]", text), (tag, currency, text) + + +def test_code_mode_still_spaced() -> None: + assert formatter("en", money="code").money("5", "CZK") == "CZK\xa05.00" + assert formatter("de", money="code").money("5", "CZK") == "5,00\xa0CZK" + + +# -- Durations: digits and plural ------------------------------------------------ + +def test_long_duration_keeps_every_digit() -> None: + assert formatter("en-US").duration("1" * 40, "D") == f"{int('1' * 40):,} days" + assert formatter("en-US").duration("0." + "1" * 40, "D") == "0." + "1" * 40 + " days" + + +@pytest.mark.parametrize(("tag", "value", "unit", "expected"), [ + ("en", "0.0001", "D", "0.0001 days"), + ("en", "1.23456", "Y", "1.23456 years"), + ("en", "1", "D", "1 day"), + ("en", "1.0", "D", "1.0 days"), + ("en", "12", "MO", "12 months"), + ("cs", "1", "Y", "1 rok"), + ("cs", "2", "Y", "2 roky"), + ("cs", "5", "Y", "5 let"), + ("cs", "2.5", "Y", "2,5 roku"), + ("de", "1.50", "W", "1,50 Wochen"), +]) +def test_duration_digits_and_plural(tag: str, value: str, unit: str, expected: str) -> None: + assert formatter(tag).duration(value, unit) == expected + + +# -- Fractions beyond Python's int/str digit limit: the plural follows the first digits ---- + +LONG = "2" * 4400 + + +def test_a_duration_with_a_4400_digit_fraction_renders() -> None: + output = render(FRONT + f"# A\n\n{{{{duration: 1.{LONG}, unit=D}}}}", format="text").output + assert f"1.{LONG} days" in output + + +def test_a_money_amount_in_name_mode_with_a_4400_digit_fraction_renders() -> None: + output = render(FRONT + f"# A\n\n{{{{money: 1.{LONG}, currency=EUR}}}}", format="text", + overrides={"values.money": "name"}).output + assert f"1.{LONG} euros" in output + + +def test_a_4400_digit_fraction_uses_the_many_category_in_czech() -> None: + # The category comes from the first 30 fraction digits, so it is the + # "many" of "2,5 roku", not "other" (" let"). + output = render(FRONT + f"# A\n\n{{{{duration: 1.{LONG}, unit=Y}}}}", format="text", locale="cs-CZ").output + assert output.rstrip().endswith(" roku") + + +@pytest.mark.parametrize("unit", ["D", "Y", "W"]) +@pytest.mark.parametrize("locale", ["cs", "sk", "lt", "pl", "ru", "en"]) +def test_a_huge_fraction_has_the_unit_form_of_its_first_digits(locale: str, unit: str) -> None: + fmt = formatter(locale) + huge = fmt.format_duration("1." + "3" * 4400, unit).text + short = fmt.format_duration("1." + "3" * 40, unit).text + assert huge.replace("3" * 4400, "") == short.replace("3" * 40, "") + + +def test_a_huge_fraction_is_many_in_czech() -> None: + fmt = formatter("cs") + assert fmt.format_duration("1." + "3" * 4400, "D").text.endswith(" dne") + assert fmt.format_duration("1." + "3" * 40, "D").text.endswith(" dne") + + +def test_a_huge_fraction_of_zeros_ending_in_a_digit_keeps_its_fraction() -> None: + # Cut to 30 digits it is "1.000...0": still a fraction (v != 0), so "many" in Czech. + value = "1." + "0" * 4399 + "1" + assert formatter("cs").format_duration(value, "D").text.rsplit(" ", 1)[-1] == "dne" + + +def test_a_name_mode_money_amount_with_a_huge_fraction_uses_the_form_of_its_first_digits() -> None: + fmt = formatter("cs", money="name") + huge = fmt.money("1." + "3" * 4400, "EUR") + short = fmt.money("1." + "3" * 40, "EUR") + assert huge.replace("3" * 4400, "") == short.replace("3" * 40, "") + assert huge.endswith("eura") # "many", not the "other" form "eur" + + +def test_an_integer_part_beyond_the_limit_falls_back_to_other() -> None: + # No category can be computed (not even from the first digits): "other" ("dnů" in Czech). + other = formatter("cs").format_duration("1.5", "D").text.rsplit(" ", 1)[-1] + assert other == "dne" + assert formatter("cs").format_duration("1" * 4400, "D").text.endswith(" dnů") + + +# -- CLDR duration patterns and the English fallback ----------------------------- + +@pytest.mark.parametrize("tag", ["pt-BR", "ja", "zh", "ko", "it", "tr", "nl", "cs", "de", "sk", "pl", "en"]) +@pytest.mark.parametrize("length", ["long", "short", "narrow"]) +@pytest.mark.parametrize("value", ["1", "2", "5", "1.5"]) +def test_no_cldr_unit_id_is_ever_printed(tag: str, length: str, value: str) -> None: + f = formatter(tag, duration=length) + for unit in DURATION_UNITS: + text = f.duration(value, unit) + assert "duration-" not in text + assert value.replace(".", f.locale.number_symbols["latn"]["decimal"]) in text + + +def test_pt_br_long_falls_back_to_the_short_pattern() -> None: + pt = formatter("pt-BR") + assert pt.duration("30", "D") == "30 dias" + assert pt.duration("12", "MO") == "12 meses" + assert pt.duration("1", "Y") == "1 ano" + + +def test_fallback_goes_to_the_other_length_first() -> None: + # cs has a long and a narrow pattern: narrow is asked, narrow is used. + assert formatter("cs", duration="narrow").duration("30", "D") == "30 d." + # zh has only "short": long and narrow fall back to it, still Chinese. + assert formatter("zh", duration="long").duration("30", "D") == "30天" + assert formatter("zh", duration="narrow").duration("30", "D") == "30天" + + +def test_a_fallback_inside_the_locale_is_not_signalled() -> None: + # pt-BR long has no pattern but short has: Portuguese, no English. + assert formatter("pt-BR").format_duration("30", "D") == DurationText("30 dias", False) + assert formatter("en").format_duration("1", "D") == DurationText("1 day", False) + + +def test_english_is_the_last_resort_and_is_signalled(monkeypatch: pytest.MonkeyPatch) -> None: + from legaldown_render.resolve import values + + real = values._unit_patterns + # A Czech locale without any duration pattern (Babel's data has none such). + monkeypatch.setattr(values, "_unit_patterns", + lambda locale, unit, length: {} if locale.language == "cs" else real(locale, unit, length)) + f = formatter("cs") + assert f.format_duration("5", "D") == DurationText("5 days", True) + assert f.duration("1", "D") == "1 day" + assert f.duration("1.5", "Y") == "1,5 years" # the number keeps the locale's separator + + +def test_fallback_warning_is_emitted_once_by_the_resolver(monkeypatch: pytest.MonkeyPatch) -> None: + from legaldown_render.resolve import values + + real = values._unit_patterns + monkeypatch.setattr(values, "_unit_patterns", + lambda locale, unit, length: {} if locale.language == "cs" else real(locale, unit, length)) + source = ("---\ntitle: T\nlanguage: cs\n---\n" + "Lhůta {{duration: 5, unit=D}} a {{duration: 6, unit=D}} a {{duration: 2, unit=MO}}.\n") + result = render(source, format="text") + fallbacks = [d for d in result.diagnostics if d.rule == "render-locale-fallback"] + assert [d.message for d in fallbacks] == [ + "No CLDR pattern for 'duration-day' in 'cs'; English used.", + "No CLDR pattern for 'duration-month' in 'cs'; English used.", + ] + assert all(d.level == "warning" for d in fallbacks) + assert "5 days" in result.output + + +def test_normal_locales_emit_no_fallback_warning() -> None: + for language in ("cs", "pt-BR", "ja", "en"): + source = f"---\ntitle: T\nlanguage: {language}\n---\nX {{{{duration: 5, unit=D}}}}.\n" + result = render(source, format="text") + assert not [d for d in result.diagnostics if d.rule == "render-locale-fallback"], language + + +# -- Locales --------------------------------------------------------------------- + def test_locales() -> None: cs = formatter("cs-CZ") assert cs.date("2026-06-01") == "1. června 2026" diff --git a/tests/test_wp10_review_fixes.py b/tests/test_wp10_review_fixes.py deleted file mode 100644 index 2c6ca00..0000000 --- a/tests/test_wp10_review_fixes.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Review fixes after wave 2 (WP10): links are an allow-list and a refused one is -reported, control characters never reach a diagnostic, style templates are -validated only where a setting is used, and small wording fixes.""" -from __future__ import annotations - -import os -import subprocess -import sys -from pathlib import Path - -import pytest - -from legaldown_render import StyleError, render -from legaldown_render.style import load_style -from legaldown_render.urls import displayed_url, is_link_target, printable - -SRC = Path(__file__).resolve().parents[1] / "src" -FRONT = "---\ntitle: T\n---\n\n" - - -def html(body: str, **settings: object) -> str: - return render(FRONT + body, format="html", **settings).output - - -def text(body: str, **settings: object) -> str: - return render(FRONT + body, format="text", **settings).output - - -# -- 1: links are an allow-list --------------------------------------------------- - -HANDLERS = [ - "ms-msdt:/id PCWDiagnostic", "ms-word:ofe|u|https://x/a.docx", "ms-officecmd:{}", "search-ms:query=x", - "ms-appinstaller:?source=https://x/a.appx", "vscode://file/x", "intent://x#Intent;end", "blob:https://x/1", - "about:blank", "jar:https://x/a.jar!/", "view-source:https://x", "mhtml:file://x", "ms-its:x.chm::/a.htm", - "C:\\x", "C:/x", "localhost:8080", "magnet:?xt=urn:btih:x", "javascript:alert(1)", "file:///etc/passwd", - "data:text/html,x", " MS-MSDT:x", -] -ALLOWED = [ - "http://e/x", "https://e/x", "mailto:a@e", "tel:+1", "ftp://e/x", "ftps://e/x", "sftp://e/x", "sip:a@e", - "sips:a@e", "sms:+1", "urn:isbn:1", "geo:1,2", "news:a.b", "a/b.pdf", "../a.pdf", "#top", "?q=1", -] - - -@pytest.mark.parametrize("href", HANDLERS) -def test_a_protocol_handler_or_pseudo_scheme_is_not_a_link(href: str) -> None: - assert not is_link_target(href) - body = f"# A\n\nSee [click](<{href}>) now.\n" - result = render(FRONT + body, format="html") - assert "See click now." in result.output - assert "")[1] - assert "See click now." in text(body) - assert "<" not in text(body).split("See click now.")[0].split("\n")[-1] - - -def test_the_scheme_is_read_as_a_browser_reads_it() -> None: - # markdown-it percent-encodes a tab in a URL it parses ("ms%09-msdt:x" is a - # relative path to a browser); the check still holds for a URL it is given - # directly, as a browser would read it. - for href in ("ms\t-msdt:x", "jav\nascript:x", "\x01javascript:x", "\u0000 vscode://x"): - assert not is_link_target(href), repr(href) - - -@pytest.mark.parametrize("href", ALLOWED) -def test_an_allowed_scheme_or_a_relative_url_is_a_link(href: str) -> None: - assert is_link_target(href) - assert f'click' in html(f"# A\n\nSee [click](<{href}>).\n") - - -def test_a_refused_link_is_reported_once_as_a_warning() -> None: - body = "# A\n\n[a](ms-msdt:x) and [b](ms-msdt:x) and [c](vscode://x) and [d](http://ok) and [e]().\n" - result = render(FRONT + body, format="html") - refused = [d for d in result.diagnostics if d.rule == "render-link-refused"] - assert [d.level for d in refused] == ["warning", "warning"] - assert "Link to 'ms-msdt:x' is not rendered as a link" in refused[0].message - assert "its text is shown" in refused[0].message - assert "vscode://x" in refused[1].message - assert result.ok # a Warning does not refuse, even under --strict's Errors - assert render(FRONT + body, format="text").diagnostics == result.diagnostics - - -def test_an_empty_or_allowed_link_is_not_reported() -> None: - result = render(FRONT + "# A\n\n[a]() [b](<>) [c](https://e) [d](#x) [e](rel/path)\n") - assert not [d for d in result.diagnostics if d.rule == "render-link-refused"] - - -# -- 3, 4, 5: what a diagnostic shows of a document ------------------------------ - - -def test_printable_escapes_controls_and_format_characters() -> None: - assert printable("a\x1b[2J\x07\n\x7f\x85") == "a\\x1b[2J\\x07\\x0a\\x7f\\x85" - assert printable("\u202e\u200b\ufeff\u2028\u2029") == "\\u202e\\u200b\\ufeff\\u2028\\u2029" - assert printable("ž é €") == "ž é €" - - -EVIL = "https://e/%1b[2J%1b]0;pwned%07%0aerror: [x] forged%e2%80%ae" - - -def test_a_control_character_in_an_image_path_is_escaped_in_the_message() -> None: - result = render(FRONT + f"# A\n\n![x](<{EVIL}>)\n") - messages = [d.message for d in result.diagnostics if d.rule == "path-not-relative"] - assert messages - for message in messages: - assert "\x1b" not in message and "\x07" not in message and "\n" not in message - assert any("\\x1b[2J\\x1b]0;pwned\\x07\\x0aerror: [x] forged\\u202e" in m for m in messages) - - -def test_a_control_character_in_a_link_is_escaped_in_the_message() -> None: - result = render(FRONT + f"# A\n\n[x]()\n") - message = next(d.message for d in result.diagnostics if d.rule == "render-link-refused") - assert "\x1b" not in message and "\x07" not in message and "\n" not in message and "\u202e" not in message - assert "\\x1b" in message - - -def test_the_cli_writes_no_control_character_from_a_document_to_standard_error(tmp_path: Path) -> None: - source = tmp_path / "evil.lgd" - source.write_text(FRONT + f"# A\n\n![x](<{EVIL}>) [y]()\n", encoding="utf-8") - env = dict(os.environ, PYTHONPATH=str(SRC)) - done = subprocess.run([sys.executable, "-m", "legaldown_render.cli", str(source), "-f", "text"], - capture_output=True, env=env, check=False) - err = done.stderr.decode("utf-8") - assert "path-not-relative" in err and "render-link-refused" in err - assert "\x1b" not in err and "\x07" not in err and "\u202e" not in err - assert "\r" not in err - assert not any(line.startswith("error: [x]") for line in err.splitlines()) - - -def test_a_path_that_starts_another_is_not_covered_by_the_validators_report() -> None: - # Whatever the validator reports for one path, a different path whose text - # it contains is still the renderer's to report. - result = render(FRONT + "# A\n\n![a](/x.png) ![b](/x.png2)\n") - messages = [d.message for d in result.diagnostics if d.rule == "path-not-relative"] - assert any("'/x.png2'" in m for m in messages) - assert any("'/x.png'" in m for m in messages) - - -@pytest.mark.parametrize("path", ["javascript:alert(1)", "data:text/html,x", "https://e/p.png"]) -def test_the_not_relative_message_is_generic(path: str) -> None: - result = render(FRONT + f"# A\n\n![x](<{path}>)\n") - message = next(d.message for d in result.diagnostics if d.rule == "path-not-relative") - assert "never fetched" not in message - assert "file references are relative paths with no URI scheme. Its alt text is shown instead." in message - - -# -- 8: a link with no text shows the URL as written ---------------------------- - - -def test_displayed_url_undoes_the_encoding_unless_it_shows_a_control() -> None: - assert displayed_url(" https://e/a%20b ") == "https://e/a b" - assert displayed_url("https://e/a%0ab") == "https://e/a%0ab" - assert displayed_url("https://e/%E2%80%AE") == "https://e/%E2%80%AE" - - -def test_a_link_with_no_text_shows_the_written_url() -> None: - body = "# A\n\nSee []().\n" - assert 'href="https://e/a%20b/%C3%A9">https://e/a b/é' in html(body) - assert "See https://e/a b/é." in text(body) - - -# -- 6: a style is validated where its settings are used ------------------------ - -def _problems(overrides: dict[str, object]) -> list[str]: - with pytest.raises(StyleError) as caught: - load_style("default", overrides=overrides) - return caught.value.problems - - -NO_N = [{"label": "x", "ref": "x"}] - - -@pytest.mark.parametrize(("overrides", "key"), [ - ({"enumeration.levels": NO_N}, "enumeration.levels[0].ref"), - ({"enumeration.levels": NO_N, "enumeration.enabled": False, "enumeration.ordered": "enumerate"}, - "enumeration.levels[0].ref"), - ({"numbering.levels": NO_N}, "numbering.levels[0].ref"), - ({"paragraphs.numbered": True, "paragraphs.format": {"label": "x", "ref": "x"}}, "paragraphs.format.ref"), -]) -def test_a_ref_that_is_in_use_must_have_its_counter(overrides: dict[str, object], key: str) -> None: - assert any(p.startswith(key) and "{n}" in p for p in _problems(overrides)) - - -@pytest.mark.parametrize("overrides", [ - {"enumeration.levels": NO_N, "enumeration.enabled": False}, - {"numbering.levels": NO_N, "numbering.scheme": "none"}, - {"paragraphs.format": {"label": "x", "ref": "x"}}, - {"paragraphs.format": {"label": "x", "ref": ""}}, -]) -def test_a_ref_that_is_not_in_use_may_leave_out_its_counter(overrides: dict[str, object]) -> None: - load_style("default", overrides=overrides) - - -def test_a_typo_is_flagged_even_where_the_setting_is_not_used() -> None: - problems = _problems({"numbering.scheme": "none", "numbering.levels": [{"label": "{n}", "ref": "{m}"}]}) - assert any(p.startswith("numbering.levels[0].ref") and "{m}" in p and "unknown field" in p for p in problems) - - -def test_a_plain_label_may_contain_braced_words() -> None: - style = load_style("default", overrides={"labels.drafting_note": "Note {see below}", "labels.colon": "{x}:"}) - assert style.labels.drafting_note == "Note {see below}" - assert style.labels.colon == "{x}:" - - -def test_the_labels_that_substitute_still_check_their_fields() -> None: - assert _problems({"labels.condition": "{condtion}"}) - assert _problems({"labels.attachment_file": "{fil}"}) - - -# -- 7: a blank of invisible characters is no visual blank ------------------------ - - -@pytest.mark.parametrize("blank", ["​", "", " ​‍ ", " ", "‮⁠"]) -@pytest.mark.parametrize("fmt", ["html", "text"]) -def test_a_blank_of_invisible_characters_falls_back_to_tbd(blank: str, fmt: str) -> None: - source = FRONT.replace("title: T", "title: T\nplaceholders:\n - id: fee\n type: text") + "# A\n\nFee {{placeholder: fee}}.\n" - result = render(source, format=fmt, overrides={"placeholders.blank": blank}) - assert "[TBD: fee]" in result.output - - -def test_a_blank_with_a_visible_character_is_kept() -> None: - source = FRONT.replace("title: T", "title: T\nplaceholders:\n - id: fee\n type: text") + "# A\n\nFee {{placeholder: fee}}.\n" - assert "Fee ​__​." in render(source, format="text", overrides={"placeholders.blank": "​__​"}).output diff --git a/tests/test_wp11_review_fixes.py b/tests/test_wp11_review_fixes.py deleted file mode 100644 index 9cf45af..0000000 --- a/tests/test_wp11_review_fixes.py +++ /dev/null @@ -1,253 +0,0 @@ -"""Review fixes after wave 3 (WP11): the plural of a huge fraction is computed -from its first digits, a list item opening with a table, code or quote keeps -that block intact in plain text, the ``-o`` extension rule has no holes, and a -term text worked out inside a cycle is not kept.""" -from __future__ import annotations - -import os -import subprocess -import sys -import time -from pathlib import Path - -import pytest -from babel import Locale - -from legaldown_render import render -from legaldown_render.resolve.values import Formatter -from legaldown_render.style.model import Values -from legaldown_render.writers import file_extension, format_for_path - -SRC = Path(__file__).resolve().parents[1] / "src" -FRONT = "---\ntitle: T\n---\n\n" -FEATURES = str(Path(__file__).resolve().parent / "documents" / "features.lgd") - - -def text(body: str, **settings: object) -> str: - return render(FRONT + body, format="text", **settings).output - - -# -- 1: the plural category of a huge fraction ----------------------------------- - - -def _formatter(locale: str, **values: str) -> Formatter: - return Formatter(Locale.parse(locale), Values(**values)) - - -@pytest.mark.parametrize("unit", ["D", "Y", "W"]) -@pytest.mark.parametrize("locale", ["cs", "sk", "lt", "pl", "ru", "en"]) -def test_a_huge_fraction_has_the_unit_form_of_its_first_digits(locale: str, unit: str) -> None: - formatter = _formatter(locale) - huge = formatter.format_duration("1." + "3" * 4400, unit).text - short = formatter.format_duration("1." + "3" * 40, unit).text - assert huge.replace("3" * 4400, "") == short.replace("3" * 40, "") - - -def test_a_huge_fraction_is_many_in_czech() -> None: - formatter = _formatter("cs") - assert formatter.format_duration("1." + "3" * 4400, "D").text.endswith(" dne") - assert formatter.format_duration("1." + "3" * 40, "D").text.endswith(" dne") - - -def test_a_huge_fraction_of_zeros_ending_in_a_digit_keeps_its_fraction() -> None: - # Cut to 30 digits it is "1.000...0": still a fraction (v != 0), so "many" in Czech. - value = "1." + "0" * 4399 + "1" - assert _formatter("cs").format_duration(value, "D").text.rsplit(" ", 1)[-1] == "dne" - - -def test_a_name_mode_money_amount_with_a_huge_fraction_uses_the_form_of_its_first_digits() -> None: - formatter = _formatter("cs", money="name") - huge = formatter.money("1." + "3" * 4400, "EUR") - short = formatter.money("1." + "3" * 40, "EUR") - assert huge.replace("3" * 4400, "") == short.replace("3" * 40, "") - assert huge.endswith("eura") # "many", not the "other" form "eur" - - -def test_an_integer_part_beyond_the_limit_falls_back_to_other() -> None: - # No category can be computed (not even from the first digits): "other" ("dnů" in Czech). - other = _formatter("cs").format_duration("1.5", "D").text.rsplit(" ", 1)[-1] - assert other == "dne" - assert _formatter("cs").format_duration("1" * 4400, "D").text.endswith(" dnů") - - -# -- 2: an item that opens with a table, code or a quote ------------------------- - - -def test_an_item_that_opens_with_a_table_keeps_a_valid_table() -> None: - out = text("# A\n\n1. | x | y |\n |---|---|\n | 1 | 2 |\n") - assert "\n1.\n | x | y |\n |---|---|\n | 1 | 2 |\n" in out - assert "1. |" not in out - - -def test_an_item_that_opens_with_code_indents_it_under_the_item_text() -> None: - out = text("# A\n\n1. ```\n code\n more\n ```\n2. next\n") - assert "\n1.\n code\n more\n2. next\n" in out - - -def test_an_item_that_opens_with_a_quote_puts_the_quote_on_its_own_lines() -> None: - out = text("# A\n\n1. > one\n >\n > two\n2. next\n") - assert "\n1.\n > one\n >\n > two\n2. next\n" in out - assert "1. >" not in out - - -def test_blocks_after_a_non_paragraph_first_block_are_set_apart() -> None: - out = text("# A\n\n1. | x |\n |---|\n | 1 |\n\n then text\n") - assert "\n1.\n | x |\n |---|\n | 1 |\n\n then text\n" in out - - -def test_an_item_that_opens_with_a_paragraph_is_unchanged() -> None: - out = text("# A\n\n1. first\n > quote\n2. second\n | x |\n |---|\n | 1 |\n") - assert "\n1. first\n\n > quote\n2. second\n\n | x |\n |---|\n | 1 |\n" in out - - -# -- 3: the extension rule of -o -------------------------------------------------- - - -@pytest.mark.parametrize(("name", "extension"), [ - ("x.pdf", "pdf"), (".pdf", "pdf"), (".md", "md"), ("x.pdf.", "pdf"), ("x.docx ", "docx"), - ("x.pdf. .", "pdf"), ("X.PDF", "pdf"), ("a.b.txt", "txt"), ("contract", ""), ("", ""), (".", ""), - ("..", ""), ("x.", ""), -]) -def test_the_extension_of_a_file_name(name: str, extension: str) -> None: - assert file_extension(name) == extension - - -@pytest.mark.parametrize(("name", "expected"), [ - (".txt", "text"), ("x.txt.", "text"), ("x.html ", "html"), (".htm", "html"), ("x.unknown", None), -]) -def test_the_format_a_file_name_implies(name: str, expected: str | None) -> None: - assert format_for_path(name) == expected - - -def _run(*arguments: str, cwd: Path) -> subprocess.CompletedProcess[str]: - env = {**os.environ, "PYTHONPATH": str(SRC)} - return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *arguments], - capture_output=True, text=True, cwd=cwd, env=env, check=False) - - -@pytest.mark.parametrize("name", [".pdf", ".md", "x.pdf.", "x.docx ", "x.docx. .", "dir.pdf", "x.MD"]) -def test_an_unwritten_extension_is_refused_however_it_is_spelled(tmp_path: Path, name: str) -> None: - result = _run(FEATURES, "-o", name, "-q", cwd=tmp_path) - assert result.returncode == 2, result.stderr - assert "cannot tell the output format" in result.stderr - assert list(tmp_path.iterdir()) == [] - - -@pytest.mark.parametrize("name", ["x.pdf.", ".md", "x.docx "]) -def test_format_overrides_an_unwritten_extension(tmp_path: Path, name: str) -> None: - result = _run(FEATURES, "-o", name, "-f", "text", "-q", cwd=tmp_path) - assert result.returncode == 0, result.stderr - assert not (tmp_path / name).read_text(encoding="utf-8").lstrip().startswith("<") - - -@pytest.mark.parametrize("name", ["contract", "contract.v2", "report.2025-10-27", ".out", "x.", "x.."]) -def test_a_name_without_a_known_extension_still_writes_html(tmp_path: Path, name: str) -> None: - result = _run(FEATURES, "-o", name, "-q", cwd=tmp_path) - assert result.returncode == 0, result.stderr - assert " None: - result = _run(FEATURES, "-o", name, "-q", cwd=tmp_path) - assert result.returncode == 0, result.stderr - assert marker not in (tmp_path / name).read_text(encoding="utf-8").lower() - - -# -- 4: a term text worked out inside a cycle is not kept ------------------------ - -CYCLE = ('# A\n\nUsed first: {{term: ca}} and {{term: cb}}.\n\n' - '"Alpha {{term: cb}}" {{def: ca}}\n\n"Beta {{term: ca}}" {{def: cb}}\n') - - -def test_a_term_reference_in_a_cycle_does_not_depend_on_the_order_of_use() -> None: - # WP12: a term shown by {{term:}} is its text with that term open (a reference back to it - # shows its id), whichever term is used first. (The definition site renders with no term - # open, so it can read longer: "Beta Alpha Beta ca" there, "Beta Alpha cb" in a reference.) - ca_first = text(CYCLE) - cb_first = text(CYCLE.replace("{{term: ca}} and {{term: cb}}", "{{term: cb}} and {{term: ca}}")) - assert "Used first: Alpha Beta ca and Beta Alpha cb." in ca_first - assert "Used first: Beta Alpha cb and Alpha Beta ca." in cb_first - - -def _chain(n: int, *, cycle: bool, heading: bool = False) -> str: - """Terms t0..tn, each reading the next one twice; the last one reads t0 when *cycle*.""" - defs = [f'"T{i} {{{{term: t{i + 1}}}}} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] - defs.append(f'"T{n}{" {{term: t0}}" if cycle else ""}" {{{{def: t{n}}}}}') - title = "About {{term: t0}}" if heading else "A" - return f"# {title}\n\nUsed: {{{{term: t0}}}}\n\n" + "\n\n".join(defs) + "\n" - - -@pytest.mark.parametrize("heading", [False, True]) -@pytest.mark.parametrize("scheme", ["decimal", "none"]) -@pytest.mark.parametrize("cycle", [False, True]) -def test_a_branching_chain_of_terms_is_worked_out_once_per_term(cycle: bool, scheme: str, heading: bool) -> None: - # Each term reads the next twice: worked out afresh at each use, 20 terms is 2**20 resolutions. - # The text of t0 is itself 2**n words long (4**n with a cycle, which reads the chain round - # twice), so the chains cannot be longer. Under the none scheme a heading is read as text - # before the targets are known (§13.3). - started = time.perf_counter() - out = text(_chain(9 if cycle else 20, cycle=cycle, heading=heading), overrides={"numbering.scheme": scheme}) - assert time.perf_counter() - started < 2 - assert "{{" not in out - - -def test_a_chain_without_a_cycle_reads_as_each_term_does() -> None: - lines = text(_chain(3, cycle=False)).splitlines() - used = next(line for line in lines if line.startswith("Used: ")) - # t1 as used inside t0 is the same text as at t1's own definition. - site = next(line for line in lines if line.startswith("T1 ")) - assert used == f"Used: T0 {site} {site}" - - -# -- 4b: a term text is capped ---------------------------------------------------- - - -def _warnings(body: str, **settings: object) -> list[str]: - return [d.message for d in render(FRONT + body, format="text", **settings).diagnostics - if d.rule == "render-term-too-long"] - - -@pytest.mark.parametrize(("n", "cycle"), [(40, False), (14, True)]) -@pytest.mark.parametrize("scheme", ["decimal", "none"]) -def test_a_branching_chain_of_terms_is_capped_and_reported(n: int, cycle: bool, scheme: str) -> None: - started = time.perf_counter() - result = render(FRONT + _chain(n, cycle=cycle, heading=True), format="text", - overrides={"numbering.scheme": scheme}) - assert time.perf_counter() - started < 2 - assert len(result.output) < 100_000 - messages = [d.message for d in result.diagnostics if d.rule == "render-term-too-long"] - assert messages - assert len(messages) == len(set(messages)) # one per id - assert all("longer than 1000 characters" in message and "shows the definition id" in message - for message in messages) - assert "{{" not in result.output - - -def test_a_term_over_the_cap_shows_its_id_and_one_warning_names_it() -> None: - long_term = "word " * 250 - body = f'# A\n\n"{long_term}" {{{{def: big}}}}\n\nSee {{{{term: big}}}} and {{{{term: big}}}}.\n' - out = text(body) - assert "See big and big." in out - assert _warnings(body) == [ - "The term of 'big' is longer than 1000 characters once its references are resolved; " - "'{{term: big}}' shows the definition id."] - - -def test_a_label_still_wins_over_the_cap() -> None: - body = '# A\n\n"' + "word " * 250 + '" {{def: big}}\n\nSee {{term: big, label=Big}}.\n' - assert "See Big." in text(body) - - -def test_a_term_at_the_cap_is_shown_whole() -> None: - term = "x" * 1000 - body = f'# A\n\n"{term}" {{{{def: edge}}}}\n\nSee {{{{term: edge}}}}.\n' - assert f"See {term}." in text(body) - assert _warnings(body) == [] - - -def test_a_normal_nested_term_is_unaffected() -> None: - body = ('# A\n\n"Company" {{def: company}}\n\n"Affiliate of the {{term: company}}" {{def: affiliate}}\n\n' - 'See {{term: affiliate}}.\n') - assert "See Affiliate of the Company." in text(body) - assert _warnings(body) == [] diff --git a/tests/test_wp12_final_fixes.py b/tests/test_wp12_final_fixes.py deleted file mode 100644 index 704c927..0000000 --- a/tests/test_wp12_final_fixes.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Final review fixes (WP12): terms that name each other are resolved within a -budget (``render-term-limit``), a cached term text is reused only where it would -read the same, long chains of terms do not exhaust the stack, and nothing from a -document reaches stderr as a control character.""" -from __future__ import annotations - -import os -import random -import subprocess -import sys -import time -from pathlib import Path - -import pytest - -from legaldown_render import render -from legaldown_render.resolve.resolver import Resolver - -SRC = Path(__file__).resolve().parents[1] / "src" -FRONT = "---\ntitle: T\n---\n\n" -LIMIT = ("Defined terms refer to each other too deeply or too often to resolve fully; " - "some {{term:}} references show their definition id.") - - -def messages(result, rule: str) -> list[str]: # noqa: ANN001 - return [d.message for d in result.diagnostics if d.rule == rule] - - -# -- 1: terms that all name each other --------------------------------------------- - - -def _tangle(k: int) -> str: - defs = [f'"T{i} {" ".join(f"{{{{term: t{j}}}}}" for j in range(k))}" {{{{def: t{i}}}}}' for i in range(k)] - return "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n" - - -@pytest.mark.parametrize("k", [3, 9, 12, 20, 40]) -def test_k_terms_that_each_name_all_k_render_quickly(k: int) -> None: - started = time.perf_counter() - result = render(FRONT + _tangle(k), format="text") - assert time.perf_counter() - started < 2 - assert "{{" not in result.output - if k >= 9: - assert messages(result, "render-term-limit") == [LIMIT] - - -def test_a_small_cycle_is_resolved_in_full_without_the_warning() -> None: - assert messages(render(FRONT + _tangle(3), format="text"), "render-term-limit") == [] - - -# -- 3: a long chain of terms ------------------------------------------------------ - - -@pytest.mark.parametrize("scheme", ["decimal", "none"]) -@pytest.mark.parametrize("heading", [False, True]) -def test_a_chain_of_500_terms_does_not_exhaust_the_stack(scheme: str, heading: bool) -> None: - n = 500 - defs = [f'"T{i} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + [f'"End" {{{{def: t{n}}}}}'] - title = "About {{term: t0}}" if heading else "A" - body = f"# {title}\n\nUsed: {{{{term: t0}}}}\n\n" + "\n\n".join(defs) + "\n" - started = time.perf_counter() - result = render(FRONT + body, format="text", overrides={"numbering.scheme": scheme}) - assert time.perf_counter() - started < 2 - assert messages(result, "render-term-limit") == [LIMIT] - used = next(line for line in result.output.splitlines() if line.startswith("Used: ")) - assert used.startswith("Used: T0 T1 T2 ") - assert "{{" not in result.output - - -def test_a_chain_inside_the_depth_limit_is_resolved_in_full() -> None: - n = 20 - defs = [f'"T{i} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + [f'"End" {{{{def: t{n}}}}}'] - result = render(FRONT + "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n", format="text") - assert f"Used: {' '.join(f'T{i}' for i in range(n))} End" in result.output - assert messages(result, "render-term-limit") == [] - - -# -- 2: a cached text is reused only where it would read the same ------------------ - - -def _oracle(self: Resolver, cache: object, definition_id: str, work, *, key: str | None = None) -> str: # noqa: ANN001 - """The term text with the open terms alone deciding, nothing kept.""" - self._terms_open.append(definition_id) - try: - return work() - finally: - self._terms_open.pop() - - -def _graph(rng: random.Random) -> str: - ids = [f"t{i}" for i in range(rng.randint(2, 6))] - defs = [] - for identifier in ids: - parts = [identifier.upper()] - for _ in range(rng.randint(0, 3)): - target = rng.choice(ids) - parts.append(f"{{{{term: {target}, label=L{target}}}}}" if rng.random() < 0.15 - else f"{{{{term: {target}}}}}") - defs.append(f'"{" ".join(parts)}" {{{{def: {identifier}}}}}') - rng.shuffle(defs) - heading = rng.choice(["A", f"About {{{{term: {rng.choice(ids)}}}}}"]) - use = " ".join(f"{{{{term: {rng.choice(ids)}}}}}" for _ in range(rng.randint(1, 4))) - return f"# {heading}\n\nUsed: {use}\n\n" + "\n\n".join(defs) + "\n" - - -def test_cached_term_texts_equal_the_texts_worked_out_afresh(monkeypatch: pytest.MonkeyPatch) -> None: - rng = random.Random(20261027) - cached_original = Resolver._while_open - for _ in range(150): - source = FRONT + _graph(rng) - for scheme in ("decimal", "none"): - overrides = {"numbering.scheme": scheme} - monkeypatch.setattr(Resolver, "_while_open", cached_original) - cached = render(source, format="text", overrides=overrides) - monkeypatch.setattr(Resolver, "_while_open", _oracle) - fresh = render(source, format="text", overrides=overrides) - assert cached.output == fresh.output, source - assert [(d.rule, d.message) for d in cached.diagnostics] == \ - [(d.rule, d.message) for d in fresh.diagnostics], source - - -def test_a_text_that_named_a_term_is_not_reused_where_that_term_is_open() -> None: - # ca's text names cb. Worked out with nothing open it is "Alpha Beta ca"; with cb open it is - # "Alpha cb", and that is what a reference inside cb must read. - body = ('# A\n\nFirst {{term: ca}}. Then {{term: cb}}.\n\n"Alpha {{term: cb}}" {{def: ca}}\n\n' - '"Beta {{term: ca}}" {{def: cb}}\n') - out = render(FRONT + body, format="text").output - assert "First Alpha Beta ca. Then Beta Alpha cb." in out - - -# -- 4: control characters never reach stderr ---------------------------------------- - - -def _run(source: str, tmp_path: Path, *arguments: str) -> subprocess.CompletedProcess[bytes]: - document = tmp_path / "d.lgd" - document.write_text(source, encoding="utf-8") - env = {**os.environ, "PYTHONPATH": str(SRC)} - return subprocess.run([sys.executable, "-m", "legaldown_render.cli", str(document), "-f", "text", *arguments], - capture_output=True, cwd=tmp_path, env=env, check=False) - - -def test_no_control_character_from_the_document_reaches_stderr(tmp_path: Path) -> None: - body = ("# A {#a}\n\n{{include: a\x1b]0;pwn\x07.lgd}}\n\n{{ref: x\x1b[2Jy}}\n\n{{term: q\x1b[31m}}\n\n" - "See {{ref: a}}.\n") - result = _run(FRONT + body, tmp_path) - errors = result.stderr.decode("utf-8") - assert errors.count("\n") >= 3 # the three findings, one line each - assert not any(ord(c) < 0x20 and c != "\n" or 0x7F <= ord(c) < 0xA0 for c in errors) - assert "\\x1b" in errors # written out, not dropped - assert "pwn" in errors diff --git a/tests/test_wp13_budget.py b/tests/test_wp13_budget.py deleted file mode 100644 index f9e9aed..0000000 --- a/tests/test_wp13_budget.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Review fixes (WP13): the term budget charges only repeat work, and every -command-line value echoed to stderr is escaped.""" -from __future__ import annotations - -import os -import subprocess -import sys -import time -from pathlib import Path - -import pytest - -from legaldown_render import render - -SRC = Path(__file__).resolve().parents[1] / "src" -FRONT = "---\ntitle: T\n---\n\n" - - -def rules(result, rule: str) -> list[str]: # noqa: ANN001 - return [d.message for d in result.diagnostics if d.rule == rule] - - -# -- 1: acyclic terms are free ------------------------------------------------------ - - -def _independent(n: int, *, heading: bool) -> str: - parts = [] - for i in range(n): - if heading: - parts.append(f"## H{i} {{{{term: t{i}}}}}\n\n") - parts.append(f'"Term {i}" {{{{def: t{i}}}}} used {{{{term: t{i}}}}}\n\n') - return "# A\n\n" + "".join(parts) - - -@pytest.mark.parametrize("scheme", ["decimal", "none"]) -@pytest.mark.parametrize("heading", [False, True]) -def test_3000_independent_terms_show_their_text_and_no_warning(scheme: str, heading: bool) -> None: - result = render(FRONT + _independent(3000, heading=heading), format="text", overrides={"numbering.scheme": scheme}) - assert rules(result, "render-term-limit") == [] - lines = result.output.splitlines() - assert "Term 2999 used Term 2999" in lines - assert "Term 0 used Term 0" in lines - assert not any(line.endswith(" used t2999") for line in lines) - if heading and scheme == "none": - assert any(line.endswith("H2999 Term 2999") for line in lines) - - -# -- 2: a work budget on repeat computations ------------------------------------------ - - -def _ring(k: int, n: int) -> str: - """k terms in a cycle, each holding n references to the next.""" - defs = [f'"T{i} {" ".join(f"{{{{term: t{(i + 1) % k}}}}}" for _ in range(n))}" {{{{def: t{i}}}}}' for i in range(k)] - return "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n" - - -def test_a_ring_of_terms_with_many_references_costs_a_small_multiple_of_the_document() -> None: - document = FRONT + _ring(12, 2000) - started = time.perf_counter() - result = render(document, format="text", overrides={"numbering.scheme": "none"}) - cyclic = time.perf_counter() - started - # The same document with the references pointing at a term that names nothing. - flat = document.replace("{{term: t0}}", "{{term: leaf}}") - for i in range(12): - flat = flat.replace(f"{{{{term: t{i}}}}}", "{{term: leaf}}") - flat += '\n"Leaf" {{def: leaf}}\n' - started = time.perf_counter() - render(flat, format="text", overrides={"numbering.scheme": "none"}) - acyclic = time.perf_counter() - started - assert rules(result, "render-term-limit") - assert cyclic < 3 * acyclic + 1 - - -@pytest.mark.parametrize("k", [7, 12, 40]) -def test_all_to_all_terms_stay_fast(k: int) -> None: - defs = [f'"T{i} {" ".join(f"{{{{term: t{j}}}}}" for j in range(k))}" {{{{def: t{i}}}}}' for i in range(k)] - started = time.perf_counter() - result = render(FRONT + "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n", format="text") - assert time.perf_counter() - started < 2 - assert "{{" not in result.output - - -def test_a_chain_of_500_terms_stays_fast_and_reports_the_limit() -> None: - defs = [f'"T{i} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(500)] + ['"End" {{def: t500}}'] - started = time.perf_counter() - result = render(FRONT + "# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n", format="text") - assert time.perf_counter() - started < 2 - assert len(rules(result, "render-term-limit")) == 1 - - -# -- 3: command-line values are escaped on stderr ------------------------------------- - - -def _cli(*arguments: str, cwd: Path) -> subprocess.CompletedProcess[bytes]: - env = {**os.environ, "PYTHONPATH": str(SRC)} - return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *arguments], - capture_output=True, cwd=cwd, env=env, check=False) - - -def _clean(errors: bytes) -> str: - text = errors.decode("utf-8") - assert not any((ord(c) < 0x20 and c != "\n") or 0x7F <= ord(c) < 0xA0 for c in text), text - return text - - -ESC = "x\x1b]0;pwn\x07y" - - -def test_an_input_file_name_is_escaped(tmp_path: Path) -> None: - result = _cli(ESC + ".lgd", cwd=tmp_path) - assert result.returncode == 1 - assert "cannot read" in _clean(result.stderr) and "\\x1b" in _clean(result.stderr) - - -def test_an_answers_path_is_escaped(tmp_path: Path) -> None: - document = tmp_path / "d.lgd" - document.write_text(FRONT + "# A\n", encoding="utf-8") - result = _cli(str(document), "--answers", ESC + ".yaml", cwd=tmp_path) - assert result.returncode == 1 - assert "\\x1b" in _clean(result.stderr) - - -def test_an_output_path_is_escaped(tmp_path: Path) -> None: - document = tmp_path / "d.lgd" - document.write_text(FRONT + "# A\n", encoding="utf-8") - result = _cli(str(document), "-o", "no-such-dir\x1b[2J/out.html", cwd=tmp_path) - assert result.returncode == 1 - assert "\\x1b" in _clean(result.stderr) - refused = _cli(str(document), "-o", "x\x1b[2J.pdf", cwd=tmp_path) - assert refused.returncode == 2 - assert "\\x1b" in _clean(refused.stderr) - - -def test_a_style_name_is_escaped(tmp_path: Path) -> None: - document = tmp_path / "d.lgd" - document.write_text(FRONT + "# A\n", encoding="utf-8") - for arguments in (["-s", ESC], ["-s", ESC, "--print-style"]): - result = _cli(str(document), *arguments, cwd=tmp_path) - assert result.returncode == 2 - assert "\\x1b" in _clean(result.stderr) - - -def test_a_locale_is_escaped(tmp_path: Path) -> None: - document = tmp_path / "d.lgd" - document.write_text(FRONT + "# A\n", encoding="utf-8") - result = _cli(str(document), "--locale", "zz\x1b[31m", cwd=tmp_path) - _clean(result.stderr) - - -def test_a_set_key_is_escaped_in_argparse_errors(tmp_path: Path) -> None: - document = tmp_path / "d.lgd" - document.write_text(FRONT + "# A\n", encoding="utf-8") - for setting in ("bad\x1b[31m", "no\x1b[31m.such=1"): - result = _cli(str(document), "--set", setting, cwd=tmp_path) - assert result.returncode == 2 - assert "\\x1b" in _clean(result.stderr) - - -def test_an_unrecognized_argument_is_escaped(tmp_path: Path) -> None: - result = _cli("--bogus\x1b[31m", cwd=tmp_path) - assert result.returncode == 2 - assert "\\x1b" in _clean(result.stderr) diff --git a/tests/test_wp1_values.py b/tests/test_wp1_values.py deleted file mode 100644 index 8549a9c..0000000 --- a/tests/test_wp1_values.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Value formatting (§10, §13.5): exact amounts, durations that keep their -written digits, no raw CLDR unit id, and spaced alphabetic currency symbols.""" -from __future__ import annotations - -import pytest - -from legaldown_render import render -from legaldown_render.resolve import DURATION_UNITS, Formatter, parse_locale -from legaldown_render.resolve.values import DurationText -from legaldown_render.style.model import Values - - -def formatter(tag: str, **values: str) -> Formatter: - locale = parse_locale(tag) - assert locale is not None - return Formatter(locale, Values(**values)) - - -# -- R2: no crash, no rounding on long amounts (§10.3) ----------------------- - -@pytest.mark.parametrize("digits", [29, 40, 60]) -def test_long_integers_format_exactly(digits: int) -> None: - value = "1234567890" * 6 - value = value[:digits] - grouped = f"{int(value):,}" - en = formatter("en-US") - assert en.number(value) == grouped - assert en.money(value, "USD") == f"${grouped}.00" - assert formatter("en-US", money="code").money(value, "USD") == f"USD\xa0{grouped}.00" - assert formatter("en-US", money="name").money(value, "USD") == f"{grouped}.00 US dollars" - - -def test_long_fraction_keeps_every_digit() -> None: - fraction = "1234567891" * 4 - en = formatter("en-US") - assert en.number("0." + fraction) == "0." + fraction - assert en.money("1." + fraction, "EUR") == "€1." + fraction - assert en.number("1" * 29 + ".5") == f"{int('1' * 29):,}.5" - - -def test_long_duration_keeps_every_digit() -> None: - assert formatter("en-US").duration("1" * 40, "D") == f"{int('1' * 40):,} days" - assert formatter("en-US").duration("0." + "1" * 40, "D") == "0." + "1" * 40 + " days" - - -def test_long_amount_renders_end_to_end() -> None: - source = "---\ntitle: T\n---\nPay {{money: " + "9" * 29 + "}} {{money: 12345678901234567890123456789.1234567890123456789}}.\n" - output = render(source, format="text", locale="en-US").output - assert f"{int('9' * 29):,}" in output - assert "12,345,678,901,234,567,890,123,456,789.1234567890123456789" in output - - -# -- R3: durations keep their digits; the plural follows the displayed value - - -@pytest.mark.parametrize(("tag", "value", "unit", "expected"), [ - ("en", "0.0001", "D", "0.0001 days"), - ("en", "1.23456", "Y", "1.23456 years"), - ("en", "1", "D", "1 day"), - ("en", "1.0", "D", "1.0 days"), - ("en", "12", "MO", "12 months"), - ("cs", "1", "Y", "1 rok"), - ("cs", "2", "Y", "2 roky"), - ("cs", "5", "Y", "5 let"), - ("cs", "2.5", "Y", "2,5 roku"), - ("de", "1.50", "W", "1,50 Wochen"), -]) -def test_duration_digits_and_plural(tag: str, value: str, unit: str, expected: str) -> None: - assert formatter(tag).duration(value, unit) == expected - - -# -- R1: never a raw CLDR unit id --------------------------------------------- - -@pytest.mark.parametrize("tag", ["pt-BR", "ja", "zh", "ko", "it", "tr", "nl", "cs", "de", "sk", "pl", "en"]) -@pytest.mark.parametrize("length", ["long", "short", "narrow"]) -@pytest.mark.parametrize("value", ["1", "2", "5", "1.5"]) -def test_no_cldr_unit_id_is_ever_printed(tag: str, length: str, value: str) -> None: - f = formatter(tag, duration=length) - for unit in DURATION_UNITS: - text = f.duration(value, unit) - assert "duration-" not in text - assert value.replace(".", f.locale.number_symbols["latn"]["decimal"]) in text - - -def test_pt_br_long_falls_back_to_the_short_pattern() -> None: - pt = formatter("pt-BR") - assert pt.duration("30", "D") == "30 dias" - assert pt.duration("12", "MO") == "12 meses" - assert pt.duration("1", "Y") == "1 ano" - - -def test_fallback_goes_to_the_other_length_first() -> None: - # cs has a long and a narrow pattern: narrow is asked, narrow is used. - assert formatter("cs", duration="narrow").duration("30", "D") == "30 d." - # zh has only "short": long and narrow fall back to it, still Chinese. - assert formatter("zh", duration="long").duration("30", "D") == "30天" - assert formatter("zh", duration="narrow").duration("30", "D") == "30天" - - -def test_a_fallback_inside_the_locale_is_not_signalled() -> None: - # pt-BR long has no pattern but short has: Portuguese, no English. - assert formatter("pt-BR").format_duration("30", "D") == DurationText("30 dias", False) - assert formatter("en").format_duration("1", "D") == DurationText("1 day", False) - - -def test_english_is_the_last_resort_and_is_signalled(monkeypatch: pytest.MonkeyPatch) -> None: - from legaldown_render.resolve import values - - real = values._unit_patterns - # A Czech locale without any duration pattern (Babel's data has none such). - monkeypatch.setattr(values, "_unit_patterns", - lambda locale, unit, length: {} if locale.language == "cs" else real(locale, unit, length)) - f = formatter("cs") - assert f.format_duration("5", "D") == DurationText("5 days", True) - assert f.duration("1", "D") == "1 day" - assert f.duration("1.5", "Y") == "1,5 years" # the number keeps the locale's separator - - -def test_fallback_warning_is_emitted_once_by_the_resolver(monkeypatch: pytest.MonkeyPatch) -> None: - from legaldown_render.resolve import values - - real = values._unit_patterns - monkeypatch.setattr(values, "_unit_patterns", - lambda locale, unit, length: {} if locale.language == "cs" else real(locale, unit, length)) - source = ("---\ntitle: T\nlanguage: cs\n---\n" - "Lhůta {{duration: 5, unit=D}} a {{duration: 6, unit=D}} a {{duration: 2, unit=MO}}.\n") - result = render(source, format="text") - fallbacks = [d for d in result.diagnostics if d.rule == "render-locale-fallback"] - assert [d.message for d in fallbacks] == [ - "No CLDR pattern for 'duration-day' in 'cs'; English used.", - "No CLDR pattern for 'duration-month' in 'cs'; English used.", - ] - assert all(d.level == "warning" for d in fallbacks) - assert "5 days" in result.output - - -def test_normal_locales_emit_no_fallback_warning() -> None: - for language in ("cs", "pt-BR", "ja", "en"): - source = f"---\ntitle: T\nlanguage: {language}\n---\nX {{{{duration: 5, unit=D}}}}.\n" - result = render(source, format="text") - assert not [d for d in result.diagnostics if d.rule == "render-locale-fallback"], language - - -# -- R4: an alphabetic symbol is spaced from the digits ----------------------- - -@pytest.mark.parametrize(("tag", "currency", "expected"), [ - ("en", "CZK", "CZK\xa05.00"), - ("en", "CHF", "CHF\xa05.00"), - ("en", "USD", "$5.00"), - ("en", "EUR", "€5.00"), - ("pt-BR", "BRL", "R$\xa05,00"), - ("en", "CAD", "CA$5.00"), - ("cs", "CZK", "5,00\xa0Kč"), -]) -def test_symbol_mode_spacing(tag: str, currency: str, expected: str) -> None: - assert formatter(tag).money("5", currency) == expected - - -def test_symbol_after_the_digits_is_spaced_too() -> None: - # Whatever the locale puts after the amount, a letter never touches a digit. - import re - - for tag in ("en", "de", "fr", "cs", "pl", "sv", "ja", "ru", "pt-PT", "es", "nl", "it"): - for currency in ("CZK", "CHF", "USD", "EUR", "SEK", "PLN", "XAF"): - text = formatter(tag).money("5", currency) - assert not re.search(r"[^\W\d_]\d|\d[^\W\d_]", text), (tag, currency, text) - - -def test_code_mode_still_spaced() -> None: - assert formatter("en", money="code").money("5", "CZK") == "CZK\xa05.00" - assert formatter("de", money="code").money("5", "CZK") == "5,00\xa0CZK" - - -# -- money keeps every written fraction digit, padded to the minor units ------ - -@pytest.mark.parametrize(("amount", "currency", "expected"), [ - ("1.1230", "EUR", "€1.1230"), - ("1.500", "EUR", "€1.500"), - ("1.50", "EUR", "€1.50"), - ("5", "EUR", "€5.00"), - ("5.5", "EUR", "€5.50"), - ("5", "JPY", "¥5"), - ("0.125", "JPY", "¥0.125"), - ("5.0", "JPY", "¥5.0"), - ("1234.5670", "USD", "$1,234.5670"), -]) -def test_money_fraction_digits_symbol(amount: str, currency: str, expected: str) -> None: - assert formatter("en-US").money(amount, currency) == expected - - -def test_money_fraction_digits_code_and_name() -> None: - code = formatter("en-US", money="code") - name = formatter("en-US", money="name") - assert code.money("1.500", "EUR") == "EUR\xa01.500" - assert code.money("5", "EUR") == "EUR\xa05.00" - assert code.money("0.125", "JPY") == "JPY\xa00.125" - assert code.money("5", "JPY") == "JPY\xa05" - assert name.money("1.500", "EUR") == "1.500 euros" - assert name.money("5", "EUR") == "5.00 euros" - assert name.money("0.125", "JPY") == "0.125 Japanese yen" - assert name.money("5", "JPY") == "5 Japanese yen" - - -def test_money_trailing_zeros_in_other_locales_and_long_amounts() -> None: - assert formatter("cs-CZ").money("1.500", "CZK") == "1,500\xa0Kč" - assert formatter("de", money="code").money("1.1230", "EUR") == "1,1230\xa0EUR" - assert formatter("en-US").money("1" * 40 + ".1230", "EUR") == f"€{int('1' * 40):,}.1230" diff --git a/tests/test_wp7_small.py b/tests/test_wp7_small.py deleted file mode 100644 index 1df7b50..0000000 --- a/tests/test_wp7_small.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Small fixes (WP7): a label filled with an empty ``{section}`` drops the field -and its dot, so a preamble item reads "1", not ".1" (R5); the print stylesheet -breaks before the attachments container, not inside it (W2).""" -from __future__ import annotations - -import os -import subprocess -import sys -from pathlib import Path - -from legaldown_render import render -from legaldown_render.resolve.numbering import fill - -FRONT = "---\ntitle: T\n---\n" -PREAMBLE_AND_SECTION = ( - "\nPreamble.\n\n- first pre\n- second pre\n\n# A {#a}\n\n- in a\n" -) - - -def text(body: str, **settings: object) -> str: - return render(FRONT + body, format="text", **settings).output - - -def html(body: str, **settings: object) -> str: - return render(FRONT + body, format="html", **settings).output - - -# -- R5: an empty section drops its dot (numbering.fill) ----------------------------------------- - -def test_fill_drops_an_empty_section_with_one_following_dot() -> None: - assert fill("{section}.{n}", n="1", section="") == "1" - assert fill("{section}.{n}", n="1", section="2") == "2.1" - # Only ONE dot goes with the field. - assert fill("{section}..{n}", n="1", section="") == ".1" - - -def test_fill_keeps_other_text_of_an_empty_section_as_written() -> None: - # No dot right after the field: the field alone is removed, as before. - assert fill("({section}){n}", n="a", section="") == "()a" - assert fill("{section}", n="1", section="") == "" - - -def test_fill_without_a_section_value_keeps_the_dot() -> None: - # The rule applies only when the section value is given and empty. - assert fill("{section}.{n}", n="1") == ".1" - - -# -- R5: preamble list items agree with their designation ---------------------------------------- - -def test_a_preamble_list_is_numbered_without_a_leading_dot_in_text() -> None: - output = text(PREAMBLE_AND_SECTION, style="continental") - assert "1 first pre" in output - assert "2 second pre" in output - assert not any(line.startswith(".") for line in output.splitlines()) - - -def test_a_preamble_list_is_numbered_without_a_leading_dot_in_html() -> None: - output = html(PREAMBLE_AND_SECTION, style="continental") - assert '1

first pre

' in output - assert '2

second pre

' in output - assert 'class="ld-label">.1<' not in output - - -def test_a_list_in_a_section_keeps_its_section_prefix() -> None: - output = text(PREAMBLE_AND_SECTION, style="continental") - assert "1.1 in a" in output - html_output = html(PREAMBLE_AND_SECTION, style="continental") - assert '1.1

in a

' in html_output - - -# -- W2: print breaks before the attachments container -------------------------------------------- - -SRC = Path(__file__).resolve().parents[1] / "src" -ATTACHED = ( - "---\ntitle: T\nattachments:\n" - " - id: one\n title: First attachment\n file: one.pdf\n" - " - id: two\n title: Second attachment\n file: two.pdf\n" - "---\n\n# A {#a}\n\nBody.\n" -) - - -def _cli(*args: str, stdin: str) -> subprocess.CompletedProcess[str]: - env = dict(os.environ, PYTHONPATH=str(SRC), PYTHONUTF8="1") - return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *args], input=stdin, - capture_output=True, text=True, env=env, check=False) - - -def test_a_standalone_page_break_render_has_the_attachment_break_rules() -> None: - done = _cli("-", "--set", "attachments.separator=page-break", stdin=ATTACHED) - assert done.returncode == 0, done.stderr - assert "" in done.stdout - assert 'class="ld-attachments ld-separator-page-break"' in done.stdout - # The break goes before the container (heading included), then before each later attachment. - assert ".ld-attachments.ld-separator-page-break { break-before: page; }" in done.stdout - assert ".ld-separator-page-break .ld-attachment + .ld-attachment { break-before: page; }" in done.stdout - assert ".ld-attachments-heading { break-after: avoid; }" in done.stdout - # The old rule broke before every attachment, the first one included, inside the container. - assert ".ld-separator-page-break .ld-attachment { break-before: page; }" not in done.stdout - - -def test_without_a_page_break_separator_the_container_is_not_broken() -> None: - done = _cli("-", "--set", "attachments.separator=none", stdin=ATTACHED) - assert done.returncode == 0, done.stderr - # The rules are scoped by the separator class, which no element carries here. - assert 'class="ld-attachments ld-separator-none"' in done.stdout - assert 'ld-separator-page-break"' not in done.stdout diff --git a/tests/test_wp9_review_fixes.py b/tests/test_wp9_review_fixes.py deleted file mode 100644 index 1a01418..0000000 --- a/tests/test_wp9_review_fixes.py +++ /dev/null @@ -1,200 +0,0 @@ -"""Review fixes after wave 1 (WP9): a ``{{ref:}}`` or placeholder inside a -defined term as ``{{term:}}`` shows it, a fraction beyond Python's int/str digit -limit renders, ``-o`` rejects only the extensions of formats the renderer does -not write, in-process ``main()`` leaves the streams alone, and output has LF -line endings.""" -from __future__ import annotations - -import io -import os -import subprocess -import sys -from pathlib import Path - -import pytest -from conftest import DOCUMENTS - -from legaldown_render import render -from legaldown_render.cli import main - -SRC = Path(__file__).resolve().parents[1] / "src" -FEATURES = str(DOCUMENTS / "features.lgd") -FRONT = "---\ntitle: T\n---\n\n" - - -def text(body: str, **settings: object) -> str: - return render(FRONT + body, format="text", **settings).output - - -# -- 1: {{term:}} shows what the definition shows ------------------------------- - - -def test_a_term_reference_keeps_a_ref_inside_the_term() -> None: - output = text('# H {#h}\n\n"Fee per {{ref: h}}" {{def: a}}\n\n{{term: a}}\n') - assert output.count("Fee per 1") == 2 - - -def test_a_term_reference_before_the_heading_keeps_the_ref() -> None: - output = text('{{term: a}}\n\n# H {#h}\n\n"Fee per {{ref: h}}" {{def: a}}\n') - assert output.count("Fee per 1") == 2 - - -def test_a_term_reference_before_its_definition_and_its_target() -> None: - output = text('# First\n\nSee {{term: a}}.\n\n# H {#h}\n\n"Fee per {{ref: h}}" {{def: a}}\n') - assert "See Fee per 2." in output - - -def test_a_term_reference_keeps_a_ref_to_a_subsection() -> None: - body = '# A\n\n"Fee per {{ref: s}}" {{def: a}}\n\n{{term: a}}\n\n## Sub {#s}\n\nx\n' - output = text(body) - assert output.count("Fee per 1.1") == 2 - - -def test_a_term_reference_in_a_textual_heading_and_in_the_body() -> None: - # The none scheme reads a heading as text before every target is known; the - # term text must not be fixed from that early pass. - body = '# Fee {{term: a}}\n\n"Fee per {{ref: h}}" {{def: a}}\n\n# H {#h}\n\n{{term: a}}\n' - output = render(FRONT + body, format="text", overrides={"numbering.scheme": "none"}).output - assert output.rstrip().endswith("Fee per H") - assert "[BROKEN REF" not in output - - -def test_a_term_reference_shows_the_placeholder_as_the_definition_does() -> None: - body = '# A\n\n"Fee {{placeholder: p}}" {{def: a}} {{term: a}} {{placeholder: p, type=money}}\n' - result = render(FRONT + body, format="text") - line = next(row for row in result.output.splitlines() if "INVALID PLACEHOLDER" in row) - assert line.count("Fee [INVALID PLACEHOLDER: p]") == 2 - assert "[_____]" not in result.output - - -def test_a_definition_that_defines_a_term_twice_uses_the_first() -> None: - body = '# H {#h}\n\n"First {{ref: h}}" {{def: t}}\n\n"Second" {{def: t}}\n\n{{term: t}}\n' - assert text(body).rstrip().endswith("First 1") - - -def test_a_term_with_a_ref_adds_no_duplicate_diagnostics() -> None: - # The term holds an image with a remote path: one report however often it is resolved. - body = '# H {#h}\n\n"Fee ![x](http://e.com/i.png) {{ref: h}}" {{def: a}}\n\n{{term: a}} {{term: a}}\n' - diagnostics = render(FRONT + body, format="text").diagnostics - keys = [(d.rule, d.message) for d in diagnostics] - assert len(keys) == len(set(keys)) - assert sum(d.rule == "path-not-relative" for d in diagnostics) == 1 - - -def test_a_term_that_holds_a_definition_does_not_take_its_anchor() -> None: - body = '# A\n\nSee {{term: outer}}.\n\n"Outer "Inner" {{def: inner}}" {{def: outer}}\n\n{{term: inner}}\n' - html = render(FRONT + body, format="html").output - assert html.count('id="def:inner"') == 1 - assert html.count('id="def:outer"') == 1 - - -@pytest.mark.parametrize("output_format", ["text", "html"]) -def test_cycles_through_refs_still_terminate(output_format: str) -> None: - body = ('# H {#h}\n\n"A {{term: b}} {{ref: h}}" {{def: a}} x.\n\n"B {{term: a}}" {{def: b}} y.\n\n' - '"C {{term: c}}" {{def: c}}\n\nSee {{term: a}}, {{term: b}}, {{term: c}}.\n') - out = render(FRONT + body, format=output_format).output - assert "{{" not in out - # Which term shows what inside a cycle follows the order they are first needed in. - assert "A b 1" in out and "C c" in out - - -# -- 2: a fraction beyond Python's int/str digit limit --------------------------- - -LONG = "2" * 4400 - - -def test_a_duration_with_a_4400_digit_fraction_renders() -> None: - output = render(FRONT + f"# A\n\n{{{{duration: 1.{LONG}, unit=D}}}}", format="text").output - assert f"1.{LONG} days" in output - - -def test_a_money_amount_in_name_mode_with_a_4400_digit_fraction_renders() -> None: - output = render(FRONT + f"# A\n\n{{{{money: 1.{LONG}, currency=EUR}}}}", format="text", - overrides={"values.money": "name"}).output - assert f"1.{LONG} euros" in output - - -def test_a_4400_digit_fraction_uses_the_many_category_in_czech() -> None: - # Wave 3 (WP11): the category comes from the first 30 fraction digits, so it is the - # "many" of "2,5 roku", not "other" (" let"). - output = render(FRONT + f"# A\n\n{{{{duration: 1.{LONG}, unit=Y}}}}", format="text", locale="cs-CZ").output - assert output.rstrip().endswith(" roku") - - -# -- 4: only the program reconfigures the streams ------------------------------- - - -def _wrapper(encoding: str, **options: object) -> io.TextIOWrapper: - return io.TextIOWrapper(io.BytesIO(), encoding=encoding, **options) # type: ignore[arg-type] - - -def test_in_process_main_leaves_the_streams_alone(monkeypatch: pytest.MonkeyPatch) -> None: - out, err = _wrapper("cp1252", newline="\r\n"), _wrapper("cp1252") - monkeypatch.setattr(sys, "stdout", out) - monkeypatch.setattr(sys, "stderr", err) - assert main([FEATURES, "-f", "text", "-q"]) == 0 - assert main(["--list-styles"]) == 0 - assert (sys.stdout.encoding, sys.stdout.errors) == ("cp1252", "strict") - assert (sys.stderr.encoding, sys.stderr.errors) == ("cp1252", "strict") - out.flush() - # Not even the newline translation of the caller's stream changed. - assert b"\r\n" in out.buffer.getvalue() # type: ignore[attr-defined] - - -def test_main_as_the_program_reconfigures_the_streams(monkeypatch: pytest.MonkeyPatch) -> None: - out, err = _wrapper("cp1252", newline="\r\n"), _wrapper("cp1252") - monkeypatch.setattr(sys, "stdout", out) - monkeypatch.setattr(sys, "stderr", err) - monkeypatch.setattr(sys, "argv", ["legaldown-render", "--list-styles"]) - assert main() == 0 - out.flush() - assert (out.encoding, err.errors) == ("utf-8", "backslashreplace") - data = out.buffer.getvalue() # type: ignore[attr-defined] - assert data and b"\r" not in data and data.endswith(b"\n") - - -def test_the_console_script_calls_main_without_argv() -> None: - # The script entry point calls main() with no argument, so it is "the program". - pyproject = (SRC.parent / "pyproject.toml").read_text(encoding="utf-8") - assert 'legaldown-render = "legaldown_render.cli:main"' in pyproject - - -# -- 5: LF line endings everywhere ------------------------------------------------ - - -def _run(args: list[str]) -> subprocess.CompletedProcess[bytes]: - env = dict(os.environ, PYTHONPATH=str(SRC), PYTHONIOENCODING="cp1252", PYTHONUTF8="0") - return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *args], capture_output=True, env=env, - check=False) - - -@pytest.mark.parametrize("fmt", ["html", "text"]) -def test_standard_output_has_lf_line_endings(fmt: str) -> None: - done = _run([FEATURES, "-f", fmt, "-q"]) - assert done.returncode == 0, done.stderr - assert b"\n" in done.stdout and b"\r" not in done.stdout - - -@pytest.mark.parametrize("fmt", ["html", "text"]) -def test_an_output_file_has_lf_line_endings(tmp_path: Path, fmt: str) -> None: - target = tmp_path / "out.bin" - assert main([FEATURES, "-q", "-f", fmt, "-o", str(target)]) == 0 - data = target.read_bytes() - assert b"\n" in data and b"\r" not in data - assert data == render(Path(FEATURES).read_text(encoding="utf-8"), format=fmt).output.encode("utf-8") - - -def test_an_output_file_is_written_without_newline_translation( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - # Windows' text mode would turn "\n" into CRLF unless the file is opened - # with newline="\n": that is what the CLI asks for. - calls: list[dict[str, object]] = [] - real = Path.write_text - - def spy(self: Path, data: str, *args: object, **kwargs: object) -> int: - calls.append(kwargs) - return real(self, data, *args, **kwargs) # type: ignore[arg-type] - - monkeypatch.setattr(Path, "write_text", spy) - assert main([FEATURES, "-q", "-o", str(tmp_path / "out.txt")]) == 0 - assert calls == [{"encoding": "utf-8", "newline": "\n"}] From e47f98f11493c0779b02512dff6d89bcf01b5b80 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:34:03 +0000 Subject: [PATCH 031/118] Bound the term-work tests by counted work, not wall-clock time The tests of the term cache, work budget, depth limit and length cap asserted that a render finished within two seconds (or within three times the time of a flat document), which can fail on a slow runner. They now count the directives the resolver resolves, in the body and in the text-only pass for headings, and bound that count: linear in the number of terms for chains, quadratic for chains read round in a cycle, a fixed ceiling for terms that all name each other, and a small multiple of the flat document for a ring of terms with many references. Doubling a chain at most doubles the work. A render that exceeds its bound is stopped, so a lost protection fails at once instead of hanging. The same adversarial documents are used, with the same output-size bounds. --- tests/test_terms_budget.py | 128 ++++++++++++++++++++++++++----------- 1 file changed, 89 insertions(+), 39 deletions(-) diff --git a/tests/test_terms_budget.py b/tests/test_terms_budget.py index 13d34e9..2dd8c80 100644 --- a/tests/test_terms_budget.py +++ b/tests/test_terms_budget.py @@ -8,11 +8,18 @@ depth of nested terms, a reference shows its definition id and the Warning ``render-term-limit`` says so. A term text is also capped in length (``render-term-too-long``). + +The tests of cost do not time the render, which depends on the machine. They +count the directives the resolver resolves, in the text-only pass for headings +as in the body (the ``resolver_work`` fixture), and bound that count, so a lost +cache, budget or depth limit fails the same way everywhere. Each bound sits +25 to 60 % above the work measured when the test was written, far below what +the document costs without the protection. """ from __future__ import annotations import random -import time +from collections.abc import Callable import pytest from conftest import FRONT, text @@ -20,9 +27,40 @@ from legaldown_render import RenderResult, render from legaldown_render.resolve.resolver import Resolver +WorkCounter = Callable[..., tuple[RenderResult, int]] + LIMIT = ("Defined terms refer to each other too deeply or too often to resolve fully; " "some {{term:}} references show their definition id.") +#: The most directives a document of k terms that all name each other may cost: +#: the work budget is 200 000 inline nodes, about half of them directives. +TANGLE_DIRECTIVES = 130_000 + + +@pytest.fixture +def resolver_work(monkeypatch: pytest.MonkeyPatch) -> WorkCounter: + """``work(body, limit, **settings)`` renders *body* as plain text and + returns the result with the number of directives the resolver resolved. + A render that resolves more than *limit* is stopped and fails, so that a + lost protection fails at once instead of running for hours.""" + state = {"directives": 0, "limit": 0} + real = Resolver._directive + + def counting(self: Resolver, directive: object): # type: ignore[no-untyped-def] + state["directives"] += 1 + if state["directives"] > state["limit"]: + pytest.fail(f"more than {state['limit']} directives resolved: the work is not bounded", pytrace=False) + return real(self, directive) # type: ignore[arg-type] + + monkeypatch.setattr(Resolver, "_directive", counting) + + def work(body: str, limit: int, **settings: object) -> tuple[RenderResult, int]: + state.update(directives=0, limit=limit) + result = render(FRONT + body, format="text", **settings) # type: ignore[arg-type] + return result, state["directives"] + + return work + def messages(result: RenderResult, rule: str) -> list[str]: return [d.message for d in result.diagnostics if d.rule == rule] @@ -36,6 +74,13 @@ def _chain(n: int, *, cycle: bool, heading: bool = False) -> str: return f"# {title}\n\nUsed: {{{{term: t0}}}}\n\n" + "\n\n".join(defs) + "\n" +def _chain_directives(n: int, *, cycle: bool) -> int: + """A bound on the directives a chain of *n* terms takes: about 6 n; read + round in a cycle each term has a text for each set of open terms, and the + work is about 2.5 n squared.""" + return 4 * n * n + 30 if cycle else 8 * n + 30 + + def _line(n: int, *, heading: bool) -> str: """Terms t0..tn, each reading the next one once.""" defs = [f'"T{i} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + [f'"End" {{{{def: t{n}}}}}'] @@ -69,15 +114,24 @@ def _independent(n: int, *, heading: bool) -> str: @pytest.mark.parametrize("heading", [False, True]) @pytest.mark.parametrize("scheme", ["decimal", "none"]) @pytest.mark.parametrize("cycle", [False, True]) -def test_a_branching_chain_of_terms_is_worked_out_once_per_term(cycle: bool, scheme: str, heading: bool) -> None: +def test_a_branching_chain_of_terms_is_worked_out_once_per_term( + resolver_work: WorkCounter, cycle: bool, scheme: str, heading: bool) -> None: # Each term reads the next twice: worked out afresh at each use, 20 terms is 2**20 resolutions. # The text of t0 is itself 2**n words long (4**n with a cycle, which reads the chain round # twice), so the chains cannot be longer. Under the none scheme a heading is read as text # before the targets are known (§13.3). - started = time.perf_counter() - out = text(_chain(9 if cycle else 20, cycle=cycle, heading=heading), overrides={"numbering.scheme": scheme}) - assert time.perf_counter() - started < 2 - assert "{{" not in out + n = 9 if cycle else 20 + result, _ = resolver_work(_chain(n, cycle=cycle, heading=heading), _chain_directives(n, cycle=cycle), + overrides={"numbering.scheme": scheme}) + assert "{{" not in result.output + + +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +def test_doubling_a_chain_of_terms_doubles_the_work(resolver_work: WorkCounter, scheme: str) -> None: + directives = {n: resolver_work(_chain(n, cycle=False, heading=True), _chain_directives(n, cycle=False), + overrides={"numbering.scheme": scheme})[1] for n in (10, 20, 40)} + assert directives[20] <= 2.2 * directives[10] + assert directives[40] <= 2.2 * directives[20] def test_a_chain_without_a_cycle_reads_as_each_term_does() -> None: @@ -90,24 +144,25 @@ def test_a_chain_without_a_cycle_reads_as_each_term_does() -> None: @pytest.mark.parametrize(("n", "cycle"), [(40, False), (14, True)]) @pytest.mark.parametrize("scheme", ["decimal", "none"]) -def test_a_branching_chain_of_terms_is_capped_and_reported(n: int, cycle: bool, scheme: str) -> None: - started = time.perf_counter() - result = render(FRONT + _chain(n, cycle=cycle, heading=True), format="text", - overrides={"numbering.scheme": scheme}) - assert time.perf_counter() - started < 2 +def test_a_branching_chain_of_terms_is_capped_and_reported( + resolver_work: WorkCounter, n: int, cycle: bool, scheme: str) -> None: + result, _ = resolver_work(_chain(n, cycle=cycle, heading=True), _chain_directives(n, cycle=cycle), + overrides={"numbering.scheme": scheme}) assert len(result.output) < 100_000 - messages = [d.message for d in result.diagnostics if d.rule == "render-term-too-long"] - assert messages - assert len(messages) == len(set(messages)) # one per id + too_long = messages(result, "render-term-too-long") + assert too_long + assert len(too_long) == len(set(too_long)) # one per id assert all("longer than 1000 characters" in message and "shows the definition id" in message - for message in messages) + for message in too_long) assert "{{" not in result.output @pytest.mark.parametrize("scheme", ["decimal", "none"]) @pytest.mark.parametrize("heading", [False, True]) -def test_3000_independent_terms_show_their_text_and_no_warning(scheme: str, heading: bool) -> None: - result = render(FRONT + _independent(3000, heading=heading), format="text", overrides={"numbering.scheme": scheme}) +def test_3000_independent_terms_show_their_text_and_no_warning( + resolver_work: WorkCounter, scheme: str, heading: bool) -> None: + result, directives = resolver_work(_independent(3000, heading=heading), 4 * 3000 + 100, + overrides={"numbering.scheme": scheme}) assert messages(result, "render-term-limit") == [] lines = result.output.splitlines() assert "Term 2999 used Term 2999" in lines @@ -115,6 +170,9 @@ def test_3000_independent_terms_show_their_text_and_no_warning(scheme: str, head assert not any(line.endswith(" used t2999") for line in lines) if heading and scheme == "none": assert any(line.endswith("H2999 Term 2999") for line in lines) + _, half = resolver_work(_independent(1500, heading=heading), 4 * 1500 + 100, + overrides={"numbering.scheme": scheme}) + assert directives <= 2.2 * half # -- A cached text is reused only where it would read the same ----------------- @@ -163,10 +221,8 @@ def test_cached_term_texts_equal_the_texts_worked_out_afresh(monkeypatch: pytest @pytest.mark.parametrize("k", [3, 7, 9, 12, 20, 40]) -def test_k_terms_that_each_name_all_k_render_quickly(k: int) -> None: - started = time.perf_counter() - result = render(FRONT + _tangle(k), format="text") - assert time.perf_counter() - started < 2 +def test_k_terms_that_each_name_all_k_stay_within_the_work_budget(resolver_work: WorkCounter, k: int) -> None: + result, _ = resolver_work(_tangle(k), 150 if k == 3 else TANGLE_DIRECTIVES) assert "{{" not in result.output if k >= 9: assert messages(result, "render-term-limit") == [LIMIT] @@ -176,21 +232,17 @@ def test_a_small_cycle_is_resolved_in_full_without_the_warning() -> None: assert messages(render(FRONT + _tangle(3), format="text"), "render-term-limit") == [] -def test_a_ring_of_terms_with_many_references_costs_a_small_multiple_of_the_document() -> None: - document = FRONT + _ring(12, 2000) - started = time.perf_counter() - result = render(document, format="text", overrides={"numbering.scheme": "none"}) - cyclic = time.perf_counter() - started +def test_a_ring_of_terms_with_many_references_costs_a_small_multiple_of_the_document( + resolver_work: WorkCounter) -> None: + document = _ring(12, 2000) # The same document with the references pointing at a term that names nothing. - flat = document.replace("{{term: t0}}", "{{term: leaf}}") + flat = document for i in range(12): flat = flat.replace(f"{{{{term: t{i}}}}}", "{{term: leaf}}") flat += '\n"Leaf" {{def: leaf}}\n' - started = time.perf_counter() - render(flat, format="text", overrides={"numbering.scheme": "none"}) - acyclic = time.perf_counter() - started + _, acyclic = resolver_work(flat, 100_000, overrides={"numbering.scheme": "none"}) + result, _ = resolver_work(document, 10 * acyclic, overrides={"numbering.scheme": "none"}) assert messages(result, "render-term-limit") - assert cyclic < 3 * acyclic + 1 # -- A long chain of terms: the depth limit ------------------------------------ @@ -198,18 +250,16 @@ def test_a_ring_of_terms_with_many_references_costs_a_small_multiple_of_the_docu @pytest.mark.parametrize("scheme", ["decimal", "none"]) @pytest.mark.parametrize("heading", [False, True]) -def test_a_chain_of_500_terms_does_not_exhaust_the_stack(scheme: str, heading: bool) -> None: - n = 500 - defs = [f'"T{i} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + [f'"End" {{{{def: t{n}}}}}'] - title = "About {{term: t0}}" if heading else "A" - body = f"# {title}\n\nUsed: {{{{term: t0}}}}\n\n" + "\n\n".join(defs) + "\n" - started = time.perf_counter() - result = render(FRONT + body, format="text", overrides={"numbering.scheme": scheme}) - assert time.perf_counter() - started < 2 +def test_a_chain_of_500_terms_does_not_exhaust_the_stack( + resolver_work: WorkCounter, scheme: str, heading: bool) -> None: + result, directives = resolver_work(_line(500, heading=heading), 3 * 500 + 50, + overrides={"numbering.scheme": scheme}) assert messages(result, "render-term-limit") == [LIMIT] used = next(line for line in result.output.splitlines() if line.startswith("Used: ")) assert used.startswith("Used: T0 T1 T2 ") assert "{{" not in result.output + _, half = resolver_work(_line(250, heading=heading), 3 * 250 + 50, overrides={"numbering.scheme": scheme}) + assert directives <= 2.2 * half def test_a_chain_inside_the_depth_limit_is_resolved_in_full() -> None: From a98cf245a77abd83ade9973e60b989ae2f45f931 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:34:19 +0000 Subject: [PATCH 032/118] List every test module in the architecture notes The Testing table names each module under tests/ with what it checks, replaces the entry for the removed links module, and notes the shared helpers in conftest.py. --- docs/architecture.md | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index f73c800..2c0016a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -300,11 +300,26 @@ line public (`Document.line_of`, U2). ## Testing +Each module is named for its topic. Shared helpers (the minimal document header, the `text` and +`html` renderers, and running the command as a program) are in `tests/conftest.py`. + | Suite | What it checks | |---|---| | `tests/test_golden.py` | The test documents under each built-in style, compared byte for byte with `tests/golden/`. Regenerate with `pytest --update-golden` and review the diff | -| `tests/test_render.py` | Numbering schemes, designations, fallbacks, every failure marker, template view, labels by language, HTML safety, unique anchors, determinism, strict mode, locales | -| `tests/test_wp3_links.py` | Which images are shown and which links are links: non-relative image paths, the link scheme allow-list, empty URLs and empty link text | -| `tests/test_style.py`, `tests/test_values.py` | Style layering and validation; counters and locale formatting | -| `tests/test_cli.py` | The command | +| `tests/test_render.py` | Numbering schemes, designations, fallbacks, every failure marker, template view, labels by language, HTML safety, unique anchors, determinism, strict mode, locales; the renderer's reading of the validator's structure (lists, quotes, comments, markers) | +| `tests/test_values.py` | Money, dates and durations in a locale: exact digits, plural of the value shown, currency display, CLDR fallback to English | +| `tests/test_numbering.py` | Counter formats, label templates and the preamble's `{section}` | +| `tests/test_build.py` | Building the tree: comment-only items and quotes, emphasis next to a directive, spacing where an inline comment was dropped | +| `tests/test_terms.py` | What `{{term:}}` shows, references inside terms, terms that name themselves or each other, the cap on a term text | +| `tests/test_terms_budget.py` | Reuse of worked-out term texts, the work budget and depth limit for terms that name each other. Bounds the work by counting resolved directives, never by timing | +| `tests/test_links.py` | Which images are shown and which links are links: non-relative image paths, the link scheme allow-list, empty URLs and empty link text, and how a refusal is reported | +| `tests/test_text_writer.py` | The plain-text layout: attachment separators, tables, quotes, list items and indent, blanked labels | +| `tests/test_html.py` | The HTML writer's print stylesheet for attachments | +| `tests/test_labels.py` | The built-in labels of parties, per language | +| `tests/test_style.py` | Built-in styles, `extends`, dotted overrides, problem reporting, dumping a style, typing of command-line values | +| `tests/test_style_validation.py` | Template field validation, `extends` cycles, self-contained dumps, `version` and `extends` not overridable, YAML errors in `--set`, the empty-blank fallback | +| `tests/test_cli.py` | The command's options, exit codes and diagnostics | +| `tests/test_cli_io.py` | Encodings of the streams, input that is not UTF-8, unwritable output, LF line endings, the extension rule of `-o` | +| `tests/test_errors.py` | The exception hierarchy and pickling of errors | +| `tests/test_escaping.py` | Control characters from a document or the command line never reach a diagnostic or standard error unescaped | | `tests/conformance/` | Every example and fixture in the specification repository renders, every internal link has a target, and each invalid fixture renders its rule's failure marker. Needs `$LEGALDOWN_SPEC_DIR` or a sibling `../LegalDown` checkout | From 3f08186595c90092b6fc295d76a892a7944cde81 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:37:10 +0000 Subject: [PATCH 033/118] Test that first computations of acyclic terms are free of the work budget 800 independent terms, each holding 120 emphasis runs, together hold more inline nodes than the budget allows for repeat computations. Each term is worked out once, so no render-term-limit warning is reported and every {{term:}} shows its real text. The test counts no time. --- tests/test_terms_budget.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_terms_budget.py b/tests/test_terms_budget.py index 2dd8c80..1428763 100644 --- a/tests/test_terms_budget.py +++ b/tests/test_terms_budget.py @@ -25,7 +25,7 @@ from conftest import FRONT, text from legaldown_render import RenderResult, render -from legaldown_render.resolve.resolver import Resolver +from legaldown_render.resolve.resolver import _MAX_TERM_WORK, Resolver WorkCounter = Callable[..., tuple[RenderResult, int]] @@ -174,6 +174,22 @@ def test_3000_independent_terms_show_their_text_and_no_warning( overrides={"numbering.scheme": scheme}) assert directives <= 2.2 * half +def test_first_computations_of_acyclic_terms_are_free_of_the_budget() -> None: + # Together the term texts hold more inline nodes than the work budget allows for repeat + # computations, but each term is worked out only once, so none of that work is charged. + terms, runs = 800, 120 + nodes_per_text = 1 + 2 * runs + (runs - 1) # the text, each emphasis with its words, the spaces + assert terms * nodes_per_text > _MAX_TERM_WORK + defs = [f'"T{i} {" ".join(f"*e{j}*" for j in range(runs))}" {{{{def: t{i}}}}}\n\nUsed {{{{term: t{i}}}}}\n\n' + for i in range(terms)] + result = render(FRONT + "# A\n\n" + "".join(defs), format="text") + assert messages(result, "render-term-limit") == [] + lines = set(result.output.splitlines()) + assert {f"Used T{i} " + " ".join(f"e{j}" for j in range(runs)) for i in (0, terms // 2, terms - 1)} <= lines + assert sum(line.startswith("Used T") for line in result.output.splitlines()) == terms + assert not any(line.startswith("Used t") for line in lines) + + # -- A cached text is reused only where it would read the same ----------------- def _oracle(self: Resolver, cache: object, definition_id: str, work, *, key: str | None = None) -> str: # noqa: ANN001 From 49c3fc4e6ec3bcd5e0501b116bc6f10c284fbfb2 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:49:21 +0000 Subject: [PATCH 034/118] Check files out with LF line endings The goldens and test documents are compared byte for byte, and the CLI writes LF on every platform, so a Windows checkout must not convert them to CRLF. --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf From f71ba397f8e57dd4c66df6c618b7814521b84037 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:49:21 +0000 Subject: [PATCH 035/118] Make the tests independent of the platform's default encoding and line endings Every read_text/write_text in the style and CLI tests names utf-8 (Windows would use cp1252). run_cli_text now encodes stdin and decodes the streams itself instead of using subprocess text mode, which uses the locale's encoding and writes CRLF to the child's stdin on Windows. Goldens are written with newline="\n" and compared as bytes, so a CRLF checkout fails instead of passing through universal newlines. The -o tests that create files named with a trailing dot or space are skipped on Windows, where Win32 strips them and the test would not exercise the name. No test expectation changed. --- tests/conftest.py | 21 ++++++++++++++------- tests/test_cli.py | 2 +- tests/test_cli_io.py | 13 ++++++++++--- tests/test_golden.py | 6 ++++-- tests/test_style.py | 14 +++++++------- tests/test_style_validation.py | 30 +++++++++++++++--------------- 6 files changed, 51 insertions(+), 35 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a77cca9..c01c435 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,11 +30,12 @@ def html(body: str, **settings: object) -> str: def run_cli(*args: str, stdin: bytes | str | None = None, cwd: Path | None = None, - narrow: bool = False) -> subprocess.CompletedProcess[bytes]: + narrow: bool = False, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[bytes]: """Run ``legaldown-render`` as a program on this checkout's sources. - *narrow* gives it a Windows-like console, where neither stream is UTF-8.""" - env = dict(os.environ, PYTHONPATH=str(SRC)) + *narrow* gives it a Windows-like console, where neither stream is UTF-8; *env* is + the environment it starts from (default: this process's).""" + env = dict(os.environ if env is None else env, PYTHONPATH=str(SRC)) if narrow: env.update(PYTHONIOENCODING="cp1252", PYTHONUTF8="0") data = stdin.encode("utf-8") if isinstance(stdin, str) else stdin @@ -43,10 +44,16 @@ def run_cli(*args: str, stdin: bytes | str | None = None, cwd: Path | None = Non def run_cli_text(*args: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]: - """Like :func:`run_cli`, with UTF-8 text for the streams.""" - env = dict(os.environ, PYTHONPATH=str(SRC), PYTHONUTF8="1") - return subprocess.run([sys.executable, "-m", "legaldown_render.cli", *args], input=stdin, - capture_output=True, text=True, env=env, check=False) + """Like :func:`run_cli`, with UTF-8 text for the streams. + + The streams are encoded and decoded here rather than by ``subprocess``'s text + mode, which would use the locale's encoding (cp1252 on Windows) and turn the + ``\\n`` of *stdin* into CRLF there.""" + env = dict(os.environ, PYTHONUTF8="1") + done = run_cli(*args, stdin=stdin, env=env) + return subprocess.CompletedProcess( + done.args, done.returncode, + done.stdout.decode("utf-8").replace("\r\n", "\n"), done.stderr.decode("utf-8").replace("\r\n", "\n")) def pytest_addoption(parser: pytest.Parser) -> None: diff --git a/tests/test_cli.py b/tests/test_cli.py index c3565d7..36b2ac3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -17,7 +17,7 @@ def test_render_to_file_infers_the_format(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: out = tmp_path / "out.txt" assert main([FEATURES, "-o", str(out)]) == 0 - assert out.read_text().startswith("Services Agreement") + assert out.read_text(encoding="utf-8").startswith("Services Agreement") err = capsys.readouterr().err assert "[ref-broken]" in err # A validator finding names its line, as the validator's CLI prints it. diff --git a/tests/test_cli_io.py b/tests/test_cli_io.py index 58da13c..a01d484 100644 --- a/tests/test_cli_io.py +++ b/tests/test_cli_io.py @@ -235,7 +235,13 @@ def test_unknown_output_extension_without_format_is_a_usage_error( assert list(tmp_path.iterdir()) == [] -@pytest.mark.parametrize("name", ["x.docx", "x.pdf.", ".md", "x.docx "]) +# Win32 drops trailing dots and spaces from a file name, so "x.pdf." names the file x.pdf there +# and the test would not exercise the name it says it does. +_TRAILING = pytest.mark.skipif(sys.platform == "win32", reason="Windows strips trailing dots and spaces from names") + + +@pytest.mark.parametrize("name", ["x.docx", pytest.param("x.pdf.", marks=_TRAILING), ".md", + pytest.param("x.docx ", marks=_TRAILING)]) def test_explicit_format_is_honoured_whatever_the_extension(tmp_path: Path, name: str) -> None: target = tmp_path / name assert main([FEATURES, "-q", "-f", "text", "-o", str(target)]) == 0 @@ -249,8 +255,9 @@ def test_output_without_an_extension_keeps_the_html_default(tmp_path: Path) -> N assert target.read_text(encoding="utf-8").startswith("") -@pytest.mark.parametrize("name", ["contract", "contract.v2", "report.2025-10-27", ".out", "out.", "notes.final", - "x.json", "x.."]) +@pytest.mark.parametrize("name", ["contract", "contract.v2", "report.2025-10-27", ".out", + pytest.param("out.", marks=_TRAILING), "notes.final", "x.json", + pytest.param("x..", marks=_TRAILING)]) def test_any_other_name_keeps_the_html_default(tmp_path: Path, name: str) -> None: target = tmp_path / name assert main([FEATURES, "-q", "-o", str(target)]) == 0 diff --git a/tests/test_golden.py b/tests/test_golden.py index 69a72f6..18fa394 100644 --- a/tests/test_golden.py +++ b/tests/test_golden.py @@ -25,7 +25,9 @@ def test_golden(document: str, style: str, output_format: str, update_golden: bo extension = "txt" if output_format == "text" else output_format golden = GOLDEN / f"{document}.{style}.{extension}" if update_golden or not golden.exists(): - golden.write_text(output, encoding="utf-8") + # LF on every platform: the CLI writes LF, and .gitattributes checks the goldens out as LF. + golden.write_text(output, encoding="utf-8", newline="\n") if not update_golden: pytest.fail(f"{golden.name} did not exist and was written; review it and run again") - assert output == golden.read_text(encoding="utf-8") + # Bytes, not read_text: universal newlines would hide a golden checked out with CRLF. + assert output == golden.read_bytes().decode("utf-8") diff --git a/tests/test_style.py b/tests/test_style.py index b3f0a09..7874bc2 100644 --- a/tests/test_style.py +++ b/tests/test_style.py @@ -42,7 +42,7 @@ def test_every_problem_is_reported_with_its_path() -> None: def test_file_extends_a_builtin(tmp_path: Path) -> None: path = tmp_path / "house.yaml" - path.write_text("version: 1\nextends: continental\nlocale: cs-CZ\ndefinitions: {style: small-caps}\n") + path.write_text("version: 1\nextends: continental\nlocale: cs-CZ\ndefinitions: {style: small-caps}\n", encoding="utf-8") style = load_style(path) assert style.name == "house" assert style.paragraphs.numbered is True # from continental @@ -51,11 +51,11 @@ def test_file_extends_a_builtin(tmp_path: Path) -> None: def test_relative_extends_and_cycles(tmp_path: Path) -> None: - (tmp_path / "base.yaml").write_text("version: 1\nnumbering: {scheme: none}\n") - (tmp_path / "child.yaml").write_text("version: 1\nextends: base.yaml\n") + (tmp_path / "base.yaml").write_text("version: 1\nnumbering: {scheme: none}\n", encoding="utf-8") + (tmp_path / "child.yaml").write_text("version: 1\nextends: base.yaml\n", encoding="utf-8") assert load_style(tmp_path / "child.yaml").numbering.scheme == "none" - (tmp_path / "a.yaml").write_text("version: 1\nextends: b.yaml\n") - (tmp_path / "b.yaml").write_text("version: 1\nextends: a.yaml\n") + (tmp_path / "a.yaml").write_text("version: 1\nextends: b.yaml\n", encoding="utf-8") + (tmp_path / "b.yaml").write_text("version: 1\nextends: a.yaml\n", encoding="utf-8") with pytest.raises(StyleError, match="circular"): load_style(tmp_path / "a.yaml") @@ -64,7 +64,7 @@ def test_unknown_builtin_and_bad_version(tmp_path: Path) -> None: with pytest.raises(StyleError, match="no built-in style named 'fancy'"): load_style("fancy") path = tmp_path / "future.yaml" - path.write_text("version: 2\n") + path.write_text("version: 2\n", encoding="utf-8") with pytest.raises(StyleError, match="style format 1"): load_style(path) @@ -72,7 +72,7 @@ def test_unknown_builtin_and_bad_version(tmp_path: Path) -> None: def test_dump_round_trips(tmp_path: Path) -> None: style = load_style("continental", overrides={"locale": "de-DE"}) path = tmp_path / "dumped.yaml" - path.write_text(dump_style(style)) + path.write_text(dump_style(style), encoding="utf-8") # The dump is flattened: the same style, without the record of what it extended. assert load_style(path) == replace(style, extends=None) diff --git a/tests/test_style_validation.py b/tests/test_style_validation.py index 73069da..405806d 100644 --- a/tests/test_style_validation.py +++ b/tests/test_style_validation.py @@ -53,21 +53,21 @@ def test_a_blank_with_a_visible_character_is_kept() -> None: def test_a_cycle_through_a_parent_directory_is_found(tmp_path: Path) -> None: (tmp_path / "dir").mkdir() - (tmp_path / "dir" / "a.yaml").write_text("version: 1\nextends: ../dir/a.yaml\n") + (tmp_path / "dir" / "a.yaml").write_text("version: 1\nextends: ../dir/a.yaml\n", encoding="utf-8") with pytest.raises(StyleError, match="circular"): load_style(tmp_path / "dir" / "a.yaml") def test_a_cycle_through_two_spellings_of_one_file_is_found(tmp_path: Path) -> None: (tmp_path / "x").mkdir() - (tmp_path / "x" / "a.yaml").write_text("version: 1\nextends: ../x/b.yaml\n") - (tmp_path / "x" / "b.yaml").write_text("version: 1\nextends: ./a.yaml\n") + (tmp_path / "x" / "a.yaml").write_text("version: 1\nextends: ../x/b.yaml\n", encoding="utf-8") + (tmp_path / "x" / "b.yaml").write_text("version: 1\nextends: ./a.yaml\n", encoding="utf-8") with pytest.raises(StyleError, match="circular"): load_style(tmp_path / "x" / "../x/a.yaml") def test_a_cycle_through_a_symlink_is_found(tmp_path: Path) -> None: - (tmp_path / "a.yaml").write_text("version: 1\nextends: link.yaml\n") + (tmp_path / "a.yaml").write_text("version: 1\nextends: link.yaml\n", encoding="utf-8") try: (tmp_path / "link.yaml").symlink_to(tmp_path / "a.yaml") except OSError: @@ -77,24 +77,24 @@ def test_a_cycle_through_a_symlink_is_found(tmp_path: Path) -> None: def test_the_same_base_used_twice_is_not_a_cycle(tmp_path: Path) -> None: - (tmp_path / "base.yaml").write_text("version: 1\nnumbering: {scheme: none}\n") - (tmp_path / "mid.yaml").write_text("version: 1\nextends: base.yaml\n") - (tmp_path / "top.yaml").write_text("version: 1\nextends: mid.yaml\n") + (tmp_path / "base.yaml").write_text("version: 1\nnumbering: {scheme: none}\n", encoding="utf-8") + (tmp_path / "mid.yaml").write_text("version: 1\nextends: base.yaml\n", encoding="utf-8") + (tmp_path / "top.yaml").write_text("version: 1\nextends: mid.yaml\n", encoding="utf-8") assert load_style(tmp_path / "top.yaml").numbering.scheme == "none" # -- A dump is self-contained ---------------------------------------------------- def test_a_dump_does_not_extend_anything(tmp_path: Path) -> None: - (tmp_path / "base.yaml").write_text("version: 1\nnumbering: {scheme: none}\n") - (tmp_path / "child.yaml").write_text("version: 1\nextends: base.yaml\nlocale: cs-CZ\n") + (tmp_path / "base.yaml").write_text("version: 1\nnumbering: {scheme: none}\n", encoding="utf-8") + (tmp_path / "child.yaml").write_text("version: 1\nextends: base.yaml\nlocale: cs-CZ\n", encoding="utf-8") style = load_style(tmp_path / "child.yaml") text = dump_style(style) assert "extends" not in text elsewhere = tmp_path / "elsewhere" / "deep" elsewhere.mkdir(parents=True) dumped = elsewhere / "dumped.yaml" - dumped.write_text(text) + dumped.write_text(text, encoding="utf-8") reloaded = load_style(dumped) assert reloaded.numbering.scheme == "none" assert reloaded.locale == "cs-CZ" @@ -105,21 +105,21 @@ def test_a_dump_does_not_extend_anything(tmp_path: Path) -> None: def test_a_dumped_builtin_reloads_equal(name: str, tmp_path: Path) -> None: style = load_style(name) path = tmp_path / "dump.yaml" - path.write_text(dump_style(style)) - assert "extends" not in path.read_text() + path.write_text(dump_style(style), encoding="utf-8") + assert "extends" not in path.read_text(encoding="utf-8") # The dump is flattened: it is the same style, minus the record of what it extended. assert load_style(path) == replace(style, extends=None) def test_print_style_of_a_relative_extends_reloads_from_anywhere(tmp_path: Path) -> None: - (tmp_path / "base.yaml").write_text("version: 1\nplaceholders: {blank: '[__]'}\n") - (tmp_path / "child.yaml").write_text("version: 1\nextends: base.yaml\n") + (tmp_path / "base.yaml").write_text("version: 1\nplaceholders: {blank: '[__]'}\n", encoding="utf-8") + (tmp_path / "child.yaml").write_text("version: 1\nextends: base.yaml\n", encoding="utf-8") done = run_cli_text("--print-style", "--style", str(tmp_path / "child.yaml")) assert done.returncode == 0, done.stderr assert "extends" not in done.stdout elsewhere = tmp_path / "other" elsewhere.mkdir() - (elsewhere / "dump.yaml").write_text(done.stdout) + (elsewhere / "dump.yaml").write_text(done.stdout, encoding="utf-8") assert load_style(elsewhere / "dump.yaml").placeholders.blank == "[__]" From c3f4ff56bc124376ed94d2924c8e6cd9bced6ab5 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:49:21 +0000 Subject: [PATCH 036/118] Test that plain-text table cells with pipes and backslashes read back unchanged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cells such as 'a | b', 'a \| b', 'ends\' written by the text writer are re-parsed with the validator's reader and compared with the cell's text (§13.8). --- tests/test_text_writer.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_text_writer.py b/tests/test_text_writer.py index 9ffef26..22e6292 100644 --- a/tests/test_text_writer.py +++ b/tests/test_text_writer.py @@ -3,6 +3,7 @@ or a quote), list indent, and the labels a style may blank.""" from __future__ import annotations +import legaldown import pytest from legaldown_render import render @@ -74,6 +75,17 @@ def test_a_pipe_in_a_cell_is_escaped() -> None: assert "| a | b |\n|---|---|\n| x \\| y | z |" in output +@pytest.mark.parametrize("cell", ["a | b", "a \\| b", "a \\\\| b", "a\\ b", "ends\\", "x\\|y"]) +def test_a_cell_with_pipes_and_backslashes_is_read_back_as_it_was(cell: str) -> None: + # The text writer's table is read by the validator's reader (the tool a + # LegalDown reader is expected to use), so a cell's plain text, pipes and + # backslashes included, must come back from it unchanged. + markup = cell.replace("\\", "\\\\").replace("|", "\\|") # the cell's text as GFM table markup + output = text(f"# A\n\n| H | K |\n|---|---|\n| {markup} | z |\n") + tables = [block for _, _, block in legaldown.parse(output).iter_blocks() if block.kind == "table"] + assert [table.rows for table in tables] == [[[cell, "z"]]] + + def test_alignment_survives_in_the_separator_row() -> None: output = text("# A\n\n| a | b | c | d |\n|:--|:-:|--:|---|\n| 1 | 2 | 3 | 4 |\n") assert "| a | b | c | d |\n|:---|:---:|---:|---|\n| 1 | 2 | 3 | 4 |" in output From f234f1f4f7a41cbf1a51eb2066e607dfb59e7527 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:49:21 +0000 Subject: [PATCH 037/118] Run the tests on Windows and gate publishing on CI ci.yml gains a windows-latest (Python 3.12) test job, is callable (workflow_call), and names the specification release once (LEGALDOWN_SPEC_REF). Its concurrency group includes github.workflow, which is the caller's name in a called workflow, so a publish run and push CI on one ref stay apart; only pull-request runs cancel. publish.yml calls ci.yml and build needs it, so nothing is built or uploaded unless CI passes on the commit. PUBLISHING.md describes this instead of recommending it. --- .github/PUBLISHING.md | 23 +++++++++++++++-------- .github/workflows/ci.yml | 27 ++++++++++++++++++++++----- .github/workflows/publish.yml | 14 ++++++++++++-- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/.github/PUBLISHING.md b/.github/PUBLISHING.md index e3fafa2..110cb07 100644 --- a/.github/PUBLISHING.md +++ b/.github/PUBLISHING.md @@ -54,9 +54,10 @@ version number can never be reused once published. 3. Commit, then publish a GitHub Release with the tag `vX.Y.Z`. Its notes are where this project records what changed in a release. -The workflow builds the sdist and wheel, runs `twine check --strict`, **verifies the tag matches -`__version__`** (a mismatch fails the run rather than publishing the wrong version), smoke-tests the -built wheel by installing it and invoking the CLI, and uploads to PyPI. +The workflow first runs the CI workflow on the tagged commit. Only if that passes does it build the +sdist and wheel, run `twine check --strict`, **verify the tag matches `__version__`** (a mismatch +fails the run rather than publishing the wrong version), smoke-test the built wheel by installing it +and invoking the CLI, and upload to PyPI. ## Dry run @@ -89,11 +90,17 @@ markdown-it-py) come from PyPI. overrides the installed validator with `main`, so it tests main regardless of the range). The tests fail on any `legaldown.LegaldownDeprecationWarning`, so a release never calls what the validator's next minor version removes. -- **Recommendation: gate the upload on the tests.** The `publish` workflow builds and uploads - without running the test suite, and `ci.yml` runs separately, so a release can reach PyPI from - a commit whose tests fail. Add the lint, test, and conformance jobs (or one called workflow) - as `needs:` of the `build` job in `publish.yml`, or require a passing CI run on the tagged - commit before the Release is published. This repository's workflows are not changed here. +- **The upload is gated on the tests.** `publish.yml` first calls [`ci.yml`](workflows/ci.yml) + (`on: workflow_call`) as the job `ci`, and `build` needs it, so the lint, test (Linux on Python + 3.11-3.13 and Windows on 3.12) and conformance jobs must pass on the commit being published + before anything is built, and `testpypi` and `pypi` need `build`. The `validator-main` job is + informational (`continue-on-error`) and never blocks a release. The caller grants the called + workflow `contents: read`; a called workflow cannot hold more permissions than its caller gives. + In the called workflow `github.workflow` is the caller's name, so its concurrency group + (`ci-${{ github.workflow }}-${{ github.ref }}`) is distinct from the push and pull-request runs + and from `publish.yml`'s own group. +- The conformance job checks out the specification release named once, in `LEGALDOWN_SPEC_REF` at the + top of `ci.yml`. Change it there when the renderer moves to a later specification. - The license is declared once, as the SPDX expression `license = "MIT"` in `pyproject.toml` (PEP 639), so the deprecated `License :: OSI Approved :: MIT License` classifier is not listed. - The package ships a `py.typed` marker, so type checkers use its annotations directly. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6567e7a..0aff2ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,13 +4,25 @@ on: push: branches: [main] pull_request: + # Called by publish.yml, so a release is only built once these jobs pass. + workflow_call: permissions: contents: read +env: + # The LegalDown specification release the conformance job renders; the one + # place to change it when the renderer moves to a later specification. + LEGALDOWN_SPEC_REF: v0.2 + +# In a called workflow, github.workflow is the *caller's* name ("Publish"), not +# "CI". That keeps this group apart from the push/pull-request CI runs on the +# same ref, so a dispatched or release publish neither cancels nor is cancelled +# by them, and it never collides with publish.yml's own group (two workflows in +# one group would wait for each other forever). concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: true + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: lint: @@ -26,12 +38,17 @@ jobs: - run: ruff check . test: - name: Tests (Python ${{ matrix.python-version }}) - runs-on: ubuntu-latest + name: Tests (${{ matrix.os }}, Python ${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-latest] python-version: ["3.11", "3.12", "3.13"] + include: + # Windows: path handling, console encodings and line endings differ. + - os: windows-latest + python-version: "3.12" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -50,7 +67,7 @@ jobs: uses: actions/checkout@v4 with: repository: ForLegalAI/LegalDown - ref: v0.2 + ref: ${{ env.LEGALDOWN_SPEC_REF }} path: .legaldown-spec - uses: actions/setup-python@v5 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 42f9a3d..9bee4c5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,8 +5,11 @@ name: Publish # setup that must exist before the first run. # # Triggers: -# - publishing a GitHub Release → build, verify, upload to PyPI -# - manual dispatch → build, verify, upload to TestPyPI +# - publishing a GitHub Release → CI, build, verify, upload to PyPI +# - manual dispatch → CI, build, verify, upload to TestPyPI +# +# The CI workflow (ci.yml) runs first on the commit being published, and nothing +# is built or uploaded unless it passes. on: release: @@ -28,8 +31,15 @@ concurrency: cancel-in-progress: false jobs: + ci: + name: CI + uses: ./.github/workflows/ci.yml + permissions: + contents: read + build: name: Build and verify distribution + needs: ci runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 308553622498d1c1d4c1e5f0d91d846d2e708996 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:54:16 +0000 Subject: [PATCH 038/118] Read the command's standard error in the console encoding in the escaping tests Only standard output is forced to UTF-8 with LF line endings. Standard error keeps the console's encoding and the platform's line ending, so on Windows the tests decoded cp1252 bytes as UTF-8 and counted the CR of each CRLF as a control character. They now decode with the preferred encoding, as the program does, and treat CRLF as one line end; a lone CR still fails. --- tests/test_escaping.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_escaping.py b/tests/test_escaping.py index 696bdc0..151ce9b 100644 --- a/tests/test_escaping.py +++ b/tests/test_escaping.py @@ -5,6 +5,7 @@ line of its own.""" from __future__ import annotations +import locale from pathlib import Path from conftest import FRONT, run_cli @@ -44,8 +45,15 @@ def test_a_control_character_in_a_link_is_escaped_in_the_message() -> None: # -- Standard error of the command ----------------------------------------------- +def _stderr(raw: bytes) -> str: + """Standard error as text. Only standard output is forced to UTF-8 with LF; standard + error keeps the console's encoding (backslash-escaping what it cannot show) and the + platform's line ending, so it is read the same way here and a CRLF is one line end.""" + return raw.decode(locale.getpreferredencoding(False), errors="replace").replace("\r\n", "\n") + + def _clean(errors: bytes) -> str: - text = errors.decode("utf-8") + text = _stderr(errors) assert not any((ord(c) < 0x20 and c != "\n") or 0x7F <= ord(c) < 0xA0 for c in text), text return text @@ -54,7 +62,7 @@ def test_the_cli_writes_no_control_character_from_a_document_to_standard_error(t source = tmp_path / "evil.lgd" source.write_text(FRONT + f"# A\n\n![x](<{EVIL}>) [y]()\n", encoding="utf-8") done = run_cli(str(source), "-f", "text") - err = done.stderr.decode("utf-8") + err = _stderr(done.stderr) assert "path-not-relative" in err and "render-link-refused" in err assert "\x1b" not in err and "\x07" not in err and "\u202e" not in err assert "\r" not in err From 1efc63a4b8f00c6547a5354ef57ffa3aca71fbcf Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:58:39 +0000 Subject: [PATCH 039/118] Record the source line of every block, list item and heading in the render tree Section, Paragraph, ListItem, Table, Quote, DraftingNote and CodeBlock carry the first line of what they were built from (Document.line_of; an item's marker line, by the pre-order number the builder already tracks). A block inside a quote has the quote's line, and one inside an item the item's, since the validator addresses nothing deeper. The field is not part of equality, so trees compare as before. tree.iter_located_* walk the inlines with their line. --- src/legaldown_render/build.py | 56 +++++++++++++++++++++++++---------- src/legaldown_render/tree.py | 47 +++++++++++++++++++++++++---- 2 files changed, 82 insertions(+), 21 deletions(-) diff --git a/src/legaldown_render/build.py b/src/legaldown_render/build.py index 2b9ecfc..5bc701a 100644 --- a/src/legaldown_render/build.py +++ b/src/legaldown_render/build.py @@ -197,6 +197,14 @@ def __init__(self, document: Document, result: ValidationResult) -> None: # Each text is lexed once in a render. self.lex = cache(lex) self.depth = 0 # quotes and list items the builder is inside + # Where the builder is, for the lines of the nodes it builds (§16.9): + # the top-level block (section index or None for the preamble, block + # index) when it is at one that a list's items can be addressed in + # (``Document.line_of``), and the line of the innermost unit it is in. + # A block in a quote, or past the first in an item, is not addressable; + # it takes the line of the quote or the item. + self.at: tuple[int | None, int] | None = None + self.line: int | None = None nonce = secrets.token_hex(8) # A sentinel with its wrappers; only a complete one matches, so a # wrapper character in the source is never taken. @@ -368,6 +376,9 @@ def blocks(self, blocks: list[ModelBlock], section: int | None, *, markers: bool if block.kind == "html": continue # never emitted: a comment (§8.6) or raw HTML (§8.7) placed = self.placed.get((section, index), []) if markers else [] + if markers: + self.at = (section, index) + self.line = self.document.line_of(section, index) built = self.block(block, placed, top_level=top_level) if built is not None: out.append(built) @@ -380,14 +391,14 @@ def block(self, block: ModelBlock, placed: list[PlacedMarker], *, top_level: boo inlines = self.text(_paragraph_source(block)) if not inlines and not identifier: return None # only a comment - return Paragraph(inlines, anchor_id=identifier, condition=condition, top_level=top_level) + return Paragraph(inlines, anchor_id=identifier, condition=condition, top_level=top_level, line=self.line) case "unordered_list" | "ordered_list": return self.list(block, placed) case "quote": return self.quote(block) case "code": code = code_content(block) - return CodeBlock(code.text, code.info) + return CodeBlock(code.text, code.info, line=self.line) case "table": width = len(block.headers) # Every row as wide as the header: short rows padded, extra @@ -397,6 +408,7 @@ def block(self, block: ModelBlock, placed: list[PlacedMarker], *, top_level: boo header=tuple(self.text(cell) for cell in block.headers), rows=tuple(tuple(self.text(cell) for cell in row) for row in rows), align=tuple(block.align), + line=self.line, ) case "rule": return Rule() @@ -404,7 +416,7 @@ def block(self, block: ModelBlock, placed: list[PlacedMarker], *, top_level: boo # A heading in a list item or a quote: not a section (§4.1), # so it has no number and no anchor. It shows as a bold line. inlines = self.text(block.text) - return Paragraph((Strong(inlines),)) if inlines else None + return Paragraph((Strong(inlines),), line=self.line) if inlines else None raise InternalError(f"unexpected block kind '{block.kind}' in the validator's model") def list(self, block: ModelBlock, placed: list[PlacedMarker]) -> List | None: @@ -430,28 +442,36 @@ def _list(self, block: ModelBlock, markers: dict[int | None, PlacedMarker], numb try: items: list[ListItem] = [] for item in list_items(block): - marker = markers.get(next(numbers)) + number = next(numbers) + marker = markers.get(number) children = list(item.blocks) identifier = condition = "" if marker is not None: identifier, condition = marker.identifier, marker.condition children[0] = replace(children[0], text=_cut_marker(children[0].text, marker)) blocks: list[Block] = [] - for child in children: - if child.kind in ("ordered_list", "unordered_list"): - nested = self._list(child, markers, numbers) - if nested is not None: - blocks.append(nested) - elif child.kind == "html": - continue # never emitted (§8.6, §8.7) - elif (built := self.block(child, [], top_level=False)) is not None: - blocks.append(built) + outer = self.line + # The item's marker line; in a quote an item is not addressable. + item_line = self.document.line_of(*self.at, number) if self.at else outer + self.line = item_line + try: + for child in children: + if child.kind in ("ordered_list", "unordered_list"): + nested = self._list(child, markers, numbers) + if nested is not None: + blocks.append(nested) + elif child.kind == "html": + continue # never emitted (§8.6, §8.7) + elif (built := self.block(child, [], top_level=False)) is not None: + blocks.append(built) + finally: + self.line = outer # Only a comment: the item is removed. Defensive: an item a # marker belongs to (an anchor or a condition) never vanishes. if item.blocks and all(child.kind == "html" for child in item.blocks) \ and not identifier and not condition: continue - items.append(ListItem(blocks=tuple(blocks), anchor_id=identifier, condition=condition)) + items.append(ListItem(blocks=tuple(blocks), anchor_id=identifier, condition=condition, line=item_line)) finally: self.depth -= 1 return List(ordered=block.kind == "ordered_list", items=tuple(items)) if items else None @@ -477,14 +497,17 @@ def quote(self, block: ModelBlock) -> Block | None: # The quotes and list items the quote is in, before entering it. read = drafting_note_blocks if drafting else quote_blocks children = read(block, depth=self.depth) + line, at = self.line, self.at self.depth += 1 + self.at = None # what is in a quote has the quote's line try: blocks = self.blocks(children, None, markers=False) finally: self.depth -= 1 + self.at = at if drafting: - return DraftingNote(blocks) - return Quote(blocks) if blocks else None + return DraftingNote(blocks, line=line) + return Quote(blocks, line=line) if blocks else None # -- document ------------------------------------------------------------- @@ -497,6 +520,7 @@ def tree(self) -> RenderTree: identifier=self.result.index.sections[index].identifier, condition=section.condition, blocks=self.blocks(section.blocks, index), + line=document.line_of(index), ) for index, section in enumerate(document.sections) ) diff --git a/src/legaldown_render/tree.py b/src/legaldown_render/tree.py index 7e88575..a98051f 100644 --- a/src/legaldown_render/tree.py +++ b/src/legaldown_render/tree.py @@ -186,6 +186,12 @@ class Paragraph: label: str | None = None anchor: str | None = None condition_label: str | None = None + #: Its first line in the document's file, counted from 1 (§16.9), which the + #: renderer's diagnostics name; None when the document has no source to + #: name a line of. Not part of equality: the same content is equal + #: wherever it stood. A block inside a list item or a quote has the item's + #: or the quote's line: the validator addresses nothing deeper. + line: int | None = field(default=None, compare=False) @dataclass(frozen=True, slots=True) @@ -196,6 +202,8 @@ class ListItem: label: str | None = None anchor: str | None = None condition_label: str | None = None + #: The item's marker line (see Paragraph.line). + line: int | None = field(default=None, compare=False) @dataclass(frozen=True, slots=True) @@ -209,6 +217,8 @@ class List: @dataclass(frozen=True, slots=True) class Quote: blocks: tuple[Block, ...] + #: Its first line (see Paragraph.line); the blocks inside it have it too. + line: int | None = field(default=None, compare=False) @dataclass(frozen=True, slots=True) @@ -217,12 +227,16 @@ class DraftingNote: blocks: tuple[Block, ...] label: str | None = None + #: Its first line (see Paragraph.line); the blocks inside it have it too. + line: int | None = field(default=None, compare=False) @dataclass(frozen=True, slots=True) class CodeBlock: text: str info: str = "" + #: Its first line (see Paragraph.line). + line: int | None = field(default=None, compare=False) @dataclass(frozen=True, slots=True) @@ -231,6 +245,8 @@ class Table: rows: tuple[tuple[tuple[Inline, ...], ...], ...] #: Per column: "left", "center", "right", or "". align: tuple[str, ...] + #: Its first line (see Paragraph.line); every cell has it too. + line: int | None = field(default=None, compare=False) @dataclass(frozen=True, slots=True) @@ -263,6 +279,8 @@ class Section: designation: str | None = None anchor: str | None = None condition_label: str | None = None + #: The heading's first line (see Paragraph.line). + line: int | None = field(default=None, compare=False) @dataclass(frozen=True, slots=True) @@ -355,22 +373,41 @@ def iter_inlines(inlines: tuple[Inline, ...]) -> Iterator[Inline]: def iter_block_inlines(block: Block) -> Iterator[Inline]: """Every inline inside *block*, depth first.""" + for inline, _ in iter_located_block_inlines(block): + yield inline + + +def iter_located_block_inlines(block: Block, line: int | None = None) -> Iterator[tuple[Inline, int | None]]: + """Every inline inside *block*, depth first, with the line of the unit + holding it: that of the nearest enclosing node that has one, else *line*.""" + line = getattr(block, "line", None) or line match block: case Paragraph(inlines=inlines): - yield from iter_inlines(inlines) + yield from ((inline, line) for inline in iter_inlines(inlines)) case List(items=items): for item in items: for child in item.blocks: - yield from iter_block_inlines(child) + yield from iter_located_block_inlines(child, item.line or line) case Quote(blocks=blocks) | DraftingNote(blocks=blocks): for child in blocks: - yield from iter_block_inlines(child) + yield from iter_located_block_inlines(child, line) case Table(header=header, rows=rows): for cell in header: - yield from iter_inlines(cell) + yield from ((inline, line) for inline in iter_inlines(cell)) for row in rows: for cell in row: - yield from iter_inlines(cell) + yield from ((inline, line) for inline in iter_inlines(cell)) + + +def iter_located_body_inlines(tree: RenderTree) -> Iterator[tuple[Inline, int | None]]: + """Every inline in the document's body (the preamble, then each section's + heading and blocks), with its line (:func:`iter_located_block_inlines`).""" + for block in tree.preamble: + yield from iter_located_block_inlines(block) + for section in tree.sections: + yield from ((inline, section.line) for inline in iter_inlines(section.title)) + for block in section.blocks: + yield from iter_located_block_inlines(block, section.line) def iter_tree_inlines(tree: RenderTree) -> Iterator[Inline]: From e4bbc666bcbd8f2399d53e3d690c7ddba98cb5a4 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:58:39 +0000 Subject: [PATCH 040/118] Name the file and line of every diagnostic the renderer reports render-ref-ambiguous, ref-not-enumerated, render-not-processed, render-link-refused, path-not-relative, render-term-limit, render-term-too-long and render-locale-fallback now carry the line of the block, list item or heading being resolved (Resolver.line), or of the frontmatter key they are about (diagnostics.key_line, one helper over source_map.key that the file references will reuse; None without a source map). A term's text is worked out at its definition whichever reference asks first. Granularity is the block's: a {{ref:}} or {{term:}} paragraph is lifted by the validator, so no directive level without validator#93. Diagnostic.file is RenderOptions.filename (new, default empty), set by render_file and the command from the file's name, as the validator's Document.filename is; the validator's diagnostics get it too. A repeated finding (same rule, level and message) is reported once at its first line, in the resolver and in api.render. The renderer's path-not-relative is left out when the validator reported the same path at the same line or at none (it used to match the quoted path alone), in diagnostics.covered_by. tests/test_links.py: the mocked validator report now sits at the image's own line (6, it was 5), since a report at another line is another finding. Docs: CONFORMANCE.md line numbers and path-not-relative limitations, architecture.md, README.md. --- CONFORMANCE.md | 32 +- README.md | 6 +- docs/architecture.md | 30 +- src/legaldown_render/api.py | 35 +- src/legaldown_render/cli.py | 3 +- src/legaldown_render/diagnostics.py | 52 +++ src/legaldown_render/resolve/resolver.py | 285 +++++++++----- tests/test_diagnostic_lines.py | 458 +++++++++++++++++++++++ tests/test_links.py | 5 +- 9 files changed, 784 insertions(+), 122 deletions(-) create mode 100644 src/legaldown_render/diagnostics.py create mode 100644 tests/test_diagnostic_lines.py diff --git a/CONFORMANCE.md b/CONFORMANCE.md index cc22c1b..54105ff 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -52,13 +52,33 @@ It is checked against the specification's own examples and fixtures corpus (`tes - **Assembly reads no other files.** A template with include fragments, LegalDown attachment files, or `translations` is refused when rendered with answers, as §17.6 requires below Full. A template with Errors is refused too, since §15.7.2 defines no output for one. -- **Line numbers.** The validator's diagnostics name their line, and the CLI prints it. The - renderer's own (`render-ref-ambiguous`, `ref-not-enumerated`, `render-not-processed`, - `render-link-refused`) do not - yet ([validator#93](https://github.com/ForLegalAI/legaldown-validator/issues/93)). +- **Line numbers (§16.9).** Every diagnostic names its line, the validator's and the renderer's + own (`render-ref-ambiguous`, `ref-not-enumerated`, `render-not-processed`, + `render-link-refused`, `path-not-relative`, `render-term-limit`, `render-term-too-long`, + `render-locale-fallback`), and the CLI prints it as `file:line:`. The renderer's line is the + **first line of the block it is in**, as `Document.line_of` gives it, whatever line the + directive itself is on: a paragraph's, a table's header row, a quote's first `>`, a code + block's opening fence, a heading's (a section title); for a list item, the line of **its marker** + (nested items too), also for the blocks the item holds; a block inside a quote has the quote's + line, since the validator addresses nothing deeper. A `{{ref:}}` or `{{term:}}` paragraph is + lifted by the validator and written back, so the renderer cannot place the directive in it; the + validator's own diagnostics for a directive can therefore name a later line of a paragraph than + the renderer's do, until [validator#93](https://github.com/ForLegalAI/legaldown-validator/issues/93). + Findings about frontmatter are at the line of their key: the `language` key for + `render-locale-fallback` of the document's language, and the `title`, `subtitle`, `version`, + `effective_date`, `sides[].label` and `sides[].parties[]` key (a party's details, name and + signatories are all at the party's key) or `attachments[].title` for a directive in that value + (also where the body shows it, as `{{attach:}}` or `{{party:}}` do). A finding in a term's text + is at the term's definition, whichever reference asks for it first; `render-term-limit` is at the + definition being worked out when the limit was reached. The same finding (rule, level and + message) is reported once, at its first line: the same refused link, or the same non-relative + image path, on two lines is one diagnostic, at the first of them. A document with no source to + name a line of (built from a dict) reports no line. With `answers`, the lines are those of the + assembled document (§15.8), not of the template file. - **`path-not-relative` for images.** The renderer reports the Core rule `path-not-relative` - (§2.3, §16.2) for an image path, because the validator (0.4) does not check image paths; it - leaves its own report out when the validator names the same path. The Error has no line number. + (§2.3, §16.2) for an image path, because the validator (0.4) does not check image paths. It + leaves its own report out when the validator names the same path at the same line (or no line), + and keeps it for a path the validator did not name, or the same path at another line. The containment rule (`path-outside-root`) is not checked: no document root is configured. - **Nesting limit.** The validator reads lists 64 levels deep and quotes 16; deeper, an item's or quote's content is its text. The renderer follows it, and refuses (`DocumentError`) a document diff --git a/README.md b/README.md index d4e9262..b7154d6 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,9 @@ legaldown-render --print-style --style continental # every setting, `--locale` sets how dates, money, and durations are formatted. The words the renderer generates (such as "Represented by" or "Signatures") follow the document's `language` (§13.7), whatever the locale. -Diagnostics go to stderr, one per line with its rule id. The exit code is 0 when the document +Diagnostics go to stderr, one per line, as `file:line: level: [rule] message` (the line is that of +the block or heading they are in, or of the frontmatter key they are about; see CONFORMANCE.md). +The exit code is 0 when the document rendered, even if it has errors (they show as markers), 1 when rendering was refused (strict mode, or a template that cannot be assembled with its answers) or the document or answers file cannot be read (a document that is not UTF-8 included) or the output file cannot be written, 2 for an invalid @@ -101,6 +103,8 @@ with open("contract.html", "w", encoding="utf-8", newline="\n") as output: ``` `render()` also accepts a `RenderOptions`. Pass `answers={...}` to render a template filled in. +Each diagnostic has its `line` and its `file`: `render_file()` and the command name the file (its +name, without the directory); for text given to `render()`, pass `filename=`. Problems in the document never raise. Only an invalid style (`StyleError`), an unknown output format (`OptionsError`), an unreadable document (`DocumentError`), strict mode, or a template that cannot be assembled with its answers (`RenderRefused`) do. All of them are `RenderError`s, and diff --git a/docs/architecture.md b/docs/architecture.md index 2c0016a..81118df 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -83,6 +83,7 @@ never decides a structural or LegalDown question itself: | Whether a quote is a drafting note, and its content | The validator's `is_drafting_note()` and `drafting_note_blocks()` | | Which list item a marker belongs to | `PlacedMarker.item`: the item's number among the list's items, in pre-order | | A lifted definition, `{{ref:}}` or `{{term:}}` block's source | The validator's `render_block()` | +| The line of a heading, a top-level block or a list item (§16.9) | `Document.line_of()` | These are all the validator's public API: the model and the result from `legaldown`, and, for assembly, `parse_template` (`Template.form().assemble()`), `AssemblyError`, and the answers @@ -190,6 +191,7 @@ clamped to 1–5, as the validator numbers them. src/legaldown_render/ ├── __init__.py public API, __version__, SPEC_VERSION, CONFORMANCE_LEVEL ├── api.py render(), render_file(), RenderOptions, RenderResult +├── diagnostics.py frontmatter key lines for diagnostics, and when a diagnostic repeats another ├── cli.py the legaldown-render command ├── errors.py RenderError, StyleError, OptionsError, DocumentError, RenderRefused, InternalError ├── tree.py render tree nodes (source and resolved inlines, blocks, document) @@ -240,9 +242,30 @@ Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can ev `ref-not-enumerated` and `path-not-relative` (an Error for an image path that is not relative, §2.3, reported here because validator 0.4 does not check image paths), specification rule ids, plus renderer-specific ids prefixed `render-`: `render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, or 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), a `{{term:}}` shows its id), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's -diagnostics carry their line (§16.9), and the CLI prints it as `file:line:`. The renderer's own -have none yet: they are about a directive within a block, and the validator makes only a block's -line public (`Document.line_of`, U2). +diagnostics carry their line (§16.9), and so do the renderer's own, and the CLI prints it as +`file:line:`. + +**Lines.** The builder asks `Document.line_of` for the line of every section heading, top-level +block and list item (an item's pre-order number, which the builder already tracks for markers) and +stores it as `line` on `Section`, `Paragraph`, `ListItem`, `Table`, `Quote`, `DraftingNote` and +`CodeBlock` (not part of equality, so trees compare as before). A block inside a quote has the +quote's line; one inside a list item, the item's. The resolver tracks the line of what it is +resolving (`Resolver.line`, set with `_at`) and every renderer diagnostic carries it, so the +granularity is the block's (or the item's, or the heading's): a `{{ref:}}` or `{{term:}}` paragraph +is lifted by the validator and written back, so the renderer cannot place a directive within a +block without the validator's internal `Locator`, which would be a second bridge to its internals +(ADR 0007); it waits for +[validator#93](https://github.com/ForLegalAI/legaldown-validator/issues/93). A finding about +frontmatter is at its key (`diagnostics.key_line`, which wraps the public `source_map.key` and +returns None without a source map; one helper for every frontmatter reference). A term's text is +worked out at the term's definition (`Resolver.term_lines`, recorded by `_survey`) whichever +reference asks first. `Diagnostic.file` is `RenderOptions.filename`, which `render_file` and the CLI +set to the file name as the validator does (`Document.filename`, without the directory). + +**Repeats.** `Resolver._warn` and `api.render` leave out a diagnostic whose rule, level and message +a reported one has already, wherever it is (`diagnostics.repeats`), so a repeated finding is +reported at its first line only. The renderer's `path-not-relative` is left out when the validator +reported the same path at the same line, or at none (`diagnostics.covered_by`). ## Security @@ -312,6 +335,7 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_build.py` | Building the tree: comment-only items and quotes, emphasis next to a directive, spacing where an inline comment was dropped | | `tests/test_terms.py` | What `{{term:}}` shows, references inside terms, terms that name themselves or each other, the cap on a term text | | `tests/test_terms_budget.py` | Reuse of worked-out term texts, the work budget and depth limit for terms that name each other. Bounds the work by counting resolved directives, never by timing | +| `tests/test_diagnostic_lines.py` | The line and file of every renderer diagnostic: in each kind of block (paragraph, list item, nested item, table, quote, heading, preamble) and at frontmatter keys, the repeats left out, the tree nodes' lines, and `file:line:` on the command line | | `tests/test_links.py` | Which images are shown and which links are links: non-relative image paths, the link scheme allow-list, empty URLs and empty link text, and how a refusal is reported | | `tests/test_text_writer.py` | The plain-text layout: attachment separators, tables, quotes, list items and indent, blanked labels | | `tests/test_html.py` | The HTML writer's print stylesheet for attachments | diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index 1ece903..9d14c66 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -13,7 +13,7 @@ from __future__ import annotations from collections.abc import Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any @@ -21,6 +21,7 @@ from legaldown import AssemblyError, Diagnostic, Document, ValidationResult, parse, parse_template, validate from .build import build_tree, normalize_source +from .diagnostics import key_line, repeats from .errors import DocumentError, OptionsError, RenderRefused, StyleError from .resolve import Formatter, parse_locale, resolve_tree from .style import Style, load_style @@ -57,6 +58,14 @@ class RenderOptions: answers: Mapping[str, Any] | None = None #: HTML only: a complete page (True) or just the ``
`` (False). standalone: bool = True + #: The document's file name, which every diagnostic of the result names as + #: its ``file`` (§16.9), the validator's and the renderer's alike, as + #: ``Document.filename`` does for the validator: the name without its + #: directory. ``render_file`` and the command line set it from the path + #: they read; for text given to :func:`render` it is empty unless set. + #: With ``answers``, the lines of the diagnostics are those of the + #: assembled document (§15.8), not of the template file. + filename: str = "" @dataclass(frozen=True, slots=True) @@ -105,15 +114,17 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any source = normalize_source(source) assembled: list[Diagnostic] = [] if options.answers is not None: - source, assembled = _assemble(source, options.answers) - document = _parse(source) + source, assembled = _assemble(source, options.answers, options.filename) + document = _parse(source, options.filename) result = validate(document, final=options.final) diagnostics = assembled + list(result.diagnostics) if options.strict and any(d.level == "error" for d in diagnostics): raise RenderRefused(diagnostics) tree, extra = _resolve(document, result, style) - diagnostics += [d for d in extra if d not in diagnostics] + # The renderer's findings that the validator's or the assembly's already + # name (the same rule, level and message), wherever they are, are not repeated. + diagnostics += [d for d in extra if not repeats(d, diagnostics)] if options.strict and any(d.level == "error" for d in diagnostics): raise RenderRefused(diagnostics) @@ -133,17 +144,22 @@ def render_file(path: str | Path, options: RenderOptions | None = None, /, **set except UnicodeDecodeError as error: raise DocumentError(f"The document cannot be read: {path} is not valid UTF-8 " f"(byte 0x{data[error.start]:02x} at offset {error.start})") from None + if options is None: + options = RenderOptions(**settings) + settings = {} + if not options.filename and not settings: + options = replace(options, filename=Path(path).name) return render(source, options, **settings) -def _assemble(source: str, answers: Mapping[str, Any]) -> tuple[str, list[Diagnostic]]: +def _assemble(source: str, answers: Mapping[str, Any], filename: str = "") -> tuple[str, list[Diagnostic]]: """*source* assembled with *answers* (§15.7), and the assembly's Warnings (such as ``answer-unknown``). Raises RenderRefused when it cannot be assembled: the template has Errors, for which §15.7.2 defines no output, or the answers do (``answer-invalid``, ``answer-missing``), or the template needs files a renderer below Full does not read (includes, LegalDown attachments, translations; §17.6).""" - findings = validate(_parse(source)).diagnostics + findings = validate(_parse(source, filename)).diagnostics errors = sum(1 for d in findings if d.level == "error") if errors: raise RenderRefused(findings, f"Assembly refused: the template has {errors} error(s), " @@ -161,9 +177,9 @@ def _assemble(source: str, answers: Mapping[str, Any]) -> tuple[str, list[Diagno return result.output, list(result.diagnostics) -def _parse(source: str) -> Document: +def _parse(source: str, filename: str = "") -> Document: try: - return parse(source) + return parse(source, filename=filename) except (ValueError, yaml.YAMLError) as error: raise DocumentError(f"The document cannot be read: {error}") from error @@ -185,6 +201,9 @@ def _resolve(document: Document, result: ValidationResult, style: Style) -> tupl message=f"No formatting locale is set and the document language '{tree.language}' is not a " f"known locale; values are formatted for 'en'. Set one with --locale or the style's " f"'locale'.", + # Document-level: at the frontmatter's `language` key. + line=key_line(document, "language"), + file=document.filename, )) locale = parse_locale("en") assert locale is not None diff --git a/src/legaldown_render/cli.py b/src/legaldown_render/cli.py index 993e704..a24a419 100644 --- a/src/legaldown_render/cli.py +++ b/src/legaldown_render/cli.py @@ -192,7 +192,8 @@ def main(argv: list[str] | None = None) -> int: return 1 options = RenderOptions(format=output_format, style=args.style, overrides=overrides, - strict=args.strict, final=args.final, answers=answers, standalone=not args.fragment) + strict=args.strict, final=args.final, answers=answers, standalone=not args.fragment, + filename="" if args.input == "-" else Path(args.input).name) try: result = render(source, options) except StyleError as error: diff --git a/src/legaldown_render/diagnostics.py b/src/legaldown_render/diagnostics.py new file mode 100644 index 0000000..916e002 --- /dev/null +++ b/src/legaldown_render/diagnostics.py @@ -0,0 +1,52 @@ +"""Where the renderer's own diagnostics point, and when one repeats another +(§16.9: a diagnostic names its file and the line of what it reports). + +The validator's diagnostics carry a line, and the renderer's own do too, as +far as the validator makes lines public: the first line of the block (or the +list item, or the heading) the finding is in, ``Document.line_of``; and, for a +finding about a frontmatter value, the line of its key (:func:`key_line`). +Directives inside a block have no public line of their own (a ``{{ref:}}`` or +``{{term:}}`` paragraph is lifted by the validator and written back, so even +its text has moved), so a block's several findings share its line. +""" +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +from legaldown import Diagnostic, Document + + +def key_line(document: Document, *path: Any) -> int | None: + """The line in the document's file of the frontmatter value at *path* + (``key_line(document, "attachments", 0, "file")``), or of the nearest key + above it that is written, as ``SourceMap.key`` says; None when the document + has no source to name a line of (it was built from a dict, or changed since + it was parsed) or has no frontmatter.""" + source_map = document.source_map + if source_map is None or not source_map.fits(document): + return None + line = source_map.key(*path) + return line if isinstance(line, int) else None + + +def repeats(found: Diagnostic, known: Iterable[Diagnostic]) -> bool: + """True if *known* holds *found*'s finding already: the same rule, level + and message, wherever it was. The same finding in two places is reported + once, at the first (a link to the same refused URL on two lines is one + ``render-link-refused``, at the line of the first).""" + return any(d.rule == found.rule and d.level == found.level and d.message == found.message for d in known) + + +def covered_by(found: Diagnostic, path: str, validator: Iterable[Diagnostic]) -> bool: + """True if the validator already reported *found*, a ``path-not-relative`` + about *path*: its diagnostic of the rule quotes the path (``''``) and + is at the same line, or has none. A path the validator did not name is + still the renderer's to report, even where the validator reported another + at the same line, and so is the same path where it stands elsewhere.""" + quoted = f"'{path}'" + return any( + d.rule == found.rule and quoted in d.message + and (d.line is None or found.line is None or d.line == found.line) + for d in validator + ) diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 6954803..b94e67b 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -22,16 +22,23 @@ ``{{term:}}`` may show, ``render-term-limit`` for terms that refer to each other too deeply or too often, and ``path-not-relative`` for an image whose path is not relative (§2.3), which the validator does not check yet. + +Every one of them names the file and the line it is at (§16.9): the first line +of the block, list item or heading being resolved, or the line of a frontmatter +key (``diagnostics.key_line``), tracked in :attr:`Resolver.line`. A finding in +a term's text is at the term's definition, since that is where the text is +worked out whatever reference asked for it first. """ from __future__ import annotations import unicodedata -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import dataclass, replace from typing import Any, NamedTuple from urllib.parse import unquote -from legaldown import Diagnostic, Document, ValidationResult +from legaldown import Diagnostic, Document, Party, ValidationResult from legaldown.grammar import ( ALWAYS, DECISION_QUESTION_TYPES, @@ -50,6 +57,7 @@ from legaldown.syntax import Directive from ..build import plain_inlines +from ..diagnostics import covered_by, key_line, repeats from ..style import Style, effective_labels from ..style.model import LevelFormat from ..tree import ( @@ -83,6 +91,7 @@ TermRef, Text, Value, + iter_located_body_inlines, iter_tree_inlines, map_tree_inlines, plain_text, @@ -183,6 +192,12 @@ def __init__( self.labels = effective_labels(style.labels, tree.language) self.textual = style.numbering.scheme == "none" self.diagnostics: list[Diagnostic] = [] + # The line (§16.9) of what is being resolved, which a diagnostic + # names: see _at. + self.line: int | None = None + # A definition id -> the line of its first {{def:}} in the body (see + # _survey), where its term's text is worked out. + self.term_lines: dict[str, int | None] = {} self.targets: dict[str, _Target] = {} self.used_anchors: set[str] = set() # For render-ref-ambiguous: every designation a numbered unit — a @@ -234,20 +249,16 @@ def run(self) -> RenderTree: # In document order, so that the first of several definitions of a # term is the one that keeps its anchor. preamble = tuple(self._resolve_block(block) for block in preamble) - sections = tuple( - replace( - section, - title=resolve(section.title), - blocks=tuple(self._resolve_block(block) for block in section.blocks), - condition_label=self._condition(section.condition), - ) - for section in sections - ) - attachments = tuple(replace(a, title=resolve(a.title)) for a in attachments) + sections = tuple(self._resolve_section(section) for section in sections) + attachments = tuple(self._resolve_attachment(index, a) for index, a in enumerate(attachments)) + with self._at(key_line(self.document, "title")): + title = resolve(self.tree.title) + with self._at(key_line(self.document, "subtitle")): + subtitle = resolve(self.tree.subtitle) resolved = replace( self.tree, - title=resolve(self.tree.title), - subtitle=resolve(self.tree.subtitle), + title=title, + subtitle=subtitle, locale=self.formatter.tag, preamble=preamble, sections=sections, @@ -273,6 +284,19 @@ def run(self) -> RenderTree: # definition may live in an attachment or an amended original. return map_tree_inlines(resolved, self._settle_term_target) + def _resolve_section(self, section: Section) -> Section: + with self._at(section.line): + return replace( + section, + title=self._resolve_inlines(section.title), + blocks=tuple(self._resolve_block(block) for block in section.blocks), + condition_label=self._condition(section.condition), + ) + + def _resolve_attachment(self, index: int, attachment: AttachmentPart) -> AttachmentPart: + with self._at(key_line(self.document, "attachments", index, "title")): + return replace(attachment, title=self._resolve_inlines(attachment.title)) + def _contents( self, sections: tuple[Section, ...], attachments: tuple[AttachmentPart, ...], ) -> tuple[ContentsEntry, ...]: @@ -321,6 +345,24 @@ def _survey(self) -> None: if directive.name == "placeholder" and directive.positional and not directive.malformed: types.setdefault(directive.positional, set()).add(self._placeholder_type(directive)) self.inconsistent_placeholders = {pid for pid, found in types.items() if len(found) > 1} + for inline, line in iter_located_body_inlines(self.tree): + if isinstance(inline, DefinitionSource): + self.term_lines.setdefault(self._definition_id(inline), line) + + @contextmanager + def _at(self, line: int | None) -> Iterator[None]: + """Name *line* in the diagnostics reported meanwhile; None for a + finding with no line (a document built without a source).""" + outer, self.line = self.line, line + try: + yield + finally: + self.line = outer + + def _here(self, line: int | None) -> AbstractContextManager[None]: + """:meth:`_at` for a unit that may have no line of its own, which + then has the line of the unit around it.""" + return self._at(self.line if line is None else line) # -- anchors -------------------------------------------------------------- @@ -365,7 +407,8 @@ def _number_sections(self) -> list[Section]: self.section_presences.append(presences[level]) parts = [int(part) for part in indexed.number.split(".")] if self.textual: - label, designation = None, self._title_text(section.title) + with self._at(section.line): + label, designation = None, self._title_text(section.title) else: designation = "" for position, counter in enumerate(parts): @@ -470,12 +513,14 @@ def _structure(self, block: Block, *, section: Section | None, depth: int, prese parent=section.designation if section else "", presence=presence, quoted=quoted) case Quote(blocks=blocks): - return Quote(tuple(self._structure(child, section=section, depth=depth, presence=presence, - quoted=True) for child in blocks)) + return replace(block, blocks=tuple(self._structure(child, section=section, depth=depth, + presence=presence, quoted=True) + for child in blocks)) case DraftingNote(blocks=blocks): - return DraftingNote(tuple(self._structure(child, section=section, depth=depth, presence=presence, - quoted=True) for child in blocks), - label=self.labels.drafting_note) + return replace(block, blocks=tuple(self._structure(child, section=section, depth=depth, + presence=presence, quoted=True) + for child in blocks), + label=self.labels.drafting_note) case _: return block @@ -534,28 +579,33 @@ def _structure_list(self, block: List, *, section: Section | None, depth: int, p # -- blocks --------------------------------------------------------------- def _resolve_block(self, block: Block) -> Block: + with self._here(getattr(block, "line", None)): + return self._resolved_block(block) + + def _resolved_block(self, block: Block) -> Block: resolve = self._resolve_inlines match block: case Paragraph(): return replace(block, inlines=resolve(block.inlines), condition_label=self._condition(block.condition)) case List(items=items): - return replace(block, items=tuple( - replace(item, blocks=tuple(self._resolve_block(child) for child in item.blocks), - condition_label=self._condition(item.condition)) - for item in items - )) + return replace(block, items=tuple(self._resolve_item(item) for item in items)) case Quote(blocks=blocks): - return Quote(tuple(self._resolve_block(child) for child in blocks)) + return replace(block, blocks=tuple(self._resolve_block(child) for child in blocks)) case DraftingNote(blocks=blocks): return replace(block, blocks=tuple(self._resolve_block(child) for child in blocks)) - case Table(header=header, rows=rows, align=align): - return Table(tuple(resolve(cell) for cell in header), - tuple(tuple(resolve(cell) for cell in row) for row in rows), align) + case Table(header=header, rows=rows): + return replace(block, header=tuple(resolve(cell) for cell in header), + rows=tuple(tuple(resolve(cell) for cell in row) for row in rows)) case CodeBlock(): return block case _: return block + def _resolve_item(self, item: ListItem) -> ListItem: + with self._here(item.line): + return replace(item, blocks=tuple(self._resolve_block(child) for child in item.blocks), + condition_label=self._condition(item.condition)) + def _condition(self, condition: str) -> str | None: if not condition: return None @@ -720,22 +770,26 @@ def _term_text(self, definition_id: str) -> str | None: known = self._targets_known def work() -> str: - if not known: - text = self._title_text(source.children) - else: - anchors = set(self.used_anchors) - try: - text = plain_text(self._resolve_inlines(source.children)) - finally: - self.used_anchors = anchors - if len(text) > _MAX_TERM_TEXT: - self._warn( - "render-term-too-long", - f"The term of '{printable(definition_id)}' is longer than {_MAX_TERM_TEXT} characters once its " - f"references are resolved; '{{{{term: {printable(definition_id)}}}}}' shows the definition id.", - ) - return definition_id - return text + # Worked out where the term is defined, whichever reference asks + # first, so what it reports is at the definition. + with self._here(self.term_lines.get(definition_id)): + if not known: + text = self._title_text(source.children) + else: + anchors = set(self.used_anchors) + try: + text = plain_text(self._resolve_inlines(source.children)) + finally: + self.used_anchors = anchors + if len(text) > _MAX_TERM_TEXT: + self._warn( + "render-term-too-long", + f"The term of '{printable(definition_id)}' is longer than {_MAX_TERM_TEXT} characters once " + f"its references are resolved; '{{{{term: {printable(definition_id)}}}}}' shows the " + f"definition id.", + ) + return definition_id + return text return self._while_open(self.term_texts if known else self.term_texts_early, definition_id, work) @@ -832,7 +886,17 @@ def _named(self, kind: str, name: str, lookup: dict[str, str], directive: Direct return FailureMarker(f"[INVALID {kind.upper()}: {name}]") if name not in lookup: return FailureMarker(f"[UNKNOWN {kind.upper()}: {name}]") - return Value(kind, directive.params.get("label") or self._plain_value(lookup[name])) + key = self._side_key(name) if kind == "side" else self._party_key(name) + return Value(kind, directive.params.get("label") or self._plain_value(lookup[name], *key)) + + def _side_key(self, name: str) -> tuple[Any, ...]: + """The frontmatter path of side *name*, or ``()``.""" + return next((("sides", i) for i, side in enumerate(self.metadata.sides) if side.name == name), ()) + + def _party_key(self, name: str) -> tuple[Any, ...]: + """The frontmatter path of party *name*, or ``()``.""" + return next((("sides", i, "parties", j) for i, side in enumerate(self.metadata.sides) + for j, party in enumerate(side.parties) if party.name == name), ()) def _duration(self, directive: Directive, value: str) -> Inline: unit = directive.params.get("unit") @@ -898,7 +962,9 @@ def _attach(self, directive: Directive, attachment_id: str) -> Inline: else: declared = next((a for a in self.metadata.attachments if a.id == attachment_id), None) href = declared.file if declared and declared.file else None - return Value("attach", directive.params.get("label") or self._plain_value(title), href) + key = next((("attachments", i, "title") for i, a in enumerate(self.metadata.attachments) + if a.id == attachment_id), ()) + return Value("attach", directive.params.get("label") or self._plain_value(title, *key), href) def _choose(self, directive: Directive, question_id: str) -> Inline: invalid = FailureMarker(f"[INVALID CHOICE: {question_id}]" if question_id else "[INVALID CHOICE]") @@ -931,12 +997,15 @@ def _def(self, directive: Directive, identifier: str) -> Inline | None: # -- frontmatter ---------------------------------------------------------- - def _plain_value(self, text: str) -> str: + def _plain_value(self, text: str, *key: Any) -> str: """A frontmatter string with its directives (placeholders, §3.10) - resolved, as plain text.""" + resolved, as plain text. *key* is the path of the frontmatter value in + the document, when the text is shown elsewhere (a title or a name + in the body): what it reports is at that key, not where it is shown.""" if "{{" not in text: return text - return plain_text(self._resolve_inlines(plain_inlines(text))) + with self._at(key_line(self.document, *key)) if key else nullcontext(): + return plain_text(self._resolve_inlines(plain_inlines(text))) def _frontmatter(self, text: str) -> tuple[Inline, ...]: return self._resolve_inlines(plain_inlines(text)) @@ -945,9 +1014,11 @@ def _header(self) -> tuple[tuple[str, tuple[Inline, ...]], ...]: rows: list[tuple[str, tuple[Inline, ...]]] = [] metadata = self.metadata if self.style.title_block.effective_date and metadata.effective_date: - rows.append((self.labels.effective_date or "", self._date_or_text(metadata.effective_date))) + with self._at(key_line(self.document, "effective_date")): + rows.append((self.labels.effective_date or "", self._date_or_text(metadata.effective_date))) if self.style.title_block.version and metadata.version: - rows.append((self.labels.version or "", self._frontmatter(metadata.version))) + with self._at(key_line(self.document, "version")): + rows.append((self.labels.version or "", self._frontmatter(metadata.version))) return tuple(rows) def _date_or_text(self, value: str) -> tuple[Inline, ...]: @@ -957,31 +1028,35 @@ def _date_or_text(self, value: str) -> tuple[Inline, ...]: def _sides(self) -> tuple[SideInfo, ...]: sides: list[SideInfo] = [] - for side in self.metadata.sides: - parties: list[PartyInfo] = [] - for party in side.parties: - details: list[tuple[str, tuple[Inline, ...]]] = [] - if party.identification_number: - details.append((self.labels.identification_number or "", self._frontmatter(party.identification_number))) - if party.date_of_birth: - details.append((self.labels.date_of_birth or "", self._date_or_text(party.date_of_birth))) - if party.address: - details.append((self.labels.address or "", self._frontmatter(party.address))) - for custom in party.custom_fields: - details.append((custom.label, self._frontmatter(custom.value))) - for rep in party.representatives: - name, title = self._frontmatter(rep.name), self._frontmatter(rep.title) - value = name + (Text(", "),) + title if name and title else name or title - if value: - details.append((self.labels.represented_by or "", value)) - parties.append(PartyInfo( - name=self._frontmatter(party.legal_name or party.label or party.name), - details=tuple(details), - )) + for side_index, side in enumerate(self.metadata.sides): + parties = tuple(self._party_info(side_index, party_index, party) + for party_index, party in enumerate(side.parties)) label = self.result.index.side_lookup.get(side.name) or side.label or side.name - sides.append(SideInfo(self._frontmatter(label), tuple(parties))) + with self._at(key_line(self.document, "sides", side_index, "label")): + sides.append(SideInfo(self._frontmatter(label), parties)) return tuple(sides) + def _party_info(self, side_index: int, party_index: int, party: Party) -> PartyInfo: + with self._at(key_line(self.document, "sides", side_index, "parties", party_index)): + details: list[tuple[str, tuple[Inline, ...]]] = [] + if party.identification_number: + details.append((self.labels.identification_number or "", self._frontmatter(party.identification_number))) + if party.date_of_birth: + details.append((self.labels.date_of_birth or "", self._date_or_text(party.date_of_birth))) + if party.address: + details.append((self.labels.address or "", self._frontmatter(party.address))) + for custom in party.custom_fields: + details.append((custom.label, self._frontmatter(custom.value))) + for rep in party.representatives: + name, title = self._frontmatter(rep.name), self._frontmatter(rep.title) + value = name + (Text(", "),) + title if name and title else name or title + if value: + details.append((self.labels.represented_by or "", value)) + return PartyInfo( + name=self._frontmatter(party.legal_name or party.label or party.name), + details=tuple(details), + ) + def _attachments(self) -> tuple[AttachmentPart, ...]: """Attachment placeholders (§13.8), in declared order. At the Rendering level the files are not read, so each shows its title.""" @@ -1005,24 +1080,34 @@ def _signatures(self) -> tuple[SignatureParty, ...]: if not (self.style.signatures.enabled and self.metadata.include_signatures): return () blocks: list[SignatureParty] = [] - for side in self.metadata.sides: - label = self._frontmatter(self.result.index.side_lookup.get(side.name) or side.label or side.name) - for party in side.parties: - # Where signature blocks are generated, legal_name MUST appear (§3.6). - blocks.append(SignatureParty( - side=label, - name=self._frontmatter(party.legal_name or party.label or party.name), - signatories=tuple( - (self._frontmatter(rep.name), self._frontmatter(rep.title)) for rep in party.representatives - ), - )) + for side_index, side in enumerate(self.metadata.sides): + with self._at(key_line(self.document, "sides", side_index, "label")): + label = self._frontmatter(self.result.index.side_lookup.get(side.name) or side.label or side.name) + for party_index, party in enumerate(side.parties): + with self._at(key_line(self.document, "sides", side_index, "parties", party_index)): + # Where signature blocks are generated, legal_name MUST appear (§3.6). + blocks.append(SignatureParty( + side=label, + name=self._frontmatter(party.legal_name or party.label or party.name), + signatories=tuple( + (self._frontmatter(rep.name), self._frontmatter(rep.title)) + for rep in party.representatives + ), + )) return tuple(blocks) # -- diagnostics ---------------------------------------------------------- + def _found(self, rule: str, level: str, message: str) -> Diagnostic: + """A diagnostic of the renderer's, at the line being resolved, in the + document's file (§16.9).""" + return Diagnostic(rule=rule, level=level, message=message, line=self.line, file=self.document.filename) + def _warn(self, rule: str, message: str) -> None: - diagnostic = Diagnostic(rule=rule, level="warning", message=message) - if diagnostic not in self.diagnostics: + """Report a Warning, unless the same one (rule and message) is + reported already: wherever it recurs, it stands at its first line.""" + diagnostic = self._found(rule, "warning", message) + if not repeats(diagnostic, self.diagnostics): self.diagnostics.append(diagnostic) def _link_refused(self, href: str) -> None: @@ -1040,25 +1125,23 @@ def _link_refused(self, href: str) -> None: def _not_relative(self, path: str) -> None: """Report an image *path* that is not relative as the Core Error ``path-not-relative`` (§2.3, §16.2), unless the validator has - reported it. The renderer has no line for an image, so a path the - validator named is the one it has covered. markdown-it has - percent-encoded the path ("C:\\x" reads "C:%5Cx"), so it is decoded - to name the path as written.""" + reported it (:func:`~legaldown_render.diagnostics.covered_by`: its + report of the path, at this line). markdown-it has percent-encoded + the path ("C:\\x" reads "C:%5Cx"), so it is decoded to name the path + as written.""" path = unquote(path) + shown = printable(path) + diagnostic = self._found( + "path-not-relative", "error", + f"Image path '{shown}' is not relative (§2.3): file references are relative paths with no " + f"URI scheme. Its alt text is shown instead.", + ) # Assumption: the validator quotes the path in its message as '', # as written. Matching the quoted form keeps a path that is a prefix of # another ("/a" and "/ab") from being counted as already reported. - if any(d.rule == "path-not-relative" and f"'{path}'" in d.message for d in self.result.diagnostics): + if covered_by(diagnostic, path, self.result.diagnostics) or repeats(diagnostic, self.diagnostics): return - path = printable(path) - diagnostic = Diagnostic( - rule="path-not-relative", - level="error", - message=f"Image path '{path}' is not relative (§2.3): file references are relative paths with no " - f"URI scheme. Its alt text is shown instead.", - ) - if diagnostic not in self.diagnostics: - self.diagnostics.append(diagnostic) + self.diagnostics.append(diagnostic) _HANDLERS: dict[str, Callable[[Resolver, Directive, str], Inline | None]] = { diff --git a/tests/test_diagnostic_lines.py b/tests/test_diagnostic_lines.py new file mode 100644 index 0000000..ff7bb51 --- /dev/null +++ b/tests/test_diagnostic_lines.py @@ -0,0 +1,458 @@ +"""Where the renderer's own diagnostics point (§16.9): the file, and the line +of the block, list item or heading they are in, or of the frontmatter key they +are about; how a repeat is left out; how the command line prints them. + +The validator names the first line of a block for anything in it, and so do +these: a paragraph's findings are all at its first line, a table's at its +header row, a quote's at its first ``>``, a list item's at its marker. What +the validator makes public of a directive's own line is not used (see +``legaldown_render.diagnostics``).""" +from __future__ import annotations + +from pathlib import Path + +import pytest +from conftest import run_cli_text + +import legaldown_render.api as api +from legaldown_render import RenderOptions, render, render_file +from legaldown_render.cli import main +from legaldown_render.diagnostics import key_line + +HEAD = "---\ntitle: T\n---\n" + + +def lines_of(result, rule: str) -> list[int | None]: # type: ignore[no-untyped-def] + return [d.line for d in result.diagnostics if d.rule == rule] + + +def at(source: str, needle: str) -> int: + """The line (from 1) of the first line of *source* holding *needle*.""" + return next(number for number, line in enumerate(source.split("\n"), 1) if needle in line) + + +# One finding that the renderer reports, whatever the block it stands in: an +# include is not processed (§17.5), a link's scheme is refused (§8.7), an image +# path is not relative (§2.3). +TRIGGERS = { + "render-not-processed": "{{include: part.lgd}}", + "render-link-refused": "[the link](ms-msdt:x)", + "path-not-relative": "![the logo](/etc/passwd)", +} + +#: Containers for a trigger: the text of a section with the trigger at MARK, and the word that +#: marks the line its finding is expected at. +CONTAINERS = { + "paragraph": ("# A\n\nFirst line FIRST\nsecond line\nthird MARK line\n", "FIRST"), + "heading": ("# Heading MARK FIRST\n\nText.\n", "FIRST"), + "list item": ("# A\n\n- one\n- two FIRST MARK\n- three\n", "FIRST"), + "item, second line": ("# A\n\n- one\n- two FIRST\n continued MARK\n- three\n", "FIRST"), + "nested item": ("# A\n\n- one\n - nested FIRST MARK\n - more\n- three\n", "FIRST"), + "nested item, third level": ("# A\n\n- a\n - b\n - c FIRST MARK\n- d\n", "FIRST"), + "item, second block": ("# A\n\n- one FIRST\n\n another MARK paragraph\n", "FIRST"), + "ordered item": ("# A\n\n1. one\n2. two FIRST MARK\n", "FIRST"), + "table": ("# A\n\n| a FIRST | b |\n|---|---|\n| x | y |\n| z | MARK |\n", "FIRST"), + "quote": ("# A\n\n> quoted FIRST\n> second MARK\n", "FIRST"), + "quote, second paragraph": ("# A\n\n> quoted FIRST\n>\n> second MARK\n", "FIRST"), + "list in a quote": ("# A\n\n> intro FIRST\n>\n> - item MARK\n> - item\n", "FIRST"), + "quote in a list item": ("# A\n\n- one\n- two FIRST\n\n > quoted MARK\n", "FIRST"), + "drafting note": ("# A\n\n> [!DRAFTING] FIRST\n> note MARK\n", "FIRST"), + "definition paragraph": ('# A\n\n"Thing" {{def: thing}} is FIRST MARK\n', "FIRST"), +} + + +@pytest.mark.parametrize("rule", sorted(TRIGGERS)) +@pytest.mark.parametrize("container", sorted(CONTAINERS)) +def test_a_finding_is_at_the_first_line_of_its_block(container: str, rule: str) -> None: + body, first = CONTAINERS[container] + source = HEAD + "\nBefore.\n\n" + body.replace("MARK", TRIGGERS[rule]) + expected = at(source, first) + for fmt in ("html", "text"): + result = render(source, format=fmt) + assert lines_of(result, rule) == [expected], (container, fmt, result.diagnostics) + + +def test_a_finding_in_the_preamble_is_at_its_block() -> None: + source = HEAD + "\nFirst preamble paragraph.\n\nSecond one\nhas a {{include: x.lgd}}.\n\n# A\n\nBody.\n" + assert lines_of(render(source), "render-not-processed") == [at(source, "Second one")] + + +def test_a_finding_in_a_list_in_the_preamble_is_at_its_item() -> None: + source = HEAD + "\n- one\n- two {{include: x.lgd}}\n\n# A\n" + assert lines_of(render(source), "render-not-processed") == [at(source, "- two")] + + +def test_a_finding_in_a_section_title_is_at_its_heading() -> None: + source = HEAD + "\nText.\n\n## Second {{include: x.lgd}}\n\nBody.\n" + assert lines_of(render(source), "render-not-processed") == [at(source, "## Second")] + + +def test_a_finding_in_a_heading_under_the_none_scheme_is_reported_once_at_its_heading() -> None: + # The `none` scheme reads each heading as text before the references are known. + source = HEAD + "\n# One\n\n## Second {{include: x.lgd}}\n\nSee {{ref: second}}.\n" + result = render(source, overrides={"numbering.scheme": "none"}) + assert lines_of(result, "render-not-processed") == [at(source, "## Second")] + + +def test_a_heading_inside_a_list_item_has_its_items_line() -> None: + source = HEAD + "\n# A\n\n- one\n- ## two {{include: x.lgd}}\n" + assert lines_of(render(source), "render-not-processed") == [at(source, "- ## two")] + + +def test_each_block_has_its_own_line() -> None: + source = (HEAD + "\n# A\n\nOne [x](ms-msdt:one).\n\n- item [y](ms-msdt:two)\n\n> quote [z](ms-msdt:three)\n\n" + "| h [w](ms-msdt:four) |\n|---|\n| c |\n\n# B\n\nLast [v](ms-msdt:five).\n") + expected = [at(source, f"ms-msdt:{word}") for word in ("one", "two", "three", "four", "five")] + assert lines_of(render(source), "render-link-refused") == expected + + +def test_a_table_is_at_its_header_row_whichever_cell_holds_the_finding() -> None: + source = HEAD + "\n# A\n\n| a | b |\n|---|---|\n| x | y |\n| z | {{include: x.lgd}} |\n" + assert lines_of(render(source), "render-not-processed") == [at(source, "| a | b |")] + + +# -- ref-not-enumerated, render-ref-ambiguous ------------------------------------------------ + + +def test_ref_not_enumerated_is_at_its_block() -> None: + source = (HEAD + "\n# A {#a}\n\n- first {#x}\n- second\n\nSee\nthe item {{ref: x}}.\n") + result = render(source, format="text", overrides={"enumeration.enabled": False}) + assert lines_of(result, "ref-not-enumerated") == [at(source, "See")] + + +def test_ref_not_enumerated_in_a_list_item_is_at_the_item() -> None: + source = (HEAD + "\n# A {#a}\n\n- first {#x}\n- second\n see {{ref: x}}\n- third\n") + result = render(source, format="text", overrides={"enumeration.enabled": False}) + assert lines_of(result, "ref-not-enumerated") == [at(source, "- second")] + + +def test_render_ref_ambiguous_is_at_its_block() -> None: + source = HEAD + "\n# A\n\n- a {#x}\n\nPara.\n\n- b {#y}\n\n> Quoted\n> See {{ref: x}}.\n" + assert lines_of(render(source, format="text"), "render-ref-ambiguous") == [at(source, "> Quoted")] + + +def test_ref_findings_are_the_block_not_the_directive() -> None: + # A {{ref:}} paragraph is lifted by the validator and written back, so the + # renderer names its block's first line, not the line the directive is on. + source = HEAD + "\n# A\n\n- a {#x}\n\nPara.\n\n- b {#y}\n\nFirst line\nof a paragraph, See {{ref: x}}.\n" + assert lines_of(render(source, format="text"), "render-ref-ambiguous") == [at(source, "First line")] + + +# -- render-term-limit, render-term-too-long ---------------------------------------------- + + +def _chain(n: int) -> str: + defs = [f'"T{i} {{{{term: t{i + 1}}}}} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + defs.append(f'"T{n}" {{{{def: t{n}}}}}') + return HEAD + "\n# About {{term: t0}}\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n" + + +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +def test_a_term_that_is_too_long_is_reported_at_its_definition(scheme: str) -> None: + # The text is worked out at the definition (for a heading under `none`, too), + # whichever reference asks for it first: t0 is used in the title and in the body. + source = _chain(14) + result = render(source, format="text", overrides={"numbering.scheme": scheme}) + found = [d for d in result.diagnostics if d.rule == "render-term-too-long"] + assert found + for d in found: + identifier = d.message.split("'")[1] + assert d.line == at(source, "{{def: " + identifier + "}}"), d + + +def _tangle(k: int) -> str: + defs = [f'"T{i} {" ".join(f"{{{{term: t{j}}}}}" for j in range(k))}" {{{{def: t{i}}}}}' for i in range(k)] + return HEAD + "\n# A\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n" + + +def test_the_term_limit_is_reported_at_a_line_of_a_definition() -> None: + source = _tangle(12) + result = render(source, format="text") + [line] = lines_of(result, "render-term-limit") + assert line is not None + assert source.split("\n")[line - 1].startswith('"T') + + +# -- render-locale-fallback -------------------------------------------------------------- + + +def test_a_duration_with_no_pattern_in_the_locale_is_at_its_block(monkeypatch: pytest.MonkeyPatch) -> None: + from legaldown_render.resolve import values + + real = values._unit_patterns + monkeypatch.setattr(values, "_unit_patterns", + lambda locale, unit, length: {} if locale.language == "cs" else real(locale, unit, length)) + source = ("---\ntitle: T\nlanguage: cs\n---\n\n# Lhůty\n\nPrvní\n\n- druhá {{duration: 5, unit=D}}\n\n" + "Třetí {{duration: 6, unit=D}} a {{duration: 2, unit=MO}}.\n") + result = render(source, format="text") + # One per unit, at the first place it is used. + assert [(d.line, d.message.split("'")[1]) for d in result.diagnostics if d.rule == "render-locale-fallback"] \ + == [(at(source, "druhá"), "duration-day"), (at(source, "Třetí"), "duration-month")] + + +def test_an_unknown_document_language_is_at_the_language_key() -> None: + source = "---\ntitle: T\nlanguage: xx\nversion: '1'\n---\n\n# A\n" + assert lines_of(render(source), "render-locale-fallback") == [at(source, "language:")] + + +def test_the_language_key_line_is_the_keys_not_the_frontmatters_first() -> None: + source = "---\ntitle: T\nparties: []\nlanguage: xx\n---\n\n# A\n" + assert lines_of(render(source), "render-locale-fallback") == [4] + + +# -- Frontmatter keys --------------------------------------------------------------------- + + +def test_a_finding_in_a_frontmatter_value_is_at_its_key() -> None: + source = ('---\ntitle: "T {{include: t.lgd}}"\nsubtitle: "Sub {{include: s.lgd}}"\n' + 'version: "v {{include: v.lgd}}"\neffective_date: "e {{include: e.lgd}}"\n---\n\n# A\n') + result = render(source, overrides={"title_block.version": True, "title_block.effective_date": True}) + found = {d.message.split("{{include: ")[1][:5]: d.line for d in result.diagnostics + if d.rule == "render-not-processed"} + assert found == {"t.lgd": 2, "s.lgd": 3, "v.lgd": 4, "e.lgd": 5} + + +def test_a_finding_in_an_attachment_title_is_at_its_title_key() -> None: + source = ("---\ntitle: T\nattachments:\n - id: first\n title: One\n - id: second\n" + " title: \"Two {{include: x.lgd}}\"\n---\n\n# A\n\nSee {{attach: first}} and {{attach: second}}.\n") + assert lines_of(render(source), "render-not-processed") == [at(source, "title: \"Two")] + + +SIDES = """--- +title: T +document_type: contract +sides: + - name: seller + label: "Seller {{include: l.lgd}}" + parties: + - name: acme + legal_name: Acme + type: legal_entity + address: "Street {{include: a.lgd}}" + - name: buyer + label: Buyer + parties: + - name: bob + legal_name: Bob + type: natural_person + address: "Road {{include: b.lgd}}" +--- + +# A +""" + + +def test_a_finding_in_a_party_or_a_side_is_at_its_key() -> None: + result = render(SIDES) + found = {d.message.split("{{include: ")[1][:5]: d.line for d in result.diagnostics + if d.rule == "render-not-processed"} + # A party's detail is at the party; a side's label, at its label key. + assert found == {"l.lgd": at(SIDES, 'label: "Seller'), "a.lgd": at(SIDES, "name: acme"), + "b.lgd": at(SIDES, "name: bob")} + + +def test_signature_blocks_are_not_a_second_report() -> None: + result = render(SIDES, overrides={"signatures.enabled": True}) + assert len(lines_of(result, "render-not-processed")) == 3 + + +def test_key_line_is_general_and_none_without_a_source() -> None: + from legaldown import Document, parse + + source = ("---\ntitle: T\namends:\n file: ../a.lgd\n title: X\nsupersedes: ../b.lgd\n" + "translations:\n cs: ../cs.lgd\nattachments:\n - id: a\n title: A\n file: ../c.lgd\n---\n\n# A\n") + document = parse(source) + assert key_line(document, "amends", "file") == at(source, "file: ../a.lgd") + assert key_line(document, "supersedes") == at(source, "supersedes:") + assert key_line(document, "translations", "cs") == at(source, "cs:") + assert key_line(document, "attachments", 0, "file") == at(source, "file: ../c.lgd") + # A key that is not written is the nearest one above it that is. + assert key_line(document, "amends", "date") == at(source, "amends:") + assert key_line(parse("# A\n"), "language") is None + assert key_line(Document(), "title") is None + document.sections.clear() # changed since it was parsed: no longer fits its source + assert key_line(document, "title") is None + + +def test_a_document_without_a_source_has_no_lines() -> None: + # The tree built from a dict has no source map; its findings name no line. + from legaldown import Document + + document = Document.from_dict({"metadata": {"title": "T"}, "sections": [ + {"title": "A", "level": 1, "blocks": [{"kind": "paragraph", "text": "x {{include: p.lgd}}"}]}]}) + from legaldown import validate + + from legaldown_render.build import build_tree + from legaldown_render.resolve import Formatter, parse_locale, resolve_tree + from legaldown_render.style import load_style + + result = validate(document) + tree = build_tree(document, result) + assert tree.sections[0].line is None + style = load_style(None) + _, found = resolve_tree(tree, document, result, style, Formatter(parse_locale("en"), style.values)) + assert [d.line for d in found if d.rule == "render-not-processed"] == [None] + + +# -- The nodes ----------------------------------------------------------------------------- + + +def test_the_tree_nodes_carry_their_lines() -> None: + source = HEAD + ("\nPreamble.\n\n# A\n\nPara.\n\n- one\n - nested\n- two\n\n> quote\n\n> [!DRAFTING] note\n\n" + "```\ncode\n```\n\n| a |\n|---|\n| b |\n") + tree = render(source).tree + assert tree.preamble[0].line == at(source, "Preamble") + section = tree.sections[0] + assert section.line == at(source, "# A") + paragraph, bullets, quote, note, code, table = section.blocks + assert [b.line for b in (paragraph, quote, note, code, table)] == [ + at(source, "Para."), at(source, "> quote"), at(source, "[!DRAFTING]"), at(source, "```"), at(source, "| a |")] + assert [item.line for item in bullets.items] == [at(source, "- one"), at(source, "- two")] + assert bullets.items[0].blocks[1].items[0].line == at(source, "- nested") + + +def test_a_line_is_not_part_of_equality() -> None: + from legaldown_render.tree import Paragraph, Table + + assert Paragraph((), line=3) == Paragraph((), line=9) + assert Table((), (), (), line=1) == Table((), (), ()) + + +def test_lines_survive_resolving() -> None: + source = HEAD + "\n# A\n\n> quote {{ref: a}}\n\n| h |\n|---|\n| {{ref: a}} |\n\n> [!DRAFTING] note\n" + section = render(source).tree.sections[0] + assert [block.line for block in section.blocks] == [at(source, "> quote"), at(source, "| h |"), + at(source, "[!DRAFTING]")] + assert section.line == at(source, "# A") + + +# -- Repeats ------------------------------------------------------------------------------- + + +def test_the_same_finding_on_two_lines_is_reported_once_at_the_first() -> None: + source = HEAD + "\n# A\n\nOne ![a](/x.png).\n\nTwo ![b](/x.png).\n\nThree [l](ms-msdt:x) [m](ms-msdt:x)\n\nFour [l](ms-msdt:x)\n" + result = render(source) + assert lines_of(result, "path-not-relative") == [at(source, "One")] + assert lines_of(result, "render-link-refused") == [at(source, "Three")] + + +def test_different_findings_of_one_rule_each_have_their_line() -> None: + source = HEAD + "\n# A\n\nOne ![a](/x.png).\n\nTwo ![b](/y.png).\n" + assert lines_of(render(source), "path-not-relative") == [at(source, "One"), at(source, "Two")] + + +def _with_validator_report(monkeypatch: pytest.MonkeyPatch, line: int | None, path: str = "https://x/p.png") -> None: + from legaldown import Diagnostic + from legaldown import validate as real_validate + + def validate(document, **kwargs): # type: ignore[no-untyped-def] + result = real_validate(document, **kwargs) + result.diagnostics.append(Diagnostic(rule="path-not-relative", level="error", line=line, + message=f"Image path '{path}' is not relative (§2.3).")) + return result + + monkeypatch.setattr(api, "validate", validate) + + +def test_the_validators_report_of_a_path_at_its_line_covers_the_renderers(monkeypatch: pytest.MonkeyPatch) -> None: + source = HEAD + "\n# A\n\n![a](https://x/p.png) ![b](/other.png)\n" + _with_validator_report(monkeypatch, at(source, "![a]")) + errors = [d for d in render(source).diagnostics if d.rule == "path-not-relative"] + # The validator's for the path it named, ours for the other one at the same line. + assert [(d.line, "/other.png" in d.message) for d in errors] == [(at(source, "![a]"), False), + (at(source, "![a]"), True)] + + +def test_a_report_with_no_line_covers_the_path_wherever_it_is(monkeypatch: pytest.MonkeyPatch) -> None: + source = HEAD + "\n# A\n\n![a](https://x/p.png)\n" + _with_validator_report(monkeypatch, None) + assert len([d for d in render(source).diagnostics if d.rule == "path-not-relative"]) == 1 + + +def test_the_same_path_at_another_line_is_not_covered(monkeypatch: pytest.MonkeyPatch) -> None: + source = HEAD + "\n# A\n\n![a](https://x/p.png)\n\n\nOther ![b](https://x/p.png)\n" + _with_validator_report(monkeypatch, at(source, "![a]")) + errors = [d for d in render(source).diagnostics if d.rule == "path-not-relative"] + # The validator's line, and ours at the first image that is not the line it named. + assert [d.line for d in errors] == [at(source, "![a]"), at(source, "Other")] + + +# -- The file ------------------------------------------------------------------------------ + +REFS = HEAD + "\n# A\n\nSee {{include: x.lgd}} and ![i](/etc/passwd).\n\n
raw
\n" + + +def test_findings_name_the_file_when_it_is_given() -> None: + result = render(REFS, filename="contract.lgd") + assert {d.rule for d in result.diagnostics} >= {"render-not-processed", "path-not-relative", "raw-html"} + assert {d.file for d in result.diagnostics} == {"contract.lgd"} + assert {d.file for d in render(REFS).diagnostics} == {""} + + +def test_render_file_names_the_file(tmp_path: Path) -> None: + path = tmp_path / "deal.lgd" + path.write_text(REFS, encoding="utf-8") + found = {(d.rule, d.file) for d in render_file(path).diagnostics} + assert ("render-not-processed", "deal.lgd") in found + assert ("raw-html", "deal.lgd") in found # the validator's, too + # Options given: the file name is added to them; one already named stays. + assert {d.file for d in render_file(path, RenderOptions(format="text")).diagnostics} == {"deal.lgd"} + named = render_file(path, RenderOptions(filename="other.lgd")) + assert {d.file for d in named.diagnostics} == {"other.lgd"} + assert {d.file for d in render_file(path, format="text").diagnostics} == {"deal.lgd"} + with pytest.raises(TypeError): + render_file(path, RenderOptions(), format="text") + + +def test_the_locale_fallback_names_the_file() -> None: + source = "---\ntitle: T\nlanguage: xx\n---\n\n# A\n" + [found] = [d for d in render(source, filename="x.lgd").diagnostics if d.rule == "render-locale-fallback"] + assert (found.file, found.line) == ("x.lgd", 3) + + +def test_assembly_names_the_file_too() -> None: + template = ("---\ntitle: T\nquestions:\n q:\n type: text\n prompt: Q\n---\n\n# A\n\n" + "Hello {{placeholder: q}} {{include: x.lgd}}\n") + result = render(template, answers={"q": "world"}, filename="t.lgd") + assert {d.file for d in result.diagnostics} == {"t.lgd"} + # The lines are those of the assembled document (the answers are filled in). + [found] = [d for d in result.diagnostics if d.rule == "render-not-processed"] + assert found.line is not None + + +# -- The command line ------------------------------------------------------------------------ + + +def test_the_command_line_prints_the_file_and_line(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + path = tmp_path / "deal.lgd" + path.write_text(REFS, encoding="utf-8") + assert main([str(path), "-f", "text", "-o", str(tmp_path / "out.txt")]) == 0 + lines = capsys.readouterr().err.splitlines() + wanted = at(REFS, "See") + assert f"{path}:{wanted}: warning: [render-not-processed] " in " ".join(lines) + assert any(line.startswith(f"{path}:{wanted}: error: [path-not-relative] ") for line in lines) + # The validator's own line is printed the same way. + assert any(line.startswith(f"{path}:{at(REFS, '
')}: warning: [raw-html] ") for line in lines) + # Each diagnostic once, with one location: no `file:line` twice over. + assert not any(f"{path}:{wanted}:" in line.split(": ", 1)[1] for line in lines) + assert len(lines) == len(set(lines)) + + +def test_the_command_line_prints_a_frontmatter_line(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + path = tmp_path / "x.lgd" + path.write_text("---\ntitle: T\nlanguage: xx\n---\n\n# A\n", encoding="utf-8") + assert main([str(path), "-o", str(tmp_path / "out.html")]) == 0 + assert f"{path}:3: info: [render-locale-fallback] " in capsys.readouterr().err + + +def test_standard_input_prints_its_name_and_line() -> None: + done = run_cli_text("-", "-f", "text", stdin=REFS) + assert f":{at(REFS, 'See')}: warning: [render-not-processed] " in done.stderr + + +def test_strict_prints_the_lines_it_refuses_with(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + path = tmp_path / "x.lgd" + path.write_text(REFS, encoding="utf-8") + assert main([str(path), "--strict"]) == 1 + err = capsys.readouterr().err + assert f"{path}:{at(REFS, 'See')}: error: [path-not-relative] " in err diff --git a/tests/test_links.py b/tests/test_links.py index d79107a..c5f4912 100644 --- a/tests/test_links.py +++ b/tests/test_links.py @@ -100,7 +100,8 @@ def test_the_validators_own_report_is_not_repeated(monkeypatch: pytest.MonkeyPat def validate(document, **kwargs): # type: ignore[no-untyped-def] result = real_validate(document, **kwargs) - found = Diagnostic(rule="path-not-relative", level="error", line=5, + # Line 6 is the paragraph's: a report at another line is another finding. + found = Diagnostic(rule="path-not-relative", level="error", line=6, message="Image path 'https://x/p.png' is not relative (§2.3).") result.diagnostics.append(found) return result @@ -109,7 +110,7 @@ def validate(document, **kwargs): # type: ignore[no-untyped-def] result = render(FRONT + "# A\n\n![a](https://x/p.png) ![b](/other.png)\n") errors = [d for d in result.diagnostics if d.rule == "path-not-relative"] assert len(errors) == 2 # the validator's, and ours for the path it did not name - assert any(d.line == 5 for d in errors) + assert all(d.line == 6 for d in errors) assert any("/other.png" in d.message for d in errors) From 63683b876f2139e369fff0264367674a11148964 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:07:20 +0000 Subject: [PATCH 041/118] Read the lines of a document once instead of checking its shape for every one Document.line_of and SourceMap.key each compare the whole document's shape with its source (SourceMap.fits) on every call, so asking for every block's line and every party's key made a render quadratic in the document's size (the 3,000 terms test took 39 s instead of 2 s). The builder now takes Document.layout() once and reads block, heading and list item lines from it (items in the validator's pre-order), and the resolver keeps one diagnostics.KeyLines per document, which checks the fit once. diagnostics.key_line stays for a single lookup. Tests: the builder's lines equal Document.line_of for every heading, block and item of a rich document, and a render of 2,000 paragraphs and 500 party, side and attachment references calls positions.document_shape at most three times. --- src/legaldown_render/build.py | 49 +++++++++++-- src/legaldown_render/diagnostics.py | 33 ++++++--- src/legaldown_render/resolve/resolver.py | 26 ++++--- tests/test_diagnostic_lines.py | 93 ++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 27 deletions(-) diff --git a/src/legaldown_render/build.py b/src/legaldown_render/build.py index 5bc701a..062668f 100644 --- a/src/legaldown_render/build.py +++ b/src/legaldown_render/build.py @@ -28,10 +28,11 @@ import re import secrets -from collections.abc import Iterator +from collections.abc import Iterable, Iterator from dataclasses import dataclass, replace from functools import cache from itertools import count +from typing import Any from urllib.parse import unquote from legaldown import Block as ModelBlock @@ -205,6 +206,14 @@ def __init__(self, document: Document, result: ValidationResult) -> None: # it takes the line of the quote or the item. self.at: tuple[int | None, int] | None = None self.line: int | None = None + # The lines of the parts of the document, read once: ``Document.line_of`` + # checks the whole document against its source on every call. None + # without a source, or when the document no longer fits it. + self.layout = document.layout() + # The lines of the items of the top-level list being built, in the + # pre-order the validator numbers them (see list()); None for a list + # whose items are not addressable. + self.item_lines: list[int] | None = None nonce = secrets.token_hex(8) # A sentinel with its wrappers; only a complete one matches, so a # wrapper character in the source is never taken. @@ -378,7 +387,7 @@ def blocks(self, blocks: list[ModelBlock], section: int | None, *, markers: bool placed = self.placed.get((section, index), []) if markers else [] if markers: self.at = (section, index) - self.line = self.document.line_of(section, index) + self.line = self._block_line(section, index) built = self.block(block, placed, top_level=top_level) if built is not None: out.append(built) @@ -425,7 +434,37 @@ def list(self, block: ModelBlock, placed: list[PlacedMarker]) -> List | None: number among all the list's items, as the validator numbers them. None for a list whose every item is only a comment.""" markers = {marker.item: marker for marker in placed} - return self._list(block, markers, count()) + outer = self.item_lines + self.item_lines = self._item_lines(*self.at) if self.at else None + try: + return self._list(block, markers, count()) + finally: + self.item_lines = outer + + # -- lines (§16.9) -------------------------------------------------------- + + def _block_line(self, section: int | None, index: int) -> int | None: + """The first line of top-level block *index* of the preamble (*section* + None) or of a section: what ``Document.line_of`` gives.""" + if self.layout is None: + return None + spans = self.layout.preamble if section is None else self.layout.sections[section].blocks + return spans[index].start + + def _item_lines(self, section: int | None, index: int) -> list[int] | None: + """The marker lines of the items of top-level list *index*, nested items + included, in pre-order as ``Document.line_of`` numbers them.""" + if self.layout is None: + return None + spans = self.layout.preamble if section is None else self.layout.sections[section].blocks + + def walk(items: Iterable[Any]) -> Iterator[int]: + for item in items: + yield item.start + for inner in item.blocks: + yield from walk(inner.items) + + return list(walk(spans[index].items)) def _list(self, block: ModelBlock, markers: dict[int | None, PlacedMarker], numbers: Iterator[int]) -> List | None: """*block*'s items, numbered from *numbers* in pre-order, to find the @@ -452,7 +491,7 @@ def _list(self, block: ModelBlock, markers: dict[int | None, PlacedMarker], numb blocks: list[Block] = [] outer = self.line # The item's marker line; in a quote an item is not addressable. - item_line = self.document.line_of(*self.at, number) if self.at else outer + item_line = self.item_lines[number] if self.item_lines is not None else outer self.line = item_line try: for child in children: @@ -520,7 +559,7 @@ def tree(self) -> RenderTree: identifier=self.result.index.sections[index].identifier, condition=section.condition, blocks=self.blocks(section.blocks, index), - line=document.line_of(index), + line=None if self.layout is None else self.layout.sections[index].heading.start, ) for index, section in enumerate(document.sections) ) diff --git a/src/legaldown_render/diagnostics.py b/src/legaldown_render/diagnostics.py index 916e002..95ce419 100644 --- a/src/legaldown_render/diagnostics.py +++ b/src/legaldown_render/diagnostics.py @@ -17,17 +17,30 @@ from legaldown import Diagnostic, Document +class KeyLines: + """The lines of a document's frontmatter keys: ``KeyLines(document)(*path)`` + is the line in the document's file of the value at *path* + (``("attachments", 0, "file")``), or of the nearest key above it that is + written, as ``SourceMap.key`` says; None when the document has no source to + name a line of (it was built from a dict, or changed since it was parsed) + or has no frontmatter. + + Whether the document still fits its source map is worked out once, when + this is made (``SourceMap.fits`` reads the whole document), so a renderer + keeps one for the document it renders and asks it for every reference.""" + + def __init__(self, document: Document) -> None: + source_map = document.source_map + self._map = source_map if source_map is not None and source_map.fits(document) else None + + def __call__(self, *path: Any) -> int | None: + line = self._map.key(*path) if self._map is not None else None + return line if isinstance(line, int) else None + + def key_line(document: Document, *path: Any) -> int | None: - """The line in the document's file of the frontmatter value at *path* - (``key_line(document, "attachments", 0, "file")``), or of the nearest key - above it that is written, as ``SourceMap.key`` says; None when the document - has no source to name a line of (it was built from a dict, or changed since - it was parsed) or has no frontmatter.""" - source_map = document.source_map - if source_map is None or not source_map.fits(document): - return None - line = source_map.key(*path) - return line if isinstance(line, int) else None + """``KeyLines(document)(*path)``, for a single lookup.""" + return KeyLines(document)(*path) def repeats(found: Diagnostic, known: Iterable[Diagnostic]) -> bool: diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index b94e67b..c8d1802 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -25,7 +25,7 @@ Every one of them names the file and the line it is at (§16.9): the first line of the block, list item or heading being resolved, or the line of a frontmatter -key (``diagnostics.key_line``), tracked in :attr:`Resolver.line`. A finding in +key (``diagnostics.KeyLines``), tracked in :attr:`Resolver.line`. A finding in a term's text is at the term's definition, since that is where the text is worked out whatever reference asked for it first. """ @@ -57,7 +57,7 @@ from legaldown.syntax import Directive from ..build import plain_inlines -from ..diagnostics import covered_by, key_line, repeats +from ..diagnostics import KeyLines, covered_by, repeats from ..style import Style, effective_labels from ..style.model import LevelFormat from ..tree import ( @@ -195,6 +195,8 @@ def __init__( # The line (§16.9) of what is being resolved, which a diagnostic # names: see _at. self.line: int | None = None + # The lines of frontmatter keys (once the document is checked against its source). + self.key_line = KeyLines(document) # A definition id -> the line of its first {{def:}} in the body (see # _survey), where its term's text is worked out. self.term_lines: dict[str, int | None] = {} @@ -251,9 +253,9 @@ def run(self) -> RenderTree: preamble = tuple(self._resolve_block(block) for block in preamble) sections = tuple(self._resolve_section(section) for section in sections) attachments = tuple(self._resolve_attachment(index, a) for index, a in enumerate(attachments)) - with self._at(key_line(self.document, "title")): + with self._at(self.key_line("title")): title = resolve(self.tree.title) - with self._at(key_line(self.document, "subtitle")): + with self._at(self.key_line("subtitle")): subtitle = resolve(self.tree.subtitle) resolved = replace( self.tree, @@ -294,7 +296,7 @@ def _resolve_section(self, section: Section) -> Section: ) def _resolve_attachment(self, index: int, attachment: AttachmentPart) -> AttachmentPart: - with self._at(key_line(self.document, "attachments", index, "title")): + with self._at(self.key_line("attachments", index, "title")): return replace(attachment, title=self._resolve_inlines(attachment.title)) def _contents( @@ -1004,7 +1006,7 @@ def _plain_value(self, text: str, *key: Any) -> str: in the body): what it reports is at that key, not where it is shown.""" if "{{" not in text: return text - with self._at(key_line(self.document, *key)) if key else nullcontext(): + with self._at(self.key_line(*key)) if key else nullcontext(): return plain_text(self._resolve_inlines(plain_inlines(text))) def _frontmatter(self, text: str) -> tuple[Inline, ...]: @@ -1014,10 +1016,10 @@ def _header(self) -> tuple[tuple[str, tuple[Inline, ...]], ...]: rows: list[tuple[str, tuple[Inline, ...]]] = [] metadata = self.metadata if self.style.title_block.effective_date and metadata.effective_date: - with self._at(key_line(self.document, "effective_date")): + with self._at(self.key_line("effective_date")): rows.append((self.labels.effective_date or "", self._date_or_text(metadata.effective_date))) if self.style.title_block.version and metadata.version: - with self._at(key_line(self.document, "version")): + with self._at(self.key_line("version")): rows.append((self.labels.version or "", self._frontmatter(metadata.version))) return tuple(rows) @@ -1032,12 +1034,12 @@ def _sides(self) -> tuple[SideInfo, ...]: parties = tuple(self._party_info(side_index, party_index, party) for party_index, party in enumerate(side.parties)) label = self.result.index.side_lookup.get(side.name) or side.label or side.name - with self._at(key_line(self.document, "sides", side_index, "label")): + with self._at(self.key_line("sides", side_index, "label")): sides.append(SideInfo(self._frontmatter(label), parties)) return tuple(sides) def _party_info(self, side_index: int, party_index: int, party: Party) -> PartyInfo: - with self._at(key_line(self.document, "sides", side_index, "parties", party_index)): + with self._at(self.key_line("sides", side_index, "parties", party_index)): details: list[tuple[str, tuple[Inline, ...]]] = [] if party.identification_number: details.append((self.labels.identification_number or "", self._frontmatter(party.identification_number))) @@ -1081,10 +1083,10 @@ def _signatures(self) -> tuple[SignatureParty, ...]: return () blocks: list[SignatureParty] = [] for side_index, side in enumerate(self.metadata.sides): - with self._at(key_line(self.document, "sides", side_index, "label")): + with self._at(self.key_line("sides", side_index, "label")): label = self._frontmatter(self.result.index.side_lookup.get(side.name) or side.label or side.name) for party_index, party in enumerate(side.parties): - with self._at(key_line(self.document, "sides", side_index, "parties", party_index)): + with self._at(self.key_line("sides", side_index, "parties", party_index)): # Where signature blocks are generated, legal_name MUST appear (§3.6). blocks.append(SignatureParty( side=label, diff --git a/tests/test_diagnostic_lines.py b/tests/test_diagnostic_lines.py index ff7bb51..6c52077 100644 --- a/tests/test_diagnostic_lines.py +++ b/tests/test_diagnostic_lines.py @@ -456,3 +456,96 @@ def test_strict_prints_the_lines_it_refuses_with(tmp_path: Path, capsys: pytest. assert main([str(path), "--strict"]) == 1 err = capsys.readouterr().err assert f"{path}:{at(REFS, 'See')}: error: [path-not-relative] " in err + + +# -- The cost of lines --------------------------------------------------------------------- + +RICH = HEAD + """ +Preamble one. + +- pre item + - pre nested + +# One {#one} + +Para. + +1. first + - nested a + - deeper + - nested b + + second block of nested b +2. second + +> quote +> +> - in a quote + +| a | b | +|---|---| +| x | y | + +``` +code +``` + +--- + +## Two + +- x +- y + 1. z +""" + + +def test_the_builders_lines_are_those_of_line_of() -> None: + from legaldown import parse, validate + from legaldown.models import LIST_KINDS + + from legaldown_render.build import build_tree + + document = parse(RICH) + tree = build_tree(document, validate(document)) + + def items(block): # type: ignore[no-untyped-def] + for item in block.items: + yield item + for child in item.blocks: + if hasattr(child, "items"): + yield from items(child) + + for section_index, (blocks, built) in enumerate( + [(document.preamble, tree.preamble)] + [(s.blocks, t.blocks) for s, t in + zip(document.sections, tree.sections, strict=True)]): + owner = None if section_index == 0 else section_index - 1 + if owner is not None: + assert tree.sections[owner].line == document.line_of(owner) + assert len(blocks) == len(built) + for index, (block, node) in enumerate(zip(blocks, built, strict=True)): + if block.kind in LIST_KINDS: + found = [item.line for item in items(node)] + assert found == [document.line_of(owner, index, n) for n in range(len(found))] + elif block.kind != "rule": # a rule has no line: nothing reports on it + assert node.line == document.line_of(owner, index), (owner, index, block.kind) + + +def test_a_render_checks_the_document_against_its_source_a_constant_number_of_times( + monkeypatch: pytest.MonkeyPatch) -> None: + # Document.line_of and SourceMap.key each re-read the whole document's shape + # (positions.document_shape) to see it still fits its source, so asking for + # every block's line, or every party's key, one by one is quadratic. + import legaldown.positions as positions + + calls = [] + real = positions.document_shape + monkeypatch.setattr(positions, "document_shape", lambda document: (calls.append(1), real(document))[1]) + front = ("---\ntitle: T\nsides:\n - name: a\n parties:\n - name: p\n legal_name: P\n" + " type: legal_entity\n - name: b\n parties:\n - name: q\n legal_name: Q\n" + " type: legal_entity\nattachments:\n - id: x\n title: X\n---\n") + body = "# A\n\n" + "\n\n".join(f"Paragraph {i}." for i in range(2000)) + body += "\n\n- one\n- two\n - three\n\n" + " ".join("{{party: p}} {{side: b}} {{attach: x}}" for _ in range(500)) + result = render(front + body + "\n", format="text") + assert result.output + assert len(calls) <= 3, len(calls) From 15e4bd0f3cbab1445d347a0f5cfdbba882d0dbdb Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:18:17 +0000 Subject: [PATCH 042/118] =?UTF-8?q?Check=20every=20file=20reference=20agai?= =?UTF-8?q?nst=20the=20document=20root=20(=C2=A72.3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Include paths, attachments[].file, amends.file, supersedes.file, translations values and image paths are now all checked, lexically (no resolving, no file reads): path-not-relative for a URI scheme, drive or absolute path, and path-outside-root when a document root is configured, reported at the block's line or the frontmatter key's. Without a root nothing changes for ../ paths. RenderOptions gains document_root and document_path (render_file and the command set the path; text and standard input sit at the root; a document outside the root is an OptionsError), and the command gains --root DIR. Percent-encoding is decoded only for what is used as a URL (an image's src, the link an {{attach:}} makes); a backslash is read as a separator. A failing image shows its alt text, an {{attach:}} its text with no link; include markers and attachment paths stay text. supersedes as a plain string is a description, not a path (§3.2), so it is not checked. No existing test changed. --- src/legaldown_render/api.py | 63 ++- src/legaldown_render/cli.py | 14 +- src/legaldown_render/resolve/resolver.py | 144 +++++-- src/legaldown_render/urls.py | 64 ++- tests/test_file_references.py | 506 +++++++++++++++++++++++ 5 files changed, 734 insertions(+), 57 deletions(-) create mode 100644 tests/test_file_references.py diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index 9d14c66..aec6883 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -12,9 +12,10 @@ """ from __future__ import annotations +import os from collections.abc import Mapping from dataclasses import dataclass, field, replace -from pathlib import Path +from pathlib import Path, PurePath from typing import Any import yaml @@ -66,6 +67,23 @@ class RenderOptions: #: With ``answers``, the lines of the diagnostics are those of the #: assembled document (§15.8), not of the template file. filename: str = "" + #: The document root (§2.3): a boundary directory that no file reference of + #: the document (an ``{{include:}}``, ``attachments[].file``, + #: ``amends.file``, ``supersedes.file``, a ``translations`` value, an image) + #: may leave. A path that does is reported as ``path-outside-root``, an + #: Error, and never used. None (the default) configures no root, as §2.3 + #: defines none: ``../`` is then valid, only a path that is not relative is + #: reported. A renderer for documents it does not control SHOULD set one + #: (§2.3). The check is lexical: nothing is read or resolved, and a symbolic + #: link is not followed. + document_root: str | Path | None = None + #: Where the document is, to say where in the root it sits: the base of a + #: path is the document's directory. Used only with ``document_root``, + #: and without it the document is at the root, as text given to :func:`render` + #: is. ``render_file`` and the command line set it from the path they read, + #: and it names the file of the diagnostics when ``filename`` is empty. A + #: document outside the root is an :class:`OptionsError`. + document_path: str | Path | None = None @dataclass(frozen=True, slots=True) @@ -95,8 +113,8 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any Problems in the document never raise: they render as failure markers and are listed in ``diagnostics``. Raises :class:`StyleError` for an invalid - style or setting, :class:`OptionsError` for an unknown output format, - :class:`DocumentError` for a document that cannot be + style or setting, :class:`OptionsError` for an unknown output format or a + document outside its document root, :class:`DocumentError` for a document that cannot be read, and :class:`RenderRefused` in strict mode when the document has errors, or when a template cannot be assembled with ``answers``. """ @@ -106,6 +124,9 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any raise TypeError("pass either a RenderOptions or keyword settings, not both") if options.format not in FORMATS: raise OptionsError(f"unknown output format '{options.format}' (known: {', '.join(FORMATS)})") + base = _document_base(options) + if options.document_path is not None and not options.filename: + options = replace(options, filename=PurePath(options.document_path).name) overrides = dict(options.overrides) if options.locale: overrides["locale"] = options.locale @@ -121,7 +142,7 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any if options.strict and any(d.level == "error" for d in diagnostics): raise RenderRefused(diagnostics) - tree, extra = _resolve(document, result, style) + tree, extra = _resolve(document, result, style, base) # The renderer's findings that the validator's or the assembly's already # name (the same rule, level and message), wherever they are, are not repeated. diagnostics += [d for d in extra if not repeats(d, diagnostics)] @@ -149,9 +170,38 @@ def render_file(path: str | Path, options: RenderOptions | None = None, /, **set settings = {} if not options.filename and not settings: options = replace(options, filename=Path(path).name) + if options.document_path is None and not settings: + options = replace(options, document_path=path) return render(source, options, **settings) +def _document_base(options: RenderOptions) -> str | None: + """The directory of the document relative to the document root, ``/``-separated + (``"."`` at the root), or None when no root is configured (§2.3). Lexical: + the paths are made absolute against the working directory and compared as + text, with no file system access. Raises OptionsError for an empty root or + path, and for a document that is not inside the root (or, on Windows, is on + another drive).""" + if options.document_root is None: + return None + if os.fspath(options.document_root) == "": + raise OptionsError("document_root is empty: name the directory file references may not leave") + if options.document_path is not None and os.fspath(options.document_path) == "": + raise OptionsError("document_path is empty: name the document's file, or leave it out") + root = os.path.abspath(options.document_root) + directory = root if options.document_path is None else os.path.dirname(os.path.abspath(options.document_path)) + try: + relative = os.path.relpath(directory, root) + except ValueError: # Windows: another drive + raise OptionsError(f"the document {options.document_path} is outside the document root " + f"{options.document_root}: it is on another drive") from None + parts = PurePath(relative).parts + if parts and parts[0] == "..": + raise OptionsError(f"the document {options.document_path} is outside the document root " + f"{options.document_root}: the root must contain the document") + return "/".join(parts) or "." + + def _assemble(source: str, answers: Mapping[str, Any], filename: str = "") -> tuple[str, list[Diagnostic]]: """*source* assembled with *answers* (§15.7), and the assembly's Warnings (such as ``answer-unknown``). Raises RenderRefused when it @@ -184,7 +234,8 @@ def _parse(source: str, filename: str = "") -> Document: raise DocumentError(f"The document cannot be read: {error}") from error -def _resolve(document: Document, result: ValidationResult, style: Style) -> tuple[RenderTree, list[Diagnostic]]: +def _resolve(document: Document, result: ValidationResult, style: Style, + base: str | None) -> tuple[RenderTree, list[Diagnostic]]: diagnostics: list[Diagnostic] = [] # Raw HTML is never emitted; the validator reports it (raw-html, §8.7). tree = build_tree(document, result) @@ -207,7 +258,7 @@ def _resolve(document: Document, result: ValidationResult, style: Style) -> tupl )) locale = parse_locale("en") assert locale is not None - resolved, found = resolve_tree(tree, document, result, style, Formatter(locale, style.values)) + resolved, found = resolve_tree(tree, document, result, style, Formatter(locale, style.values), base) return resolved, diagnostics + found diff --git a/src/legaldown_render/cli.py b/src/legaldown_render/cli.py index a24a419..7a9db4a 100644 --- a/src/legaldown_render/cli.py +++ b/src/legaldown_render/cli.py @@ -3,7 +3,7 @@ Exit codes: 0 — rendered (document errors, if any, are shown as markers and listed on stderr); 1 — not rendered: refused under ``--strict``, or the document cannot be read (including input that is not UTF-8) or the output cannot -be written; 2 — a usage or style problem; 70 — an internal error (a bug to +be written; 2 — a usage, style or document-root problem; 70 — an internal error (a bug to report). Documents and output are always UTF-8 (the HTML says ````), @@ -22,7 +22,7 @@ from . import __version__ from .api import RenderOptions, render -from .errors import DocumentError, InternalError, RenderRefused, StyleError +from .errors import DocumentError, InternalError, OptionsError, RenderRefused, StyleError from .style import builtin_styles, dump_style, load_style, parse_override from .urls import printable from .writers import FORMATS, file_extension, format_for_path @@ -89,6 +89,11 @@ def _parser() -> argparse.ArgumentParser: "written; one pair of matching outer quotes is removed (wrap the value in the " "other kind of quote to keep quotation marks). Others read it as YAML, e.g. true or 2") parser.add_argument("--locale", help="formatting locale, e.g. en-US or cs-CZ (default: the style's, else the document language)") + parser.add_argument("--root", metavar="DIR", + help="the document root (§2.3): a file reference (an include, an attachment, amends, " + "supersedes, a translation, an image) that leaves DIR is an error " + "(path-outside-root), and the file is not used. The document must be inside DIR " + "(standard input counts as at DIR itself). Without it, ../ is allowed") parser.add_argument("--fragment", action="store_true", help="HTML: write only the
, without page and stylesheet") parser.add_argument("--strict", action="store_true", help="refuse to render a document that has errors") parser.add_argument("--answers", metavar="FILE", @@ -193,10 +198,11 @@ def main(argv: list[str] | None = None) -> int: options = RenderOptions(format=output_format, style=args.style, overrides=overrides, strict=args.strict, final=args.final, answers=answers, standalone=not args.fragment, - filename="" if args.input == "-" else Path(args.input).name) + filename="" if args.input == "-" else Path(args.input).name, + document_root=args.root, document_path=None if args.input == "-" else args.input) try: result = render(source, options) - except StyleError as error: + except (StyleError, OptionsError) as error: _complain(f"legaldown-render: {error}") return 2 except RenderRefused as error: diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index c8d1802..1783bed 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -20,8 +20,13 @@ ``render-link-refused`` for a link whose URL scheme is not one links may use (§8.7), ``render-term-too-long`` for a term whose text is longer than a ``{{term:}}`` may show, ``render-term-limit`` for terms that refer to each other too deeply or too often, -and ``path-not-relative`` for an image whose path is not relative (§2.3), which -the validator does not check yet. +and the two file-reference rules of §2.3, which the validator does not check yet: +``path-not-relative`` for a path with a URI scheme or an absolute one, and +``path-outside-root`` for one that leaves the document root, when a root is +configured. They apply to every file reference: an ``{{include:}}``, +``attachments[].file``, ``amends.file``, ``supersedes.file``, a ``translations`` +value, and an image. The files are never read at this level; what the renderer +does with a path that fails is in :meth:`Resolver._reference`. Every one of them names the file and the line it is at (§16.9): the first line of the block, list item or heading being resolved, or the line of a frontmatter @@ -38,7 +43,7 @@ from typing import Any, NamedTuple from urllib.parse import unquote -from legaldown import Diagnostic, Document, Party, ValidationResult +from legaldown import Amends, Diagnostic, Document, Party, ValidationResult from legaldown.grammar import ( ALWAYS, DECISION_QUESTION_TYPES, @@ -97,13 +102,20 @@ plain_text, unlinked, ) -from ..urls import is_empty_path, is_link_target, is_relative_path, printable +from ..urls import file_reference_problem, is_empty_path, is_link_target, printable from .numbering import RENUMBERED, extend, fill, format_counter, heading_levels from .values import DURATION_UNITS, Formatter #: Definition anchors are prefixed with a character no identifier can hold, #: so they can never collide with a section or item anchor (§5.6). DEFINITION_ANCHOR_PREFIX = "def:" +#: What the renderer does with a file reference that fails §2.3 (the rest of +#: a message of ``path-not-relative`` or ``path-outside-root``). An attachment's +#: path is shown as the placeholder's text and is not linked to, wherever it is +#: reported from (the frontmatter, a ``{{attach:}}``), so that it is one message. +_NEVER_READ = "It is never read." +_ATTACHMENT_EFFECT = "It is shown as text, not linked to, and never read." + #: The attachments heading's anchor, with a colon for the same reason. ATTACHMENTS_ANCHOR = "ld:attachments" @@ -182,8 +194,12 @@ def __init__( result: ValidationResult, style: Style, formatter: Formatter, + base: str | None = None, ) -> None: self.tree = tree + # The directory of the document relative to the document root (§2.3), + # or None when there is no root: the containment rule is then not checked. + self.base = base self.document = document self.metadata = document.metadata self.result = result @@ -238,6 +254,7 @@ def __init__( # -- entry point ---------------------------------------------------------- def run(self) -> RenderTree: + self._references() self._survey() attachments = self._attachments() sections = self._number_sections() @@ -651,12 +668,15 @@ def _resolve_inline(self, inline: Inline) -> Inline | tuple[Inline, ...] | None: # No path, no picture: would request the page # itself. §2.3 says nothing of an empty path, so no rule fires. return alt - if not is_relative_path(src): - # §2.3: an image path is relative and has no URI scheme, so a - # remote or absolute image is never fetched or embedded. Its - # alt text stands in; the validator (0.4) does not check - # image paths, so the renderer reports the Core rule. - self._not_relative(src) + # markdown-it has percent-encoded the path ("C:\\x" reads + # "C:%5Cx"); it is read as the URL it is, and shown as written. + if self._reference("Image path", src, url=True, shown=unquote(src), + effect="Its alt text is shown instead."): + # §2.3: an image path is relative and within the document + # root, so a remote, absolute or outside image is never + # fetched or embedded. Its alt text stands in; the validator + # (0.4) does not check file references, so the renderer + # reports the Core rule. return alt return replace(inline, children=alt, title=self._resolve_inlines(_attribute_text(title))) case _: @@ -959,13 +979,20 @@ def _attach(self, directive: Directive, attachment_id: str) -> Inline: if title is None: return FailureMarker(f"[UNKNOWN ATTACHMENT: {attachment_id}]") anchor = self.attachment_anchors.get(attachment_id) + index = next((i for i, a in enumerate(self.metadata.attachments) if a.id == attachment_id), None) + key = ("attachments", index, "title") if index is not None else () if anchor is not None: href = "#" + anchor else: - declared = next((a for a in self.metadata.attachments if a.id == attachment_id), None) - href = declared.file if declared and declared.file else None - key = next((("attachments", i, "title") for i, a in enumerate(self.metadata.attachments) - if a.id == attachment_id), ()) + file = self.metadata.attachments[index].file if index is not None else "" + href = file or None + # A link to the file is a URL: its path is read as one (%2e%2e is + # "..") and must be relative and within the root (§2.3), or the + # attachment's title is text. A path with the same problem as + # written is the same finding, reported once (see _references). + if href and self._reference(self._attachment_label(attachment_id), href, url=True, + effect=_ATTACHMENT_EFFECT): + href = None return Value("attach", directive.params.get("label") or self._plain_value(title, *key), href) def _choose(self, directive: Directive, question_id: str) -> Inline: @@ -986,6 +1013,9 @@ def _choose(self, directive: Directive, question_id: str) -> Inline: return Choice(question_id, options, self.style.template_view.choice_separator) def _include(self, directive: Directive, path: str) -> Inline: + # The path is a file reference (§2.3) whether or not the file is read: + # the marker shows it as text, never as something to fetch. + self._reference("Include path", path, effect="It is shown as text and never read.") # Includes are a Full-level capability (§17.4): never dropped silently. self._warn( "render-not-processed", @@ -1124,26 +1154,65 @@ def _link_refused(self, href: str) -> None: f"sips, sms, urn, geo, news and relative links are; its text is shown.", ) - def _not_relative(self, path: str) -> None: - """Report an image *path* that is not relative as the Core Error - ``path-not-relative`` (§2.3, §16.2), unless the validator has - reported it (:func:`~legaldown_render.diagnostics.covered_by`: its - report of the path, at this line). markdown-it has percent-encoded - the path ("C:\\x" reads "C:%5Cx"), so it is decoded to name the path - as written.""" - path = unquote(path) - shown = printable(path) - diagnostic = self._found( - "path-not-relative", "error", - f"Image path '{shown}' is not relative (§2.3): file references are relative paths with no " - f"URI scheme. Its alt text is shown instead.", - ) - # Assumption: the validator quotes the path in its message as '', - # as written. Matching the quoted form keeps a path that is a prefix of - # another ("/a" and "/ab") from being counted as already reported. - if covered_by(diagnostic, path, self.result.diagnostics) or repeats(diagnostic, self.diagnostics): - return - self.diagnostics.append(diagnostic) + def _reference(self, label: str, path: str, *, effect: str, url: bool = False, + shown: str | None = None) -> bool: + """Check *path*, a file reference of the document (§2.3; *label* says + which, e.g. "Include path"): True if it is not relative or leaves the + document root, which is reported as the Core Error ``path-not-relative`` + or ``path-outside-root`` (§16.2), with *effect* saying what the renderer + does instead (it never reads the file). *url* says the path is read as + a URL (see :func:`~legaldown_render.urls.file_reference_problem`); + *shown* is the path as the document wrote it, if that is not *path*. + + A report is left out when the validator made it already + (:func:`~legaldown_render.diagnostics.covered_by`: its report of the + path, at this line) or this resolver did (the same message). The + validator quotes the path in its message as '', as written; + matching the quoted form keeps a path that is a prefix of another + ("/a" and "/ab") from being counted as already reported.""" + problem = file_reference_problem(path, base=self.base, url=url) + if problem is None: + return False + shown = path if shown is None else shown + if problem == "not-relative": + rule, what = "path-not-relative", ("is not relative (§2.3): file references are relative paths " + "with no URI scheme.") + else: + rule, what = "path-outside-root", ("is outside the document root (§2.3): a file reference may " + "not leave the root.") + diagnostic = self._found(rule, "error", f"{label} '{printable(shown)}' {what} {effect}") + if not (covered_by(diagnostic, shown, self.result.diagnostics) or repeats(diagnostic, self.diagnostics)): + self.diagnostics.append(diagnostic) + return True + + def _attachment_label(self, attachment_id: str) -> str: + return f"Attachment '{printable(attachment_id)}' file path" + + def _references(self) -> None: + """Check the file references in the frontmatter (§2.3): each + ``attachments[].file``, ``amends.file``, ``supersedes.file``, and + ``translations`` value, at the line of its key. They are shown, never + read: an attachment's path is the placeholder's text, and the + amended and translated documents are not loaded below Full (§17.4). + ``supersedes`` as a plain string describes the superseded document + (§3.2) and is not a path.""" + metadata = self.metadata + references: list[tuple[str, str, tuple[Any, ...], str]] = [] + for index, attachment in enumerate(metadata.attachments): + references.append((self._attachment_label(attachment.id), attachment.file, + ("attachments", index, "file"), _ATTACHMENT_EFFECT)) + if metadata.amends is not None: + references.append(("Amended document path", metadata.amends.file, ("amends", "file"), _NEVER_READ)) + if isinstance(metadata.supersedes, Amends): + references.append(("Superseded document path", metadata.supersedes.file, ("supersedes", "file"), + _NEVER_READ)) + for language, path in metadata.translations.items(): + references.append((f"Translation '{printable(language)}' path", path, ("translations", language), + _NEVER_READ)) + for label, path, key, effect in references: + if path: + with self._at(self.key_line(*key)): + self._reference(label, path, effect=effect) _HANDLERS: dict[str, Callable[[Resolver, Directive, str], Inline | None]] = { @@ -1246,9 +1315,12 @@ def resolve_tree( result: ValidationResult, style: Style, formatter: Formatter, + base: str | None = None, ) -> tuple[RenderTree, list[Diagnostic]]: - """The resolved tree, and the diagnostics only rendering can produce.""" - resolver = Resolver(tree, document, result, style, formatter) + """The resolved tree, and the diagnostics only rendering can produce. + *base* is the document's directory relative to the document root (§2.3), + or None when no root is configured.""" + resolver = Resolver(tree, document, result, style, formatter, base) resolved = resolver.run() return resolved, resolver.diagnostics diff --git a/src/legaldown_render/urls.py b/src/legaldown_render/urls.py index 6b89a8f..a6f7c32 100644 --- a/src/legaldown_render/urls.py +++ b/src/legaldown_render/urls.py @@ -1,6 +1,6 @@ """Reading URLs the way the renderer needs to: which schemes a link may use -(§8.7), whether an image path is a relative one (§2.3), and how to quote one in -a message. +(§8.7), whether a file reference is a relative path within the document root +(§2.3), and how to quote one in a message. The checks are syntactic, like the validator's (§2.3). They are shared by the resolver, which decides what an image is, and by both writers, which decide @@ -93,13 +93,55 @@ def is_empty_path(path: str) -> bool: return not _as_read(unquote(path)) +def file_reference_problem(path: str, *, base: str | None = None, url: bool = False) -> str | None: + """What is wrong with *path* as a file reference (§2.3), or None: + + * ``"not-relative"`` (``path-not-relative``): it has a URI scheme + (``https://``, ``file://``, ``data:``), which includes a Windows drive + path (``C:\\x``, ``C:/x``, a scheme as far as a parser can tell), or + starts with ``/`` or ``\\`` (so also ``//host``); + * ``"outside-root"`` (``path-outside-root``): there is a document root + (*base* is not None) and the path leaves it. + + *base* is the directory of the document relative to the root, ``/``-separated + (``"."`` for a document at the root, ``"contracts/2025"``), or None where no + root is configured: §2.3 defines no default root, so ``../shared/a.lgd`` is + then valid. Both checks are lexical, as §2.3 says: nothing is resolved or + read, so a symbolic link is not followed, and a path that does not exist is + as good as any. An empty path names nothing and has no problem. + + How the path is read: a backslash is a separator (the renderer's reading, + so that a path that goes up on Windows is not missed; it matters only with a + root, a leading one is already not relative), and so are ``.. `` and ``. `` + with trailing spaces, which Windows reads as ``..`` and ``.``. What a + browser ignores before the scheme (tabs, newlines, control characters and + spaces at either end) is ignored. *url* says the path is consumed as a URL + (an image's source, a link's target), where percent-encoding counts as the + character it encodes (``%2e%2e`` is ``..``, ``%2f`` a separator), once; a + path from frontmatter or an ``{{include:}}`` is a file name and is read as + written, in which ``%2e%2e`` is a directory with that name.""" + read = _as_read(unquote(path) if url else path) + if not read: + return None + if _SCHEME_RE.match(read) or read[0] in "/\\": + return "not-relative" + if base is None: + return None + depth = 0 if base == "." else len([part for part in base.split("/") if part]) + for part in read.replace("\\", "/").split("/"): + part = part.rstrip(" ") + if part == "..": + depth -= 1 + if depth < 0: + return "outside-root" + elif part not in ("", "."): + depth += 1 + return None + + def is_relative_path(path: str) -> bool: - """True if *path*, an image's, is a relative path (§2.3). Not relative: a - path with a URI scheme (``https://``, ``file://``, ``data:``), which - includes a Windows drive path (``C:\\x``, ``C:/x``, a scheme as far as a - parser can tell), and one that starts with ``/`` or ``\\`` (so also - ``//host``). A leading ``../`` is relative: §2.3 defines no default - document root. A percent-encoded character counts as the character, since - a path may be encoded by the time it is read.""" - read = _as_read(unquote(path)) - return not read or (_SCHEME_RE.match(read) is None and read[0] not in "/\\") + """True if *path*, an image's, is a relative path (§2.3): see + :func:`file_reference_problem`, which also reads a path as a URL. A leading + ``../`` is relative here: it is the containment rule, with a document root + configured, that keeps a path within it.""" + return file_reference_problem(path, url=True) != "not-relative" diff --git a/tests/test_file_references.py b/tests/test_file_references.py new file mode 100644 index 0000000..0af5ba8 --- /dev/null +++ b/tests/test_file_references.py @@ -0,0 +1,506 @@ +"""File references (§2.3): every path a document uses to name another file (an +``{{include:}}``, ``attachments[].file``, ``amends.file``, ``supersedes.file``, a +``translations`` value, an image) must be relative (``path-not-relative``) and, +when a document root is configured, stay within it (``path-outside-root``); the +checks are lexical; what the renderer does with a path that fails; the options +and the ``--root`` setting that configure the root; the lines of the findings.""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest +from conftest import run_cli_text + +import legaldown_render.api as api +from legaldown_render import OptionsError, RenderOptions, RenderRefused, render, render_file +from legaldown_render.cli import main +from legaldown_render.urls import file_reference_problem + +NOT_RELATIVE = "path-not-relative" +OUTSIDE = "path-outside-root" +RULES = (NOT_RELATIVE, OUTSIDE) + +# -- The check itself ------------------------------------------------------------------ + + +@pytest.mark.parametrize(("path", "base", "url", "expected"), [ + ("a/b.lgd", None, False, None), + ("", None, False, None), + ("../x.lgd", None, False, None), + ("../x.lgd", ".", False, "outside-root"), + ("../x.lgd", "sub", False, None), + ("../../x.lgd", "sub", False, "outside-root"), + ("a/../x.lgd", ".", False, None), + ("a/../../x.lgd", ".", False, "outside-root"), + ("./a/./b", ".", False, None), + ("a//b", ".", False, None), + ("..", ".", False, "outside-root"), + ("..", "sub", False, None), + ("a/b/..", ".", False, None), + ("/etc/passwd", ".", False, "not-relative"), + ("/etc/passwd", None, False, "not-relative"), + ("//host/share", None, False, "not-relative"), + ("\\x", None, False, "not-relative"), + ("\\\\host\\x", None, False, "not-relative"), + ("https://x.example/a", None, False, "not-relative"), + ("HTTPS://x.example/a", None, False, "not-relative"), + ("file:///etc/passwd", None, False, "not-relative"), + ("data:text/plain,x", None, False, "not-relative"), + ("C:\\x", None, False, "not-relative"), + ("C:/x", None, False, "not-relative"), + ("c:x", None, False, "not-relative"), + ("ht\ttps://x", None, False, "not-relative"), + (" /etc/passwd", None, False, "not-relative"), + ("a/b:c", None, False, None), + # A backslash is read as a separator. + ("a\\b", ".", False, None), + ("..\\x", ".", False, "outside-root"), + ("..\\x", "sub", False, None), + ("a\\..\\..\\x", ".", False, "outside-root"), + # Encoding: decoded for a URL, as written for a path. + ("%2e%2e/x", ".", False, None), + ("%2e%2e/x", ".", True, "outside-root"), + ("..%2fx", ".", True, "outside-root"), + ("..%5cx", ".", True, "outside-root"), + ("%2fetc", None, True, "not-relative"), + ("%2fetc", None, False, None), + ("%2e%2e/x", "sub", True, None), + # One decoding only: %252e is a name, %2e after it. + ("%252e%252e/x", ".", True, None), + # Spaces after the name are the Windows reading of the same segment. + (".. /x", ".", False, "outside-root"), +]) +def test_the_check(path: str, base: str | None, url: bool, expected: str | None) -> None: + assert file_reference_problem(path, base=base, url=url) == expected + + +# -- Each kind of reference ------------------------------------------------------------ + + +def _frontmatter(extra: str) -> str: + return f"---\ntitle: T\n{extra}---\n\n# A\n\nBody.\n" + + +def _kind_source(kind: str, path: str) -> tuple[str, str]: + """A document with *path* in a reference of *kind*, and the text of the + line the finding is expected at.""" + quoted = json.dumps(path) + if kind == "include": + return _frontmatter("").replace("Body.", f"Before {{{{include: {path}}}}} after."), "Before" + if kind == "image": + # Markdown reads two backslashes as one. + return _frontmatter("").replace("Body.", f"Before ![the logo](<{path.replace(chr(92), chr(92) * 2)}>) after."), "Before" + if kind == "attachment": + return _frontmatter(f"attachments:\n - id: annex\n title: Annex\n file: {quoted}\n"), " file:" + if kind == "amends": + return _frontmatter(f"amends:\n title: Original\n file: {quoted}\n"), " file:" + if kind == "supersedes": + return _frontmatter(f"supersedes:\n title: Old\n file: {quoted}\n"), " file:" + if kind == "translation": + return _frontmatter(f"translations:\n cs: {quoted}\n"), " cs:" + raise AssertionError(kind) + + +KINDS = ["include", "image", "attachment", "amends", "supersedes", "translation"] +#: The kinds whose path is read as a URL. +URL_KINDS = {"image"} + + +def line_of(source: str, needle: str) -> int: + return next(n for n, line in enumerate(source.split("\n"), 1) if needle in line) + + +def expected_line(kind: str, path: str) -> int: + source, marker = _kind_source(kind, path) + return line_of(source, marker) + + +def findings(result) -> list[tuple[str, int | None]]: # type: ignore[no-untyped-def] + return [(d.rule, d.line) for d in result.diagnostics if d.rule in RULES] + + +def reported(kind: str, path: str, tmp_path: Path, where: str | None, **settings: object) -> list[tuple[str, int | None]]: + """The file-reference findings for *path* in *kind*, with the document root + *tmp_path* (``where`` is the document's directory in it, ``""`` for the + root itself) or none.""" + source, _ = _kind_source(kind, path) + if where is not None: + settings.update(document_root=tmp_path, document_path=tmp_path / where / "doc.lgd") + return findings(render(source, **settings)) + + +# (path, no root, document at the root, document in the directory "sub") +COMMON = [ + ("annex/a.lgd", None, None, None), + ("a.lgd", None, None, None), + ("../a.lgd", None, OUTSIDE, None), + ("a/../a.lgd", None, None, None), + ("a/../../a.lgd", None, OUTSIDE, None), + ("../../a.lgd", None, OUTSIDE, OUTSIDE), + ("../sub/a.lgd", None, OUTSIDE, None), + ("/etc/passwd", NOT_RELATIVE, NOT_RELATIVE, NOT_RELATIVE), + ("//host/share/a.lgd", NOT_RELATIVE, NOT_RELATIVE, NOT_RELATIVE), + ("https://x.example/a.lgd", NOT_RELATIVE, NOT_RELATIVE, NOT_RELATIVE), + ("file:///etc/passwd", NOT_RELATIVE, NOT_RELATIVE, NOT_RELATIVE), + ("C:\\x\\a.lgd", NOT_RELATIVE, NOT_RELATIVE, NOT_RELATIVE), + ("C:/x/a.lgd", NOT_RELATIVE, NOT_RELATIVE, NOT_RELATIVE), + ("\\\\host\\share\\a.lgd", NOT_RELATIVE, NOT_RELATIVE, NOT_RELATIVE), + ("\\a.lgd", NOT_RELATIVE, NOT_RELATIVE, NOT_RELATIVE), + # A backslash is a separator. + ("a\\b.lgd", None, None, None), + ("..\\a.lgd", None, OUTSIDE, None), + ("..\\..\\a.lgd", None, OUTSIDE, OUTSIDE), +] + + +@pytest.mark.parametrize("kind", KINDS) +@pytest.mark.parametrize(("path", "no_root", "at_root", "in_sub"), COMMON, ids=[c[0] for c in COMMON]) +def test_each_kind_of_reference(kind: str, path: str, no_root: str | None, at_root: str | None, + in_sub: str | None, tmp_path: Path) -> None: + line = expected_line(kind, path) + for where, expected in ((None, no_root), ("", at_root), ("sub", in_sub)): + found = reported(kind, path, tmp_path, where) + assert found == ([(expected, line)] if expected else []), (kind, path, where, found) + + +@pytest.mark.parametrize("kind", KINDS) +def test_encoded_traversal_is_read_as_a_url_only_where_it_is_one(kind: str, tmp_path: Path) -> None: + # A directory may be called "%2e%2e"; an image's path is a URL, in which it is "..". + path = "%2e%2e/%2e%2e/a.lgd" + expected = [(OUTSIDE, expected_line(kind, path))] if kind in URL_KINDS else [] + assert reported(kind, path, tmp_path, "") == expected + assert reported(kind, path, tmp_path, None) == [] + # Encoded separators too. + encoded = "..%2f..%5ca.lgd" + assert reported(kind, encoded, tmp_path, "") == ([(OUTSIDE, expected_line(kind, encoded))] + if kind in URL_KINDS else []) + + +@pytest.mark.parametrize("kind", ["include", "image"]) +def test_a_disguised_scheme_is_a_scheme(kind: str, tmp_path: Path) -> None: + for path in ("ht\ttps://x.example/a", "%68ttps://x.example/a" if kind in URL_KINDS else "https:%2f%2fx.example"): + assert [rule for rule, _ in reported(kind, path, tmp_path, None)] == [NOT_RELATIVE], path + + +# -- What happens to the path ------------------------------------------------------------ + + +def test_an_image_outside_the_root_is_its_alt_text(tmp_path: Path) -> None: + source = _frontmatter("").replace("Body.", "Before ![the logo](../../etc/logo.png) after.") + for fmt in ("html", "text"): + result = render(source, format=fmt, document_root=tmp_path, document_path=tmp_path / "doc.lgd") + assert " None: + source = _frontmatter("").replace("Body.", "{{include: ../../outside/part.lgd}}") + for fmt in ("html", "text"): + result = render(source, format=fmt, document_root=tmp_path) + assert "[NOT PROCESSED: include ../../outside/part.lgd]" in result.output + assert {d.rule for d in result.diagnostics} >= {"render-not-processed", OUTSIDE} + + +def test_attachments_keep_their_path_as_text_and_never_link_to_a_bad_one(tmp_path: Path) -> None: + front = ("attachments:\n - id: annex\n title: Annex A\n file: ../../secret/annex.lgd\n" + " - id: fine\n title: Annex B\n file: annex/b.lgd\n") + source = _frontmatter(front).replace("Body.", "See {{attach: annex}} and {{attach: fine}}.") + for mode, overrides in (("placeholder", {}), ("omit", {"attachments.render": "omit"})): + result = render(source, document_root=tmp_path, overrides=overrides) + assert (OUTSIDE, line_of(source, " file: ../../")) in findings(result), mode + assert 'href="../../secret' not in result.output + assert "See Annex A and" in render(source, document_root=tmp_path, overrides=overrides, format="text").output + if mode == "placeholder": + # The placeholder shows the path, as text. + assert "../../secret/annex.lgd" in result.output + omitted = render(source, document_root=tmp_path, overrides={"attachments.render": "omit"}) + assert "secret" not in omitted.output + # A path that is fine is a link; with no root, so is the one that leaves the directory. + assert 'Annex B' in omitted.output + assert 'href="../../secret/annex.lgd"' in render(source, overrides={"attachments.render": "omit"}).output + + +def test_the_attachment_link_is_read_as_a_url(tmp_path: Path) -> None: + # As written the path names a directory called "%2e%2e"; as the link's URL it leaves the root. + source = _frontmatter("attachments:\n - id: annex\n title: Annex\n file: '%2e%2e/%2e%2e/a.lgd'\n" + ).replace("Body.", "See {{attach: annex}}.") + result = render(source, document_root=tmp_path, overrides={"attachments.render": "omit"}) + assert "href=" not in result.output + assert [rule for rule, _ in findings(result)] == [OUTSIDE] + assert findings(render(source, document_root=tmp_path)) == [] + assert findings(render(source, overrides={"attachments.render": "omit"})) == [] + + +def test_the_same_attachment_path_is_one_finding(tmp_path: Path) -> None: + source = _frontmatter("attachments:\n - id: annex\n title: Annex\n file: /etc/passwd\n" + ).replace("Body.", "See {{attach: annex}}.") + result = render(source, overrides={"attachments.render": "omit"}) + assert findings(result) == [(NOT_RELATIVE, line_of(source, " file:"))] + assert "href=" not in result.output + + +def test_a_markdown_link_is_not_a_file_reference(tmp_path: Path) -> None: + # §8.7 links follow the link allow-list; §2.3 is about references to files. + source = _frontmatter("").replace("Body.", "See [the file](/etc/passwd) and [up](../../a.html).") + result = render(source, document_root=tmp_path) + assert findings(result) == [] + assert 'the file' in result.output and 'up' in result.output + + +def test_the_same_path_in_two_places_is_one_finding_at_the_first(tmp_path: Path) -> None: + source = _frontmatter("").replace("Body.", "One {{include: /x.lgd}}.\n\nTwo {{include: /x.lgd}}.\n\nThree {{include: /y.lgd}}.") + assert findings(render(source)) == [(NOT_RELATIVE, line_of(source, "One")), (NOT_RELATIVE, line_of(source, "Three"))] + + +def test_a_path_that_breaks_both_rules_is_not_relative() -> None: + assert [rule for rule, _ in findings(render(_frontmatter("").replace("Body.", "{{include: /../x}}"), + document_root="."))] == [NOT_RELATIVE] + + +def test_the_validators_report_of_an_include_is_not_repeated(monkeypatch: pytest.MonkeyPatch) -> None: + from legaldown import Diagnostic + from legaldown import validate as real_validate + + source = _frontmatter("").replace("Body.", "Before {{include: https://x.example/p.lgd}}") + line = line_of(source, "Before") + + def validate(document, **kwargs): # type: ignore[no-untyped-def] + result = real_validate(document, **kwargs) + result.diagnostics.append(Diagnostic(rule=NOT_RELATIVE, level="error", line=line, + message="Include path 'https://x.example/p.lgd' is not relative.")) + return result + + monkeypatch.setattr(api, "validate", validate) + assert findings(render(source)) == [(NOT_RELATIVE, line)] + assert len([d for d in render(source).diagnostics if d.rule == NOT_RELATIVE]) == 1 + + +def test_a_finding_names_its_file(tmp_path: Path) -> None: + source, _ = _kind_source("amends", "/etc/passwd") + [found] = [d for d in render(source, filename="deal.lgd").diagnostics if d.rule == NOT_RELATIVE] + assert found.file == "deal.lgd" + assert found.level == "error" and "'/etc/passwd'" in found.message + + +# -- Strict --------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("kind", KINDS) +def test_strict_refuses_a_path_outside_the_root(kind: str, tmp_path: Path) -> None: + source, _ = _kind_source(kind, "../../a.lgd") + with pytest.raises(RenderRefused) as refused: + render(source, strict=True, document_root=tmp_path) + assert OUTSIDE in {d.rule for d in refused.value.diagnostics} + + +@pytest.mark.parametrize("kind", [k for k in KINDS if k != "include"]) +def test_strict_without_a_root_accepts_a_path_that_leaves_the_directory(kind: str) -> None: + # §2.3 defines no default root. (An include is a warning of its own, not an error.) + source, _ = _kind_source(kind, "../../a.lgd") + assert render(source, strict=True).ok + + +@pytest.mark.parametrize("kind", KINDS) +def test_strict_refuses_an_absolute_path(kind: str) -> None: + source, _ = _kind_source(kind, "/etc/passwd") + with pytest.raises(RenderRefused) as refused: + render(source, strict=True) + assert NOT_RELATIVE in {d.rule for d in refused.value.diagnostics} + + +# -- The document root -------------------------------------------------------------------- + + +def test_the_document_sits_at_the_root_when_it_has_no_path(tmp_path: Path) -> None: + source, _ = _kind_source("amends", "../a.lgd") + assert [rule for rule, _ in findings(render(source, document_root=tmp_path))] == [OUTSIDE] + assert [rule for rule, _ in findings(render(source, document_root=str(tmp_path)))] == [OUTSIDE] + assert findings(render(source)) == [] + + +def test_a_document_in_a_subdirectory_may_go_up_to_the_root(tmp_path: Path) -> None: + source, _ = _kind_source("amends", "../shared/a.lgd") + deep = tmp_path / "a" / "b" / "doc.lgd" + assert findings(render(source, document_root=tmp_path, document_path=deep)) == [] + source, _ = _kind_source("amends", "../../../a.lgd") + assert [rule for rule, _ in findings(render(source, document_root=tmp_path, document_path=deep))] == [OUTSIDE] + + +def test_the_root_may_be_given_with_dots_and_the_path_relative(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + (tmp_path / "docs" / "sub").mkdir(parents=True) + monkeypatch.chdir(tmp_path) + source, _ = _kind_source("amends", "../a.lgd") + assert findings(render(source, document_root="docs", document_path="docs/sub/doc.lgd")) == [] + assert findings(render(source, document_root="docs/sub/..", document_path="./docs/sub/doc.lgd")) == [] + assert [r for r, _ in findings(render(source, document_root="docs", document_path="docs/doc.lgd"))] == [OUTSIDE] + + +def test_a_symbolic_link_is_not_followed(tmp_path: Path) -> None: + # The check is lexical: it neither resolves links nor reads anything. + outside = tmp_path / "outside" + root = tmp_path / "root" + outside.mkdir() + root.mkdir() + try: + (root / "link").symlink_to(outside, target_is_directory=True) + except (OSError, NotImplementedError): + pytest.skip("symbolic links are not available") + source, _ = _kind_source("amends", "link/a.lgd") + assert findings(render(source, document_root=root)) == [] + + +def test_a_root_that_does_not_exist_is_fine(tmp_path: Path) -> None: + source, _ = _kind_source("amends", "../a.lgd") + assert [r for r, _ in findings(render(source, document_root=tmp_path / "nope"))] == [OUTSIDE] + + +def test_a_document_outside_the_root_is_refused(tmp_path: Path) -> None: + root = tmp_path / "root" + with pytest.raises(OptionsError, match="outside the document root"): + render("# A\n", document_root=root, document_path=tmp_path / "doc.lgd") + with pytest.raises(OptionsError): + render("# A\n", document_root=root, document_path=root / ".." / "doc.lgd") + # A directory whose name starts with dots is not the parent. + render("# A\n", document_root=root, document_path=root / "..x" / "doc.lgd") + # Both are checked even when the document has nothing to check. + with pytest.raises(OptionsError): + render("# A\n", document_root=root, document_path=tmp_path / "other" / "doc.lgd", format="text") + + +def test_an_empty_root_or_path_is_refused() -> None: + with pytest.raises(OptionsError, match="document_root"): + render("# A\n", document_root="") + with pytest.raises(OptionsError, match="document_path"): + render("# A\n", document_root=".", document_path="") + + +@pytest.mark.skipif(os.name != "nt", reason="drive letters are a Windows matter") +def test_a_document_on_another_drive_is_refused() -> None: # pragma: no cover - Windows only + with pytest.raises(OptionsError, match="drive"): + render("# A\n", document_root="C:\\docs", document_path="D:\\docs\\doc.lgd") + + +def test_a_path_without_a_root_is_not_checked_for_the_document(tmp_path: Path) -> None: + # document_path alone configures no root. + source, _ = _kind_source("amends", "../../../a.lgd") + assert findings(render(source, document_path=tmp_path / "doc.lgd")) == [] + + +def test_the_document_path_names_the_file_unless_one_is_given(tmp_path: Path) -> None: + source, _ = _kind_source("amends", "/x") + result = render(source, document_path=tmp_path / "deal.lgd") + assert {d.file for d in result.diagnostics} == {"deal.lgd"} + named = render(source, document_path=tmp_path / "deal.lgd", filename="other.lgd") + assert {d.file for d in named.diagnostics} == {"other.lgd"} + + +def test_options_default_to_no_root() -> None: + options = RenderOptions() + assert options.document_root is None and options.document_path is None + + +# -- render_file ----------------------------------------------------------------------- + + +def test_render_file_sets_the_document_path(tmp_path: Path) -> None: + sub = tmp_path / "sub" + sub.mkdir() + path = sub / "deal.lgd" + # Up one directory is the root; up two escapes it. + source, _ = _kind_source("amends", "../a.lgd") + path.write_text(source, encoding="utf-8") + assert findings(render_file(path, document_root=tmp_path)) == [] + assert findings(render_file(path, RenderOptions(document_root=tmp_path))) == [] + assert [r for r, _ in findings(render_file(path, document_root=sub))] == [OUTSIDE] + # Without a root: no check. + path.write_text(_kind_source("amends", "../../../a.lgd")[0], encoding="utf-8") + assert findings(render_file(path)) == [] + escaping = findings(render_file(path, document_root=tmp_path)) + assert [r for r, _ in escaping] == [OUTSIDE] + assert {d.file for d in render_file(path, document_root=tmp_path).diagnostics} == {"deal.lgd"} + + +def test_render_file_refuses_a_document_outside_the_root(tmp_path: Path) -> None: + (tmp_path / "root").mkdir() + path = tmp_path / "deal.lgd" + path.write_text("# A\n", encoding="utf-8") + with pytest.raises(OptionsError): + render_file(path, document_root=tmp_path / "root") + + +def test_render_file_keeps_a_document_path_that_was_given(tmp_path: Path) -> None: + (tmp_path / "sub").mkdir() + path = tmp_path / "deal.lgd" + path.write_text(_kind_source("amends", "../a.lgd")[0], encoding="utf-8") + # The document is said to be in sub/, so ../ stays in the root. + assert findings(render_file(path, document_root=tmp_path, document_path=tmp_path / "sub" / "x.lgd")) == [] + + +# -- The command line -------------------------------------------------------------------- + + +def _write(tmp_path: Path, body_kind: str, path: str, name: str = "deal.lgd", directory: str = "") -> Path: + target = tmp_path / directory / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(_kind_source(body_kind, path)[0], encoding="utf-8") + return target + + +def test_root_reports_a_path_that_escapes(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + document = _write(tmp_path, "include", "../../outside/part.lgd") + source, _ = _kind_source("include", "../../outside/part.lgd") + assert main([str(document), "-f", "text", "--root", str(tmp_path)]) == 0 + err = capsys.readouterr().err + assert f"{document}:{line_of(source, 'Before')}: error: [path-outside-root] " in err + assert main([str(document), "-f", "text"]) == 0 + assert "path-outside-root" not in capsys.readouterr().err + + +def test_root_with_strict_refuses(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + document = _write(tmp_path, "amends", "../../a.lgd") + assert main([str(document), "--strict", "--root", str(tmp_path)]) == 1 + assert "[path-outside-root]" in capsys.readouterr().err + assert main([str(document), "--strict"]) == 0 + + +def test_root_is_the_boundary_for_a_document_in_a_subdirectory(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + document = _write(tmp_path, "amends", "../shared/a.lgd", directory="contracts") + assert main([str(document), "-q", "--root", str(tmp_path)]) == 0 + assert main([str(document), "--root", str(tmp_path / "contracts")]) == 0 + assert "[path-outside-root]" in capsys.readouterr().err + + +def test_root_refuses_a_document_outside_it(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + (tmp_path / "root").mkdir() + document = _write(tmp_path, "amends", "a.lgd") + assert main([str(document), "--root", str(tmp_path / "root")]) == 2 + assert "outside the document root" in capsys.readouterr().err + + +def test_root_with_standard_input_puts_the_document_at_the_root(tmp_path: Path) -> None: + source, _ = _kind_source("amends", "../a.lgd") + done = run_cli_text("-", "-f", "text", "--root", str(tmp_path), stdin=source) + assert done.returncode == 0 + assert f":{line_of(source, ' file:')}: error: [path-outside-root] " in done.stderr + assert "path-outside-root" not in run_cli_text("-", "-f", "text", stdin=source).stderr + + +def test_not_relative_is_reported_without_a_root(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + document = _write(tmp_path, "translation", "https://x.example/cs.lgd") + assert main([str(document), "-f", "text"]) == 0 + assert f"{document}:{line_of(document.read_text(encoding='utf-8'), ' cs:')}: error: [path-not-relative] " \ + in capsys.readouterr().err + + +def test_root_is_in_the_help() -> None: + done = run_cli_text("--help") + assert "--root" in done.stdout and "DIR" in done.stdout From 852308111da2f3a95009faec6edf2dcdf6e5b999 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:18:17 +0000 Subject: [PATCH 043/118] Run the spec's file-reference fixtures with their document root The conformance tests honour requires_config.document_root and check that path-not-relative and path-outside-root are reported at the expected lines; without a root the outside-root fixture renders with no path error. --- tests/conformance/test_spec_corpus.py | 34 ++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/tests/conformance/test_spec_corpus.py b/tests/conformance/test_spec_corpus.py index 93dfb79..32b94dd 100644 --- a/tests/conformance/test_spec_corpus.py +++ b/tests/conformance/test_spec_corpus.py @@ -12,7 +12,7 @@ import pytest from conftest import spec_root -from legaldown_render import DocumentError, render +from legaldown_render import DocumentError, render, render_file pytestmark = pytest.mark.conformance @@ -82,6 +82,38 @@ def test_invalid_fixtures_render_their_markers(rule: str) -> None: assert MARKERS[rule] in output, f"{case.name} does not render {MARKERS[rule]!r}" +#: The file-reference rules (§2.3, §16.2): the renderer reports them itself, since the validator does not. +FILE_REFERENCE_RULES = ("path-not-relative", "path-outside-root") + + +@pytest.mark.parametrize("rule", FILE_REFERENCE_RULES) +def test_file_reference_fixtures_report_their_rule_at_its_line(rule: str) -> None: + """A case may ask for a configuration (``requires_config``). The document + root is the one setting this renderer can make; a case that needs more is + skipped.""" + cases = _entries(SPEC / "fixtures" / "invalid" / rule) + assert cases, f"no fixture for {rule}" + for case in cases: + expected = json.loads(case.with_suffix(".expected.json").read_text(encoding="utf-8")) + config = expected.get("requires_config", {}) + if set(config) - {"document_root"}: + pytest.skip(f"{case.name} needs {sorted(config)}") + settings = {"document_root": case.parent / config["document_root"]} if "document_root" in config else {} + result = render_file(case, **settings) + found = {(d.rule, d.line) for d in result.diagnostics} + for diagnostic in expected["diagnostics"]: + assert (diagnostic["rule"], diagnostic["line"]) in found, f"{case.name}: {sorted(found)}" + assert not result.ok + + +def test_the_outside_root_fixture_is_valid_without_a_root() -> None: + """§2.3: no default root, so the containment check does not run.""" + for case in _entries(SPEC / "fixtures" / "invalid" / "path-outside-root"): + result = render_file(case) + assert "path-outside-root" not in {d.rule for d in result.diagnostics} + assert not [d for d in result.diagnostics if d.rule.startswith("path-")], case.name + + def test_raw_html_fixture_emits_no_html() -> None: for case in _entries(SPEC / "fixtures" / "invalid" / "raw-html"): result = render(case.read_text(encoding="utf-8"), standalone=False) From c09041c0985e71fdfb35520a0ec574e8128503a5 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:18:17 +0000 Subject: [PATCH 044/118] Document the file-reference checks and the document root CONFORMANCE (the images and file-reference row, the include row, the containment sentence that said it was not checked), architecture (security, diagnostics, test table) and README (--root, document_root, document_path). --- CONFORMANCE.md | 38 ++++++++++++++++++++++++++++++-------- README.md | 18 ++++++++++++++++-- docs/architecture.md | 44 ++++++++++++++++++++++++++++++++------------ 3 files changed, 78 insertions(+), 22 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 54105ff..9be547a 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -17,7 +17,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes |---|---| | Section numbering generated at render time, configurable per job (§13.1) | ✅ `decimal`, `legal-outline`, `mixed`, `none`, and custom level formats; per job with `--set numbering.scheme=…` | | Resolution of every §11.1 directive except `{{include:}}`, with every failure marker (§6.3, §7.3, §13.3–§13.5, §15.8) | ✅ | -| `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`) | +| `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | @@ -40,7 +40,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `raw-html` Warning; raw HTML never emitted (§8.7) | ✅ Nothing is emitted: an HTML block is dropped whole, and an inline tag is dropped with its text kept. The Warning is the validator's, once per block or text that holds raw HTML other than comments (see "Text after a comment", below) | | Automatic term recognition (§7.4, MAY) | ❌ Not supported | | Links (§8.7) | ⚠️ Partial, a documented deviation for safety with untrusted documents: links with schemes outside the allow-list render as text with a `render-link-refused` Warning. A link is a hyperlink in HTML and shows `text ` in plain text when its URL is relative, a fragment, or has one of the schemes `http`, `https`, `mailto`, `tel`, `ftp`, `ftps`, `sftp`, `sip`, `sips`, `sms`, `urn`, `geo`, `news` (an allow-list, judged as a browser reads the URL). Any other scheme (`javascript:`, `file:`, `data:`, protocol handlers such as `ms-msdt:`, `vscode:`, `search-ms:`, and pseudo-schemes such as `C:\x` or `localhost:8080`), or an empty URL, is not a link: its text is shown alone, and the Warning `render-link-refused` is reported for a non-empty URL. A link with no text shows its URL as written; the text writer omits `` for an autolink | -| Images and file-reference paths (§8.7, §2.3) | ✅ A relative path (`../` included, as §2.3 defines no default document root) renders `` in HTML and its alt text in plain text. A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is never fetched or embedded: the alt text is shown and the Error `path-not-relative` is reported, which `--strict` refuses. An empty path shows the alt text with no diagnostic | +| Images and file-reference paths (§8.7, §2.3) | ✅ Every file reference (`{{include:}}`, `attachments[].file`, `amends.file`, `supersedes.file`, a `translations` value, an image) must be relative, and within the document root when one is configured (`--root DIR`, `document_root=`; see "File references", below). A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is the Error `path-not-relative`; a path that leaves the root is the Error `path-outside-root`; `--strict` refuses both. Such an image shows its alt text and is never fetched or embedded; an `{{attach:}}` shows its text with no link; an include marker and an attachment's path stay as text, and no file is ever read. A relative path (`../` included, as §2.3 defines no default root) renders `` in HTML and its alt text in plain text. An empty image path shows the alt text with no diagnostic | | Plain-text layout (§13.6, optional) | ✅ `attachments.separator` (§13.8): `none` prints no `---`; `rule` and `page-break` both print one before the attachments heading and between consecutive attachments, so the heading stays with the first (text has no pages). One quote stays one quote (blank lines keep the `>`), and the blocks of a list item are set apart by blank lines and indented under the item's label, whatever its condition label; an item that opens with a nested list, table, code block or quote has its label on a line of its own, so that block stays valid. A pipe in a table cell is backslash-escaped, and the separator row marks column alignment (`:---`, `:---:`, `---:`). A label a style blanks drops its `label: ` prefix in the title block, party details and signature blocks (a signature row with neither label nor value is omitted); HTML keeps an empty `
` so its grid layout and the `
` stay valid | | Extended tables through raw HTML (§9.2, MAY) | ❌ Not supported; the HTML table is dropped with the Warning | | Visual blank for a placeholder (§13.5 rule 6) | ✅ `placeholders.blank`; a blank that is empty, or only whitespace or invisible format characters (U+200B, U+FEFF), renders `[TBD: id]` | @@ -54,7 +54,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes A template with Errors is refused too, since §15.7.2 defines no output for one. - **Line numbers (§16.9).** Every diagnostic names its line, the validator's and the renderer's own (`render-ref-ambiguous`, `ref-not-enumerated`, `render-not-processed`, - `render-link-refused`, `path-not-relative`, `render-term-limit`, `render-term-too-long`, + `render-link-refused`, `path-not-relative`, `path-outside-root`, `render-term-limit`, `render-term-too-long`, `render-locale-fallback`), and the CLI prints it as `file:line:`. The renderer's line is the **first line of the block it is in**, as `Document.line_of` gives it, whatever line the directive itself is on: a paragraph's, a table's header row, a quote's first `>`, a code @@ -75,11 +75,33 @@ It is checked against the specification's own examples and fixtures corpus (`tes image path, on two lines is one diagnostic, at the first of them. A document with no source to name a line of (built from a dict) reports no line. With `answers`, the lines are those of the assembled document (§15.8), not of the template file. -- **`path-not-relative` for images.** The renderer reports the Core rule `path-not-relative` - (§2.3, §16.2) for an image path, because the validator (0.4) does not check image paths. It - leaves its own report out when the validator names the same path at the same line (or no line), - and keeps it for a path the validator did not name, or the same path at another line. - The containment rule (`path-outside-root`) is not checked: no document root is configured. +- **File references (§2.3).** The renderer reports `path-not-relative` and `path-outside-root` + (§2.3, §16.2) for every file reference itself, because the validator (0.4) checks neither rule. + It leaves its own report out when the validator names the same path at the same line (or no + line), and keeps it for a path the validator did not name, or the same path at another line. + A path with both problems (`/../x`) is `path-not-relative`. The reports are at the line of the + block holding the `{{include:}}` or image, or of the frontmatter key (`attachments[].file`, + `amends.file`, `supersedes.file`, `translations.`), like the rest. + The checks are lexical, as §2.3 says: no path is resolved against the file system, no symbolic + link is followed, and no file is read or its existence checked (that is Full, §17.4). + The containment rule runs only where a root is set, as §2.3 says (it defines no default root, + and a renderer for documents it does not control SHOULD set one): `--root DIR`, or + `RenderOptions.document_root`. The root is the boundary, and the base of a path is the directory + of the document relative to it, so `../shared/a.lgd` is within the root from `/contracts/x.lgd` + and outside it from `/x.lgd`. `render_file` and the command line say where the document is + (`RenderOptions.document_path`); text given to `render()`, or standard input, is at the root. A + document outside the root, or on another Windows drive, is an `OptionsError` (exit 2). + How a path is read: a backslash is a separator, so `..\..\x` leaves the root as `../../x` does + (the renderer's reading; a leading `\` is not relative anyway); `..` and `.` followed by spaces + are `..` and `.` (Windows reads them so); tabs and newlines inside a scheme, and spaces and + control characters at either end, are ignored as a browser ignores them. Percent-encoding is + decoded once, only for what is used as a URL, an image's source and the link `{{attach:}}` would + make to an attachment's file: there `%2e%2e/` is `../`. A path in the frontmatter or an + `{{include:}}` is a file name, read as written, where `%2e%2e` is a directory with that name. + `supersedes` as a plain string describes the superseded document (§3.2) and is not a path. + Markdown links (`[text](/etc/passwd)`, §8.7) are not file references and stay governed by the + link allow-list (`render-link-refused`): they are links the reader may follow, not files the + renderer would read. - **Nesting limit.** The validator reads lists 64 levels deep and quotes 16; deeper, an item's or quote's content is its text. The renderer follows it, and refuses (`DocumentError`) a document in which the enclosing lists and quotes plus one text's inline formatting (emphasis, links) nest diff --git a/README.md b/README.md index b7154d6..3078948 100644 --- a/README.md +++ b/README.md @@ -61,11 +61,20 @@ legaldown-render contract.lgd --style continental --locale cs-CZ -o smlouva.html legaldown-render contract.lgd --set numbering.scheme=legal-outline --set enumeration.enabled=false legaldown-render contract.lgd --set contents.enabled=true --set contents.depth=3 # table of contents legaldown-render contract.lgd --strict # refuse if the document has errors +legaldown-render contract.lgd --root . --strict # file references may not leave this directory legaldown-render contract.lgd --final --strict # refuse if blanks or template constructs remain legaldown-render template.lgd --answers answers.yaml -o nda.html # fill in a template, then render it legaldown-render --print-style --style continental # every setting, as YAML ``` +`--root DIR` sets the document root (§2.3): a file reference that leaves it (an `{{include:}}` +path, an attachment's, `amends`, `supersedes`, a translation's, an image's) is the Error +`path-outside-root`, and the file is not used. Paths that are not relative (a URI scheme, an absolute +path, a Windows drive) are the Error `path-not-relative` with or without a root. Nothing is ever +read or fetched, and the check is lexical, so symbolic links are not followed. The document must be +inside DIR (exit 2 otherwise); standard input counts as at DIR itself. Set a root when you render documents +you do not control: without one, `../` is allowed (§2.3 defines no default). + `--locale` sets how dates, money, and durations are formatted. The words the renderer generates (such as "Represented by" or "Signatures") follow the document's `language` (§13.7), whatever the locale. @@ -75,7 +84,7 @@ The exit code is 0 when the document rendered, even if it has errors (they show as markers), 1 when rendering was refused (strict mode, or a template that cannot be assembled with its answers) or the document or answers file cannot be read (a document that is not UTF-8 included) or the output file cannot be written, 2 for an invalid -style, setting, or usage, and 70 for an internal error (a bug to report). +style, setting, or usage (a document outside `--root` too), and 70 for an internal error (a bug to report). Documents are read as UTF-8, and output is UTF-8 with LF line endings, on standard output and in `-o` files, whatever the console's encoding and the platform's newline. The command reconfigures @@ -105,8 +114,13 @@ with open("contract.html", "w", encoding="utf-8", newline="\n") as output: `render()` also accepts a `RenderOptions`. Pass `answers={...}` to render a template filled in. Each diagnostic has its `line` and its `file`: `render_file()` and the command name the file (its name, without the directory); for text given to `render()`, pass `filename=`. +`document_root=` is the `--root` of the command line, and `document_path=` says where the document +is in it (`render_file()` sets it from its path; text given to `render()` is at the root); a +document outside the root raises `OptionsError`. For example, a hosting service that renders +uploaded text: +`render(text, document_root="/srv/docs/tenant-1")`. Problems in the document never raise. Only an invalid style (`StyleError`), an unknown output -format (`OptionsError`), an unreadable document (`DocumentError`), strict mode, or a template that +format or a document outside its root (`OptionsError`), an unreadable document (`DocumentError`), strict mode, or a template that cannot be assembled with its answers (`RenderRefused`) do. All of them are `RenderError`s, and `StyleError` and `OptionsError` are also `ValueError`s. They can be pickled, so they cross process boundaries (`multiprocessing`, `concurrent.futures`) intact. `overrides=` takes dotted settings and diff --git a/docs/architecture.md b/docs/architecture.md index 81118df..e401ef7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -233,14 +233,15 @@ turns into an error. raises `RenderRefused` instead, carrying the diagnostics. - **Configuration problems** raise before rendering, with every problem named: an invalid style, setting, or locale raises `StyleError` (this includes a `{field}` that a text template does not - accept or lacks, a `--set` value that is not valid YAML, and an `extends` cycle); an unknown output format raises `OptionsError`. Both are + accept or lacks, a `--set` value that is not valid YAML, and an `extends` cycle); an unknown output format, or a document that is not inside the configured document root, raises `OptionsError`. Both are `RenderError`s (and `ValueError`s, as they always were). Every renderer error pickles, so it survives `multiprocessing`. - **Internal inconsistencies** raise `InternalError`, and the CLI exits with 70. Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can evaluate: -`ref-not-enumerated` and `path-not-relative` (an Error for an image path that is not relative, §2.3, -reported here because validator 0.4 does not check image paths), specification rule ids, plus renderer-specific ids prefixed `render-`: +`ref-not-enumerated`, `path-not-relative` and `path-outside-root` (Errors for a file reference that is +not relative, or that leaves the document root, §2.3; reported here because validator 0.4 checks +neither), specification rule ids, plus renderer-specific ids prefixed `render-`: `render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, or 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), a `{{term:}}` shows its id), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's diagnostics carry their line (§16.9), and so do the renderer's own, and the CLI prints it as `file:line:`. @@ -264,8 +265,8 @@ set to the file name as the validator does (`Document.filename`, without the dir **Repeats.** `Resolver._warn` and `api.render` leave out a diagnostic whose rule, level and message a reported one has already, wherever it is (`diagnostics.repeats`), so a repeated finding is -reported at its first line only. The renderer's `path-not-relative` is left out when the validator -reported the same path at the same line, or at none (`diagnostics.covered_by`). +reported at its first line only. The renderer's `path-not-relative` and `path-outside-root` are left +out when the validator reported the same path at the same line, or at none (`diagnostics.covered_by`). ## Security @@ -297,13 +298,31 @@ reported the same path at the same line, or at none (`diagnostics.covered_by`). API returns are not all escaped: only a path or URL quoted by the renderer's own rules (`path-not-relative`, `render-link-refused`) is, at its source; a caller that shows other messages must do the same. -- An image path must be relative with no URI scheme (§2.3), so a remote resource is never fetched - or embedded. Resolution, not a writer, enforces it: an image whose path has a scheme (`https:`, - `file:`, `data:`, a Windows drive such as `C:\x`) or starts with `/` or `\` (`//host`) is - replaced by its alt text and reported as the Core Error `path-not-relative`, which `--strict` - refuses. An image with an empty path shows its alt text too, with no diagnostic (§2.3 has no - rule for it). `../` stays allowed: §2.3 defines no default document root. The writers' image check - is a second line. +- Every file reference (§2.3) must be relative with no URI scheme, and within the document root + when one is configured, so a remote resource is never fetched or embedded and no path leads + outside the tree the document belongs to. The references are `{{include:}}` paths, + `attachments[].file`, `amends.file`, `supersedes.file` (the object form; a plain string + describes the document and is no path), `translations` values, and image paths. Resolution, not a + writer, enforces it (`Resolver._reference`, one place for every kind, over + `urls.file_reference_problem`): an image whose path has a scheme (`https:`, `file:`, `data:`, a + Windows drive such as `C:\x`) or starts with `/` or `\` (`//host`), or leaves the root, is replaced by + its alt text; an `{{attach:}}` whose file fails is its text with no link; an include marker and + an attachment placeholder's path stay as text. Each is reported as the Core Error + `path-not-relative` or `path-outside-root`, which `--strict` refuses. An image with an empty path + shows its alt text too, with no diagnostic (§2.3 has no rule for it). The checks are lexical, + as §2.3 says: no `resolve()`, no symbolic link followed, no file read or tested for existence. + The root is `RenderOptions.document_root` (`--root DIR`); with none, `../` is allowed, §2.3 + defining no default root (a service rendering documents it does not control SHOULD set one). A + path's base is the document's directory relative to the root (`api._document_base`: the lexical + `relpath` of the directory of `document_path` to the root, `/`-separated; the document at the + root when it has no path, as text given to `render()` and standard input are); a document + outside the root, or on another Windows drive, is an `OptionsError`, since the boundary would + mean nothing. A backslash is read as a separator (the renderer's reading, so a path that goes up + on Windows is not missed), `..`/`.` with trailing spaces are `..`/`.`, and percent-encoding is + decoded once, only for what is used as a URL (an image's `src`, the link an `{{attach:}}` + makes): `%2e%2e` is `..` there, and a directory name in a frontmatter or include path. The + writers' image check is a second line. Markdown links are not file references (§8.7): they stay + governed by the allow-list above, since the renderer reads nothing at their targets. - At Rendering level, the only files read are the document, the style template, and the answers file the CLI reads with `--answers` (the Python API takes answers as a mapping). Attachment files and includes are never opened. @@ -337,6 +356,7 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_terms_budget.py` | Reuse of worked-out term texts, the work budget and depth limit for terms that name each other. Bounds the work by counting resolved directives, never by timing | | `tests/test_diagnostic_lines.py` | The line and file of every renderer diagnostic: in each kind of block (paragraph, list item, nested item, table, quote, heading, preamble) and at frontmatter keys, the repeats left out, the tree nodes' lines, and `file:line:` on the command line | | `tests/test_links.py` | Which images are shown and which links are links: non-relative image paths, the link scheme allow-list, empty URLs and empty link text, and how a refusal is reported | +| `tests/test_file_references.py` | File references (§2.3): the lexical check (relative, `../`, scheme, drive, backslash, encoding), each kind of reference with and without a root and from a subdirectory, what is shown instead, the options and `--root`, strict, and the lines | | `tests/test_text_writer.py` | The plain-text layout: attachment separators, tables, quotes, list items and indent, blanked labels | | `tests/test_html.py` | The HTML writer's print stylesheet for attachments | | `tests/test_labels.py` | The built-in labels of parties, per language | From 29edc2e9fb16b985b585e74243b11813b8bff78d Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:24:58 +0000 Subject: [PATCH 045/118] =?UTF-8?q?Designate=20sections=20under=20the=20no?= =?UTF-8?q?ne=20scheme=20by=20their=20heading=20as=20rendered=20(=C2=A713.?= =?UTF-8?q?3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reference to a section under the none scheme shows its heading text. The text was a text-only reading that left every {{ref:}} out, so a heading with a reference was designated without it. Each section is now designated by the plain text of its heading with the references and terms in it resolved. The designations are worked out once each by a depth-first walk in document order (Resolver._designate), after the sections, items and paragraphs are numbered on their text-only designations. A reference to an item or paragraph under another heading is composed again on that heading's designation, from the pieces the target records (_Target.section_index and parts). Where any designation differs, the structure is numbered again on the final ones; the anchors are the same, since they depend on identifiers and order alone, and render-ref-ambiguous reads the final designations. Documents whose headings and the terms they use hold no {{ref:}} skip all of this. A heading that refers back to itself (render-designation-cycle), or reads through more than 32 headings, is longer than 1000 characters once resolved, or is nested too deeply (render-designation-limit) is designated by its text-only reading. A reference shows at most its first 1000 characters. The walk reports nothing itself and gives out no anchor: its terms are kept in a cache of their own, and what it could not do is reported at the heading. A term met inside an open term, or while designations are worked out, at 60 nested inline levels now shows its label or id with render-term-limit. Terms read inside terms stack the 100 levels the builder allows in one block, and a chain of them exhausted the Python stack (a RecursionError) in decimal and none documents alike; one block's own nesting never reaches the limit. Tests: test_none_designations.py is new. The none-scheme test that pinned the old behaviour (test_none_scheme_designation_skips_nested_references) is renamed and expects the reference to be shown. The term-cache equivalence test now also draws headings that refer to later headings and to terms. --- src/legaldown_render/resolve/resolver.py | 380 +++++++++++++++++---- tests/test_none_designations.py | 416 +++++++++++++++++++++++ tests/test_render.py | 6 +- tests/test_terms_budget.py | 42 ++- 4 files changed, 779 insertions(+), 65 deletions(-) create mode 100644 tests/test_none_designations.py diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index c8d1802..91740b1 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -151,6 +151,42 @@ #: one term, so that a pathological tangle of cycles cannot fill memory. _TERM_TEXTS_PER_TERM = 32 +#: The most inline levels (emphasis, links, a term's text, a heading read for a +#: reference) resolved inside one another where one unit is read inside another, +#: which is where the Python stack can run out: the builder allows 100 levels in +#: one block (``build.MAX_NESTING``), but terms read inside terms, and headings +#: read through the references in other headings, stack one block's levels on +#: another's. A ``{{term:}}`` met inside an open term, or while designations are +#: worked out, that is nested this deep shows its label or id +#: (``render-term-limit``); a heading this deep stops reading others +#: (``render-designation-limit``). A single block's nesting, up to the +#: builder's limit, never reaches it. +_MAX_NEST = 60 + +#: Under the ``none`` scheme (§13.3) a section is designated by its heading with +#: the references in it resolved. The most headings read inside one another (a +#: heading that refers to a heading that refers to...), so that a long chain +#: does not exhaust the stack; and the longest designation: a heading that +#: reads longer once its references are resolved (``# H1 {{ref: h2}} {{ref: +#: h2}}`` down a chain would otherwise grow as 2**n) is designated by its text +#: without them, and a reference never shows more than this many characters. +_MAX_DESIGNATION_DEPTH = 32 +_MAX_DESIGNATION_TEXT = 1000 + +_DESIGNATION_LIMIT = ( + f"Headings and the terms in them refer to each other too deeply or too often (more than " + f"{_MAX_DESIGNATION_DEPTH} headings, or terms or formatting nested too far), or a heading reads longer than " + f"{_MAX_DESIGNATION_TEXT} characters once its references are resolved; under the none scheme such a heading is " + f"designated by its text as written, without the references in it, and a reference shows at most its first " + f"{_MAX_DESIGNATION_TEXT} characters." +) + +#: What a reference can be resolved from: before the targets are known, only the +#: headings' own text; then, under the ``none`` scheme, the designations of the +#: headings that refer to each other being worked out; then everything (see +#: ``Resolver.run``). +_EARLY, _DESIGNATING, _FINAL = range(3) + class _TermText(NamedTuple): """A term's text, the terms it named, and those it was cut short at @@ -172,6 +208,12 @@ class _Target: enumerated: bool = True #: The numbered unit the anchor is on, for render-ref-ambiguous. slot: _Slot | None = None + #: The section the anchor is in (a section's own index), None in the preamble. + section_index: int | None = None + #: For an item or paragraph: the (``ref`` template, counter text) of each + #: level below the section, so that its designation can be put together + #: again on another section designation (see ``_compose``). + parts: tuple[tuple[str, str], ...] = () class Resolver: @@ -216,6 +258,8 @@ def __init__( # The same for the text-only pass before the targets are known, and # for the validator's term text where a definition is not in the body. self.term_texts_early: dict[str, list[_TermText]] = {} + # And for the text-only pass that works out designations (see _designate). + self.term_texts_designating: dict[str, list[_TermText]] = {} self.term_texts_plain: dict[str, list[_TermText]] = {} # The definitions whose term text is being worked out, so that a # {{term:}} inside a term, directly or through other terms, cannot @@ -229,9 +273,24 @@ def __init__( # Inline nodes resolved inside repeat computations (see _while_open). self._term_work = 0 self._repeating = 0 - # False until every target is registered and the placeholder types are - # known (run() sets it just before the inlines are resolved). - self._targets_known = False + # What a reference can be resolved from (see run()). + self._stage = _EARLY + # Inline levels being resolved inside one another (see _MAX_NEST). + self._nest = 0 + # Under the none scheme, while designations are worked out (see + # _designate): each section's designation (None until known), its + # text-only reading, its title, the sections open inside one another, + # the sections a reference came back to, and whether a limit was hit + # (and in which section first). + self._memo: list[str | None] = [] + self._provisional: list[str] = [] + self._titles: list[tuple[Inline, ...]] = [] + self._stack: list[int] = [] + self._cycles: list[int] = [] + self._limited = False + self._limited_at: int | None = None + # The section being structured, for the targets it holds. + self._section_index: int | None = None self.attachment_anchors: dict[str, str | None] = {} self.section_presences: list[Presence] = [] @@ -240,13 +299,21 @@ def __init__( def run(self) -> RenderTree: self._survey() attachments = self._attachments() - sections = self._number_sections() - preamble = tuple(self._structure(block, section=None, depth=0, presence=ALWAYS) for block in self.tree.preamble) - sections = tuple( - replace(section, blocks=self._structure_section(section, presence)) - for section, presence in zip(sections, self.section_presences, strict=True) - ) - self._targets_known = True + anchors = set(self.used_anchors) + sections, preamble = self._structure_all(None) + if self.textual and self._reads_references(sections): + # §13.3: a section is designated by its heading as rendered, which + # may hold references to sections, items and paragraphs, all + # numbered by now under their provisional (text-only) + # designations. Work the final ones out, and where any differs + # number everything again on them: the anchors come out the same, + # since which unit gets which depends on identifiers and order alone. + designations = self._designate(sections) + if designations != [section.designation for section in sections]: + self.used_anchors = anchors + self.targets, self.designation_slots, self.section_presences = {}, {}, [] + sections, preamble = self._structure_all(designations) + self._stage = _FINAL resolve = self._resolve_inlines # In document order, so that the first of several definitions of a # term is the one that keeps its anchor. @@ -286,6 +353,133 @@ def run(self) -> RenderTree: # definition may live in an attachment or an amended original. return map_tree_inlines(resolved, self._settle_term_target) + def _structure_all(self, designations: list[str] | None) -> tuple[tuple[Section, ...], tuple[Block, ...]]: + """The preamble and the sections, numbered and labelled; under the + ``none`` scheme with the section *designations* worked out by + :meth:`_designate`, or else the headings' text alone.""" + sections = self._number_sections(designations) + self._section_index = None + preamble = tuple(self._structure(block, section=None, depth=0, presence=ALWAYS) for block in self.tree.preamble) + out: list[Section] = [] + for index, (section, presence) in enumerate(zip(sections, self.section_presences, strict=True)): + self._section_index = index + out.append(replace(section, blocks=self._structure_section(section, presence))) + self._section_index = None + return tuple(out), preamble + + # -- designations under the none scheme (§13.3) ------------------------------ + + def _reads_references(self, sections: tuple[Section, ...]) -> bool: + """True if a section's designation may depend on a ``{{ref:}}``: one + stands in a heading or in the text of a term the headings use, + directly or through other terms, or in the validator's own text of + such a term (read where the definition is not in the body). Without + one, the text-only reading of a heading is its designation.""" + lookup = self.result.index.definition_lookup + seen: set[str] = set() + pending = [section.title for section in sections] + while pending: + for inline in _walk(pending.pop()): + if not isinstance(inline, DirectiveSource): + continue + directive = inline.directive + if directive.name == "ref": + return True + if directive.name == "term" and directive.positional and directive.positional not in seen: + seen.add(directive.positional) + source = self.term_sources.get(directive.positional) + if source is not None: + pending.append(source.children) + if "{{" in (lookup.get(directive.positional) or ""): + return True + return False + + def _designate(self, sections: tuple[Section, ...]) -> list[str]: + """The designation of every section under the ``none`` scheme: its + heading as rendered, plain, with the references and terms in it + resolved (§13.3). Each is worked out once, by a depth-first walk in + document order that reads the designation of a referenced heading + where the reference stands (:meth:`_designation`). + + The walk reports nothing and gives out no anchor itself: the heading + is read again in its place, once every designation is known, and a + diagnostic about it is made there, once. The terms it works out are + kept apart from those of that later pass, which the designations + decide. What the walk could not do in full is reported here, at the + heading it concerns.""" + self._provisional = [section.designation or "" for section in sections] + self._titles = [section.title for section in sections] + self._memo = [None] * len(sections) + self._stack, self._cycles = [], [] + self._limited, self._limited_at = False, None + mark, work = len(self.diagnostics), self._term_work + self._stage = _DESIGNATING + try: + for index in range(len(sections)): + self._designation(index) + finally: + self._stage = _EARLY + del self.diagnostics[mark:] + self._term_work = work + self.term_texts_designating = {} + for index in sorted(set(self._cycles)): + with self._at(sections[index].line): + self._warn( + "render-designation-cycle", + f"The heading '{printable(sections[index].identifier or self._provisional[index])}' refers to " + f"itself, directly or through other headings or terms; under the none scheme the reference " + f"that closes the circle shows the heading without its references.", + ) + if self._limited: + with self._at(sections[self._limited_at].line if self._limited_at is not None else self.line): + self._warn("render-designation-limit", _DESIGNATION_LIMIT) + return [memo or "" for memo in self._memo] + + def _designation(self, index: int) -> str: + """The designation of section *index*, worked out the first time it is + asked for. A section that is open already (a reference that comes back + to a heading it is read from) reads as its text-only reading, and is + reported by ``render-designation-cycle``; so does one asked for past + :data:`_MAX_DESIGNATION_DEPTH` open sections or :data:`_MAX_NEST` levels + (``render-designation-limit``), which is worked out when it is next + reached by a shorter way, or in its turn. Neither is kept. A designation + that reads longer than :data:`_MAX_DESIGNATION_TEXT` characters once its + references are resolved is its text-only reading too, which is kept.""" + memo = self._memo[index] + if memo is not None: + return memo + provisional = self._provisional[index] + if index in self._stack: + self._cycles.append(index) + return provisional + if len(self._stack) >= _MAX_DESIGNATION_DEPTH or self._nest >= _MAX_NEST: + self._limit_hit() + return provisional + self._stack.append(index) + try: + text = self._title_text(self._titles[index]) + finally: + self._stack.pop() + if len(text) > _MAX_DESIGNATION_TEXT and text != provisional: + self._limit_hit(index) + text = provisional + self._memo[index] = text + return text + + def _limit_hit(self, index: int | None = None) -> None: + """Note, while designations are worked out, that a limit was reached + in section *index*, else in the one being read.""" + self._limited = True + if self._limited_at is None: + self._limited_at = index if index is not None else self._stack[-1] if self._stack else None + + def _designation_of(self, target: _Target) -> str: + """What a reference to *target* reads as while designations are worked + out: the designation of its section, with its item or paragraph pieces.""" + if target.section_index is None: + return target.designation + return _compose(target.parts, self._designation(target.section_index)) + def _resolve_section(self, section: Section) -> Section: with self._at(section.line): return replace( @@ -385,7 +579,7 @@ def _register(self, identifier: str, target: _Target) -> None: # -- structure: numbering, labels, anchors --------------------------------- - def _number_sections(self) -> list[Section]: + def _number_sections(self, designations: list[str] | None = None) -> list[Section]: """Number the sections, and record each one's presence (§15.3). The numbers are the validator's own (``result.index.sections``), @@ -395,11 +589,15 @@ def _number_sections(self) -> list[Section]: part of a number ("2.1" has two) with the n-th level format. A section's rendered level is how many parts its number has, so its number format, heading, and heading style always go together. + + Under the ``none`` scheme a section's designation is given in + *designations* once they are worked out (:meth:`_designate`); until + then it is the heading's text-only reading. """ levels = heading_levels(self.style.numbering) presences: dict[int, Presence] = {} # level -> presence of the open section out: list[Section] = [] - for section, indexed in zip(self.tree.sections, self.result.index.sections, strict=True): + for index, (section, indexed) in enumerate(zip(self.tree.sections, self.result.index.sections, strict=True)): # The heading's own level, clamped to 1-5 (§4.1), decides which # sections enclose it, and so its presence (§15.3). level = min(max(section.level, 1), 5) @@ -409,8 +607,12 @@ def _number_sections(self) -> list[Section]: self.section_presences.append(presences[level]) parts = [int(part) for part in indexed.number.split(".")] if self.textual: - with self._at(section.line): - label, designation = None, self._title_text(section.title) + label = None + if designations is not None: + designation = designations[index] + else: + with self._at(section.line): + designation = self._title_text(section.title) else: designation = "" for position, counter in enumerate(parts): @@ -423,7 +625,7 @@ def _number_sections(self) -> list[Section]: # Sections with one number are alternatives (the validator's). slot = _Slot(_SECTIONS, indexed.number, presences[level], "section", section.identifier) self._record(designation, slot) - self._register(section.identifier, _Target(designation, anchor, slot=slot)) + self._register(section.identifier, _Target(designation, anchor, slot=slot, section_index=index)) out.append(replace(section, level=min(len(parts), 5), label=label, designation=designation, anchor=anchor)) return out @@ -444,25 +646,32 @@ def _exclusive(self, first: Presence, second: Presence) -> bool: def _title_text(self, title: tuple[Inline, ...]) -> str: """A heading's text, as the ``none`` scheme designates it (§13.3). - A text-only pass: it gives out no anchors, and it leaves references - out wherever they are, since their targets are not all known yet. - Other directives show their display text; a defined term its term. + A text-only pass: it gives out no anchors. Before the designations + are known it leaves references out wherever they are, since their + targets are not all known yet, which makes it the provisional + designation; while they are worked out (:meth:`_designate`) a reference + shows the designation of its target. Other directives show their + display text; a defined term its term. """ def text(inlines: tuple[Inline, ...]) -> str: parts: list[str] = [] - for inline in inlines: - if self._repeating: - self._term_work += 1 - match inline: - case DirectiveSource(directive=directive): - if directive.name != "ref": - resolved = self._directive(directive) - parts.append(plain_text((resolved,)) if resolved is not None else "") - case DefinitionSource(children=children) | Emphasis(children=children) \ - | Strong(children=children) | Link(children=children) | Image(children=children): - parts.append(text(children)) - case _: - parts.append(plain_text((inline,))) + self._nest += 1 + try: + for inline in inlines: + if self._repeating: + self._term_work += 1 + match inline: + case DirectiveSource(directive=directive): + if directive.name != "ref" or self._stage == _DESIGNATING: + resolved = self._directive(directive) + parts.append(plain_text((resolved,)) if resolved is not None else "") + case DefinitionSource(children=children) | Emphasis(children=children) \ + | Strong(children=children) | Link(children=children) | Image(children=children): + parts.append(text(children)) + case _: + parts.append(plain_text((inline,))) + finally: + self._nest -= 1 return "".join(parts) return " ".join(text(title).split()) @@ -496,10 +705,11 @@ def _number_paragraph(self, block: Paragraph, section: Section, counter: _Counte designation = extend(base, fill(fmt.ref, n=n, section=base), textual=self.textual) label = fill(fmt.label, n=n, section=base, path=designation) slot = _Slot(counter, number, presence, "paragraph", block.anchor_id) - target = _Target(designation, None, slot=slot) + target = _Target(designation, None, slot=slot, section_index=self._section_index, + parts=((fmt.ref, n),)) self._record(designation, slot) else: - label, target = None, _Target(base, None, enumerated=False) + label, target = None, _Target(base, None, enumerated=False, section_index=self._section_index) anchor = self._anchor(block.anchor_id) if block.anchor_id: self._register(block.anchor_id, replace(target, anchor=anchor)) @@ -528,7 +738,7 @@ def _structure(self, block: Block, *, section: Section | None, depth: int, prese def _structure_list(self, block: List, *, section: Section | None, depth: int, parent: str, presence: Presence, counters: dict[tuple[str, str] | None, _Counter] | None = None, - quoted: bool = False) -> List: + quoted: bool = False, parts: tuple[tuple[str, str], ...] = ()) -> List: """*block* numbered under *parent*. Sibling lists nested in the same item share *counters*, one per way of writing a designation (counter style and reference form): a second list whose designations would @@ -557,20 +767,22 @@ def _structure_list(self, block: List, *, section: Section | None, depth: int, p n = format_counter(index, fmt.counter) designation = extend(parent, fill(fmt.ref, n=n, section=base), textual=self.textual) label = fill(fmt.label, n=n, section=base, path=designation) - target = _Target(designation, None) + item_parts = (*parts, (fmt.ref, n)) + target = _Target(designation, None, section_index=self._section_index, parts=item_parts) if not quoted: slot = _Slot(counter, index, inner, "item", item.anchor_id) target = replace(target, slot=slot) self._record(designation, slot) else: - designation, label, target = parent, None, _Target(base, None, enumerated=False) + designation, label, item_parts = parent, None, parts + target = _Target(base, None, enumerated=False, section_index=self._section_index) anchor = self._anchor(item.anchor_id) if item.anchor_id and section is not None: self._register(item.anchor_id, replace(target, anchor=anchor)) siblings: dict[tuple[str, str] | None, _Counter] = {} children = tuple( self._structure_list(child, section=section, depth=depth + 1, parent=designation, presence=inner, - counters=siblings, quoted=quoted) + counters=siblings, quoted=quoted, parts=item_parts) if isinstance(child, List) else self._structure(child, section=section, depth=depth + 1, presence=inner, quoted=quoted) for child in item.blocks @@ -617,12 +829,16 @@ def _condition(self, condition: str) -> str | None: def _resolve_inlines(self, inlines: tuple[Inline, ...]) -> tuple[Inline, ...]: out: list[Inline] = [] - for inline in inlines: - resolved = self._resolve_inline(inline) - if isinstance(resolved, tuple): - out.extend(resolved) - elif resolved is not None: - out.append(resolved) + self._nest += 1 + try: + for inline in inlines: + resolved = self._resolve_inline(inline) + if isinstance(resolved, tuple): + out.extend(resolved) + elif resolved is not None: + out.append(resolved) + finally: + self._nest -= 1 return tuple(out) def _resolve_inline(self, inline: Inline) -> Inline | tuple[Inline, ...] | None: @@ -677,6 +893,11 @@ def _ref(self, directive: Directive, target_id: str) -> Inline: target = self.targets.get(target_id) if target is None: return FailureMarker(f"[BROKEN REF: {target_id}]") + if self._stage == _DESIGNATING: + # Only the text matters: what is wrong with the reference is + # reported where the heading is read again, not for a designation + # that is still being worked out. + return CrossRef(self._reference_text(self._designation_of(target)), "") if not target.enumerated: self._warn( "ref-not-enumerated", @@ -685,8 +906,16 @@ def _ref(self, directive: Directive, target_id: str) -> Inline: ) elif target.slot is not None and (clashes := self._clashes(target.slot, target.designation)): self._warn("render-ref-ambiguous", self._ambiguity_message(target_id, target, clashes)) - text = self.style.references.format.replace("{designation}", target.designation) - return CrossRef(text, target.anchor or "") + return CrossRef(self._reference_text(target.designation), target.anchor or "") + + def _reference_text(self, designation: str) -> str: + """What a reference shows for *designation*: at most its first + :data:`_MAX_DESIGNATION_TEXT` characters, so that a heading that is long + as written cannot be pasted unbounded by thousands of references.""" + if len(designation) > _MAX_DESIGNATION_TEXT: + self._warn("render-designation-limit", _DESIGNATION_LIMIT) + designation = designation[:_MAX_DESIGNATION_TEXT].rstrip() + "\u2026" + return self.style.references.format.replace("{designation}", designation) def _record(self, designation: str, slot: _Slot) -> None: """Note that a numbered unit reads as *designation* (render-ref-ambiguous).""" @@ -757,8 +986,9 @@ def _term_text(self, definition_id: str) -> str | None: Worked out the first time it is needed and kept, which is once every target is registered and the placeholder types are known. Before that (a heading read as text by the ``none`` scheme, §13.3) a text-only pass - stands in, which leaves references out; it is kept apart from the - text of the later pass. Resolving the term gives out no anchor: a + stands in, which leaves references out, and while designations are + worked out one that reads them (:meth:`_designate`); each is kept apart + from the text of the later pass. Resolving the term gives out no anchor: a definition inside a term keeps its place in document order. The definition is open meanwhile, so a ``{{term:}}`` inside it, directly or through others, shows its label or id and goes no further (see @@ -769,13 +999,13 @@ def _term_text(self, definition_id: str) -> str | None: source = self.term_sources.get(definition_id) if source is None: return "" - known = self._targets_known + stage = self._stage def work() -> str: # Worked out where the term is defined, whichever reference asks # first, so what it reports is at the definition. with self._here(self.term_lines.get(definition_id)): - if not known: + if stage != _FINAL: text = self._title_text(source.children) else: anchors = set(self.used_anchors) @@ -793,12 +1023,13 @@ def work() -> str: return definition_id return text - return self._while_open(self.term_texts if known else self.term_texts_early, definition_id, work) + cache = (self.term_texts_early, self.term_texts_designating, self.term_texts)[stage] + return self._while_open(cache, definition_id, work) def _plain_open(self, definition_id: str, term: str) -> str | None: """The validator's *term* text of a definition as plain text, with the definition marked open while its directives resolve.""" - key = f"{definition_id}\0{int(self._targets_known)}" + key = f"{definition_id}\0{self._stage}" return self._while_open(self.term_texts_plain, definition_id, lambda: self._plain_value(term), key=key) def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, work: Callable[[], str], @@ -822,10 +1053,13 @@ def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, wor is limited. A text worked out when the cache already holds one for this term (for other open terms) is a repeat, and the inline nodes resolved inside a repeat, nested ones included, are counted; first - texts are free. Past :data:`_MAX_TERM_WORK` such nodes, or with - :data:`_MAX_TERM_DEPTH` terms open at once, the result is ``None`` (the - reference shows its label or id), and the Warning ``render-term-limit`` - says so, once. + texts are free. Past :data:`_MAX_TERM_WORK` such nodes, with + :data:`_MAX_TERM_DEPTH` terms open at once, or inside an open term (or + while designations are worked out) at :data:`_MAX_NEST` levels, the + result is ``None`` (the reference shows its label or id), and the + Warning ``render-term-limit`` says so, once. While designations are + worked out nothing is reported there: the walk notes it + (:meth:`_limit_hit`), and ``render-designation-limit`` says so. """ key = definition_id if key is None else key around = set(self._terms_open) @@ -835,15 +1069,19 @@ def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, wor self._cut |= kept.cut return kept.text repeat = bool(cache.get(key)) - if len(self._terms_open) >= _MAX_TERM_DEPTH or (repeat and self._term_work >= _MAX_TERM_WORK): + too_nested = self._nest >= _MAX_NEST and (self._terms_open or self._stage == _DESIGNATING) + if len(self._terms_open) >= _MAX_TERM_DEPTH or too_nested or (repeat and self._term_work >= _MAX_TERM_WORK): # Terms that refer to each other this deeply or this often are # not resolved further: like a cut, the reference shows its label # or id, and nothing is kept. - self._warn( - "render-term-limit", - "Defined terms refer to each other too deeply or too often to resolve fully; " - "some {{term:}} references show their definition id.", - ) + if self._stage == _DESIGNATING: + self._limit_hit() + else: + self._warn( + "render-term-limit", + "Defined terms refer to each other too deeply or too often to resolve fully; " + "some {{term:}} references show their definition id.", + ) return None self._repeating += repeat self._terms_open.append(definition_id) @@ -1220,6 +1458,26 @@ def key(self) -> tuple[object, object]: return (self.source, self.number) +def _compose(parts: tuple[tuple[str, str], ...], base: str) -> str: + """The designation of an item or paragraph whose pieces are *parts* in a + section designated *base*, under the ``none`` scheme: what + ``_structure_list`` and ``_number_paragraph`` write for it, for a *base* + that is not the one they were given.""" + designation = base + for template, n in parts: + designation = extend(designation, fill(template, n=n, section=base), textual=True) + return designation + + +def _walk(inlines: tuple[Inline, ...]) -> Iterator[Inline]: + """Every inline in *inlines*, and in the children of each.""" + for inline in inlines: + yield inline + children = getattr(inline, "children", None) + if isinstance(children, tuple): + yield from _walk(children) + + def fill_condition(template: str, value: str, *, key: str = "condition") -> str: return template.replace("{" + key + "}", value) diff --git a/tests/test_none_designations.py b/tests/test_none_designations.py new file mode 100644 index 0000000..0420e2d --- /dev/null +++ b/tests/test_none_designations.py @@ -0,0 +1,416 @@ +"""Designations under the ``none`` scheme (§13.3): a section is designated by its heading as +rendered, references and terms resolved, and its items and paragraphs follow; what happens to +headings that refer to each other in a circle, too deeply, or at too great a length.""" +from __future__ import annotations + +import random + +import pytest +from conftest import FRONT, text + +from legaldown_render import RenderResult, render +from legaldown_render.resolve.resolver import Resolver, _compose +from legaldown_render.tree import plain_text + +NONE = {"numbering.scheme": "none"} + + +def lines(body: str, **settings: object) -> list[str]: + return [line for line in text(body, overrides=NONE, **settings).splitlines() if line] + + +def rules(result: RenderResult) -> list[str]: + return [d.rule for d in result.diagnostics if d.rule.startswith("render-")] + + +def run(body: str, **overrides: object) -> RenderResult: + return render(FRONT + body, format="text", overrides={**NONE, **overrides}) + + +def designations(body: str, **overrides: object) -> list[str]: + return [s.designation for s in run(body, **overrides).tree.sections] + + +# -- What a heading designates --------------------------------------------------- + +def test_a_reference_in_a_heading_shows_the_designation_of_its_target() -> None: + out = lines("# About {{ref: ov}} here {#h}\n\n# Overview {#ov}\n\nSee {{ref: h}}.\n") + assert out[1:4] == ["About Overview here", "Overview", "See About Overview here."] + + +def test_a_reference_to_a_later_heading_is_resolved_through_a_chain() -> None: + body = "# A {{ref: b}} {#a}\n\n# B {{ref: c}} {#b}\n\n# C {#c}\n\nSee {{ref: a}}, {{ref: b}}.\n" + assert designations(body) == ["A B C", "B C", "C"] + assert "See A B C, B C." in text(body, overrides=NONE) + + +def test_a_reference_to_an_earlier_heading_too() -> None: + body = "# C {#c}\n\n# B {{ref: c}} {#b}\n\n# A {{ref: b}} {#a}\n" + assert designations(body) == ["C", "B C", "A B C"] + + +def test_a_reference_formats_the_designation_inside_the_heading() -> None: + body = "# A {{ref: b}} {#a}\n\n# B {#b}\n\nSee {{ref: a}}.\n" + assert designations(body, **{"references.format": "'{designation}'"}) == ["A 'B'", "B"] + + +def test_a_broken_reference_in_a_heading_shows_its_marker() -> None: + assert designations("# A {{ref: nope}} {#a}\n") == ["A [BROKEN REF: nope]"] + + +def test_markup_is_dropped_and_white_space_is_collapsed() -> None: + assert designations("# Pay *by* **{{ref: b}}** now {#a}\n\n# B {#b}\n") == ["Pay by B now", "B"] + + +def test_a_term_with_a_reference_in_a_heading() -> None: + body = ('# About the {{term: fee}} {#fee-h}\n\n# Overview {#ov}\n\n"Fee per {{ref: ov}}" {{def: fee}}\n\n' + "See {{ref: fee-h}}.\n") + out = lines(body) + assert "About the Fee per Overview" in out + assert "See About the Fee per Overview." in out + assert "Fee per Overview" in out # the definition, as before + + +# -- Items and paragraphs follow their section ------------------------------------- + +def test_items_follow_the_final_designation_of_their_section() -> None: + body = "# Termination {{ref: x}} {#t}\n\n- one {#it}\n - nested {#nn}\n\n# X {#x}\n\nSee {{ref: it}} and {{ref: nn}}.\n" + assert "See Termination X (a) and Termination X (a) (i)." in text(body, overrides=NONE) + + +def test_a_heading_may_refer_to_an_item_under_another_heading() -> None: + body = "# Later {{ref: it}} {#l}\n\nSee {{ref: l}}.\n\n# Home {#h}\n\n- first {#it}\n - nested {#nn}\n\n# X {{ref: nn}} {#x}\n" + assert designations(body) == ["Later Home (a)", "Home", "X Home (a) (i)"] + assert "See Later Home (a)." in text(body, overrides=NONE) + + +def test_paragraph_designations_follow_too() -> None: + body = "# T {{ref: b}} {#a}\n\nOne. {#p}\n\n# B {#b}\n\nSee {{ref: p}}.\n" + out = text(body, overrides={**NONE, "paragraphs.numbered": True, + "paragraphs.format": {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}}) + assert "See T B (a)." in out + + +def test_a_reference_to_an_unnumbered_item_shows_the_final_heading() -> None: + result = run("# A {{ref: b}} {#a}\n\n- one {#i}\n\n# B {#b}\n\nSee {{ref: i}}.\n", **{"enumeration.enabled": False}) + assert "See A B." in result.output + assert [d.rule for d in result.diagnostics if d.rule == "ref-not-enumerated"] == ["ref-not-enumerated"] + + +def test_an_empty_heading_designation_leaves_the_item_piece_alone() -> None: + assert designations('# {{def: x}} {#e}\n\n- one {#i}\n\n# T {{ref: i}} {#t}\n') == ["", "T (a)"] + + +# -- render-ref-ambiguous reads the final designations --------------------------------- + +def test_headings_that_differ_once_resolved_are_not_ambiguous() -> None: + body = "# Part {{ref: a}} {#p1}\n\n# Part {{ref: b}} {#p2}\n\n# Alpha {#a}\n\n# Beta {#b}\n\nSee {{ref: p1}}.\n" + result = run(body) + assert "See Part Alpha." in result.output + assert "render-ref-ambiguous" not in {d.rule for d in result.diagnostics} + + +def test_headings_that_are_the_same_once_resolved_are_ambiguous() -> None: + result = run("# Fee {{ref: x}} {#f1}\n\n# Fee Xylo {#f2}\n\n# Xylo {#x}\n\nSee {{ref: f1}}.\n") + assert "render-ref-ambiguous" in {d.rule for d in result.diagnostics} + + +def test_alternative_headings_keep_their_own_text_and_the_first_is_referenced() -> None: + questions = FRONT.replace("---\n", "---\nquestions:\n forum:\n type: choice\n choices:\n courts: Courts\n" + " arbitration: Arbitration\n", 1) + body = ("# Disputes {{ref: z}} {#d when=forum:courts}\n\nSee {{ref: d}}.\n\n" + "# Disputes {{ref: y}} {#d when=forum:arbitration}\n\n# Z {#z}\n\n# Y {#y}\n") + result = render(questions + body, format="text", overrides=NONE) + assert [s.designation for s in result.tree.sections] == ["Disputes Z", "Disputes Y", "Z", "Y"] + assert "See Disputes Z." in result.output + assert "render-ref-ambiguous" not in {d.rule for d in result.diagnostics} + + +# -- Anchors are given out once ------------------------------------------------------------ + +def test_a_rebuild_keeps_every_anchor() -> None: + body = ('# *The "Buyer" {{def:}}* {{ref: b}} {#a}\n\n- one {#it}\n\nSee {{ref: it}}.\n\n# B {#b}\n') + out = render(FRONT + body, format="html", standalone=False, overrides=NONE).output + assert 'id="a"' in out and 'id="it"' in out and 'id="b"' in out + assert 'id="def:buyer"' in out + assert 'href="#it"' in out + assert out.count('id="a"') == 1 + + +# -- Circles, depth, length ---------------------------------------------------------------------- + +def test_headings_that_refer_to_each_other_are_cut_at_the_first_return() -> None: + result = run("# A {{ref: b}} {#a}\n\n# B {{ref: a}} {#b}\n\nSee {{ref: a}}.\n") + assert [s.designation for s in result.tree.sections] == ["A B A", "B A"] + assert rules(result) == ["render-designation-cycle"] + assert "'a'" in next(d.message for d in result.diagnostics if d.rule == "render-designation-cycle") + + +def test_a_heading_that_refers_to_itself_or_to_its_own_item() -> None: + assert rules(run("# A {{ref: a}} {#a}\n")) == ["render-designation-cycle"] + assert rules(run("# A {{ref: i}} {#a}\n\n- one {#i}\n")) == ["render-designation-cycle"] + + +def _doubling(n: int) -> str: + return "".join(f"# H{i} {{{{ref: h{i + 1}}}}} {{{{ref: h{i + 1}}}}} {{#h{i}}}\n\n" for i in range(1, n)) + f"# H{n} {{#h{n}}}\n" + + +def test_a_doubling_chain_is_capped_and_reported_once() -> None: + result = run(_doubling(200)) + assert max(len(s.designation) for s in result.tree.sections) <= 1000 + assert rules(result) == ["render-designation-limit"] + assert len(result.output) < 200_000 + + +def test_a_long_heading_without_references_is_not_cut_but_a_reference_to_it_is() -> None: + heading = "word " * 300 + unreferenced = run(f"# {heading}{{#a}}\n\nText.\n") + assert rules(unreferenced) == [] + assert len(unreferenced.tree.sections[0].designation) > 1000 + referenced = run(f"# {heading}{{#a}}\n\nSee {{{{ref: a}}}}.\n") + assert rules(referenced) == ["render-designation-limit"] + shown = next(line for line in referenced.output.splitlines() if line.startswith("See ")) + assert shown.endswith("\u2026.") + assert 990 < len(shown) - len("See \u2026.") <= 1000 + + +def test_thousands_of_references_to_a_long_heading_paste_a_bounded_text() -> None: + result = run("# " + "x" * 5000 + " {#a}\n\n" + "{{ref: a}} " * 2000 + "\n") + assert len(result.output) < 2000 * 1100 + + +def _chain(n: int, *, forward: bool = True) -> str: + ids = range(n) if forward else range(n - 1, -1, -1) + return "".join(f"# S{i} {{{{ref: s{i + 1}}}}} {{#s{i}}}\n\n" for i in ids) + f"# S{n} {{#s{n}}}\n" + + +def test_a_long_forward_chain_is_cut_at_32_headings_and_reported() -> None: + result = run(_chain(500)) + sections = result.tree.sections + assert sections[0].designation == " ".join(f"S{i}" for i in range(33)) + assert sections[32].designation == " ".join(f"S{i}" for i in range(32, 65)) + assert rules(result) == ["render-designation-limit"] + + +def test_a_chain_inside_the_depth_limit_is_resolved_in_full() -> None: + result = run(_chain(30)) + assert result.tree.sections[0].designation == " ".join(f"S{i}" for i in range(31)) + assert rules(result) == [] + + +def _wrap(depth: int, inner: str) -> str: + out = inner + for i in range(depth): + mark = ("*", "_", "**", "__")[i % 4] + out = f"{mark}x {out} x{mark}" + return out + + +def test_nested_formatting_does_not_exhaust_the_stack() -> None: + # The builder allows 100 levels in a tree; headings read through each other multiply that. + n = 60 + body = "".join(f"# {_wrap(90, f'S{i} {{{{ref: s{i + 1}}}}}')} {{#s{i}}}\n\n" for i in range(n)) + f"# S{n} {{#s{n}}}\n" + result = run(body) + assert "{{" not in result.output + assert "render-designation-limit" in rules(result) + + +# -- Work ----------------------------------------------------------------------------------------- + +@pytest.fixture +def directives(monkeypatch: pytest.MonkeyPatch): # noqa: ANN201 + count = {"n": 0} + real = Resolver._directive + + def counting(self, directive): # noqa: ANN001, ANN202 + count["n"] += 1 + return real(self, directive) + + monkeypatch.setattr(Resolver, "_directive", counting) + + def work(body: str) -> int: + count["n"] = 0 + run(body) + return count["n"] + + return work + + +@pytest.mark.parametrize("make", [_chain, lambda n: _chain(n, forward=False), _doubling]) +def test_the_work_is_linear_in_the_headings_and_references(directives, make) -> None: # noqa: ANN001 + small, large = directives(make(2500)), directives(make(5000)) + assert large <= 2.1 * small + assert large <= 4 * 5000 + 100 # at most one directive per reference, in each of two passes + + +def test_a_chain_of_terms_in_a_heading_is_still_worked_out_once_per_term(directives) -> None: # noqa: ANN001 + n = 20 + defs = [f'"T{i} {{{{term: t{i + 1}}}}} {{{{term: t{i + 1}}}}}" {{{{def: t{i}}}}}' for i in range(n)] + defs.append(f'"T{n}" {{{{def: t{n}}}}}') + body = "# About {{term: t0}}\n\nUsed: {{term: t0}}\n\n" + "\n\n".join(defs) + "\n" + assert directives(body) <= 8 * n + 60 + + +# -- Invariants on random documents -------------------------------------------------------------------- + +def _document(rng: random.Random) -> str: + n = rng.randint(2, 6) + ids = [f"h{i}" for i in range(n)] + items = {i: [f"i{i}_{j}" for j in range(rng.randint(0, 3))] for i in range(n)} + out = [] + for i in range(n): + words = [f"W{i}"] + later = [t for j in range(i + 1, n) for t in (ids[j], *items[j])] # later ones only: no circle + for _ in range(rng.randint(0, 2)): + if later: + words.append("{{ref: " + rng.choice(later) + "}}") + out.append(f"# {' '.join(words)} {{#{ids[i]}}}\n") + if items[i]: + out.append("\n".join(f"- item {{#{x}}}" for x in items[i]) + "\n") + everything = ids + [x for group in items.values() for x in group] + out.append("See {{ref: " + rng.choice(everything) + "}}.\n") + return "\n".join(out) + + +def _anchors(result) -> list[str | None]: # noqa: ANN001 + found: list[str | None] = [] + + def walk(blocks) -> None: # noqa: ANN001 + for block in blocks: + for item in getattr(block, "items", ()): + found.append(item.anchor) + walk(item.blocks) + + for section in result.tree.sections: + found.append(section.anchor) + walk(section.blocks) + return found + + +@pytest.mark.parametrize("style", [None, "continental"]) +def test_designations_equal_headings_as_rendered_and_anchors_do_not_move( + monkeypatch: pytest.MonkeyPatch, style: str | None) -> None: + seen: list[Resolver] = [] + real = Resolver.run + monkeypatch.setattr(Resolver, "run", lambda self: (seen.append(self), real(self))[1]) + rng = random.Random(20261027) + for _ in range(60): + source = FRONT + _document(rng) + result = render(source, format="text", style=style, overrides=NONE) + for section in result.tree.sections: + assert " ".join(plain_text(section.title).split()) == section.designation, source + resolver = seen[-1] + for target in resolver.targets.values(): + base = result.tree.sections[target.section_index].designation + assert _compose(target.parts, base) == target.designation, source + plain = render(source.replace("{{ref:", "{{party:"), format="text", style=style, overrides=NONE) + assert _anchors(result) == _anchors(plain), source + + +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +@pytest.mark.parametrize("heading", [False, True]) +def test_terms_with_nested_formatting_do_not_exhaust_the_stack(scheme: str, heading: bool) -> None: + n = 40 + defs = "".join(f'"T{i} {_wrap(60, "{{term: t" + str(i + 1) + "}}")}" {{{{def: t{i}}}}}\n\n' for i in range(n)) + title = "A {{term: t0}}" if heading else "A" + body = f"# {title}\n\nUsed: {{{{term: t0}}}}\n\n{defs}" + f'"End" {{{{def: t{n}}}}}\n' + result = render(FRONT + body, format="text", overrides={"numbering.scheme": scheme}) + assert "{{" not in result.output + assert "render-term-limit" in {d.rule for d in result.diagnostics} + + +# -- Limits that are reached inside terms -------------------------------------------------------- + + +def test_a_term_cut_by_its_limits_while_designating_is_reported() -> None: + # Two terms per hop: past 32 open terms a {{term:}} shows its id, which the walk must not do silently. + n = 20 + defs = "".join(f'"A{i} {{{{term: b{i}}}}}" {{{{def: a{i}}}}}\n\n"B{i} {{{{ref: h{i + 1}}}}}" {{{{def: b{i}}}}}\n\n' + for i in range(n)) + heads = "".join(f"# H{i} {{{{term: a{i}}}}} {{#h{i}}}\n\n" for i in range(n)) + f"# H{n} {{#h{n}}}\n\n" + result = run(heads + defs) + assert rules(result) == ["render-designation-limit"] + assert "terms" in next(d.message for d in result.diagnostics if d.rule == "render-designation-limit") + assert "{{" not in result.output + + +def test_a_term_inside_deeply_nested_formatting_is_cut_inside_an_open_term_and_reported() -> None: + n = 12 + defs = "".join(f'"T{i} {_wrap(8, "{{ref: h" + str(i + 1) + "}}")}" {{{{def: t{i}}}}}\n\n' for i in range(n)) + heads = "".join(f"# H{i} {{{{term: t{i}}}}} {{#h{i}}}\n\n" for i in range(n)) + f"# H{n} {{#h{n}}}\n\n" + assert rules(run(heads + defs)) == ["render-designation-limit"] + + +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +def test_a_term_in_one_blocks_deepest_formatting_is_still_a_term(scheme: str) -> None: + # The builder allows 100 levels in a block; a term met there is not nested inside another unit. + body = f'# A\n\n{_wrap(99, "{{term: fee}}")}\n\n"Fee" {{{{def: fee}}}}\n' + result = render(FRONT + body, format="text", overrides={"numbering.scheme": scheme}) + assert " Fee " in next(line for line in result.output.splitlines() if line.startswith("x ")) + assert "render-term-limit" not in {d.rule for d in result.diagnostics} + + +# -- Where a diagnostic about a heading, and the new Warnings, are ------------------------------------ + + +def _line(source: str, needle: str) -> int: + return next(number for number, line in enumerate(source.split("\n"), 1) if needle in line) + + +def test_a_cycle_is_reported_at_the_heading_it_closes_on() -> None: + source = FRONT + "\nIntro.\n\n# A {{ref: b}} {#a}\n\nText.\n\n# B {{ref: a}} {#b}\n" + result = render(source, format="text", overrides=NONE) + assert [(d.rule, d.line) for d in result.diagnostics if d.rule == "render-designation-cycle"] == [ + ("render-designation-cycle", _line(source, "# A "))] + + +def test_a_limit_is_reported_at_the_heading_that_reached_it() -> None: + source = FRONT + "\n" + _chain(40) + result = render(source, format="text", overrides=NONE) + assert [d.line for d in result.diagnostics if d.rule == "render-designation-limit"] == [_line(source, "# S31 ")] + source = FRONT + "\n" + _doubling(40) + result = render(source, format="text", overrides=NONE) + lines = [d.line for d in result.diagnostics if d.rule == "render-designation-limit"] + assert len(lines) == 1 and lines[0] is not None + + +def test_diagnostics_inside_a_referenced_heading_are_reported_once_at_that_heading() -> None: + source = (FRONT + "\n# A [x](ms-msdt:1) ![alt](/etc/p.png) {{include: f.lgd}} {#a}\n\n" + "# B {{ref: a}} {#b}\n\n# C {{ref: b}} {#c}\n\nSee {{ref: c}} and {{ref: a}}.\n") + result = render(source, format="text", overrides=NONE) + for rule in ("render-link-refused", "path-not-relative", "render-not-processed"): + assert [d.line for d in result.diagnostics if d.rule == rule] == [_line(source, "# A ")], rule + + +def test_a_reference_inside_a_heading_is_a_link_there_and_not_in_the_table_of_contents() -> None: + result = render(FRONT + "# A {{ref: b}} {#a}\n\n# B {#b}\n", standalone=False, + overrides={**NONE, "contents.enabled": True}) + page = result.output + nav = page[page.index("")] + assert 'B' not in nav + assert nav.count(""):] + assert 'A B' in body + + +# -- No references in the headings: nothing to work out ----------------------------------------------- + + +def test_nothing_is_worked_out_for_headings_that_read_no_references(monkeypatch: pytest.MonkeyPatch) -> None: + called: list[int] = [] + real = Resolver._designate + monkeypatch.setattr(Resolver, "_designate", lambda self, sections: (called.append(1), real(self, sections))[1]) + body = '# Fee {{term: fee}} {#a}\n\n"Fee *rate*" {{def: fee}}\n\nSee {{ref: a}}.\n' + assert designations(body) == ["Fee Fee rate"] + assert called == [] + run("# A {{ref: b}} {#a}\n\n# B {#b}\n") + assert called == [1] + called.clear() + run('# Fee {{term: fee}} {#a}\n\n"Fee {{ref: a}}" {{def: fee}}\n') # a reference in a term the heading uses + assert called == [1] + + +def test_a_preamble_list_is_numbered_the_same_on_every_pass() -> None: + body = "- pre {#p}\n\n# A {{ref: b}} {#a}\n\n- one {#i}\n\n# B {#b}\n\nSee {{ref: i}}.\n" + out = text(body, overrides=NONE) + assert "See A B (a)." in out diff --git a/tests/test_render.py b/tests/test_render.py index 42e5746..836e43e 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -709,10 +709,12 @@ def test_level_six_headings_are_numbered_as_the_validator_numbers_them() -> None assert [section.designation for section in result.tree.sections] == expected -def test_none_scheme_designation_skips_nested_references() -> None: +def test_none_scheme_designation_includes_nested_references() -> None: + # §13.3: a reference shows the target's heading text as rendered, its own references resolved. body = "# Later *see {{ref: b}}* {#c}\n\nSee {{ref: c}}.\n\n# B {#b}\n" output = text(body, overrides={"numbering.scheme": "none"}) - assert "See Later see." in output + assert "\nLater see B\n" in output + assert "See Later see B." in output assert "BROKEN" not in output diff --git a/tests/test_terms_budget.py b/tests/test_terms_budget.py index 1428763..e8b20b1 100644 --- a/tests/test_terms_budget.py +++ b/tests/test_terms_budget.py @@ -203,6 +203,7 @@ def _oracle(self: Resolver, cache: object, definition_id: str, work, *, key: str def _graph(rng: random.Random) -> str: ids = [f"t{i}" for i in range(rng.randint(2, 6))] + headings = [f"h{i}" for i in range(3)] defs = [] for identifier in ids: parts = [identifier.upper()] @@ -210,11 +211,24 @@ def _graph(rng: random.Random) -> str: target = rng.choice(ids) parts.append(f"{{{{term: {target}, label=L{target}}}}}" if rng.random() < 0.15 else f"{{{{term: {target}}}}}") + if rng.random() < 0.3: + # Terms may refer to a heading, and headings to later headings and to terms, but + # never back: the designations of the none scheme are then the same however they + # are worked out (a circle through headings is read in the order it is met). + parts.append(f"{{{{ref: {headings[-1]}}}}}") defs.append(f'"{" ".join(parts)}" {{{{def: {identifier}}}}}') rng.shuffle(defs) - heading = rng.choice(["A", f"About {{{{term: {rng.choice(ids)}}}}}"]) use = " ".join(f"{{{{term: {rng.choice(ids)}}}}}" for _ in range(rng.randint(1, 4))) - return f"# {heading}\n\nUsed: {use}\n\n" + "\n\n".join(defs) + "\n" + sections = [] + for i, identifier in enumerate(headings): + title = [f"H{i}"] + if i < len(headings) - 1: + if rng.random() < 0.6: + title.append(f"{{{{term: {rng.choice(ids)}}}}}") + if rng.random() < 0.5: + title.append(f"{{{{ref: {rng.choice(headings[i + 1:])}}}}}") + sections.append(f"# {' '.join(title)} {{#{identifier}}}\n") + return "\n".join(sections) + f"\nUsed: {use}\n\n" + "\n\n".join(defs) + "\n" def test_cached_term_texts_equal_the_texts_worked_out_afresh(monkeypatch: pytest.MonkeyPatch) -> None: @@ -283,3 +297,27 @@ def test_a_chain_inside_the_depth_limit_is_resolved_in_full() -> None: result = render(FRONT + _line(n, heading=False), format="text") assert f"Used: {' '.join(f'T{i}' for i in range(n))} End" in result.output assert messages(result, "render-term-limit") == [] + + +# -- Terms inside formatting inside terms: the nesting limit ---------------------------------------- + + +def _nested(depth: int, inner: str) -> str: + out = inner + for i in range(depth): + mark = ("*", "_", "**", "__")[i % 4] + out = f"{mark}x {out} x{mark}" + return out + + +@pytest.mark.parametrize("scheme", ["decimal", "none"]) +@pytest.mark.parametrize("heading", [False, True]) +def test_a_chain_of_terms_in_deep_formatting_does_not_exhaust_the_stack(scheme: str, heading: bool) -> None: + # The builder allows 100 levels in a block, and terms read inside terms stack them (§7.3). + n = 40 + defs = "".join(f'"T{i} {_nested(60, "{{term: t" + str(i + 1) + "}}")}" {{{{def: t{i}}}}}\n\n' for i in range(n)) + title = "A {{term: t0}}" if heading else "A" + body = f"# {title}\n\nUsed: {{{{term: t0}}}}\n\n{defs}" + f'"End" {{{{def: t{n}}}}}\n' + result = render(FRONT + body, format="text", overrides={"numbering.scheme": scheme}) + assert "{{" not in result.output + assert messages(result, "render-term-limit") == [LIMIT] From e7433236abb2312e505b5ac0d877becd9d6d0ac7 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:27:41 +0000 Subject: [PATCH 046/118] Describe designations under the none scheme in the docs, and test that file reference findings are reported once CONFORMANCE.md, docs/architecture.md and docs/style-templates.md describe the heading as rendered, the cycle and limit Warnings and their lines, the nesting limit for terms and its scope, and the new test module. The rule lists (module docstring, architecture, line granularity in CONFORMANCE.md) name render-designation-cycle and render-designation-limit. --- CONFORMANCE.md | 31 +++++++++++++---- docs/architecture.md | 44 +++++++++++++++++++++--- docs/style-templates.md | 2 +- src/legaldown_render/resolve/resolver.py | 4 ++- tests/test_none_designations.py | 14 ++++++++ 5 files changed, 82 insertions(+), 13 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 9be547a..5c3bbd2 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -55,7 +55,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes - **Line numbers (§16.9).** Every diagnostic names its line, the validator's and the renderer's own (`render-ref-ambiguous`, `ref-not-enumerated`, `render-not-processed`, `render-link-refused`, `path-not-relative`, `path-outside-root`, `render-term-limit`, `render-term-too-long`, - `render-locale-fallback`), and the CLI prints it as `file:line:`. The renderer's line is the + `render-designation-cycle`, `render-designation-limit`, `render-locale-fallback`), and the CLI prints it as `file:line:`. The renderer's line is the **first line of the block it is in**, as `Document.line_of` gives it, whatever line the directive itself is on: a paragraph's, a table's header row, a quote's first `>`, a code block's opening fence, a heading's (a section title); for a list item, the line of **its marker** @@ -70,7 +70,9 @@ It is checked against the specification's own examples and fixtures corpus (`tes signatories are all at the party's key) or `attachments[].title` for a directive in that value (also where the body shows it, as `{{attach:}}` or `{{party:}}` do). A finding in a term's text is at the term's definition, whichever reference asks for it first; `render-term-limit` is at the - definition being worked out when the limit was reached. The same finding (rule, level and + definition being worked out when the limit was reached; `render-designation-cycle` is at the + heading it closes on, and `render-designation-limit` at the first heading that reached a limit + (or, for a reference longer than the cap, at the reference). The same finding (rule, level and message) is reported once, at its first line: the same refused link, or the same non-relative image path, on two lines is one diagnostic, at the first of them. A document with no source to name a line of (built from a dict) reports no line. With `answers`, the lines are those of the @@ -145,8 +147,11 @@ It is checked against the specification's own examples and fixtures corpus (`tes also in a heading read before the targets are known (§13.3). Terms that name each other can still ask for more different texts than any cache holds, so the extra work is bounded: a term's first text is free, and past 200,000 inline nodes resolved inside later texts of the same term (nested ones - included), or 32 terms open inside one another, a `{{term:}}` shows its `label=` or definition id - and the Warning `render-term-limit` is reported, once. Any number of terms that do not name each + included), 32 terms open inside one another, or 60 inline levels (emphasis, links, terms) nested + inside an open term, a `{{term:}}` shows its `label=` or definition id and the Warning + `render-term-limit` is reported, once. The nesting limit only applies where one unit is read + inside another (inside an open term, or while designations are worked out, below): a term in a + single block, however deeply formatted (the builder allows 100 levels), is never cut by it. Any number of terms that do not name each other resolve in full and never reach the limit. A term text is capped at 1,000 characters of plain text, checked as it is built (no real defined term comes near it; a chain of terms that each read the next twice would otherwise ask for a text of 2^n words): a longer one is shown by its @@ -162,9 +167,23 @@ It is checked against the specification's own examples and fixtures corpus (`tes exclusive conditions), but the condition the reference itself stands under is not considered. Lists in quotes and drafting notes are not counted, and a second use of an identifier is left to the validator's `anchor-duplicate`. +- **Designations under the `none` scheme (§13.3).** A `{{ref:}}` shows the target's heading as + rendered: the references and terms in the heading resolved, as plain text with white space + collapsed (`# About {{ref: ov}}` is "About Overview"), then the item path or paragraph number for + an item or paragraph ("Termination (a)"). A heading that refers back to itself, directly or + through other headings or terms, reads as its text without references where the circle closes + (`render-designation-cycle`). Headings read through more than 32 headings, nested more than 60 + inline levels while headings and terms are read through each other, or longer than 1,000 + characters once resolved are designated by their text as written, without the references in them + (`render-designation-limit`), and a reference shows at most the first 1,000 characters of a + designation. A heading in a cycle or past a cap, or one whose terms reach into one, may read + differently from the designation references show; the Warning says so. Alternative sections + sharing an identifier (§15.4) keep their own heading text, and a reference to the identifier + shows the first alternative's. - **Labels under the `none` scheme.** A style label that shows `{section}` (such as the default - paragraph label `{section}.{n}`) shows the section's heading text there, while `{{ref:}}` - writes the §13.3 designation ("Termination 1"), so the two can differ. + paragraph label `{section}.{n}`) shows the section's designation there (its heading text, as + above), while `{{ref:}}` to an item writes the §13.3 path after it ("Termination 1"), so the two + can differ. - **Malformed directives** (a §11.2 grammar violation) render their type's marker without a value, for example `[INVALID AMOUNT]`. The specification defines no marker for this case. - **One parser.** The renderer builds from the validator's document model only diff --git a/docs/architecture.md b/docs/architecture.md index e401ef7..ecdbc05 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -137,7 +137,7 @@ Guessing at lost structure was tried, and it traded each gap for new bugs. validator's `exclusive()`). It then labels list items by depth and top-level paragraphs, and allocates output anchors. Every anchor target is registered with its **designation**: "4.2", "4.2(b)(ii)", or "Termination (a)" - under the `none` scheme. + under the `none` scheme, where it is worked out from the headings (see below). 3. **Inlines.** It replaces every source inline with a resolved one: | Directive | Resolves to | Failure markers | @@ -170,9 +170,42 @@ so ids stay unique. A definition's anchor is given out in one place, when its de resolved. Terms in titles or alt text get none. After everything is resolved, one pass drops the link of any `{{term:}}` whose definition got no anchor, so term links always have a target. -Under the `none` scheme, a section's designation is its heading text, computed by a text-only -pass that gives out no anchors and leaves references out. Headings are numbered with their level -clamped to 1–5, as the validator numbers them. +Headings are numbered with their level clamped to 1–5, as the validator numbers them. + +**Designations under the `none` scheme (§13.3).** A section's designation is its heading as +rendered, as plain text with white space collapsed: the `{{ref:}}`s and `{{term:}}`s in it are +resolved (a reference shows `references.format` applied to its target's designation, so +`# About {{ref: ov}}` is "About Overview"), a broken reference shows its marker. An item's or +paragraph's designation is its section's with the item pieces after it ("Termination (a)"). A +heading can refer to a heading, item or paragraph anywhere, so designations are worked out in +four steps, only when a heading, or a term a heading uses, holds a `{{ref:}}` (otherwise the first +step's text is final): + +1. Every section, item and paragraph is numbered on a **provisional** designation: the heading's + text-only reading (`Resolver._title_text`, which leaves references out). Each target records + its section and the pieces after it (`_Target.section_index`, `parts`). +2. `Resolver._designate` works out every section's designation once, by a depth-first walk in + document order that memoises: a reference reads the designation of its target (an item's or + paragraph's is its pieces put together again on its section's, `_compose`), which is worked out + on the spot if it is not known yet. The walk reads headings with `_title_text`, so it gives out + no anchor, and reports nothing itself (what it resolves is read again in its place, where a + diagnostic about a heading is made once); its term texts have a cache of their own, apart from the + later pass's. +3. Where any designation differs from the provisional one, the numbering of step 1 is done again on the + final designations (labels, designations, `render-ref-ambiguous` slots). Anchors are given out + in the same order and so come out the same. +4. The inlines are resolved as for every scheme, now with every designation known. + +A heading that reads itself again (directly, through other headings or terms, or through its own +item) reads as its provisional text where the circle closes, with the Warning +`render-designation-cycle`. A walk through more than 32 headings, a designation longer than 1,000 +characters once its references are resolved (a chain of headings that each cite the next twice would +ask for 2^n), or inline levels nested past 60 while headings and terms are read through each other, +also give the heading's provisional text, with `render-designation-limit` (so does a term the walk +cut by the term limits); a `{{ref:}}` never shows more than the first 1,000 characters of a +designation. So a heading in a cycle or past a cap, or one whose terms reach into one, may read +differently from the designation references show; the Warning says so. Every other heading reads +exactly as its designation. ### 5. Write (`writers/`) @@ -242,7 +275,7 @@ Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can ev `ref-not-enumerated`, `path-not-relative` and `path-outside-root` (Errors for a file reference that is not relative, or that leaves the document root, §2.3; reported here because validator 0.4 checks neither), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, or 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), a `{{term:}}` shows its id), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters under the `none` scheme, or a reference to a heading longer than that), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's diagnostics carry their line (§16.9), and so do the renderer's own, and the CLI prints it as `file:line:`. @@ -353,6 +386,7 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_numbering.py` | Counter formats, label templates and the preamble's `{section}` | | `tests/test_build.py` | Building the tree: comment-only items and quotes, emphasis next to a directive, spacing where an inline comment was dropped | | `tests/test_terms.py` | What `{{term:}}` shows, references inside terms, terms that name themselves or each other, the cap on a term text | +| `tests/test_none_designations.py` | Designations under the `none` scheme: references and terms in headings, items and paragraphs under them, ambiguity on the final designations, cycles, the depth, length and nesting limits, the lines of the Warnings, work bounds, invariants on random documents | | `tests/test_terms_budget.py` | Reuse of worked-out term texts, the work budget and depth limit for terms that name each other. Bounds the work by counting resolved directives, never by timing | | `tests/test_diagnostic_lines.py` | The line and file of every renderer diagnostic: in each kind of block (paragraph, list item, nested item, table, quote, heading, preamble) and at frontmatter keys, the repeats left out, the tree nodes' lines, and `file:line:` on the command line | | `tests/test_links.py` | Which images are shown and which links are links: non-relative image paths, the link scheme allow-list, empty URLs and empty link text, and how a refusal is reported | diff --git a/docs/style-templates.md b/docs/style-templates.md index 780cda3..9ecbc42 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -156,7 +156,7 @@ preamble, which has no section, `{section}` is empty and it is removed together it, so `label: "{section}.{n}"` shows "1", not ".1". For example, decimal headings use `label: "{path}"` and `ref: ".{n}"`. Legal enumeration uses `label: "({n})"` and `ref: "({n})"`, so the second item of Section 2.3 is shown as "(b)" and -referenced as "2.3(b)". Under the `none` scheme, designations are heading text, and item pieces +referenced as "2.3(b)". Under the `none` scheme, designations are heading text as rendered (references and terms in it resolved), and item pieces are set off by a space ("Termination (a)", §13.3). ### Text templates and their fields diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index f75eef5..1b45545 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -19,7 +19,9 @@ and ``render-not-processed`` for constructs beyond the Rendering level (§17.5), ``render-link-refused`` for a link whose URL scheme is not one links may use (§8.7), ``render-term-too-long`` for a term whose text is longer than a -``{{term:}}`` may show, ``render-term-limit`` for terms that refer to each other too deeply or too often, +``{{term:}}`` may show, ``render-term-limit`` for terms that refer to each other too deeply or too often +(or are nested too deeply inside one another's formatting), ``render-designation-cycle`` and +``render-designation-limit`` for headings that cannot be designated in full under the ``none`` scheme (§13.3), and the two file-reference rules of §2.3, which the validator does not check yet: ``path-not-relative`` for a path with a URI scheme or an absolute one, and ``path-outside-root`` for one that leaves the document root, when a root is diff --git a/tests/test_none_designations.py b/tests/test_none_designations.py index 0420e2d..a4c68d8 100644 --- a/tests/test_none_designations.py +++ b/tests/test_none_designations.py @@ -4,6 +4,7 @@ from __future__ import annotations import random +from pathlib import Path import pytest from conftest import FRONT, text @@ -414,3 +415,16 @@ def test_a_preamble_list_is_numbered_the_same_on_every_pass() -> None: body = "- pre {#p}\n\n# A {{ref: b}} {#a}\n\n- one {#i}\n\n# B {#b}\n\nSee {{ref: i}}.\n" out = text(body, overrides=NONE) assert "See A B (a)." in out + + +def test_file_reference_findings_are_reported_once_whatever_designates_them(tmp_path: Path) -> None: + # A frontmatter path, and an image and an include in a heading that other headings refer to. + source = ("---\ntitle: T\nattachments:\n - id: annex\n title: Annex\n file: /etc/annex.lgd\n---\n\n" + "# A ![logo](../../logo.png) {{include: ../../part.lgd}} {#a}\n\n# B {{ref: a}} {#b}\n\n" + "# C {{ref: b}} {#c}\n\nSee {{ref: c}}.\n") + result = render(source, format="text", overrides=NONE, document_root=tmp_path) + found = [(d.rule, d.line) for d in result.diagnostics if d.rule.startswith("path-")] + assert sorted(found) == sorted([("path-not-relative", _line(source, "file: /etc")), + ("path-outside-root", _line(source, "# A ")), + ("path-outside-root", _line(source, "# A "))]) + assert [d.line for d in result.diagnostics if d.rule == "render-not-processed"] == [_line(source, "# A ")] From a3bc4bebe9e1b69096512a29fcaac987a1294df1 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:27:52 +0000 Subject: [PATCH 047/118] Name the file on the assembly's diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assembler does not know the file it was given, so its findings (answer-unknown, answer-invalid, answer-missing) had an empty file even when RenderOptions.filename was set. _assemble now stamps the file name on every diagnostic that names none, on the refusal and on the result (§16.9). Tests extend the assembly test with the warning and the refusal. --- src/legaldown_render/api.py | 20 ++++++++++++++------ tests/test_diagnostic_lines.py | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index aec6883..16a25ff 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -13,7 +13,7 @@ from __future__ import annotations import os -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field, replace from pathlib import Path, PurePath from typing import Any @@ -209,7 +209,7 @@ def _assemble(source: str, answers: Mapping[str, Any], filename: str = "") -> tu no output, or the answers do (``answer-invalid``, ``answer-missing``), or the template needs files a renderer below Full does not read (includes, LegalDown attachments, translations; §17.6).""" - findings = validate(_parse(source, filename)).diagnostics + findings = _named(validate(_parse(source, filename)).diagnostics, filename) errors = sum(1 for d in findings if d.level == "error") if errors: raise RenderRefused(findings, f"Assembly refused: the template has {errors} error(s), " @@ -219,12 +219,20 @@ def _assemble(source: str, answers: Mapping[str, Any], filename: str = "") -> tu result = parse_template(source).form(answers).assemble() except AssemblyError as error: raise DocumentError(f"The template cannot be assembled: {error}") from error + diagnostics = _named(result.diagnostics, filename) if not result.ok: - errors = [d for d in result.diagnostics if d.level == "error"] - raise RenderRefused(result.diagnostics, f"Assembly refused: {len(errors)} error(s) in the answers or " - "the template (§15.7).") + errors = [d for d in diagnostics if d.level == "error"] + raise RenderRefused(diagnostics, f"Assembly refused: {len(errors)} error(s) in the answers or " + "the template (§15.7).") # The source was normalized, and assembly keeps its line breaks. - return result.output, list(result.diagnostics) + return result.output, diagnostics + + +def _named(diagnostics: Iterable[Diagnostic], filename: str) -> list[Diagnostic]: + """*diagnostics* with *filename* as the ``file`` of each one that names no + file (§16.9): the assembly's findings are the document's, and the + assembler does not know the file it was given.""" + return [replace(d, file=filename) if not d.file else d for d in diagnostics] def _parse(source: str, filename: str = "") -> Document: diff --git a/tests/test_diagnostic_lines.py b/tests/test_diagnostic_lines.py index 6c52077..df3c633 100644 --- a/tests/test_diagnostic_lines.py +++ b/tests/test_diagnostic_lines.py @@ -18,6 +18,7 @@ from legaldown_render import RenderOptions, render, render_file from legaldown_render.cli import main from legaldown_render.diagnostics import key_line +from legaldown_render.errors import RenderRefused HEAD = "---\ntitle: T\n---\n" @@ -420,6 +421,23 @@ def test_assembly_names_the_file_too() -> None: assert found.line is not None +def test_an_assembly_warning_names_the_file() -> None: + template = ("---\ntitle: T\nquestions:\n q:\n type: text\n prompt: Q\n---\n\n# A\n\n" + "Hello {{placeholder: q}}\n") + result = render(template, RenderOptions(answers={"q": "world", "zzz": "1"}, filename="t.lgd")) + [unknown] = [d for d in result.diagnostics if d.rule == "answer-unknown"] + assert unknown.file == "t.lgd" + + +def test_a_refused_assembly_names_the_file_in_every_diagnostic() -> None: + template = ("---\ntitle: T\nquestions:\n q:\n type: text\n prompt: Q\n---\n\n# A\n\n" + "Hello {{placeholder: q}}\n") + with pytest.raises(RenderRefused) as refused: + render(template, RenderOptions(answers={"q": 5}, filename="t.lgd")) + assert [d.rule for d in refused.value.diagnostics] == ["answer-invalid"] + assert {d.file for d in refused.value.diagnostics} == {"t.lgd"} + + # -- The command line ------------------------------------------------------------------------ From 1fdceb41e41f1b4ab41b61ba532828d29d242234 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:33:17 +0000 Subject: [PATCH 048/118] Report a party's findings at the field they are in A directive in a party's legal_name, label, address, identification_number, date_of_birth, custom field or a representative's name or title was reported at the party's first line. Each field is now resolved at its own key (the signature blocks too); a value the source writes elsewhere, as a << merge does, is at the line it is written. tests/test_diagnostic_lines.py: the party/side test now expects the address key, not the party's line. --- CONFORMANCE.md | 6 +- src/legaldown_render/diagnostics.py | 7 ++ src/legaldown_render/resolve/resolver.py | 88 +++++++++++++++++------- tests/test_diagnostic_lines.py | 85 ++++++++++++++++++++++- 4 files changed, 155 insertions(+), 31 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 9be547a..2fc1a58 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -66,8 +66,10 @@ It is checked against the specification's own examples and fixtures corpus (`tes the renderer's do, until [validator#93](https://github.com/ForLegalAI/legaldown-validator/issues/93). Findings about frontmatter are at the line of their key: the `language` key for `render-locale-fallback` of the document's language, and the `title`, `subtitle`, `version`, - `effective_date`, `sides[].label` and `sides[].parties[]` key (a party's details, name and - signatories are all at the party's key) or `attachments[].title` for a directive in that value + `effective_date`, `sides[].label`, a field of a party (`legal_name`, `label`, `address`, + `identification_number`, `date_of_birth`, a custom field, a representative's `name` or `title`: + each at its own key, or where the source writes the value, as with a `<<` merge; the party's key + where it writes none) or `attachments[].title` for a directive in that value (also where the body shows it, as `{{attach:}}` or `{{party:}}` do). A finding in a term's text is at the term's definition, whichever reference asks for it first; `render-term-limit` is at the definition being worked out when the limit was reached. The same finding (rule, level and diff --git a/src/legaldown_render/diagnostics.py b/src/legaldown_render/diagnostics.py index 95ce419..d9b071e 100644 --- a/src/legaldown_render/diagnostics.py +++ b/src/legaldown_render/diagnostics.py @@ -37,6 +37,13 @@ def __call__(self, *path: Any) -> int | None: line = self._map.key(*path) if self._map is not None else None return line if isinstance(line, int) else None + def has(self, *path: Any) -> bool: + """True if the document's source writes the frontmatter node at *path*, + as a key or a value (a value merged in with ``<<`` is written where it + is defined). A node that is not written has the line of the key above + it when asked for (``__call__``).""" + return self._map is not None and path in self._map.keys + def key_line(document: Document, *path: Any) -> int | None: """``KeyLines(document)(*path)``, for a single lookup.""" diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 1783bed..2c2680c 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -43,7 +43,7 @@ from typing import Any, NamedTuple from urllib.parse import unquote -from legaldown import Amends, Diagnostic, Document, Party, ValidationResult +from legaldown import Amends, CustomField, Diagnostic, Document, Party, Representative, ValidationResult from legaldown.grammar import ( ALWAYS, DECISION_QUESTION_TYPES, @@ -1069,25 +1069,61 @@ def _sides(self) -> tuple[SideInfo, ...]: return tuple(sides) def _party_info(self, side_index: int, party_index: int, party: Party) -> PartyInfo: - with self._at(self.key_line("sides", side_index, "parties", party_index)): - details: list[tuple[str, tuple[Inline, ...]]] = [] - if party.identification_number: + party_path = ("sides", side_index, "parties", party_index) + details: list[tuple[str, tuple[Inline, ...]]] = [] + if party.identification_number: + with self._at_key(*party_path, "identification_number"): details.append((self.labels.identification_number or "", self._frontmatter(party.identification_number))) - if party.date_of_birth: + if party.date_of_birth: + with self._at_key(*party_path, "date_of_birth"): details.append((self.labels.date_of_birth or "", self._date_or_text(party.date_of_birth))) - if party.address: + if party.address: + with self._at_key(*party_path, "address"): details.append((self.labels.address or "", self._frontmatter(party.address))) - for custom in party.custom_fields: + for index, custom in enumerate(party.custom_fields): + with self._at_key(*self._custom_field_path(party_path, index, custom)): details.append((custom.label, self._frontmatter(custom.value))) - for rep in party.representatives: - name, title = self._frontmatter(rep.name), self._frontmatter(rep.title) - value = name + (Text(", "),) + title if name and title else name or title - if value: - details.append((self.labels.represented_by or "", value)) - return PartyInfo( - name=self._frontmatter(party.legal_name or party.label or party.name), - details=tuple(details), - ) + for index, rep in enumerate(party.representatives): + name, title = self._representative(party_path, index, rep) + value = name + (Text(", "),) + title if name and title else name or title + if value: + details.append((self.labels.represented_by or "", value)) + return PartyInfo( + name=self._party_name(party_path, party), + details=tuple(details), + ) + + def _party_name(self, party_path: tuple[Any, ...], party: Party) -> tuple[Inline, ...]: + """The name a party is shown by (§3.4): its legal name, else its label, + else its name, at the key of the one shown.""" + field = "legal_name" if party.legal_name else "label" if party.label else "name" + with self._at_key(*party_path, field): + return self._frontmatter(party.legal_name or party.label or party.name) + + def _representative(self, party_path: tuple[Any, ...], index: int, + rep: Representative) -> tuple[tuple[Inline, ...], tuple[Inline, ...]]: + """A representative's name and title (§3.4), each at its own key.""" + representative = (*party_path, "representatives", index) + with self._at_key(*representative, "name"): + name = self._frontmatter(rep.name) + with self._at_key(*representative, "title"): + title = self._frontmatter(rep.title) + return name, title + + def _custom_field_path(self, party_path: tuple[Any, ...], index: int, custom: CustomField) -> tuple[Any, ...]: + """The frontmatter path of a custom field's value (§3.4): its key in + the party when the source writes one, else its ``value`` in the + party's ``custom_fields`` list.""" + if self.key_line.has(*party_path, custom.label): + return (*party_path, custom.label) + return (*party_path, "custom_fields", index, "value") + + def _at_key(self, *path: Any) -> AbstractContextManager[None]: + """:meth:`_here` for the frontmatter value at *path*: the line it is + written at (a value merged in with ``<<`` is written where it is + defined), else the line of the nearest key above it, the party's + (§16.9).""" + return self._here(self.key_line(*path)) def _attachments(self) -> tuple[AttachmentPart, ...]: """Attachment placeholders (§13.8), in declared order. At the @@ -1116,16 +1152,16 @@ def _signatures(self) -> tuple[SignatureParty, ...]: with self._at(self.key_line("sides", side_index, "label")): label = self._frontmatter(self.result.index.side_lookup.get(side.name) or side.label or side.name) for party_index, party in enumerate(side.parties): - with self._at(self.key_line("sides", side_index, "parties", party_index)): - # Where signature blocks are generated, legal_name MUST appear (§3.6). - blocks.append(SignatureParty( - side=label, - name=self._frontmatter(party.legal_name or party.label or party.name), - signatories=tuple( - (self._frontmatter(rep.name), self._frontmatter(rep.title)) - for rep in party.representatives - ), - )) + party_path = ("sides", side_index, "parties", party_index) + # Where signature blocks are generated, legal_name MUST appear (§3.6). + blocks.append(SignatureParty( + side=label, + name=self._party_name(party_path, party), + signatories=tuple( + self._representative(party_path, index, rep) + for index, rep in enumerate(party.representatives) + ), + )) return tuple(blocks) # -- diagnostics ---------------------------------------------------------- diff --git a/tests/test_diagnostic_lines.py b/tests/test_diagnostic_lines.py index df3c633..c26ed3e 100644 --- a/tests/test_diagnostic_lines.py +++ b/tests/test_diagnostic_lines.py @@ -247,9 +247,9 @@ def test_a_finding_in_a_party_or_a_side_is_at_its_key() -> None: result = render(SIDES) found = {d.message.split("{{include: ")[1][:5]: d.line for d in result.diagnostics if d.rule == "render-not-processed"} - # A party's detail is at the party; a side's label, at its label key. - assert found == {"l.lgd": at(SIDES, 'label: "Seller'), "a.lgd": at(SIDES, "name: acme"), - "b.lgd": at(SIDES, "name: bob")} + # A party's address is at its own key; a side's label, at its label key. + assert found == {"l.lgd": at(SIDES, 'label: "Seller'), "a.lgd": at(SIDES, 'address: "Street'), + "b.lgd": at(SIDES, 'address: "Road')} def test_signature_blocks_are_not_a_second_report() -> None: @@ -257,6 +257,85 @@ def test_signature_blocks_are_not_a_second_report() -> None: assert len(lines_of(result, "render-not-processed")) == 3 +# Each field of a party, with its own include: the field's key is its line, and a +# custom field is at its key (keyed in the party) or at its ``value`` (listed). +PARTY_FIELDS = """--- +title: T +document_type: contract +sides: + - name: seller + label: Seller + parties: + - name: acme + legal_name: "Acme {{include: legal.lgd}}" + type: legal_entity + identification_number: "ID {{include: ident.lgd}}" + address: "Street {{include: addr.lgd}}" + Passport: "Pass {{include: keyed.lgd}}" + custom_fields: + - label: Extra + value: "Extra {{include: listed.lgd}}" + representatives: + - name: "Rep {{include: repname.lgd}}" + title: "Title {{include: reptitle.lgd}}" + - name: buyer + label: Buyer + parties: + - name: bob + label: "Bob {{include: label.lgd}}" + type: natural_person + date_of_birth: "{{include: dob.lgd}}" +--- + +# A +""" + + +def test_a_finding_in_a_party_field_is_at_that_field() -> None: + result = render(PARTY_FIELDS, overrides={"signatures.enabled": True}) + found = {_included(d): d.line for d in result.diagnostics if d.rule == "render-not-processed"} + assert found == { + "legal.lgd": at(PARTY_FIELDS, "legal_name:"), + "ident.lgd": at(PARTY_FIELDS, "identification_number:"), + "addr.lgd": at(PARTY_FIELDS, "address:"), + "keyed.lgd": at(PARTY_FIELDS, "Passport:"), + "listed.lgd": at(PARTY_FIELDS, "value: \"Extra"), + "repname.lgd": at(PARTY_FIELDS, 'name: "Rep'), + "reptitle.lgd": at(PARTY_FIELDS, 'title: "Title'), + "label.lgd": at(PARTY_FIELDS, "label: \"Bob"), + "dob.lgd": at(PARTY_FIELDS, "date_of_birth:"), + } + + +MERGED = """--- +title: T +document_type: contract +defaults: &d + address: "Street {{include: addr.lgd}}" +sides: + - name: seller + label: Seller + parties: + - name: acme + <<: *d + legal_name: Acme + type: legal_entity +--- + +# A +""" + + +def test_a_party_field_merged_in_is_where_the_source_writes_it() -> None: + # The address is merged in (``<<``): it is written where it is defined. + [found] = [d for d in render(MERGED).diagnostics if d.rule == "render-not-processed"] + assert found.line == at(MERGED, "address:") + + +def _included(found) -> str: # type: ignore[no-untyped-def] + return found.message.split("{{include: ")[1].split("}}")[0] + + def test_key_line_is_general_and_none_without_a_source() -> None: from legaldown import Document, parse From 9eed084d214915359ce75733eac0352b48ef7afe Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:35:05 +0000 Subject: [PATCH 049/118] Keep a repeated finding at its lowest line, list the renderer's in line order The resolver works out the body before the title block, so a finding seen first in the body and again in the title was reported at the later line. diagnostics.record now keeps one finding (rule, level, message) at the lowest line it was seen at, in Resolver._warn and Resolver._reference, and api.render lists the renderer's own diagnostics in the order of their lines (none last, stable) after the validator's. CONFORMANCE.md and docs/architecture.md say so. --- CONFORMANCE.md | 6 +++-- docs/architecture.md | 9 ++++--- src/legaldown_render/api.py | 7 +++--- src/legaldown_render/diagnostics.py | 30 ++++++++++++++++++++---- src/legaldown_render/resolve/resolver.py | 17 +++++++------- tests/test_diagnostic_lines.py | 15 ++++++++++++ 6 files changed, 63 insertions(+), 21 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 2fc1a58..7646537 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -73,8 +73,10 @@ It is checked against the specification's own examples and fixtures corpus (`tes (also where the body shows it, as `{{attach:}}` or `{{party:}}` do). A finding in a term's text is at the term's definition, whichever reference asks for it first; `render-term-limit` is at the definition being worked out when the limit was reached. The same finding (rule, level and - message) is reported once, at its first line: the same refused link, or the same non-relative - image path, on two lines is one diagnostic, at the first of them. A document with no source to + message) is reported once, at the lowest line it is seen at: the same refused link, or the same + non-relative image path, on two lines is one diagnostic, at the earlier of them, whichever the + renderer reached first (it works out the body before the title block). The renderer's own + diagnostics are listed in the order of their lines (none last), after the validator's. A document with no source to name a line of (built from a dict) reports no line. With `answers`, the lines are those of the assembled document (§15.8), not of the template file. - **File references (§2.3).** The renderer reports `path-not-relative` and `path-outside-root` diff --git a/docs/architecture.md b/docs/architecture.md index e401ef7..859f9cc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -263,9 +263,12 @@ worked out at the term's definition (`Resolver.term_lines`, recorded by `_survey reference asks first. `Diagnostic.file` is `RenderOptions.filename`, which `render_file` and the CLI set to the file name as the validator does (`Document.filename`, without the directory). -**Repeats.** `Resolver._warn` and `api.render` leave out a diagnostic whose rule, level and message -a reported one has already, wherever it is (`diagnostics.repeats`), so a repeated finding is -reported at its first line only. The renderer's `path-not-relative` and `path-outside-root` are left +**Repeats.** `Resolver._warn` and `Resolver._reference` add a diagnostic through `diagnostics.record`, +which keeps one finding (rule, level, message) once, at the lowest line it was seen at: the +resolver works out the body before the title block, so a later sighting at a lower line moves the +reported one. `api.render` lists the renderer's diagnostics in the order of their lines (`diagnostics.by_line`, +none last) after the validator's, and leaves out those the validator's or the assembly's name already +(`diagnostics.repeats`). The renderer's `path-not-relative` and `path-outside-root` are left out when the validator reported the same path at the same line, or at none (`diagnostics.covered_by`). ## Security diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index 16a25ff..5f5ddce 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -22,7 +22,7 @@ from legaldown import AssemblyError, Diagnostic, Document, ValidationResult, parse, parse_template, validate from .build import build_tree, normalize_source -from .diagnostics import key_line, repeats +from .diagnostics import by_line, key_line, repeats from .errors import DocumentError, OptionsError, RenderRefused, StyleError from .resolve import Formatter, parse_locale, resolve_tree from .style import Style, load_style @@ -144,8 +144,9 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any tree, extra = _resolve(document, result, style, base) # The renderer's findings that the validator's or the assembly's already - # name (the same rule, level and message), wherever they are, are not repeated. - diagnostics += [d for d in extra if not repeats(d, diagnostics)] + # name (the same rule, level and message), wherever they are, are not repeated; + # the rest follow in the order of their lines. + diagnostics += [d for d in by_line(extra) if not repeats(d, diagnostics)] if options.strict and any(d.level == "error" for d in diagnostics): raise RenderRefused(diagnostics) diff --git a/src/legaldown_render/diagnostics.py b/src/legaldown_render/diagnostics.py index d9b071e..60a94c1 100644 --- a/src/legaldown_render/diagnostics.py +++ b/src/legaldown_render/diagnostics.py @@ -12,6 +12,7 @@ from __future__ import annotations from collections.abc import Iterable +from dataclasses import replace from typing import Any from legaldown import Diagnostic, Document @@ -52,10 +53,31 @@ def key_line(document: Document, *path: Any) -> int | None: def repeats(found: Diagnostic, known: Iterable[Diagnostic]) -> bool: """True if *known* holds *found*'s finding already: the same rule, level - and message, wherever it was. The same finding in two places is reported - once, at the first (a link to the same refused URL on two lines is one - ``render-link-refused``, at the line of the first).""" - return any(d.rule == found.rule and d.level == found.level and d.message == found.message for d in known) + and message, wherever it was.""" + return any(_same(d, found) for d in known) + + +def record(found: Diagnostic, known: list[Diagnostic]) -> None: + """Add *found* to *known*, unless it holds the same finding already (the + same rule, level and message): the finding is reported once, at the lowest + line it was seen at (a link to the same refused URL on two lines is one + ``render-link-refused``, at the earlier line, whichever was found first).""" + for index, d in enumerate(known): + if _same(d, found): + if found.line is not None and (d.line is None or found.line < d.line): + known[index] = replace(d, line=found.line) + return + known.append(found) + + +def by_line(found: Iterable[Diagnostic]) -> list[Diagnostic]: + """*found* in the order of their lines, those with none last, and those at + one line as they were.""" + return sorted(found, key=lambda d: (d.line is None, d.line or 0)) + + +def _same(a: Diagnostic, b: Diagnostic) -> bool: + return a.rule == b.rule and a.level == b.level and a.message == b.message def covered_by(found: Diagnostic, path: str, validator: Iterable[Diagnostic]) -> bool: diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 2c2680c..6910706 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -62,7 +62,7 @@ from legaldown.syntax import Directive from ..build import plain_inlines -from ..diagnostics import KeyLines, covered_by, repeats +from ..diagnostics import KeyLines, covered_by, record from ..style import Style, effective_labels from ..style.model import LevelFormat from ..tree import ( @@ -1172,11 +1172,9 @@ def _found(self, rule: str, level: str, message: str) -> Diagnostic: return Diagnostic(rule=rule, level=level, message=message, line=self.line, file=self.document.filename) def _warn(self, rule: str, message: str) -> None: - """Report a Warning, unless the same one (rule and message) is - reported already: wherever it recurs, it stands at its first line.""" - diagnostic = self._found(rule, "warning", message) - if not repeats(diagnostic, self.diagnostics): - self.diagnostics.append(diagnostic) + """Report a Warning; the same one (rule and message) reported again is + one, at the lowest line it was seen at.""" + record(self._found(rule, "warning", message), self.diagnostics) def _link_refused(self, href: str) -> None: """Report a link whose URL does not use a scheme links may use @@ -1202,7 +1200,8 @@ def _reference(self, label: str, path: str, *, effect: str, url: bool = False, A report is left out when the validator made it already (:func:`~legaldown_render.diagnostics.covered_by`: its report of the - path, at this line) or this resolver did (the same message). The + path, at this line); one this resolver made is kept once, at the lowest + line (:func:`~legaldown_render.diagnostics.record`). The validator quotes the path in its message as '', as written; matching the quoted form keeps a path that is a prefix of another ("/a" and "/ab") from being counted as already reported.""" @@ -1217,8 +1216,8 @@ def _reference(self, label: str, path: str, *, effect: str, url: bool = False, rule, what = "path-outside-root", ("is outside the document root (§2.3): a file reference may " "not leave the root.") diagnostic = self._found(rule, "error", f"{label} '{printable(shown)}' {what} {effect}") - if not (covered_by(diagnostic, shown, self.result.diagnostics) or repeats(diagnostic, self.diagnostics)): - self.diagnostics.append(diagnostic) + if not covered_by(diagnostic, shown, self.result.diagnostics): + record(diagnostic, self.diagnostics) return True def _attachment_label(self, attachment_id: str) -> str: diff --git a/tests/test_diagnostic_lines.py b/tests/test_diagnostic_lines.py index c26ed3e..4be8813 100644 --- a/tests/test_diagnostic_lines.py +++ b/tests/test_diagnostic_lines.py @@ -416,6 +416,21 @@ def test_the_same_finding_on_two_lines_is_reported_once_at_the_first() -> None: assert lines_of(result, "render-link-refused") == [at(source, "Three")] +def test_a_finding_seen_again_at_a_lower_line_is_reported_at_the_lower() -> None: + # The body is resolved before the title, so the title's is the later sighting. + source = '---\ntitle: "T {{include: x.lgd}}"\n---\n\n# A\n\nBody {{include: x.lgd}}\n' + assert at(source, "title:") < at(source, "Body") + assert lines_of(render(source), "render-not-processed") == [at(source, "title:")] + + +def test_the_renderers_diagnostics_are_in_the_order_of_their_lines() -> None: + source = ('---\ntitle: "T {{include: t.lgd}}"\nsubtitle: "S {{include: s.lgd}}"\n---\n\n# A\n\n' + "![a](/x.png)\n\nBody {{include: b.lgd}}\n") + result = render(source) + lines = [d.line for d in result.diagnostics if d.rule in ("render-not-processed", "path-not-relative")] + assert lines == sorted(lines) and len(lines) == 4 + + def test_different_findings_of_one_rule_each_have_their_line() -> None: source = HEAD + "\n# A\n\nOne ![a](/x.png).\n\nTwo ![b](/y.png).\n" assert lines_of(render(source), "path-not-relative") == [at(source, "One"), at(source, "Two")] From be49fcfa3e4af9c0119dcc9b55f056faa0235c25 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:35:38 +0000 Subject: [PATCH 050/118] Drop the directory from RenderOptions.filename filename is documented as the name without its directory, but render(filename='a/b.lgd') kept it on every diagnostic's file. render now keeps the last path component, split at both / and \. --- README.md | 2 +- docs/architecture.md | 2 +- src/legaldown_render/api.py | 6 +++++- tests/test_diagnostic_lines.py | 7 +++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3078948..10f2c35 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ with open("contract.html", "w", encoding="utf-8", newline="\n") as output: `render()` also accepts a `RenderOptions`. Pass `answers={...}` to render a template filled in. Each diagnostic has its `line` and its `file`: `render_file()` and the command name the file (its -name, without the directory); for text given to `render()`, pass `filename=`. +name, without the directory); for text given to `render()`, pass `filename=` (a directory in it is dropped, `/` or `\`). `document_root=` is the `--root` of the command line, and `document_path=` says where the document is in it (`render_file()` sets it from its path; text given to `render()` is at the root); a document outside the root raises `OptionsError`. For example, a hosting service that renders diff --git a/docs/architecture.md b/docs/architecture.md index 859f9cc..23cf2d5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -261,7 +261,7 @@ frontmatter is at its key (`diagnostics.key_line`, which wraps the public `sourc returns None without a source map; one helper for every frontmatter reference). A term's text is worked out at the term's definition (`Resolver.term_lines`, recorded by `_survey`) whichever reference asks first. `Diagnostic.file` is `RenderOptions.filename`, which `render_file` and the CLI -set to the file name as the validator does (`Document.filename`, without the directory). +set to the file name as the validator does (`Document.filename`, without the directory; `render` drops one given, `/` or `\`). **Repeats.** `Resolver._warn` and `Resolver._reference` add a diagnostic through `diagnostics.record`, which keeps one finding (rule, level, message) once, at the lowest line it was seen at: the diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index 5f5ddce..79d4a8e 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -62,7 +62,8 @@ class RenderOptions: #: The document's file name, which every diagnostic of the result names as #: its ``file`` (§16.9), the validator's and the renderer's alike, as #: ``Document.filename`` does for the validator: the name without its - #: directory. ``render_file`` and the command line set it from the path + #: directory (a directory given is dropped, whether it is written with ``/`` + #: or ``\``). ``render_file`` and the command line set it from the path #: they read; for text given to :func:`render` it is empty unless set. #: With ``answers``, the lines of the diagnostics are those of the #: assembled document (§15.8), not of the template file. @@ -125,6 +126,9 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any if options.format not in FORMATS: raise OptionsError(f"unknown output format '{options.format}' (known: {', '.join(FORMATS)})") base = _document_base(options) + name = options.filename.replace("\\", "/").rsplit("/", 1)[-1] + if name != options.filename: + options = replace(options, filename=name) if options.document_path is not None and not options.filename: options = replace(options, filename=PurePath(options.document_path).name) overrides = dict(options.overrides) diff --git a/tests/test_diagnostic_lines.py b/tests/test_diagnostic_lines.py index 4be8813..a6163ea 100644 --- a/tests/test_diagnostic_lines.py +++ b/tests/test_diagnostic_lines.py @@ -336,6 +336,13 @@ def _included(found) -> str: # type: ignore[no-untyped-def] return found.message.split("{{include: ")[1].split("}}")[0] +@pytest.mark.parametrize("given", ["a/b.lgd", "a\\b.lgd", "/abs/dir/b.lgd", "C:\\x\\y/b.lgd"]) +def test_the_file_name_is_without_its_directory(given: str) -> None: + source = HEAD + "\n# A\n\n{{include: x.lgd}}\n" + result = render(source, filename=given) + assert {d.file for d in result.diagnostics} == {"b.lgd"} + + def test_key_line_is_general_and_none_without_a_source() -> None: from legaldown import Document, parse From 9d5a157743c4d86b698792877af5766d9f7ff4bd Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:37:42 +0000 Subject: [PATCH 051/118] Refuse control characters in a file reference, read ... as .. for the root A path with NUL or another control character (after the URL decoding that applies to an image's source) is path-not-relative, so '..%00/x.png' and an include of 'ab' no longer pass as plain names. In the walk of a path against the root a segment of '..' followed by dots or spaces counts as '..' (Windows drops trailing dots and spaces), so '... /x' and '..../x' leave the root; 'a..b.png' and '...hidden/x' are names as before. CONFORMANCE.md and docs/architecture.md describe both, and the limitations that a file name with a colon before any slash reads as a URI scheme (write ./report:v2.png) and that an attachment's file is read as a URL only where it becomes a link. --- CONFORMANCE.md | 14 ++++++++++---- docs/architecture.md | 2 +- src/legaldown_render/urls.py | 25 +++++++++++++++---------- tests/test_file_references.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 15 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 7646537..e3916b2 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -40,7 +40,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `raw-html` Warning; raw HTML never emitted (§8.7) | ✅ Nothing is emitted: an HTML block is dropped whole, and an inline tag is dropped with its text kept. The Warning is the validator's, once per block or text that holds raw HTML other than comments (see "Text after a comment", below) | | Automatic term recognition (§7.4, MAY) | ❌ Not supported | | Links (§8.7) | ⚠️ Partial, a documented deviation for safety with untrusted documents: links with schemes outside the allow-list render as text with a `render-link-refused` Warning. A link is a hyperlink in HTML and shows `text ` in plain text when its URL is relative, a fragment, or has one of the schemes `http`, `https`, `mailto`, `tel`, `ftp`, `ftps`, `sftp`, `sip`, `sips`, `sms`, `urn`, `geo`, `news` (an allow-list, judged as a browser reads the URL). Any other scheme (`javascript:`, `file:`, `data:`, protocol handlers such as `ms-msdt:`, `vscode:`, `search-ms:`, and pseudo-schemes such as `C:\x` or `localhost:8080`), or an empty URL, is not a link: its text is shown alone, and the Warning `render-link-refused` is reported for a non-empty URL. A link with no text shows its URL as written; the text writer omits `` for an autolink | -| Images and file-reference paths (§8.7, §2.3) | ✅ Every file reference (`{{include:}}`, `attachments[].file`, `amends.file`, `supersedes.file`, a `translations` value, an image) must be relative, and within the document root when one is configured (`--root DIR`, `document_root=`; see "File references", below). A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), or a Windows drive path is the Error `path-not-relative`; a path that leaves the root is the Error `path-outside-root`; `--strict` refuses both. Such an image shows its alt text and is never fetched or embedded; an `{{attach:}}` shows its text with no link; an include marker and an attachment's path stay as text, and no file is ever read. A relative path (`../` included, as §2.3 defines no default root) renders `` in HTML and its alt text in plain text. An empty image path shows the alt text with no diagnostic | +| Images and file-reference paths (§8.7, §2.3) | ✅ Every file reference (`{{include:}}`, `attachments[].file`, `amends.file`, `supersedes.file`, a `translations` value, an image) must be relative, and within the document root when one is configured (`--root DIR`, `document_root=`; see "File references", below). A path with a URI scheme (`data:` too), an absolute one (`/`, `\`, `//host`), a Windows drive path, or one with a control character is the Error `path-not-relative`; a path that leaves the root is the Error `path-outside-root`; `--strict` refuses both. Such an image shows its alt text and is never fetched or embedded; an `{{attach:}}` shows its text with no link; an include marker and an attachment's path stay as text, and no file is ever read. A relative path (`../` included, as §2.3 defines no default root) renders `` in HTML and its alt text in plain text. An empty image path shows the alt text with no diagnostic | | Plain-text layout (§13.6, optional) | ✅ `attachments.separator` (§13.8): `none` prints no `---`; `rule` and `page-break` both print one before the attachments heading and between consecutive attachments, so the heading stays with the first (text has no pages). One quote stays one quote (blank lines keep the `>`), and the blocks of a list item are set apart by blank lines and indented under the item's label, whatever its condition label; an item that opens with a nested list, table, code block or quote has its label on a line of its own, so that block stays valid. A pipe in a table cell is backslash-escaped, and the separator row marks column alignment (`:---`, `:---:`, `---:`). A label a style blanks drops its `label: ` prefix in the title block, party details and signature blocks (a signature row with neither label nor value is omitted); HTML keeps an empty `
` so its grid layout and the `
` stay valid | | Extended tables through raw HTML (§9.2, MAY) | ❌ Not supported; the HTML table is dropped with the Warning | | Visual blank for a placeholder (§13.5 rule 6) | ✅ `placeholders.blank`; a blank that is empty, or only whitespace or invisible format characters (U+200B, U+FEFF), renders `[TBD: id]` | @@ -96,12 +96,18 @@ It is checked against the specification's own examples and fixtures corpus (`tes (`RenderOptions.document_path`); text given to `render()`, or standard input, is at the root. A document outside the root, or on another Windows drive, is an `OptionsError` (exit 2). How a path is read: a backslash is a separator, so `..\..\x` leaves the root as `../../x` does - (the renderer's reading; a leading `\` is not relative anyway); `..` and `.` followed by spaces - are `..` and `.` (Windows reads them so); tabs and newlines inside a scheme, and spaces and - control characters at either end, are ignored as a browser ignores them. Percent-encoding is + (the renderer's reading; a leading `\` is not relative anyway); `..` followed by dots and + spaces (`.. `, `...`) is `..`, and `.` followed by spaces is `.` (Windows reads them so); + tabs and newlines inside a scheme, and spaces and control characters at either end, are ignored + as a browser ignores them. A path with a control character (NUL, a newline, any other of + Unicode category Cc) in it, after the decoding below, is `path-not-relative`. Percent-encoding is decoded once, only for what is used as a URL, an image's source and the link `{{attach:}}` would make to an attachment's file: there `%2e%2e/` is `../`. A path in the frontmatter or an `{{include:}}` is a file name, read as written, where `%2e%2e` is a directory with that name. + An attachment's `file` is read as a URL only where it becomes a link (`attachments.render: omit` + with `{{attach:}}`), so a percent-encoded traversal in it is reported only in that mode. + A relative file name with a colon before any `/` reads as a URI scheme (§2.3), so it is + `path-not-relative`: write `./report:v2.png`. `supersedes` as a plain string describes the superseded document (§3.2) and is not a path. Markdown links (`[text](/etc/passwd)`, §8.7) are not file references and stay governed by the link allow-list (`render-link-refused`): they are links the reader may follow, not files the diff --git a/docs/architecture.md b/docs/architecture.md index 23cf2d5..2a740f5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -321,7 +321,7 @@ out when the validator reported the same path at the same line, or at none (`dia root when it has no path, as text given to `render()` and standard input are); a document outside the root, or on another Windows drive, is an `OptionsError`, since the boundary would mean nothing. A backslash is read as a separator (the renderer's reading, so a path that goes up - on Windows is not missed), `..`/`.` with trailing spaces are `..`/`.`, and percent-encoding is + on Windows is not missed), `..` with trailing dots and spaces is `..` and `.` with trailing spaces is `.`, a control character (NUL included) makes the path `not-relative`, and percent-encoding is decoded once, only for what is used as a URL (an image's `src`, the link an `{{attach:}}` makes): `%2e%2e` is `..` there, and a directory name in a frontmatter or include path. The writers' image check is a second line. Markdown links are not file references (§8.7): they stay diff --git a/src/legaldown_render/urls.py b/src/legaldown_render/urls.py index a6f7c32..4a17f5f 100644 --- a/src/legaldown_render/urls.py +++ b/src/legaldown_render/urls.py @@ -34,6 +34,8 @@ # newlines anywhere, and C0 controls and spaces at either end. Checking the # URL as a browser will read it means "java\tscript:" cannot slip through. _URL_IGNORED_RE = re.compile(r"[\t\n\r]") +# A segment Windows reads as ``..``: it drops trailing dots and spaces (``.. ``, ``...``). +_PARENT_RE = re.compile(r"^\.\.[. ]*$") _URL_TRIMMED = "".join(chr(code) for code in range(0x21)) @@ -99,7 +101,8 @@ def file_reference_problem(path: str, *, base: str | None = None, url: bool = Fa * ``"not-relative"`` (``path-not-relative``): it has a URI scheme (``https://``, ``file://``, ``data:``), which includes a Windows drive path (``C:\\x``, ``C:/x``, a scheme as far as a parser can tell), or - starts with ``/`` or ``\\`` (so also ``//host``); + starts with ``/`` or ``\\`` (so also ``//host``), or has a control + character (NUL, a newline, ...); * ``"outside-root"`` (``path-outside-root``): there is a document root (*base* is not None) and the path leaves it. @@ -112,29 +115,31 @@ def file_reference_problem(path: str, *, base: str | None = None, url: bool = Fa How the path is read: a backslash is a separator (the renderer's reading, so that a path that goes up on Windows is not missed; it matters only with a - root, a leading one is already not relative), and so are ``.. `` and ``. `` - with trailing spaces, which Windows reads as ``..`` and ``.``. What a - browser ignores before the scheme (tabs, newlines, control characters and - spaces at either end) is ignored. *url* says the path is consumed as a URL + root, a leading one is already not relative), and so are ``.. ``, ``...`` + and ``. `` with trailing dots and spaces, which Windows reads as ``..`` and + ``.``. A path with a control character (NUL, a newline, ...) is not a file + name a renderer can name safely, and is ``not-relative``. What a + browser ignores before it reads the scheme (tabs and newlines, control + characters and spaces at either end) is ignored in finding one. *url* says the path is consumed as a URL (an image's source, a link's target), where percent-encoding counts as the character it encodes (``%2e%2e`` is ``..``, ``%2f`` a separator), once; a path from frontmatter or an ``{{include:}}`` is a file name and is read as written, in which ``%2e%2e`` is a directory with that name.""" - read = _as_read(unquote(path) if url else path) + decoded = unquote(path) if url else path + read = _as_read(decoded) if not read: return None - if _SCHEME_RE.match(read) or read[0] in "/\\": + if _SCHEME_RE.match(read) or read[0] in "/\\" or any(unicodedata.category(c) == "Cc" for c in decoded): return "not-relative" if base is None: return None depth = 0 if base == "." else len([part for part in base.split("/") if part]) for part in read.replace("\\", "/").split("/"): - part = part.rstrip(" ") - if part == "..": + if _PARENT_RE.match(part): depth -= 1 if depth < 0: return "outside-root" - elif part not in ("", "."): + elif part.rstrip(" ") not in ("", "."): depth += 1 return None diff --git a/tests/test_file_references.py b/tests/test_file_references.py index 0af5ba8..7f95258 100644 --- a/tests/test_file_references.py +++ b/tests/test_file_references.py @@ -71,6 +71,22 @@ ("%252e%252e/x", ".", True, None), # Spaces after the name are the Windows reading of the same segment. (".. /x", ".", False, "outside-root"), + # Windows drops trailing dots and spaces too. + ("... /x", ".", False, "outside-root"), + ("..../x", ".", False, "outside-root"), + ("... /x", "sub", False, None), + ("a/.. ./../x", ".", False, "outside-root"), + # Names that only look like it. + ("a..b.png", ".", False, None), + ("...hidden/x", ".", False, None), + ("..a/x", ".", False, None), + # A control character is not a file name: NUL, after the decoding a URL gets. + ("a\x00b", None, False, "not-relative"), + ("..%00/x.png", ".", True, "not-relative"), + ("..%00/x.png", ".", False, None), + ("a\x1fb/x", ".", False, "not-relative"), + ("a\x7fb/x", ".", False, "not-relative"), + ("a\x85b/x", ".", False, "not-relative"), ]) def test_the_check(path: str, base: str | None, url: bool, expected: str | None) -> None: assert file_reference_problem(path, base=base, url=url) == expected @@ -184,6 +200,19 @@ def test_a_disguised_scheme_is_a_scheme(kind: str, tmp_path: Path) -> None: assert [rule for rule, _ in reported(kind, path, tmp_path, None)] == [NOT_RELATIVE], path +@pytest.mark.parametrize("kind", ["include", "image"]) +def test_a_path_with_a_control_character_is_not_relative(kind: str) -> None: + path = {"include": "a\x00b.lgd", "image": "..%00/x.png"}[kind] + source = _frontmatter("").replace("Body.", {"include": f"{{{{include: {path}}}}}", "image": f"![x]({path})"}[kind]) + assert [rule for rule, _ in findings(render(source))] == [NOT_RELATIVE] + + +@pytest.mark.parametrize("path", [".. ./x.png", "... /x.png", "..../x.png"]) +def test_a_parent_directory_as_windows_reads_it_leaves_the_root(path: str, tmp_path: Path) -> None: + source = _frontmatter("").replace("Body.", f"![x](<{path}>)") + assert [rule for rule, _ in findings(render(source, document_root=tmp_path, document_path=tmp_path / "d.lgd"))] == [OUTSIDE] + + # -- What happens to the path ------------------------------------------------------------ From eeb6622c0aeb6b96a48b89c32523d0d30bdb0759 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:38:29 +0000 Subject: [PATCH 052/118] Refuse style values that could leave the stylesheet, and add an untrusted mode A style value written into the stylesheet (fonts, sizes, colours, margins, page size, max width) containing /*, */, Document: def _resolve(document: Document, result: ValidationResult, style: Style, - base: str | None) -> tuple[RenderTree, list[Diagnostic]]: + base: str | None, untrusted: bool = False) -> tuple[RenderTree, list[Diagnostic]]: diagnostics: list[Diagnostic] = [] # Raw HTML is never emitted; the validator reports it (raw-html, §8.7). tree = build_tree(document, result) locale = parse_locale(style.locale) if style.locale else None if style.locale and locale is None: - raise StyleError("setting", [f"locale: '{style.locale}' is not a known locale (e.g. en-US, cs-CZ)"]) + shown = "" if untrusted else f" '{style.locale}'" + raise StyleError("setting", [f"locale:{shown} is not a known locale (e.g. en-US, cs-CZ)"]) if locale is None: # No locale set: the document's language is the hint (§10.1). locale = parse_locale(tree.language) diff --git a/src/legaldown_render/cli.py b/src/legaldown_render/cli.py index 7a9db4a..d9c6e93 100644 --- a/src/legaldown_render/cli.py +++ b/src/legaldown_render/cli.py @@ -94,6 +94,10 @@ def _parser() -> argparse.ArgumentParser: "supersedes, a translation, an image) that leaves DIR is an error " "(path-outside-root), and the file is not used. The document must be inside DIR " "(standard input counts as at DIR itself). Without it, ../ is allowed") + parser.add_argument("--untrusted", action="store_true", + help="the document and the style are somebody else's (for a service rendering what its " + "users send): requires --root (exit 2 without), and the style must be a built-in " + "name, with no file, no raw CSS (html.extra_css) and only plain CSS values") parser.add_argument("--fragment", action="store_true", help="HTML: write only the
, without page and stylesheet") parser.add_argument("--strict", action="store_true", help="refuse to render a document that has errors") parser.add_argument("--answers", metavar="FILE", @@ -199,7 +203,8 @@ def main(argv: list[str] | None = None) -> int: options = RenderOptions(format=output_format, style=args.style, overrides=overrides, strict=args.strict, final=args.final, answers=answers, standalone=not args.fragment, filename="" if args.input == "-" else Path(args.input).name, - document_root=args.root, document_path=None if args.input == "-" else args.input) + document_root=args.root, document_path=None if args.input == "-" else args.input, + untrusted=args.untrusted) try: result = render(source, options) except (StyleError, OptionsError) as error: diff --git a/src/legaldown_render/style/loader.py b/src/legaldown_render/style/loader.py index 7e472da..5e08b81 100644 --- a/src/legaldown_render/style/loader.py +++ b/src/legaldown_render/style/loader.py @@ -10,6 +10,12 @@ The merged mapping is then checked against the dataclasses in :mod:`.model`. Every problem is collected, with its key path, before :class:`StyleError` is raised, so one run reports everything that is wrong with a template. + +A style is trusted input by default: the operator's own file. ``trusted=False`` +(:func:`load_style`) is for a style that comes from someone else — it reads +no file, extends only built-in styles, holds every value that reaches the +stylesheet to a strict grammar, and never quotes the style in a message +(docs/architecture.md, Security). """ from __future__ import annotations @@ -28,10 +34,17 @@ from ..errors import StyleError from . import model -from .model import Style, template_settings +from .model import Style, css_problem, template_settings STYLE_FORMAT_VERSION = 1 +#: The longest YAML text an untrusted style may be (bytes of UTF-8). +MAX_UNTRUSTED_STYLE_BYTES = 64 * 1024 + +#: How an untrusted style is named in a message, instead of its own name (which +#: is source text) or a path. +_UNTRUSTED_SOURCE = "'untrusted style'" + def builtin_styles() -> list[str]: """Names of the style templates shipped with the package.""" @@ -40,26 +53,55 @@ def builtin_styles() -> list[str]: def load_style( - style: str | Path | Style | None = None, + style: str | Path | Style | Mapping[str, Any] | None = None, *, overrides: Mapping[str, Any] | None = None, + trusted: bool = True, ) -> Style: """The validated :class:`Style` for *style* — a built-in name, a path to - a YAML file, or a :class:`Style` — with *overrides* applied. + a YAML file, a :class:`Style`, a mapping, or YAML text (a string with a + line break) — with *overrides* applied. *overrides* maps dotted keys to values, e.g. ``{"numbering.scheme": "mixed", "locale": "cs-CZ"}``; nested mappings are accepted too. + + Always: a value written into the stylesheet may not contain a comment + delimiter, `` Style: if isinstance(style, Style): - data, source = style_to_dict(style), f"'{style.name}'" + data = style_to_dict(style) + source = f"'{style.name}'" if trusted else _UNTRUSTED_SOURCE + elif isinstance(style, Mapping) or (isinstance(style, str) and ("\n" in style or "\r" in style)): + source = "'(style)'" if trusted else _UNTRUSTED_SOURCE + raw = _parse(style, source, trusted) if isinstance(style, str) else _check_raw(_copy(style), source, trusted) + data, _ = _extend(raw, source, None, f"'(style {id(raw)})'", (), trusted) else: - data, source = _load_chain(style if style is not None else "default", seen=()) + data, source = _load_chain(style if style is not None else "default", seen=(), trusted=trusted) if overrides: - expanded = _expand_dotted(overrides) + expanded = _expand_dotted(overrides, trusted) _refuse_meta_overrides(expanded) data = _merge(data, expanded) - return _build(data, source) + return _build(data, source, trusted) def style_to_dict(style: Style) -> dict[str, Any]: @@ -84,7 +126,7 @@ def dump_style(style: Style) -> str: def _load_chain( - spec: str | Path, *, seen: tuple[tuple[str, str], ...], relative_to: Path | None = None + spec: str | Path, *, seen: tuple[tuple[str, str], ...], relative_to: Path | None = None, trusted: bool = True ) -> tuple[dict, str]: """The merged raw mapping for *spec* and every template it extends. @@ -92,7 +134,13 @@ def _load_chain( A file's identity is its resolved path, so ``../dir/a.yaml`` and ``a.yaml`` — or a symbolic link — are one template and a cycle through them is found.""" - raw, source, folder, identity = _read(spec, relative_to) + raw, source, folder, identity = _read(spec, relative_to, trusted) + return _extend(raw, source, folder, identity, seen, trusted) + + +def _extend(raw: dict, source: str, folder: Path | None, identity: str, + seen: tuple[tuple[str, str], ...], trusted: bool) -> tuple[dict, str]: + """*raw* merged onto the template it extends (``default`` when it names none).""" if any(identity == earlier for earlier, _ in seen): chain = " -> ".join((*(label for _, label in seen), source)) raise StyleError(source, [f"extends: circular chain {chain}"]) @@ -103,20 +151,36 @@ def _load_chain( # Start from every built-in default, so that a partial mapping — # one heading level's alignment, say — merges field by field. return _merge(style_to_dict(Style()), raw), source - base, _ = _load_chain("default", seen=here) + base, _ = _load_chain("default", seen=here, trusted=trusted) elif isinstance(parent, str) and parent: - base, _ = _load_chain(parent, seen=here, relative_to=folder) + if not trusted and not _is_builtin_name(parent): + raise StyleError(source, ["extends: an untrusted style may extend only a built-in style, " + f"by name (built-in: {', '.join(builtin_styles())})"]) + base, _ = _load_chain(parent, seen=here, relative_to=folder, trusted=trusted) else: - raise StyleError(source, ["extends: must be a style name or a path"]) + raise StyleError(source, ["extends: must be a style name or a path" if trusted + else "extends: must be the name of a built-in style"]) return _merge(base, raw), source -def _read(spec: str | Path, relative_to: Path | None) -> tuple[dict, str, Path | None, str]: +def _is_builtin_name(name: str) -> bool: + return name in builtin_styles() + + +def _read(spec: str | Path, relative_to: Path | None, + trusted: bool = True) -> tuple[dict, str, Path | None, str]: """The raw mapping in *spec*, a label for messages, the folder that relative ``extends`` paths are resolved against, and the template's - identity (its resolved path, or the quoted name of a built-in).""" + identity (its resolved path, or the quoted name of a built-in). An + untrusted style reads built-in styles only, never a file.""" name = str(spec) is_path = isinstance(spec, Path) or name.endswith((".yaml", ".yml")) or "/" in name or "\\" in name + if not trusted: + if not isinstance(spec, str) or is_path or not _is_builtin_name(name): + known = ", ".join(builtin_styles()) + raise StyleError(_UNTRUSTED_SOURCE, [f"an untrusted style is a built-in style (built-in: {known}), " + f"a mapping, or YAML text, never a file"]) + is_path = False if not is_path: if name not in builtin_styles(): known = ", ".join(builtin_styles()) @@ -148,21 +212,59 @@ def _identity(path: Path) -> str: return os.path.abspath(path) -def _parse(text: str, source: str) -> dict: +def _parse(text: str, source: str, trusted: bool = True) -> dict: + if not trusted: + _check_untrusted_text(text) try: raw = yaml.safe_load(text) except yaml.YAMLError as error: - raise StyleError(source, [f"not valid YAML: {error}"]) from None - if raw is None: - return {} + # PyYAML's message quotes the source around the problem, which for a + # style from elsewhere would show a reader whatever the text holds. + raise StyleError(source, [f"not valid YAML: {error}" if trusted else _untrusted_yaml_problem(error)]) from None + return _check_raw({} if raw is None else raw, source, trusted) + + +def _check_raw(raw: Any, source: str, trusted: bool = True) -> dict: + """*raw*, a style template as loaded, once it is known to be a mapping of + the format version this renderer reads.""" if not isinstance(raw, dict): raise StyleError(source, ["a style template must be a YAML mapping"]) version = raw.get("version", STYLE_FORMAT_VERSION) if version != STYLE_FORMAT_VERSION: - raise StyleError(source, [f"version: this renderer reads style format {STYLE_FORMAT_VERSION}, got {version!r}"]) + got = f", got {version!r}" if trusted else "" + raise StyleError(source, [f"version: this renderer reads style format {STYLE_FORMAT_VERSION}{got}"]) return raw +def _check_untrusted_text(text: str) -> None: + """Refuse YAML text that is too large or uses aliases. An alias lets a + small text stand for an enormous structure (the "billion laughs" shape), + and nothing in a style needs one.""" + if len(text.encode("utf-8", "replace")) > MAX_UNTRUSTED_STYLE_BYTES: + raise StyleError(_UNTRUSTED_SOURCE, [f"the style text is larger than {MAX_UNTRUSTED_STYLE_BYTES // 1024} KiB"]) + try: + for event in yaml.parse(text, Loader=yaml.SafeLoader): + if isinstance(event, yaml.AliasEvent): + raise StyleError(_UNTRUSTED_SOURCE, [f"YAML aliases are not allowed (line {event.start_mark.line + 1})"]) + except yaml.YAMLError: + pass # reported, without the source, by the load that follows + + +def _untrusted_yaml_problem(error: yaml.YAMLError) -> str: + """Where a YAML text is wrong, and what, without quoting it. A scanner or + parser problem names a kind of mistake; any other (an unknown tag, an + undefined alias, a duplicate anchor) names text from the source, so it + gives the place only.""" + mark = getattr(error, "problem_mark", None) + where = f" (line {mark.line + 1}, column {mark.column + 1})" if mark is not None else "" + problem = getattr(error, "problem", None) + if problem and isinstance(error, (yaml.scanner.ScannerError, yaml.parser.ParserError)): + # Even there a quoted piece can be the source's (a tag handle). + problem = re.sub(r"'[^']*'", "'...'", problem) + return f"not valid YAML: {problem}{where}" + return f"not valid YAML{where}" + + def _merge(base: Mapping, override: Mapping) -> dict: """*override* deep-merged onto *base*; mappings merge, anything else (lists included) replaces.""" @@ -176,7 +278,7 @@ def _merge(base: Mapping, override: Mapping) -> dict: return merged -def _expand_dotted(overrides: Mapping[str, Any]) -> dict: +def _expand_dotted(overrides: Mapping[str, Any], trusted: bool = True) -> dict: """*overrides* as one nested mapping. Dotted and nested keys for the same setting merge in order, and the caller's values are copied, never modified.""" @@ -187,7 +289,8 @@ def _expand_dotted(overrides: Mapping[str, Any]) -> dict: for part in parents: target = target.setdefault(part, {}) if not isinstance(target, dict): - raise StyleError("overrides", [f"{key}: '{part}' is set to a value and cannot hold keys"]) + raise StyleError("overrides", [f"{key}: '{part}' is set to a value and cannot hold keys" if trusted + else "an override is set to a value and cannot hold keys"]) value = _copy(value) if isinstance(value, dict) and isinstance(target.get(last), dict): value = _merge(target[last], value) @@ -295,16 +398,16 @@ def _is_text(hint: Any) -> bool: _FIELD_RE = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}") -def _build(data: Mapping, source: str) -> Style: +def _build(data: Mapping, source: str, trusted: bool = True) -> Style: problems: list[str] = [] - style = _convert(data, Style, "", problems) - problems.extend(_template_problems(style)) + style = _convert(data, Style, "", problems, trusted=trusted) + problems.extend(_template_problems(style, trusted)) if problems: raise StyleError(source, problems) return style -def _template_problems(style: Style) -> list[str]: +def _template_problems(style: Style, trusted: bool = True) -> list[str]: """What is wrong with the ``{field}`` names in the style's text templates (:func:`~.model.template_settings`): an identifier-shaped field the setting does not accept (a typo would be shown literally), or a required @@ -314,14 +417,16 @@ def _template_problems(style: Style) -> list[str]: bare field for it (``labels.condition``): it already shows the condition. Where there is no such fallback, an empty template lacks its fields. :func:`~.model.template_settings` lists only the settings that substitute - fields, and drops the required fields of one the style does not use.""" + fields, and drops the required fields of one the style does not use. An + untrusted style's own field name is not quoted.""" problems = [] for key, template, fields in template_settings(style): found = set(_FIELD_RE.findall(template)) accepted = ", ".join(f"{{{name}}}" for name in sorted(fields.accepted)) for name in sorted(found - fields.accepted): takes = f"accepts {accepted}" if accepted else "takes no fields" - problems.append(f"{key}: unknown field {{{name}}}; this setting {takes}") + unknown = f"{{{name}}}" if trusted else "a {field}" + problems.append(f"{key}: unknown field {unknown}; this setting {takes}") if fields.empty_falls_back and not template: continue for name in sorted(fields.required - found): @@ -329,29 +434,42 @@ def _template_problems(style: Style) -> list[str]: return problems -def _convert(value: Any, hint: Any, path: str, problems: list[str]) -> Any: +def _got(value: Any, trusted: bool) -> str: + """`` (got 3)`` for a message about a value — nothing for an untrusted + style, whose text a message does not quote.""" + return f" (got {value!r})" if trusted else "" + + +def _convert(value: Any, hint: Any, path: str, problems: list[str], *, trusted: bool = True, + css: str | None = None) -> Any: + """*value* as *hint*, appending each problem to *problems*. *css* is the + kind of a text setting that reaches the stylesheet (:func:`~.model.css_field`), + whose value :func:`~.model.css_problem` checks; *trusted* is False for a + style from someone else, which is held to the strict grammars and never + quoted.""" label = path or "(root)" origin = get_origin(hint) if dataclasses.is_dataclass(hint): - return _convert_dataclass(value, hint, path, problems) + return _convert_dataclass(value, hint, path, problems, trusted=trusted) if origin is Literal: allowed = get_args(hint) # By type too: True == 1 and 2.0 == 2, but neither is a level. if not any(value == option and type(value) is type(option) for option in allowed): - problems.append(f"{label}: must be one of {', '.join(map(str, allowed))} (got {value!r})") + problems.append(f"{label}: must be one of {', '.join(map(str, allowed))}{_got(value, trusted)}") return allowed[0] return value if origin in (Union, types.UnionType): options = [arg for arg in get_args(hint) if arg is not type(None)] if value is None and type(None) in get_args(hint): return None - return _convert(value, options[0], path, problems) + return _convert(value, options[0], path, problems, trusted=trusted, css=css) if origin is tuple: item_hint = get_args(hint)[0] if not isinstance(value, (list, tuple)): problems.append(f"{label}: must be a list") return () - return tuple(_convert(item, item_hint, f"{path}[{index}]", problems) for index, item in enumerate(value)) + return tuple(_convert(item, item_hint, f"{path}[{index}]", problems, trusted=trusted) + for index, item in enumerate(value)) if origin is dict: key_hint, item_hint = get_args(hint) if not isinstance(value, Mapping): @@ -363,38 +481,44 @@ def _convert(value: Any, hint: Any, path: str, problems: list[str]) -> Any: try: key = int(key) except (TypeError, ValueError): - problems.append(f"{path}.{key}: key must be a number") + problems.append(f"{path}.{key}: key must be a number" if trusted + else f"{path}: a key must be a number") continue if not 1 <= key <= 5: problems.append(f"{path}.{key}: heading levels are 1 to 5 (§4.1)") continue - converted[key] = _convert(item, item_hint, f"{path}.{key}", problems) + converted[key] = _convert(item, item_hint, f"{path}.{key}", problems, trusted=trusted) return converted if hint is bool: if not isinstance(value, bool): - problems.append(f"{label}: must be true or false (got {value!r})") + problems.append(f"{label}: must be true or false{_got(value, trusted)}") return False return value if hint is int: if isinstance(value, bool) or not isinstance(value, int): - problems.append(f"{label}: must be a whole number (got {value!r})") + problems.append(f"{label}: must be a whole number{_got(value, trusted)}") return 0 return value if hint is str: # YAML reads 700 or 1.5 as numbers; they are fine as text here. if isinstance(value, (int, float)) and not isinstance(value, bool): - return str(value) - if not isinstance(value, str): - problems.append(f"{label}: must be text (got {value!r})") + value = str(value) + elif not isinstance(value, str): + problems.append(f"{label}: must be text{_got(value, trusted)}") + return "" + if css is not None and (problem := css_problem(value, css, trusted=trusted)): + problems.append(f"{label}: {problem}") return "" return value raise TypeError(f"unsupported style field type {hint!r} at {label}") # a bug in model.py -def _convert_dataclass(value: Any, cls: type, path: str, problems: list[str]) -> Any: +def _convert_dataclass(value: Any, cls: type, path: str, problems: list[str], *, trusted: bool = True) -> Any: label = path or "(root)" if isinstance(value, cls): - return value + # Checked like data, not taken as built: a settings object handed in as + # an override would otherwise skip every check below. + value = _plain(value) if cls is model.Numbering and isinstance(value, Mapping) and isinstance(value.get("levels"), (list, tuple)) \ and len(value["levels"]) > 5: problems.append(f"{path}.levels: at most 5 formats, one per heading level (§4.1); got {len(value['levels'])}") @@ -409,9 +533,15 @@ def _convert_dataclass(value: Any, cls: type, path: str, problems: list[str]) -> where = f"{path}.{key}" if path else key if key not in known: suggestion = _closest(key, list(known)) - problems.append(f"{where}: unknown key" + (f" (did you mean '{suggestion}'?)" if suggestion else "")) + if not trusted: + # The key is the style's own text: say where, and what is known instead. + hint = f"did you mean '{suggestion}'?" if suggestion else f"known: {', '.join(known)}" + problems.append(f"{path or '(root)'}: unknown key ({hint})") + else: + problems.append(f"{where}: unknown key" + (f" (did you mean '{suggestion}'?)" if suggestion else "")) continue - values[key] = _convert(item, hints[key], where, problems) + values[key] = _convert(item, hints[key], where, problems, trusted=trusted, + css=known[key].metadata.get("css")) return cls(**values) diff --git a/src/legaldown_render/style/model.py b/src/legaldown_render/style/model.py index 971834c..43b5052 100644 --- a/src/legaldown_render/style/model.py +++ b/src/legaldown_render/style/model.py @@ -15,9 +15,11 @@ """ from __future__ import annotations +import re +import unicodedata from collections.abc import Iterator from dataclasses import dataclass, field, fields, replace -from typing import Literal +from typing import Any, Literal Counter = Literal["decimal", "lower-alpha", "upper-alpha", "lower-roman", "upper-roman"] @@ -208,12 +210,94 @@ class Labels: signature_title: str | None = None +# --------------------------------------------------------------------------- +# Values that reach the stylesheet +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class CssKind: + """What a style setting that is written into the HTML stylesheet holds. + + ``grammar`` is the strict grammar an untrusted style is held to + (docs/style-templates.md, "Untrusted styles"): plain values only, no + function, no variable, nothing a reader could mistake for more than one + value. ``expects`` says what it accepts, for the message.""" + + grammar: re.Pattern[str] + expects: str + + +_NUMBER = r"(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+)" +_LENGTH = rf"(?:0|{_NUMBER}(?:em|rem|ex|ch|pt|pc|px|mm|cm|in|%))" +_FONT_NAME = r"""(?:"[\w .\-]{1,60}"|'[\w .\-]{1,60}'|-?[A-Za-z][\w\-]{0,40}(?: [\w\-]{1,40}){0,4})""" +_PAPER = r"(?:[Aa][3-5]|[Bb][4-5]|[Ll]etter|[Ll]egal|[Ll]edger)" +_ORIENTATION = r"(?:portrait|landscape)" + +CSS_KINDS: dict[str, CssKind] = { + "length": CssKind(re.compile(_LENGTH), "a length such as 11pt, 1.2em or 0"), + "line-height": CssKind(re.compile(rf"(?:normal|{_NUMBER}|{_LENGTH})"), "normal, a number such as 1.5, or a length"), + "color": CssKind( + re.compile(r"(?:#(?:[0-9A-Fa-f]{3,4}|[0-9A-Fa-f]{6}|[0-9A-Fa-f]{8})|[A-Za-z]{3,24}" + r"|(?:rgb|rgba|hsl|hsla)\([0-9.%, /]{1,40}\))"), + "a colour: #rgb, #rrggbb, a colour name, or rgb()/hsl() with plain numbers"), + "font-weight": CssKind(re.compile(r"(?:normal|bold|bolder|lighter|[1-9][0-9]{0,2}|1000)"), + "normal, bold, or a number from 1 to 1000"), + "font-family": CssKind(re.compile(rf"{_FONT_NAME}(?:, ?{_FONT_NAME}){{0,9}}"), + "font names separated by commas, with names of more than one word quoted"), + "page-size": CssKind( + re.compile(rf"(?:auto|{_ORIENTATION}|{_PAPER}(?: {_ORIENTATION})?|{_LENGTH}(?: {_LENGTH})?)"), + "a paper size such as A4 or letter (and landscape or portrait), or one or two lengths"), + "margin": CssKind(re.compile(rf"{_LENGTH}(?: {_LENGTH}){{0,3}}"), "one to four lengths separated by spaces"), + "max-width": CssKind(re.compile(rf"(?:none|{_LENGTH})"), "none or a length such as 46rem"), + # Raw CSS appended to the stylesheet: no grammar, so an untrusted style may not set it at all. + "css": CssKind(re.compile(r"(?!)"), "empty"), +} + +#: What no value placed in the stylesheet may contain, trusted or not: it +#: would open or close a comment, end the ``@import 'x'; /* c */ \\ \x07\n\u2028}" + style = Style( + typography=Typography(font_family=hostile, font_size=hostile, line_height=hostile, color=hostile, + link_color=hostile), + headings={1: HeadingStyle(size=hostile, weight=hostile)}, + page=Page(size=hostile, margin=hostile), + html=Html(max_width=hostile), + ) + css = stylesheet(style) + generated = css.split(":root {", 1)[1] + assert "<" not in generated + assert "/*" not in generated and "*/" not in generated + declarations = [line for line in css.split("\n") if line.startswith(("body {", ".ld-document {", "@page {"))] + assert declarations + for line in declarations: + assert line.count("{") == 1 + assert line.count("}") == 1 + assert "\\" not in line + assert "@" not in line.removeprefix("@page") + + +def test_comment_delimiters_removed_from_a_value_cannot_join_into_another() -> None: + from legaldown_render.writers.html import _css + + assert "/*" not in _css("a//**/*b*/*/c") and "*/" not in _css("a//**/*b*/*/c") + assert _css("a/*b*/c") == "abc" + + +def test_the_stylesheet_keeps_a_division_sign() -> None: + from legaldown_render.style import Html, Style + from legaldown_render.writers.html import stylesheet + + assert "max-width: calc(100% / 3);" in stylesheet(Style(html=Html(max_width="calc(100% / 3)"))) diff --git a/tests/test_style_validation.py b/tests/test_style_validation.py index 405806d..5501dbe 100644 --- a/tests/test_style_validation.py +++ b/tests/test_style_validation.py @@ -337,3 +337,86 @@ def test_a_plain_label_may_contain_braced_words() -> None: def test_the_labels_that_substitute_still_check_their_fields() -> None: assert _problems({"labels.condition": "{condtion}"}) assert _problems({"labels.attachment_file": "{fil}"}) + + +# -- Values that reach the stylesheet -------------------------------------------- + +#: Every setting written into the stylesheet, as a dotted key. +CSS_SETTINGS = ["typography.font_family", "typography.font_size", "typography.line_height", "typography.color", + "typography.link_color", "headings.1.size", "headings.1.weight", "page.size", "page.margin", + "html.max_width"] +CSS_TOKENS = ["/*", "*/", " None: + """A setting added to the model with a CSS kind must join the tables here.""" + from legaldown_render.style import model + + declared = [f"{section}.{item.name}" + for section, cls in (("typography", model.Typography), ("headings.1", model.HeadingStyle), + ("page", model.Page), ("html", model.Html)) + for item in dataclasses.fields(cls) if "css" in item.metadata] + assert sorted(declared) == sorted([*CSS_SETTINGS, "html.extra_css"]) + + +@pytest.mark.parametrize("trusted", [True, False]) +@pytest.mark.parametrize("token", CSS_TOKENS) +@pytest.mark.parametrize("key", CSS_SETTINGS) +def test_a_css_value_with_a_forbidden_token_is_refused_and_named(key: str, token: str, trusted: bool) -> None: + """Rejected, not stripped: removing a token can form another (``/*`` out of ``//**``).""" + with pytest.raises(StyleError) as caught: + load_style("default", overrides={key: f"1em{token}x"}, trusted=trusted) + assert any(problem.startswith(f"{key}:") for problem in caught.value.problems) + + +@pytest.mark.parametrize("trusted", [True, False]) +@pytest.mark.parametrize("extra", ["", " None: + with pytest.raises(StyleError, match=r"html\.extra_css: .* None: + css = "a > b { color: red } /* a comment */ p::before { content: '\\201c'; }\n.x { margin: 0 }" + result = render(DOC, overrides={"html.extra_css": css}) + assert css in result.output + + +@pytest.mark.parametrize(("key", "value"), [ + ("html.max_width", "calc(100% - 2rem)"), + ("html.max_width", "min(46rem, 100vw)"), + ("typography.color", "var(--ink)"), + ("typography.font_size", "clamp(10pt, 2vw, 14pt)"), + ("typography.line_height", "calc(1em + 4px)"), + ("page.margin", "max(20mm, 5%)"), +]) +def test_a_trusted_style_may_use_css_functions_the_strict_grammar_refuses(key: str, value: str) -> None: + style = load_style("default", overrides={key: value}) + assert value in render(DOC, style=style).output + with pytest.raises(StyleError, match=key): + load_style("default", overrides={key: value}, trusted=False) + + +def test_a_style_object_is_checked_for_css() -> None: + from legaldown_render.style import Typography + + style = replace(load_style("default"), typography=Typography(font_size="12pt /* ")) + with pytest.raises(StyleError, match="typography.font_size"): + load_style(style) + with pytest.raises(StyleError, match="typography.font_size"): + render(DOC, style=style) + + +def test_a_settings_object_handed_in_as_an_override_is_checked_too() -> None: + from legaldown_render.style import Typography + + with pytest.raises(StyleError, match="typography.font_size"): + load_style("default", overrides={"typography": Typography(font_size="1em\n}")}) + + +def test_builtin_styles_and_documented_values_still_load() -> None: + for name in builtin_styles(): + load_style(name) + load_style("default", overrides={"typography": {"font_family": "Georgia, serif", "justify": True}}) diff --git a/tests/test_untrusted.py b/tests/test_untrusted.py new file mode 100644 index 0000000..c9bb755 --- /dev/null +++ b/tests/test_untrusted.py @@ -0,0 +1,434 @@ +"""Rendering documents and styles from somebody else (§2.3, docs/architecture.md +Security): ``load_style(trusted=False)`` — built-in names, mappings and YAML +text only, ``extends`` only built-ins, strict per-kind CSS grammars, no raw CSS, +a size cap and no YAML aliases, and messages that never quote the style — and +the ``untrusted`` render option and ``--untrusted`` flag, which also require a +document root. What holds for every style (the forbidden tokens) is in +test_style_validation.py.""" +from __future__ import annotations + +from pathlib import Path + +import pytest +from conftest import DOCUMENTS, FEATURES, FRONT + +from legaldown_render import OptionsError, RenderOptions, StyleError, builtin_styles, load_style, render +from legaldown_render.cli import main +from legaldown_render.style import HeadingStyle, Style, Typography, style_to_dict +from legaldown_render.style.loader import MAX_UNTRUSTED_STYLE_BYTES + +SECRET = "hunter2-correct-horse" +DOC = FRONT + "\n# One {#one}\n\nText {{ref: one}}.\n" + + +def refused(style: object = None, overrides: dict | None = None) -> StyleError: + with pytest.raises(StyleError) as caught: + load_style(style, overrides=overrides, trusted=False) # type: ignore[arg-type] + return caught.value + + +# -- Where an untrusted style may come from ------------------------------------- + +@pytest.mark.parametrize("name", builtin_styles()) +def test_builtin_styles_pass_the_strict_grammar(name: str) -> None: + assert load_style(name, trusted=False) == load_style(name) + + +def test_the_default_style_passes_as_an_object_and_as_its_own_dump() -> None: + assert load_style(Style(), trusted=False) == Style() + assert load_style(style_to_dict(Style()), trusted=False) == Style() + + +def test_no_style_at_all_is_the_default() -> None: + assert load_style(None, trusted=False) == load_style("default") + + +@pytest.mark.parametrize("spec", ["style.yaml", "style.yml", "./style.yaml", "sub/style", "sub\\style"]) +def test_a_path_is_refused_even_when_the_file_exists(spec: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + (tmp_path / "sub").mkdir() + for name in ("style.yaml", "style.yml", "sub/style"): + (tmp_path / name).write_text("version: 1\nlocale: cs-CZ\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + assert refused(spec).problems == [ + f"an untrusted style is a built-in style (built-in: {', '.join(builtin_styles())}), a mapping, or YAML " + "text, never a file"] + + +def test_a_path_object_and_an_absolute_path_are_refused(tmp_path: Path) -> None: + path = tmp_path / "house.yaml" + path.write_text("version: 1\nlocale: cs-CZ\n", encoding="utf-8") + for spec in (path, str(path), "/etc/passwd", "/etc/shadow.yaml"): + error = refused(spec) + assert "never a file" in str(error) + assert str(tmp_path) not in str(error) + assert "passwd" not in str(error) + + +def test_an_unknown_name_is_refused_without_quoting_it() -> None: + assert SECRET not in str(refused(SECRET)) + + +def test_a_mapping_and_yaml_text_are_accepted() -> None: + assert load_style({"locale": "cs-CZ"}, trusted=False).locale == "cs-CZ" + assert load_style("version: 1\nlocale: cs-CZ\nnumbering: {scheme: none}\n", trusted=False).numbering.scheme == "none" + + +def test_extends_may_name_a_builtin_style() -> None: + assert load_style({"extends": "continental"}, trusted=False).paragraphs.numbered + assert load_style("extends: outline\nlocale: cs-CZ\n", trusted=False).numbering.scheme == "legal-outline" + + +@pytest.mark.parametrize("parent", ["parent.yaml", "./parent.yaml", "../parent.yaml", "ABS", "sub/x", "nope", 3, ""]) +def test_extends_a_path_or_anything_but_a_builtin_is_refused(parent: object, tmp_path: Path, + monkeypatch: pytest.MonkeyPatch) -> None: + (tmp_path / "parent.yaml").write_text("version: 1\nlocale: cs-CZ\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + if parent == "ABS": + parent = str(tmp_path / "parent.yaml") + error = refused({"extends": parent}) + assert error.problems and all(problem.startswith("extends:") for problem in error.problems) + assert str(tmp_path) not in str(error) + + +def test_a_file_cannot_be_reached_through_an_untrusted_render(tmp_path: Path) -> None: + (tmp_path / "house.yaml").write_text("version: 1\nlocale: cs-CZ\n", encoding="utf-8") + with pytest.raises(StyleError, match="never a file"): + render(DOC, style=tmp_path / "house.yaml", untrusted=True, document_root=tmp_path) + # The operator's own render still reads it. + assert render(DOC, style=tmp_path / "house.yaml").style.locale == "cs-CZ" + + +# -- The strict grammars ---------------------------------------------------------- + +#: setting -> (values that pass, values that do not) +GRAMMARS: dict[str, tuple[list[str], list[str]]] = { + "typography.font_size": ( + ["11pt", "1.2em", "0.9rem", ".5em", "12px", "10mm", "0", "100%", "1in", "2ch"], + ["11", "-1em", "1e3em", "calc(1em)", "var(--x)", "11 pt", " 11pt", "11pt ", "11pt 12pt", "1em!important", "", + "medium", "url(x)", "1.em", "１１pt", "11pt\n"]), + "headings.1.size": (["1em", "1.15em", "14pt"], ["big", "1em 2em", "calc(1em + 1px)", "-0.5em"]), + "typography.line_height": ( + ["normal", "1.5", "2", ".9", "1.2em", "18pt", "150%", "0"], + ["1.5.5", "-1", "calc(1.5)", "var(--lh)", "1.5 2", "tall", "1,5"]), + "typography.color": ( + ["#000", "#1a1a1a", "#1a1a1a80", "#abcd", "red", "currentcolor", "rgb(10, 20, 30)", "rgba(0,0,0,.5)", + "rgb(10 20 30 / 50%)", "hsl(120, 50%, 50%)"], + ["#12", "#12345", "#ggg", "#1234567", "red blue", "rgb(", "rgb(1,2,3", "var(--ink)", "rgb(var(--x))", + "url(x)", "r", "expression(x)", "rgb(a)", "red!important", "#fff;"]), + "typography.link_color": (["#1f4e8c", "blue"], ["var(--link)", "#12"]), + "headings.1.weight": ( + ["700", "400", "normal", "bold", "bolder", "lighter", "100", "1000", "350", "1"], + ["0", "1001", "heavy", "700 800", "-700", "7.5", "var(--w)", "bold!important"]), + "typography.font_family": ( + ['Georgia, "Times New Roman", serif', "serif", "Helvetica Neue, Arial", "'Source Serif', serif", + '"Palatino Linotype",Palatino,serif', "-apple-system, sans-serif", '"Čeština Sans", serif'], + ["Georgia; color: red", "serif}", 'url("x")', "Georgia, ", ", serif", '"unterminated', "serif /* c */", + "var(--font)", "a" * 100, '"a\\"b"', "Georgia,, serif", "Times (New)", "1font", "serif,\nsans"]), + "page.size": ( + ["A4", "A3", "a5", "letter", "Legal", "A4 landscape", "landscape", "portrait", "auto", "210mm 297mm", "8.5in"], + ["A4 A3", "huge", "A4 sideways", "A4;", "calc(1mm)", "210 297", "A4 landscape portrait", "A9"]), + "page.margin": ( + ["25mm", "1in", "20mm 15mm", "0", "1em 2em 3em", "1cm 2cm 3cm 4cm", "5%"], + ["25", "1cm 2cm 3cm 4cm 5cm", "auto", "20mm,15mm", "calc(1mm)", "-1cm", "var(--m)", "20mm 15mm"]), + "html.max_width": ( + ["46rem", "100%", "800px", "none", "0", "70ch"], + ["auto", "calc(100% - 2rem)", "min(46rem, 100%)", "46", "fit-content(20em)", "46rem 2em"]), +} + + +def test_the_table_covers_every_css_setting() -> None: + from test_style_validation import CSS_SETTINGS + + assert sorted(GRAMMARS) == sorted(CSS_SETTINGS) + + +@pytest.mark.parametrize(("key", "value"), [(key, value) for key, (good, _) in GRAMMARS.items() for value in good]) +def test_a_plain_value_passes_in_both_modes(key: str, value: str) -> None: + trusted = load_style("default", overrides={key: value}) + assert load_style("default", overrides={key: value}, trusted=False) == trusted + + +@pytest.mark.parametrize(("key", "value"), [(key, value) for key, (_, bad) in GRAMMARS.items() for value in bad]) +def test_an_unusual_value_is_refused_untrusted_with_the_setting_named(key: str, value: str) -> None: + error = refused("default", {key: value}) + assert [problem.split(":")[0] for problem in error.problems] == [key] + + +@pytest.mark.parametrize(("key", "value"), [(key, value) for key, (_, bad) in GRAMMARS.items() for value in bad + if not any(token in value for token in ("/*", "*/", "\\", "\n"))]) +def test_the_same_values_load_when_trusted(key: str, value: str) -> None: + """A trusted style is only held to the forbidden tokens (or a control character).""" + if any(ord(char) < 32 or ord(char) == 127 for char in value): + pytest.skip("control characters are refused in every mode") + assert load_style("default", overrides={key: value}) + + +def test_a_number_that_yaml_reads_is_text_to_the_grammar() -> None: + assert load_style({"headings": {1: {"weight": 700}}, "typography": {"line_height": 1.5}}, + trusted=False).typography.line_height == "1.5" + assert refused({"typography": {"font_size": 11}}).problems + + +def test_every_problem_is_reported_together() -> None: + error = refused({"typography": {"font_size": "x", "color": "y y"}, "page": {"size": "z z"}}) + assert [problem.split(":")[0] for problem in error.problems] == ["typography.font_size", "typography.color", + "page.size"] + + +def test_overrides_are_held_to_the_grammar_too() -> None: + assert refused("continental", {"typography": {"font_size": "calc(1em)"}}).problems + assert load_style("continental", overrides={"typography.font_size": "12pt"}, trusted=False).typography.font_size == "12pt" + + +# -- Raw CSS ---------------------------------------------------------------------- + +def test_extra_css_must_be_empty() -> None: + assert load_style({"html": {"extra_css": ""}}, trusted=False).html.extra_css == "" + for css in ("p { color: red }", " ", "/* x */"): + error = refused({"html": {"extra_css": css}}) + assert error.problems == ["html.extra_css: is not allowed in an untrusted style (it is raw CSS); leave it empty"] + assert refused("default", {"html.extra_css": "a { }"}).problems + + +def test_extra_css_still_works_for_a_trusted_style() -> None: + assert load_style("default", overrides={"html.extra_css": "p { color: red }"}).html.extra_css + + +# -- A Style object --------------------------------------------------------------- + +def test_a_style_object_is_held_to_the_grammar() -> None: + style = Style(typography=Typography(font_size="calc(1em)")) + assert load_style(style) # trusted: accepted + error = refused(style) + assert [problem.split(":")[0] for problem in error.problems] == ["typography.font_size"] + + +def test_a_style_object_with_a_heading_or_raw_css_is_refused() -> None: + assert refused(Style(headings={1: HeadingStyle(size="2 em")})).problems[0].startswith("headings.1.size:") + style = Style() + style = Style(**{**{f: getattr(style, f) for f in style_to_dict(style)}, "html": type(style.html)(extra_css="a{}")}) + assert refused(style).problems[0].startswith("html.extra_css:") + + +def test_a_settings_object_in_an_override_is_held_to_the_grammar() -> None: + assert refused("default", {"typography": Typography(color="var(--x)")}).problems + + +# -- YAML text -------------------------------------------------------------------- + +def test_yaml_aliases_are_refused() -> None: + text = "version: 1\nlabels: &l {colon: ':'}\ntypography: *l\n" + error = refused(text) + assert error.problems == ["YAML aliases are not allowed (line 3)"] + # The shape that makes a small text enormous. + bomb = "version: 1\na: &a [x, x, x, x]\nb: &b [*a, *a, *a, *a]\nc: &c [*b, *b, *b, *b]\n" + assert "aliases" in str(refused(bomb)) + + +def test_yaml_aliases_stay_valid_for_a_trusted_file(tmp_path: Path) -> None: + path = tmp_path / "s.yaml" + path.write_text("version: 1\nlabels: &l {colon: ':'}\nplaceholders: {blank: '__'}\n", encoding="utf-8") + assert load_style(path).labels.colon == ":" + + +def _text_of(size: int, *, multibyte: bool = False) -> str: + head = "version: 1\n# " + filler = ("é" if multibyte else "x") + unit = len(filler.encode()) + return head + filler * ((size - len(head.encode()) - 1) // unit) + "\n" + + +def test_the_size_cap_counts_bytes() -> None: + at_cap = _text_of(MAX_UNTRUSTED_STYLE_BYTES) + assert len(at_cap.encode()) == MAX_UNTRUSTED_STYLE_BYTES + assert load_style(at_cap, trusted=False) == Style() + assert refused(at_cap + "x").problems == ["the style text is larger than 64 KiB"] + # Fewer characters than the cap, more bytes. + wide = _text_of(MAX_UNTRUSTED_STYLE_BYTES + 100, multibyte=True) + assert len(wide) < MAX_UNTRUSTED_STYLE_BYTES < len(wide.encode()) + assert "64 KiB" in str(refused(wide)) + + +def test_a_trusted_text_has_no_cap() -> None: + assert load_style(_text_of(MAX_UNTRUSTED_STYLE_BYTES + 1000)) == Style() + + +def test_a_deeply_nested_style_is_refused_not_a_crash() -> None: + assert "nested too deeply" in str(refused("a: " + "[" * 20000 + "\n" + "]" * 20000 + "\n")) + deep: dict = {} + node = deep + for _ in range(5000): + node["a"] = {} + node = node["a"] + assert "nested too deeply" in str(refused(deep)) + assert "nested too deeply" in str(refused({"typography": deep})) + + +def test_a_non_mapping_and_a_wrong_version_are_refused_without_the_value() -> None: + assert refused(f"- {SECRET}\n- x\n").problems == ["a style template must be a YAML mapping"] + error = refused(f"version: {SECRET}\n") + assert SECRET not in str(error) + assert error.problems[0].startswith("version:") + + +# -- Messages never quote the style ----------------------------------------------- + +@pytest.mark.parametrize("text", [ + f"version: 1\napi_key: {SECRET}\ntypography: [unclosed\n", + f"version: 1\napi_key: {SECRET}\n bad: indentation\n", + f"version: 1\napi_key: {SECRET}\ntypography: !{SECRET} x\n", + f"version: 1\napi_key: {SECRET}\ntypography: *{SECRET}\n", + f"version: 1\napi_key: {SECRET}\nlabels: {{colon: \"\\{SECRET[0]}\"}}\n", + f"version: 1\napi_key: {SECRET}\nx: !!python/object:os.system {SECRET}\n", + f"version: 1\n{SECRET}: [\n", + f"version: 1\na: &{SECRET} 1\na: *{SECRET}x\n", + f"version: 1\nx: !{SECRET}!y z\n", +]) +def test_a_yaml_error_names_the_place_and_never_quotes_the_text(text: str) -> None: + error = refused(text) + assert SECRET not in str(error) + assert "api_key" not in str(error) + assert len(error.problems) == 1 + + +def test_a_yaml_error_in_a_trusted_text_still_quotes_it() -> None: + """The operator's own style: the snippet helps them find the mistake.""" + with pytest.raises(StyleError) as caught: + load_style(f"version: 1\ntypography: [unclosed\napi_key: {SECRET}\n") + assert "api_key" in str(caught.value) + + +@pytest.mark.parametrize("style", [ + {"typography": {"font_size": SECRET}}, + {"typography": {"font_size": [SECRET]}}, + {"typography": SECRET}, + {"numbering": {"scheme": SECRET}}, + {"numbering": {"levels": SECRET}}, + {"definitions": {"style": SECRET}}, + {"contents": {"enabled": SECRET}}, + {"contents": {"depth": SECRET}}, + {"version": SECRET}, + {"extends": SECRET}, + {"html": {"extra_css": SECRET}}, + {"headings": {SECRET: {"size": "1em"}}}, + {"headings": {7: {"size": SECRET}}}, + {"labels": {"condition": SECRET}}, + {"labels": {"condition": "{" + SECRET + "}"}}, +]) +def test_a_value_of_the_style_is_never_quoted(style: dict) -> None: + assert SECRET not in str(refused(style)) + + +def test_an_override_that_cannot_hold_keys_is_not_quoted() -> None: + error = refused("default", {f"typography.font_size.{SECRET}": "1em"}) + assert SECRET not in str(error) + with pytest.raises(StyleError) as caught: + load_style("default", overrides={"typography.font_size": "1em", f"typography.font_size.{SECRET}": 1}) + assert SECRET in str(caught.value) + + +def test_the_style_object_name_is_not_quoted() -> None: + error = refused(Style(name=SECRET, typography=Typography(font_size=SECRET))) + assert SECRET not in str(error) + assert error.source == "'untrusted style'" + + +def test_an_unknown_key_is_located_not_quoted() -> None: + error = refused({"typography": {"x" * 500: 1, "font_sizee": 2, "bad\nkey\x1b[31m": 3}, SECRET: 4}) + message = str(error) + assert error.problems[0].startswith("typography: unknown key (known: font_family") + assert "typography: unknown key (did you mean 'font_size'?)" in error.problems + assert error.problems[-1].startswith("(root): unknown key") + assert "xxxxx" not in message and SECRET not in message and "\x1b" not in message + assert message.count("\n") == len(error.problems) + + +def test_an_unknown_key_of_a_trusted_style_is_named() -> None: + with pytest.raises(StyleError, match="typography.font_sizee: unknown key"): + load_style({"typography": {"font_sizee": 2}}) + + +def test_a_locale_is_named_not_quoted() -> None: + with pytest.raises(StyleError) as caught: + render(DOC, style={"locale": SECRET}, untrusted=True, document_root=".") + assert SECRET not in str(caught.value) + with pytest.raises(StyleError) as caught: + render(DOC, style={"locale": SECRET}) + assert SECRET in str(caught.value) + + +# -- The render option ------------------------------------------------------------ + +def test_untrusted_needs_a_document_root(tmp_path: Path) -> None: + with pytest.raises(OptionsError, match="document_root"): + render(DOC, untrusted=True) + with pytest.raises(OptionsError, match="document_root"): + render(DOC, RenderOptions(untrusted=True, document_path=tmp_path / "a.lgd")) + result = render(DOC, untrusted=True, document_root=tmp_path) + assert result.ok + + +def test_untrusted_loads_the_style_untrusted() -> None: + with pytest.raises(StyleError, match="typography.font_size"): + render(DOC, untrusted=True, document_root=".", overrides={"typography.font_size": "calc(1em)"}) + with pytest.raises(StyleError, match="html.extra_css"): + render(DOC, untrusted=True, document_root=".", overrides={"html.extra_css": "p{}"}) + + +def test_untrusted_takes_yaml_text_and_a_builtin_name_for_the_style() -> None: + text = "extends: continental\ntypography: {font_family: 'Georgia, serif', font_size: 12pt}\n" + result = render(DOC, style=text, untrusted=True, document_root=".") + assert "font-size: 12pt" in result.output + assert render(DOC, style="outline", untrusted=True, document_root=".").style.numbering.scheme == "legal-outline" + + +@pytest.mark.parametrize("fmt", ["html", "text"]) +def test_the_defaults_leave_existing_callers_alone(fmt: str) -> None: + assert RenderOptions().untrusted is False + css = "p { color: red }" + assert render(DOC, format=fmt, overrides={"html.extra_css": css}).ok + assert css in render(DOC, overrides={"html.extra_css": css}).output + + +def test_nothing_hostile_reaches_the_page() -> None: + page = render(DOC, untrusted=True, document_root=".").output + assert page.count("") == 1 + + +# -- The command line ------------------------------------------------------------- + +def test_cli_untrusted_without_a_root_exits_2(capsys: pytest.CaptureFixture[str]) -> None: + assert main([FEATURES, "--untrusted", "-q"]) == 2 + captured = capsys.readouterr() + assert "document_root" in captured.err + assert captured.out == "" + + +def test_cli_untrusted_with_a_root_renders(capsys: pytest.CaptureFixture[str]) -> None: + assert main([FEATURES, "--untrusted", "--root", str(DOCUMENTS), "-q", "--style", "continental"]) == 0 + assert "` included. +- **A style template is trusted input, unless it is loaded untrusted.** By default a template, a + `--set` value and an `extends` path are the operator's, and the loader still keeps a value from + breaking out of the stylesheet: + - A value placed into the stylesheet (fonts, sizes, colours, margins, `page.size`, + `html.max_width`) is refused with a `StyleError` naming the setting when it contains `/*`, `*/`, + `\@`, control characters and line breaks, + and removes comment delimiters until none is left. A lone `/` stays, since `calc(100% / 3)` + needs it. - `extends` names any path the process can read, absolute or with `../`; only cycles are - rejected. (A style file is read with `yaml.safe_load`, and as UTF-8.) - A service that renders for several parties must choose the style itself, and never accept a - style file, `--set` values, or `extends` paths from a user it does not trust. Document source - has no such power: it is never emitted as HTML or CSS (§8.7). + rejected. A style file is read with `yaml.safe_load`, as UTF-8. +- **Untrusted mode** (`load_style(..., trusted=False)`, `RenderOptions.untrusted`, `--untrusted`) + is for a service that renders what its users send. The style is a built-in name, a `Style`, a + mapping, or YAML text (a string with a line break), never a file path: nothing is read from disk + but the package's own styles. `extends` may name only a built-in style. Each setting that reaches + the stylesheet must match a strict grammar declared next to the field (`style/model.py:CSS_KINDS`: + length, line-height, colour, font-weight, font-family, page size, margin, max-width), enforced in + `_convert` for a mapping and, through `style_to_dict`, for a `Style` object and for settings + objects given as overrides. The grammars allow no function, no variable and no `!important`, so + `calc(1em + 2px)` and `var(--x)` are refused. `html.extra_css` must be empty. YAML text is at most + 64 KiB (UTF-8 bytes) and may not use aliases (the "billion laughs" shape; a nesting that exhausts + the stack is a `StyleError` too). A message names the setting and the problem, and never quotes + the style, a key, a template field or a path (a YAML error gives the line and column only), so it + can be shown to the user who sent it. `RenderOptions.untrusted` also requires a `document_root` + (§2.3: a renderer for documents it does not control SHOULD set one): without it `render` raises an + `OptionsError`, and the command exits 2. The option and the flag are one switch for both checks. + Defaults are unchanged: trusted, no root required. A service should still choose the style + itself. Document source has no power over the stylesheet either way: it is never emitted as HTML + or CSS (§8.7). ## Testing diff --git a/docs/style-templates.md b/docs/style-templates.md index 780cda3..560edf9 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -26,7 +26,7 @@ Values are layered. Each layer overrides the one before it, **field by field**: The `default` style is exactly these defaults. 2. **The `extends` chain.** A style names the style it builds on, which can be a built-in name or a path relative to the file (any path the process can read, `../` and absolute paths - included). Chains are resolved base first, and cycles are rejected, however the files are + included; an untrusted style names a built-in only, see "Untrusted mode"). Chains are resolved base first, and cycles are rejected, however the files are spelled (`../dir/a.yaml`, a symbolic link). 3. **The chosen style**: `--style NAME|PATH`, or `RenderOptions(style=...)`. 4. **Per-job overrides**: `--set numbering.scheme=mixed` or `overrides={"numbering.scheme": "mixed"}`, @@ -195,11 +195,41 @@ takes the built-in one for the document language, which always passes. ### Trusted input A style template, a `--set` value, and an `extends` path are the operator's, not the document -author's. Values placed into the stylesheet (`typography`, `headings`, `page`, `html.max_width`) -are stripped of `;{}<>` and are otherwise not checked as CSS, so a value such as `12pt (` is -emitted as written and breaks the rest of the stylesheet. `html.extra_css` is raw CSS. Do not -render with styles from users you do not trust -([Security](architecture.md#security)). +author's. A value placed into the stylesheet (`typography`, `headings`, `page`, `html.max_width`) +must not contain `/*`, `*/`, ` Date: Tue, 27 Oct 2026 23:48:16 +0000 Subject: [PATCH 054/118] Ignore a newline at the end of a file reference The control-character check ran on the path before what a browser strips at either end was removed, so a YAML block or folded scalar (which ends in a newline), as in 'file: >' for an attachment, was a false path-not-relative. The path is now read as before (tabs and newlines ignored, control characters and spaces at either end stripped) and only a control character left inside it (NUL, ESC, ...) is path-not-relative. Tests: block and folded scalars give no finding; interior NUL and ESC, and '..%00/x.png' as an image, are still refused. --- CONFORMANCE.md | 5 +++-- src/legaldown_render/urls.py | 17 ++++++++--------- tests/test_file_references.py | 10 ++++++++++ 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 01c189a..8fe4552 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -101,8 +101,9 @@ It is checked against the specification's own examples and fixtures corpus (`tes (the renderer's reading; a leading `\` is not relative anyway); `..` followed by dots and spaces (`.. `, `...`) is `..`, and `.` followed by spaces is `.` (Windows reads them so); tabs and newlines inside a scheme, and spaces and control characters at either end, are ignored - as a browser ignores them. A path with a control character (NUL, a newline, any other of - Unicode category Cc) in it, after the decoding below, is `path-not-relative`. Percent-encoding is + as a browser ignores them. A control character left inside a path (NUL, ESC, any other of Unicode + category Cc), after the decoding below and what is ignored above, makes it `path-not-relative`; + a newline at the end of a YAML block scalar is ignored. Percent-encoding is decoded once, only for what is used as a URL, an image's source and the link `{{attach:}}` would make to an attachment's file: there `%2e%2e/` is `../`. A path in the frontmatter or an `{{include:}}` is a file name, read as written, where `%2e%2e` is a directory with that name. diff --git a/src/legaldown_render/urls.py b/src/legaldown_render/urls.py index 4a17f5f..c433a65 100644 --- a/src/legaldown_render/urls.py +++ b/src/legaldown_render/urls.py @@ -102,7 +102,7 @@ def file_reference_problem(path: str, *, base: str | None = None, url: bool = Fa (``https://``, ``file://``, ``data:``), which includes a Windows drive path (``C:\\x``, ``C:/x``, a scheme as far as a parser can tell), or starts with ``/`` or ``\\`` (so also ``//host``), or has a control - character (NUL, a newline, ...); + character inside it (NUL, ESC, ...); * ``"outside-root"`` (``path-outside-root``): there is a document root (*base* is not None) and the path leaves it. @@ -117,19 +117,18 @@ def file_reference_problem(path: str, *, base: str | None = None, url: bool = Fa so that a path that goes up on Windows is not missed; it matters only with a root, a leading one is already not relative), and so are ``.. ``, ``...`` and ``. `` with trailing dots and spaces, which Windows reads as ``..`` and - ``.``. A path with a control character (NUL, a newline, ...) is not a file - name a renderer can name safely, and is ``not-relative``. What a - browser ignores before it reads the scheme (tabs and newlines, control - characters and spaces at either end) is ignored in finding one. *url* says the path is consumed as a URL - (an image's source, a link's target), where percent-encoding counts as the + ``.``. What a browser ignores (tabs and newlines anywhere, control + characters and spaces at either end: a YAML block scalar ends in a newline) + is ignored. A control character left inside the path (NUL, ESC, ...) is not + a file name a renderer can name safely, and is ``not-relative``. *url* + says the path is consumed as a URL (an image's source, a link's target), where percent-encoding counts as the character it encodes (``%2e%2e`` is ``..``, ``%2f`` a separator), once; a path from frontmatter or an ``{{include:}}`` is a file name and is read as written, in which ``%2e%2e`` is a directory with that name.""" - decoded = unquote(path) if url else path - read = _as_read(decoded) + read = _as_read(unquote(path) if url else path) if not read: return None - if _SCHEME_RE.match(read) or read[0] in "/\\" or any(unicodedata.category(c) == "Cc" for c in decoded): + if _SCHEME_RE.match(read) or read[0] in "/\\" or any(unicodedata.category(c) == "Cc" for c in read): return "not-relative" if base is None: return None diff --git a/tests/test_file_references.py b/tests/test_file_references.py index 7f95258..86f49e0 100644 --- a/tests/test_file_references.py +++ b/tests/test_file_references.py @@ -86,6 +86,10 @@ ("..%00/x.png", ".", False, None), ("a\x1fb/x", ".", False, "not-relative"), ("a\x7fb/x", ".", False, "not-relative"), + ("a\x1bb/x", ".", False, "not-relative"), + # What a browser strips at either end is not inside the path. + ("a.lgd\n", ".", False, None), + ("\x00a.lgd\x00", ".", False, None), ("a\x85b/x", ".", False, "not-relative"), ]) def test_the_check(path: str, base: str | None, url: bool, expected: str | None) -> None: @@ -207,6 +211,12 @@ def test_a_path_with_a_control_character_is_not_relative(kind: str) -> None: assert [rule for rule, _ in findings(render(source))] == [NOT_RELATIVE] +@pytest.mark.parametrize("scalar", [">", "|", ">-", "|+"]) +def test_a_block_scalar_file_is_not_a_control_character(scalar: str) -> None: + source = _frontmatter(f"attachments:\n - id: annex\n title: Annex\n file: {scalar}\n annex.pdf\n") + assert findings(render(source)) == [] + + @pytest.mark.parametrize("path", [".. ./x.png", "... /x.png", "..../x.png"]) def test_a_parent_directory_as_windows_reads_it_leaves_the_root(path: str, tmp_path: Path) -> None: source = _frontmatter("").replace("Body.", f"![x](<{path}>)") From 05f815c65b0013d6ff14c4336df4ef7e8ca7cf78 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:49:24 +0000 Subject: [PATCH 055/118] Bound the nesting depth of a style, so reading one costs time linear in its size PyYAML's scanner takes time quadratic in the nesting depth of a flow collection: a 40 KB text of nested brackets took 45 seconds to refuse in untrusted mode. Styles are now read with a SafeLoader that refuses more than 32 levels (flow and block) as the scanner reaches them, trusted or not; a mapping or Style passed in is held to the same depth, and RecursionError stays only as a last resort. The nesting test counts the scanner's work instead of timing it, which also brings the suite back from 110 s to 60 s. --- docs/architecture.md | 5 +- docs/style-templates.md | 2 +- src/legaldown_render/style/loader.py | 59 +++++++++++++++++++---- tests/test_untrusted.py | 70 ++++++++++++++++++++++++++-- 4 files changed, 118 insertions(+), 18 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index aec82c1..e51af26 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -385,8 +385,9 @@ out when the validator reported the same path at the same line, or at none (`dia `_convert` for a mapping and, through `style_to_dict`, for a `Style` object and for settings objects given as overrides. The grammars allow no function, no variable and no `!important`, so `calc(1em + 2px)` and `var(--x)` are refused. `html.extra_css` must be empty. YAML text is at most - 64 KiB (UTF-8 bytes) and may not use aliases (the "billion laughs" shape; a nesting that exhausts - the stack is a `StyleError` too). A message names the setting and the problem, and never quotes + 64 KiB (UTF-8 bytes) and may not use aliases (the "billion laughs" shape). Every style, trusted or + not, may nest 32 levels at most, in YAML text (refused by the scanner, `style/loader.py:_StyleLoader`, + before anything is built, as PyYAML's cost is quadratic in the depth) and in a mapping. A message names the setting and the problem, and never quotes the style, a key, a template field or a path (a YAML error gives the line and column only), so it can be shown to the user who sent it. `RenderOptions.untrusted` also requires a `document_root` (§2.3: a renderer for documents it does not control SHOULD set one): without it `render` raises an diff --git a/docs/style-templates.md b/docs/style-templates.md index 407493a..c2e049a 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -226,7 +226,7 @@ with `RenderOptions(untrusted=True, document_root=...)` (`legaldown-render --unt Functions (`calc()`, `var()`, `min()`), `!important` and anything else are refused. The same grammar holds for a `Style` object and for overrides. - `html.extra_css` must be empty. -- YAML text is at most 64 KiB and may not use aliases. +- YAML text is at most 64 KiB and may not use aliases. (A style, trusted or not, nests 32 levels at most.) - An error names the setting and the problem and never quotes the style, a key or a path, so it can be shown to whoever sent the style. - The built-in styles and the examples in this document all pass. diff --git a/src/legaldown_render/style/loader.py b/src/legaldown_render/style/loader.py index 5e08b81..2dc5a42 100644 --- a/src/legaldown_render/style/loader.py +++ b/src/legaldown_render/style/loader.py @@ -41,11 +41,44 @@ #: The longest YAML text an untrusted style may be (bytes of UTF-8). MAX_UNTRUSTED_STYLE_BYTES = 64 * 1024 +#: How deep a style may nest, in YAML text and in a mapping. A style nests five +#: levels at most (``headings.1.size``); the limit is what keeps the cost of +#: reading one linear in its size, since PyYAML's scanner is quadratic in the +#: nesting depth of what it reads. +MAX_STYLE_DEPTH = 32 + #: How an untrusted style is named in a message, instead of its own name (which #: is source text) or a path. _UNTRUSTED_SOURCE = "'untrusted style'" +class _StyleLoader(yaml.SafeLoader): + """``yaml.SafeLoader`` that refuses a text nested more than + :data:`MAX_STYLE_DEPTH` levels, as the scanner reads it: before any + structure is built, and before the quadratic cost of a deep one is paid.""" + + def _check_depth(self) -> None: + if self.flow_level + len(self.indents) >= MAX_STYLE_DEPTH: + raise yaml.scanner.ScannerError(None, None, "the text is nested too deeply", self.get_mark()) + + def fetch_flow_collection_start(self, TokenClass: Any) -> None: # noqa: N803 + self._check_depth() + super().fetch_flow_collection_start(TokenClass) + + def add_indent(self, column: int) -> bool: + if self.indent < column: + self._check_depth() + return super().add_indent(column) + + +def _load_yaml(text: str) -> Any: + return yaml.load(text, Loader=_StyleLoader) # noqa: S506 - a SafeLoader + + +class _TooDeep(Exception): + """A mapping nested more than :data:`MAX_STYLE_DEPTH` levels.""" + + def builtin_styles() -> list[str]: """Names of the style templates shipped with the package.""" folder = resources.files(__package__).joinpath("builtin") @@ -81,10 +114,9 @@ def load_style( """ try: return _load(style, overrides, trusted) - except RecursionError: - if trusted: - raise - raise StyleError(_UNTRUSTED_SOURCE, ["the style is nested too deeply"]) from None + except (_TooDeep, RecursionError): # the second only as a last resort + raise StyleError(_UNTRUSTED_SOURCE if not trusted else "'(style)'", + ["the style is nested too deeply"]) from None def _load(style: Any, overrides: Mapping[str, Any] | None, trusted: bool) -> Style: @@ -216,7 +248,7 @@ def _parse(text: str, source: str, trusted: bool = True) -> dict: if not trusted: _check_untrusted_text(text) try: - raw = yaml.safe_load(text) + raw = _load_yaml(text) except yaml.YAMLError as error: # PyYAML's message quotes the source around the problem, which for a # style from elsewhere would show a reader whatever the text holds. @@ -243,7 +275,7 @@ def _check_untrusted_text(text: str) -> None: if len(text.encode("utf-8", "replace")) > MAX_UNTRUSTED_STYLE_BYTES: raise StyleError(_UNTRUSTED_SOURCE, [f"the style text is larger than {MAX_UNTRUSTED_STYLE_BYTES // 1024} KiB"]) try: - for event in yaml.parse(text, Loader=yaml.SafeLoader): + for event in yaml.parse(text, Loader=_StyleLoader): if isinstance(event, yaml.AliasEvent): raise StyleError(_UNTRUSTED_SOURCE, [f"YAML aliases are not allowed (line {event.start_mark.line + 1})"]) except yaml.YAMLError: @@ -315,10 +347,17 @@ def _refuse_meta_overrides(expanded: Mapping) -> None: raise StyleError("overrides", problems) -def _copy(value: Any) -> Any: - """A deep copy of *value* with every mapping made a plain dict.""" +def _copy(value: Any, depth: int = 0) -> Any: + """A deep copy of *value* with every mapping made a plain dict. A value + nested more than :data:`MAX_STYLE_DEPTH` levels raises :class:`_TooDeep`.""" + if depth > MAX_STYLE_DEPTH: + raise _TooDeep if isinstance(value, Mapping): - return {str(key): _copy(item) for key, item in value.items()} + return {str(key): _copy(item, depth + 1) for key, item in value.items()} + if isinstance(value, list): + return [_copy(item, depth + 1) for item in value] + if isinstance(value, tuple): + return tuple(_copy(item, depth + 1) for item in value) return copy.deepcopy(value) @@ -345,7 +384,7 @@ def parse_override(key: str, text: str) -> Any: return stripped[1:-1] return text try: - return yaml.safe_load(text) if text.strip() else "" + return _load_yaml(text) if text.strip() else "" except yaml.YAMLError as error: raise StyleError("overrides", [f"{key}: the value is not valid YAML: {_yaml_problem(error)}"]) from None diff --git a/tests/test_untrusted.py b/tests/test_untrusted.py index c9bb755..75b5c24 100644 --- a/tests/test_untrusted.py +++ b/tests/test_untrusted.py @@ -10,12 +10,13 @@ from pathlib import Path import pytest +import yaml from conftest import DOCUMENTS, FEATURES, FRONT from legaldown_render import OptionsError, RenderOptions, StyleError, builtin_styles, load_style, render from legaldown_render.cli import main from legaldown_render.style import HeadingStyle, Style, Typography, style_to_dict -from legaldown_render.style.loader import MAX_UNTRUSTED_STYLE_BYTES +from legaldown_render.style.loader import MAX_STYLE_DEPTH, MAX_UNTRUSTED_STYLE_BYTES SECRET = "hunter2-correct-horse" DOC = FRONT + "\n# One {#one}\n\nText {{ref: one}}.\n" @@ -253,15 +254,74 @@ def test_a_trusted_text_has_no_cap() -> None: assert load_style(_text_of(MAX_UNTRUSTED_STYLE_BYTES + 1000)) == Style() -def test_a_deeply_nested_style_is_refused_not_a_crash() -> None: - assert "nested too deeply" in str(refused("a: " + "[" * 20000 + "\n" + "]" * 20000 + "\n")) +def _flow(depth: int) -> str: + return "a: " + "[" * depth + "\n" + "]" * depth + "\n" + + +def _block(depth: int) -> str: + return "".join(" " * level + "a:\n" for level in range(depth)) + " " * depth + "b: 1\n" + + +@pytest.mark.parametrize("trusted", [True, False]) +def test_the_nesting_limit_is_exact(trusted: bool) -> None: + """A style is read to a fixed depth, in flow and in block form: the limit + that keeps the scanner's cost linear in the size of the text.""" + limit = MAX_STYLE_DEPTH + for make in (_flow, _block): + assert "nested too deeply" in str(_loaded(make(limit + 1), trusted)), make + assert "nested too deeply" not in str(_loaded(make(limit - 1), trusted)), make + mixed = "a: " + "[" * (limit - 1) + "]" * (limit - 1) + "\n" + assert "nested too deeply" not in str(_loaded(mixed, trusted)) + + +def _loaded(text: str, trusted: bool) -> object: + try: + load_style(text, trusted=trusted) + except StyleError as error: + return error + return "" + + +def test_a_deeply_nested_text_is_refused_after_a_bounded_amount_of_work(monkeypatch: pytest.MonkeyPatch) -> None: + """Counted, not timed: the scanner opens at most MAX_STYLE_DEPTH + 1 levels + however deep the text goes, so the cost is linear in its size (PyYAML's + is quadratic in the depth it is allowed to reach).""" + from legaldown_render.style import loader + + opened = [] + for name in ("fetch_flow_collection_start", "add_indent"): + original = getattr(yaml.SafeLoader, name) + monkeypatch.setattr(yaml.SafeLoader, name, lambda self, *args, _o=original: opened.append(1) or _o(self, *args)) + for text, trusted in ((_flow(20000), False), (_block(300), False), ("- " * 30000 + "x\n", False), + (_flow(200000), True)): # a style file has no size cap + opened.clear() + error = _loaded(text, trusted) + assert "the text is nested too deeply" in str(error) + assert len(opened) <= 2 * (loader.MAX_STYLE_DEPTH + 1) + + +def test_a_deeply_nested_mapping_is_refused_not_a_crash() -> None: deep: dict = {} node = deep for _ in range(5000): node["a"] = {} node = node["a"] - assert "nested too deeply" in str(refused(deep)) - assert "nested too deeply" in str(refused({"typography": deep})) + for trusted in (True, False): + assert "nested too deeply" in str(_loaded_mapping(deep, trusted)) + assert "nested too deeply" in str(_loaded_mapping({"typography": deep}, trusted)) + nested: list = [] + for _ in range(5000): + nested = [nested] + assert "nested too deeply" in str(_loaded_mapping({"enumeration": {"levels": nested}}, False)) + assert "nested too deeply" in str(refused("default", {"enumeration": {"levels": nested}})) + + +def _loaded_mapping(style: dict, trusted: bool) -> object: + try: + load_style(style, trusted=trusted) + except StyleError as error: + return error + return "" def test_a_non_mapping_and_a_wrong_version_are_refused_without_the_value() -> None: From 8b69281cebb13c96f24374b53210231fd432f023 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:58:02 +0000 Subject: [PATCH 056/118] Report a heading read from a term it uses, count term nesting from the outermost term, keep the item piece of a cut reference A heading read from the text of a term that the heading itself uses was memoised with the term cut short in it, so a reference to it showed a text that differed from the heading with no Warning. Its text is not kept now, it is worked out again where the term is not open, and render-designation-cycle reports it. The nesting limit counted the levels the outer block already had, so a term nested about 58 levels in a block, whose definition used another term, showed the inner id with render-term-limit. It now counts from where the outermost open term began (or from the heading, while designations are worked out). A reference to an item or paragraph under a heading longer than 1,000 characters cuts the heading part and keeps the item piece, and says in its own message that the reference shows the first 1,000 characters. The validator's text of a term is searched for a reference only for a term whose definition is not in the body. The docs state when designations are worked out, that a ring of more than 32 headings is reported as a limit, that a heading with 59 or more nested levels can reach the nesting limit after one hop, and that a frontmatter value shown in a heading is not designated through. Tests: the repro, the nesting at 58 and 98 levels in both schemes, the cut reference, and every heading that reads differently from its designation being reported, over random tangles of headings, items and terms. --- CONFORMANCE.md | 23 ++++-- docs/architecture.md | 27 ++++--- src/legaldown_render/resolve/resolver.py | 94 ++++++++++++++++++------ tests/test_none_designations.py | 77 +++++++++++++++++++ 4 files changed, 181 insertions(+), 40 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 13e384e..a50cf4b 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -166,12 +166,14 @@ It is checked against the specification's own examples and fixtures corpus (`tes also in a heading read before the targets are known (§13.3). Terms that name each other can still ask for more different texts than any cache holds, so the extra work is bounded: a term's first text is free, and past 200,000 inline nodes resolved inside later texts of the same term (nested ones - included), 32 terms open inside one another, or 60 inline levels (emphasis, links, terms) nested - inside an open term, a `{{term:}}` shows its `label=` or definition id and the Warning - `render-term-limit` is reported, once. The nesting limit only applies where one unit is read - inside another (inside an open term, or while designations are worked out, below): a term in a - single block, however deeply formatted (the builder allows 100 levels), is never cut by it. Any number of terms that do not name each - other resolve in full and never reach the limit. A term text is capped at 1,000 characters + included), 32 terms open inside one another, or 60 inline levels (emphasis, links, terms) added + inside the outermost open term, a `{{term:}}` shows its `label=` or definition id and the Warning + `render-term-limit` is reported, once. The nesting limit counts from where the outermost open term + began, so the levels a block already has there (the builder allows 100 in a block) are not counted: a + term in a single block, however deeply formatted, and the first term in it, are never cut by it, + and a term inside it is cut only when its own text nests 60 or more levels further. While + designations are worked out (below) the count is from the heading. Any number of terms that do not + name each other resolve in full and never reach the limit. A term text is capped at 1,000 characters of plain text, checked as it is built (no real defined term comes near it; a chain of terms that each read the next twice would otherwise ask for a text of 2^n words): a longer one is shown by its definition id in `{{term:}}` (a `label=` still wins), and the Warning `render-term-too-long` names the @@ -195,8 +197,13 @@ It is checked against the specification's own examples and fixtures corpus (`tes inline levels while headings and terms are read through each other, or longer than 1,000 characters once resolved are designated by their text as written, without the references in them (`render-designation-limit`), and a reference shows at most the first 1,000 characters of a - designation. A heading in a cycle or past a cap, or one whose terms reach into one, may read - differently from the designation references show; the Warning says so. Alternative sections + designation (an item or paragraph keeps its own piece after it), which `render-designation-limit` + also says. A ring of more than 32 headings is reported as a limit, not a cycle, and a heading with 59 + or more nested emphasis levels that holds a reference can reach the nesting limit after one hop. A + heading read from the text of a term it uses is a cycle through that term. A heading in a cycle or + past a cap, or one whose terms reach into one, may read differently from the designation + references show; the Warning says so. A heading that shows a frontmatter value (`{{party:}}`, + `{{attach:}}`) whose text holds a `{{ref:}}` is not designated through it. Alternative sections sharing an identifier (§15.4) keep their own heading text, and a reference to the identifier shows the first alternative's. - **Labels under the `none` scheme.** A style label that shows `{section}` (such as the default diff --git a/docs/architecture.md b/docs/architecture.md index 8dd6d53..292078d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -178,8 +178,11 @@ resolved (a reference shows `references.format` applied to its target's designat `# About {{ref: ov}}` is "About Overview"), a broken reference shows its marker. An item's or paragraph's designation is its section's with the item pieces after it ("Termination (a)"). A heading can refer to a heading, item or paragraph anywhere, so designations are worked out in -four steps, only when a heading, or a term a heading uses, holds a `{{ref:}}` (otherwise the first -step's text is final): +four steps. They run only when the scheme is `none` and a section heading, or the text of a term a +heading uses (directly or through other terms), holds a `{{ref:}}`; for a term whose definition is +not in the body, the validator's own text of it is searched for `{{`. Otherwise the first step's text +is final and nothing more is done. (A value of the frontmatter that a heading shows, with +`{{party:}}` or `{{attach:}}`, is not searched: a `{{ref:}}` in it is not designated through.) 1. Every section, item and paragraph is numbered on a **provisional** designation: the heading's text-only reading (`Resolver._title_text`, which leaves references out). Each target records @@ -198,14 +201,18 @@ step's text is final): A heading that reads itself again (directly, through other headings or terms, or through its own item) reads as its provisional text where the circle closes, with the Warning -`render-designation-cycle`. A walk through more than 32 headings, a designation longer than 1,000 -characters once its references are resolved (a chain of headings that each cite the next twice would -ask for 2^n), or inline levels nested past 60 while headings and terms are read through each other, +`render-designation-cycle`. A heading read from the text of a term it uses is such a circle, and what +it reads there is not kept (it is worked out again where the term is not open). A walk through more +than 32 headings, a designation longer than 1,000 characters once its references are resolved (a +chain of headings that each cite the next twice would ask for 2^n), or inline levels nested past 60 +(counted from where the outermost open term began, or from the heading where the walk started: so a +heading with 59 or more nested emphasis levels that holds a reference can reach it after one hop) also give the heading's provisional text, with `render-designation-limit` (so does a term the walk -cut by the term limits); a `{{ref:}}` never shows more than the first 1,000 characters of a -designation. So a heading in a cycle or past a cap, or one whose terms reach into one, may read -differently from the designation references show; the Warning says so. Every other heading reads -exactly as its designation. +cut by the term limits). A ring of more than 32 headings is therefore reported as a limit, not as a +cycle. A `{{ref:}}` shows at most the first 1,000 characters of a heading (the item or paragraph +piece after it is kept), and `render-designation-limit` says so for that heading. So a heading in a +cycle or past a cap, or one whose terms reach into one, may read differently from the designation +references show; the Warning says so. Every other heading reads exactly as its designation. ### 5. Write (`writers/`) @@ -275,7 +282,7 @@ Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can ev `ref-not-enumerated`, `path-not-relative` and `path-outside-root` (Errors for a file reference that is not relative, or that leaves the document root, §2.3; reported here because validator 0.4 checks neither), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters under the `none` scheme, or a reference to a heading longer than that), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's diagnostics carry their line (§16.9), and so do the renderer's own, and the CLI prints it as `file:line:`. diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 54f58d9..fd406b5 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -170,11 +170,14 @@ #: which is where the Python stack can run out: the builder allows 100 levels in #: one block (``build.MAX_NESTING``), but terms read inside terms, and headings #: read through the references in other headings, stack one block's levels on -#: another's. A ``{{term:}}`` met inside an open term, or while designations are -#: worked out, that is nested this deep shows its label or id -#: (``render-term-limit``); a heading this deep stops reading others -#: (``render-designation-limit``). A single block's nesting, up to the -#: builder's limit, never reaches it. +#: another's. Counted from where the outermost open term began (or from the +#: start of the walk that works out designations, which has no block around +#: it): the levels a block already has where its first term opens are its own. +#: A ``{{term:}}`` met inside an open term, or while designations are worked +#: out, that is nested this deep shows its label or id (``render-term-limit``); +#: a heading this deep stops reading others (``render-designation-limit``). +#: The first term of a block opens however deep the block is nested, up to the +#: builder's limit. _MAX_NEST = 60 #: Under the ``none`` scheme (§13.3) a section is designated by its heading with @@ -191,8 +194,7 @@ f"Headings and the terms in them refer to each other too deeply or too often (more than " f"{_MAX_DESIGNATION_DEPTH} headings, or terms or formatting nested too far), or a heading reads longer than " f"{_MAX_DESIGNATION_TEXT} characters once its references are resolved; under the none scheme such a heading is " - f"designated by its text as written, without the references in it, and a reference shows at most its first " - f"{_MAX_DESIGNATION_TEXT} characters." + f"designated by its text as written, without the references in it." ) #: What a reference can be resolved from: before the targets are known, only the @@ -295,6 +297,8 @@ def __init__( self._stage = _EARLY # Inline levels being resolved inside one another (see _MAX_NEST). self._nest = 0 + # The level at which the outermost open term began (see _while_open). + self._nest_base = 0 # Under the none scheme, while designations are worked out (see # _designate): each section's designation (None until known), its # text-only reading, its title, the sections open inside one another, @@ -307,6 +311,8 @@ def __init__( self._cycles: list[int] = [] self._limited = False self._limited_at: int | None = None + # The designation of each section, once numbered (for a long one, see _cut_text). + self._section_texts: list[str] = [] # The section being structured, for the targets it holds. self._section_index: int | None = None self.attachment_anchors: dict[str, str | None] = {} @@ -332,6 +338,7 @@ def run(self) -> RenderTree: self.used_anchors = anchors self.targets, self.designation_slots, self.section_presences = {}, {}, [] sections, preamble = self._structure_all(designations) + self._section_texts = [section.designation or "" for section in sections] self._stage = _FINAL resolve = self._resolve_inlines # In document order, so that the first of several definitions of a @@ -392,8 +399,10 @@ def _reads_references(self, sections: tuple[Section, ...]) -> bool: """True if a section's designation may depend on a ``{{ref:}}``: one stands in a heading or in the text of a term the headings use, directly or through other terms, or in the validator's own text of - such a term (read where the definition is not in the body). Without - one, the text-only reading of a heading is its designation.""" + such a term where its definition is not in the body (that text is + what the term shows then). Without one, the text-only reading of a + heading is its designation. A value of the frontmatter a heading + shows (``{{party:}}``, ``{{attach:}}``) is not read for one.""" lookup = self.result.index.definition_lookup seen: set[str] = set() pending = [section.title for section in sections] @@ -409,7 +418,7 @@ def _reads_references(self, sections: tuple[Section, ...]) -> bool: source = self.term_sources.get(directive.positional) if source is not None: pending.append(source.children) - if "{{" in (lookup.get(directive.positional) or ""): + elif "{{" in (lookup.get(directive.positional) or ""): return True return False @@ -463,7 +472,11 @@ def _designation(self, index: int) -> str: (``render-designation-limit``), which is worked out when it is next reached by a shorter way, or in its turn. Neither is kept. A designation that reads longer than :data:`_MAX_DESIGNATION_TEXT` characters once its - references are resolved is its text-only reading too, which is kept.""" + references are resolved is its text-only reading too, which is kept. A + heading read from the text of a term that it uses itself is a circle + through that term: what it reads here is not kept (it is worked out + again where that term is not open), and ``render-designation-cycle`` + reports it.""" memo = self._memo[index] if memo is not None: return memo @@ -471,17 +484,28 @@ def _designation(self, index: int) -> str: if index in self._stack: self._cycles.append(index) return provisional - if len(self._stack) >= _MAX_DESIGNATION_DEPTH or self._nest >= _MAX_NEST: + if len(self._stack) >= _MAX_DESIGNATION_DEPTH or self._nest - self._nest_base >= _MAX_NEST: self._limit_hit() return provisional + # A term open around this heading (the heading is read from that term's + # text) is cut short wherever the heading uses it, so what the heading + # reads depends on where it was asked for (see _while_open). + around = set(self._terms_open) + outer_cut, self._cut = self._cut, set() self._stack.append(index) try: text = self._title_text(self._titles[index]) finally: self._stack.pop() + cut, self._cut = self._cut, outer_cut | (self._cut & around) if len(text) > _MAX_DESIGNATION_TEXT and text != provisional: self._limit_hit(index) text = provisional + if cut & around: + # Not kept: it is worked out afresh where no such term is open. The + # heading reads through itself by way of the term, a circle. + self._cycles.append(index) + return text self._memo[index] = text return text @@ -497,7 +521,11 @@ def _designation_of(self, target: _Target) -> str: out: the designation of its section, with its item or paragraph pieces.""" if target.section_index is None: return target.designation - return _compose(target.parts, self._designation(target.section_index)) + base = self._designation(target.section_index) + designation = _compose(target.parts, base) + if len(designation) > _MAX_DESIGNATION_TEXT: + designation = _compose(target.parts, self._cut_text(base)) + return designation def _resolve_section(self, section: Section) -> Section: with self._at(section.line): @@ -928,15 +956,31 @@ def _ref(self, directive: Directive, target_id: str) -> Inline: ) elif target.slot is not None and (clashes := self._clashes(target.slot, target.designation)): self._warn("render-ref-ambiguous", self._ambiguity_message(target_id, target, clashes)) - return CrossRef(self._reference_text(target.designation), target.anchor or "") + designation = target.designation + if len(designation) > _MAX_DESIGNATION_TEXT: + if self.textual and target.section_index is not None: + designation = _compose(target.parts, self._cut_text(self._section_texts[target.section_index])) + else: + designation = self._cut_text(designation) + return CrossRef(self._reference_text(designation), target.anchor or "") + + def _cut_text(self, text: str) -> str: + """*text*, a heading's designation, as a reference shows it: at most + its first :data:`_MAX_DESIGNATION_TEXT` characters, so that a heading + that is long as written cannot be pasted unbounded by thousands of + references. An item or paragraph keeps its own piece after it.""" + if len(text) <= _MAX_DESIGNATION_TEXT: + return text + self._warn( + "render-designation-limit", + f"A reference to the heading '{printable(text[:40])}...' shows only its first " + f"{_MAX_DESIGNATION_TEXT} characters: under the none scheme it refers to the heading's text, " + f"which is longer.", + ) + return text[:_MAX_DESIGNATION_TEXT].rstrip() + "\u2026" def _reference_text(self, designation: str) -> str: - """What a reference shows for *designation*: at most its first - :data:`_MAX_DESIGNATION_TEXT` characters, so that a heading that is long - as written cannot be pasted unbounded by thousands of references.""" - if len(designation) > _MAX_DESIGNATION_TEXT: - self._warn("render-designation-limit", _DESIGNATION_LIMIT) - designation = designation[:_MAX_DESIGNATION_TEXT].rstrip() + "\u2026" + """What a reference shows for *designation*, formatted as the style asks.""" return self.style.references.format.replace("{designation}", designation) def _record(self, designation: str, slot: _Slot) -> None: @@ -1077,7 +1121,8 @@ def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, wor resolved inside a repeat, nested ones included, are counted; first texts are free. Past :data:`_MAX_TERM_WORK` such nodes, with :data:`_MAX_TERM_DEPTH` terms open at once, or inside an open term (or - while designations are worked out) at :data:`_MAX_NEST` levels, the + while designations are worked out) at :data:`_MAX_NEST` levels (counted + from the outermost open term), the result is ``None`` (the reference shows its label or id), and the Warning ``render-term-limit`` says so, once. While designations are worked out nothing is reported there: the walk notes it @@ -1091,7 +1136,7 @@ def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, wor self._cut |= kept.cut return kept.text repeat = bool(cache.get(key)) - too_nested = self._nest >= _MAX_NEST and (self._terms_open or self._stage == _DESIGNATING) + too_nested = self._nest - self._nest_base >= _MAX_NEST and (self._terms_open or self._stage == _DESIGNATING) if len(self._terms_open) >= _MAX_TERM_DEPTH or too_nested or (repeat and self._term_work >= _MAX_TERM_WORK): # Terms that refer to each other this deeply or this often are # not resolved further: like a cut, the reference shows its label @@ -1106,6 +1151,10 @@ def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, wor ) return None self._repeating += repeat + outermost = not self._terms_open + outer_base = self._nest_base + if outermost: + self._nest_base = self._nest self._terms_open.append(definition_id) outer = self._seen, self._cut self._seen, self._cut = set(), set() @@ -1117,6 +1166,7 @@ def _while_open(self, cache: dict[str, list[_TermText]], definition_id: str, wor finally: self._repeating -= repeat self._terms_open.pop() + self._nest_base = outer_base self._seen = outer[0] | self._seen self._cut = outer[1] | (self._cut & around) return text diff --git a/tests/test_none_designations.py b/tests/test_none_designations.py index a4c68d8..02edc63 100644 --- a/tests/test_none_designations.py +++ b/tests/test_none_designations.py @@ -428,3 +428,80 @@ def test_file_reference_findings_are_reported_once_whatever_designates_them(tmp_ ("path-outside-root", _line(source, "# A ")), ("path-outside-root", _line(source, "# A "))]) assert [d.line for d in result.diagnostics if d.rule == "render-not-processed"] == [_line(source, "# A ")] + + +def test_a_heading_read_from_a_term_it_uses_is_a_circle_and_is_reported() -> None: + # A is read from the text of t, which A uses, so t is cut short in it: what A reads there + # depends on where it was asked for, and is not kept. + body = ('# Intro {{term: t}} {#i}\n\n# A {{term: t}} {#a}\n\nSee {{ref: a}}.\n\n"x {{ref: a}}" {{def: t}}\n') + source = FRONT + body + result = render(source, format="text", overrides=NONE) + assert [d.line for d in result.diagnostics if d.rule == "render-designation-cycle"] == [_line(source, "# A ")] + + +def test_a_term_with_a_term_in_it_at_the_deepest_formatting_of_a_block() -> None: + # The levels a block has where its first term opens are its own: only those added inside count. + for depth in (58, 98): + for scheme in ("decimal", "none"): + body = f'# H\n\n{_wrap(depth, "{{term: a}}")}\n\n"A {{{{term: b}}}}" {{{{def: a}}}}\n\n"B" {{{{def: b}}}}\n' + result = render(FRONT + body, format="text", overrides={"numbering.scheme": scheme}) + assert " A B " in next(line for line in result.output.splitlines() if line.startswith("x ")), (depth, scheme) + assert "render-term-limit" not in {d.rule for d in result.diagnostics}, (depth, scheme) + + +def test_a_term_chain_inside_deep_formatting_is_cut_not_a_stack_overflow() -> None: + n = 40 + defs = "".join(f'"T{i} {_wrap(40, "{{term: t" + str(i + 1) + "}}")}" {{{{def: t{i}}}}}\n\n' for i in range(n)) + body = f'# A\n\n{_wrap(98, "{{term: t0}}")}\n\n{defs}"End" {{{{def: t{n}}}}}\n' + for scheme in ("decimal", "none"): + result = render(FRONT + body, format="text", overrides={"numbering.scheme": scheme}) + assert "render-term-limit" in {d.rule for d in result.diagnostics} + + +def test_a_reference_to_an_item_under_a_long_heading_keeps_the_item_piece() -> None: + result = run("# " + "word " * 300 + "{#a}\n\n- one {#i}\n\nSee {{ref: i}}.\n") + shown = next(line for line in result.output.splitlines() if line.startswith("See ")) + assert shown.endswith("\u2026 (a).") + assert len(shown) < 1010 + message = next(d.message for d in result.diagnostics if d.rule == "render-designation-limit") + assert message.startswith("A reference to the heading 'word word") and "refer to each other" not in message + + +def test_a_term_defined_in_the_body_is_not_searched_in_the_validators_text(monkeypatch: pytest.MonkeyPatch) -> None: + called: list[int] = [] + real = Resolver._designate + monkeypatch.setattr(Resolver, "_designate", lambda self, sections: (called.append(1), real(self, sections))[1]) + run('# Fee {{term: fee}} {#a}\n\n"Fee {{placeholder: p}}" {{def: fee}}\n\nSee {{ref: a}}.\n') + assert called == [] + + +def _tangle(rng: random.Random) -> str: + """Headings, items and terms that refer to each other at random, circles included.""" + n = rng.randint(2, 6) + terms = [f"t{i}" for i in range(rng.randint(1, 3))] + ids = [f"h{i}" for i in range(n)] + [f"h{i}i" for i in range(n)] + out = [] + for i in range(n): + parts = [f"W{i}"] + for _ in range(rng.randint(0, 2)): + parts.append("{{ref: " + rng.choice(ids) + "}}" if rng.random() < 0.6 + else "{{term: " + rng.choice(terms) + "}}") + out.append(f"# {' '.join(parts)} {{#h{i}}}\n\n- item {{#h{i}i}}\n") + for term in terms: + parts = [term.upper()] + if rng.random() < 0.7: + parts.append("{{ref: " + rng.choice(ids) + "}}") + if rng.random() < 0.3: + parts.append("{{term: " + rng.choice(terms) + "}}") + out.append(f'"{" ".join(parts)}" {{{{def: {term}}}}}\n') + return "\n".join(out) + + +def test_a_heading_that_reads_differently_from_its_designation_is_always_reported() -> None: + rng = random.Random(329) + reported = {"render-designation-cycle", "render-designation-limit", "render-term-limit"} + for _ in range(250): + source = FRONT + _tangle(rng) + result = render(source, format="text", overrides=NONE) + if any(" ".join(plain_text(s.title).split()) != s.designation for s in result.tree.sections): + assert reported & {d.rule for d in result.diagnostics}, source From 1dee684c2d56a1e504103f46d256d17d5755c83d Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:09:04 +0000 Subject: [PATCH 057/118] Close the gaps found reviewing the untrusted mode YAML numbers, booleans and timestamps are read through guarded constructors: text PyYAML cannot convert, or a number of more than 32 characters, is a style error (it raised ValueError or KeyError quoting the text, or a traceback on the command line). Enormous numbers in a mapping, as keys, values or overrides, are refused without str(). Whatever else the loader raises on an untrusted style is reported as one generic style error; a heading level out of range no longer quotes its key. values.date: an untrusted style names short, medium, long or full; a trusted pattern is formatted on fixed dates when the style loads, so one Babel refuses (GGGGGGG, QQQQQQ) is a style error instead of an exception at every render. An untrusted style's free-text settings are at most 200 characters (a long references.format multiplied the output by the number of references), and NUL and lone surrogates are refused in them and, in every mode, in the settings written into the stylesheet. A message lists 20 problems at most. --untrusted --print-style loads the style untrusted; the --untrusted help says a style may be a name or YAML text. stylesheet() writes any \@`, control characters and line breaks, and removes comment delimiters until none is left. A lone `/` stays, since `calc(100% / 3)` @@ -395,12 +402,19 @@ out when the validator reported the same path at the same line, or at none (`dia `_convert` for a mapping and, through `style_to_dict`, for a `Style` object and for settings objects given as overrides. The grammars allow no function, no variable and no `!important`, so `calc(1em + 2px)` and `var(--x)` are refused. `html.extra_css` must be empty. YAML text is at most - 64 KiB (UTF-8 bytes) and may not use aliases (the "billion laughs" shape). Every style, trusted or + 64 KiB (UTF-8 bytes), may not use aliases (the "billion laughs" shape), and a number in it is at most + 32 characters (`_StyleLoader` reads `!!int`, `!!float`, `!!bool` and timestamps through guarded + constructors, so a text PyYAML cannot convert is a `StyleError`, not its `ValueError`). Every + free-text setting is at most 200 characters (a label or format is repeated wherever the output shows + it, so an unbounded one multiplies the document's size) and `values.date` is one of the four names. + A message lists 20 problems at most and "and N more"; an exception the loader did not foresee is + reported as "the style could not be read", never with its own message. Every style, trusted or not, may nest 32 levels at most, in YAML text (refused by the scanner, `style/loader.py:_StyleLoader`, - before anything is built, as PyYAML's cost is quadratic in the depth) and in a mapping. A message names the setting and the problem, and never quotes - the style, a key, a template field or a path (a YAML error gives the line and column only), so it - can be shown to the user who sent it. `RenderOptions.untrusted` also requires a `document_root` - (§2.3: a renderer for documents it does not control SHOULD set one): without it `render` raises an + before anything is built, as PyYAML's cost is quadratic in the depth) and in a mapping. A message + names the setting and the problem, and never quotes the style, a key, a template field or a path (a + YAML error gives the line and column only), so it can be shown to the user who sent it. + `--untrusted --print-style` loads the style this way and needs no root. `RenderOptions.untrusted` + also requires a `document_root` (§2.3: a renderer for documents it does not control SHOULD set one): without it `render` raises an `OptionsError`, and the command exits 2. The option and the flag are one switch for both checks. Defaults are unchanged: trusted, no root required. A service should still choose the style itself. Document source has no power over the stylesheet either way: it is never emitted as HTML diff --git a/docs/style-templates.md b/docs/style-templates.md index c2e049a..47af8ff 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -100,7 +100,7 @@ definitions: term_style: plain # every {{term:}} reference values: - date: long # short | medium | long | full | a CLDR pattern such as "d. M. yyyy" + date: long # short | medium | long | full | a CLDR pattern such as "d. M. yyyy" (checked when the style loads; an untrusted style names one of the four) money: symbol # symbol ($10,000.00) | code (USD 10,000.00) | name (10,000.00 US dollars) duration: long # long (12 months) | short | narrow @@ -199,7 +199,10 @@ author's. A value placed into the stylesheet (`typography`, `headings`, `page`, must not contain `/*`, `*/`, ` argparse.ArgumentParser: parser.add_argument("--untrusted", action="store_true", help="the document and the style are somebody else's (for a service rendering what its " "users send): requires --root (exit 2 without), and the style must be a built-in " - "name, with no file, no raw CSS (html.extra_css) and only plain CSS values") + "name or YAML text (--style takes no file path), with no raw CSS (html.extra_css), " + "plain CSS values and short labels. With --print-style it checks the style the " + "same way and needs no --root") parser.add_argument("--fragment", action="store_true", help="HTML: write only the
, without page and stylesheet") parser.add_argument("--strict", action="store_true", help="refuse to render a document that has errors") parser.add_argument("--answers", metavar="FILE", @@ -161,7 +163,7 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.print_style: try: - print(dump_style(load_style(args.style, overrides=overrides)), end="") + print(dump_style(load_style(args.style, overrides=overrides, trusted=not args.untrusted)), end="") except StyleError as error: _complain(f"legaldown-render: {error}") return 2 diff --git a/src/legaldown_render/style/loader.py b/src/legaldown_render/style/loader.py index 2dc5a42..ce2eda9 100644 --- a/src/legaldown_render/style/loader.py +++ b/src/legaldown_render/style/loader.py @@ -21,16 +21,19 @@ import copy import dataclasses +import datetime import difflib import os import re import types +import unicodedata from collections.abc import Mapping from importlib import resources from pathlib import Path from typing import Any, Literal, Union, get_args, get_origin, get_type_hints import yaml +from babel.dates import format_date from ..errors import StyleError from . import model @@ -47,6 +50,29 @@ #: nesting depth of what it reads. MAX_STYLE_DEPTH = 32 +#: The named date formats (``values.date``); anything else is a CLDR pattern. +DATE_FORMATS = ("short", "medium", "long", "full") + +#: The longest number a style may write, in characters (30 digits and a sign or +#: an underscore or two). A style needs none above 1000, and converting a long +#: one is a cost an untrusted text should not impose. +MAX_NUMBER_CHARS = 32 + +#: A number a mapping or override may hold, in bits: far above what +#: :data:`MAX_NUMBER_CHARS` allows in YAML, and far below what makes ``str`` slow or raise. +MAX_NUMBER_BITS = 200 + +#: The longest timestamp a style may write; the longest valid one is 35. +MAX_TIMESTAMP_CHARS = 64 + +#: The most free text one untrusted setting may hold, in characters: a label or +#: a format is repeated in the output, once for every heading, reference or +#: item that shows it. +MAX_UNTRUSTED_TEXT = 200 + +#: The most problems one message lists; a style with more says how many it left out. +MAX_PROBLEMS = 20 + #: How an untrusted style is named in a message, instead of its own name (which #: is source text) or a path. _UNTRUSTED_SOURCE = "'untrusted style'" @@ -71,12 +97,44 @@ def add_indent(self, column: int) -> bool: return super().add_indent(column) +def _guarded(tag: str, kind: str) -> None: + """Make :class:`_StyleLoader` read *tag* with PyYAML's own constructor, but + refuse a text that is too long to be one, and report one that is not valid + as a :class:`yaml.constructor.ConstructorError`. PyYAML's constructors let + ``ValueError`` and ``KeyError`` out, with the text in the message, and + convert whatever number they are given.""" + original = yaml.SafeLoader.yaml_constructors[tag] + + def construct(loader: yaml.SafeLoader, node: yaml.Node) -> Any: + problem = None + if len(node.value) > MAX_NUMBER_CHARS and kind in ("integer", "number"): + problem = f"a {kind} of more than {MAX_NUMBER_CHARS} characters" + elif len(node.value) > MAX_TIMESTAMP_CHARS and kind == "timestamp": + problem = "a timestamp that is too long" + if problem is None: + try: + return original(loader, node) + except yaml.YAMLError: + raise + except Exception: # ValueError, KeyError, OverflowError, AttributeError (an impossible timestamp) + problem = f"not a valid {kind}" + raise yaml.constructor.ConstructorError(None, None, problem, node.start_mark) + + _StyleLoader.add_constructor(tag, construct) + + def _load_yaml(text: str) -> Any: return yaml.load(text, Loader=_StyleLoader) # noqa: S506 - a SafeLoader -class _TooDeep(Exception): - """A mapping nested more than :data:`MAX_STYLE_DEPTH` levels.""" +for _tag, _kind in (("int", "integer"), ("float", "number"), ("bool", "boolean"), ("timestamp", "timestamp")): + _guarded(f"tag:yaml.org,2002:{_tag}", _kind) + + +class _Refused(Exception): + """A mapping that cannot be read as a style: nested more than + :data:`MAX_STYLE_DEPTH` levels, or with a key that is an enormous number. + The message is the problem, which never quotes the style.""" def builtin_styles() -> list[str]: @@ -108,15 +166,33 @@ def load_style( aliases, never a file; it may extend only built-in styles; every value that reaches the stylesheet (fonts, sizes, colours, margins, page size) must match a strict grammar (:data:`~.model.CSS_KINDS`), also in a - :class:`Style` and in *overrides*; and ``html.extra_css`` must be empty. - A message names the setting and the problem and never quotes the style or - a path. + :class:`Style` and in *overrides*; ``html.extra_css`` must be empty; + ``values.date`` is a named format; every free-text setting is at most 200 + characters, without NUL or a lone surrogate; and a number in YAML text is at + most 32 characters. A message names the setting and the problem, never + quotes the style or a path, and lists at most 20 problems. Whatever else + the loader raises on such a style is a :class:`StyleError` too. + + Always, ``values.date`` that is not a named format must be a pattern Babel + can format, and a number or timestamp in YAML text that is not one is a + :class:`StyleError`. """ try: return _load(style, overrides, trusted) - except (_TooDeep, RecursionError): # the second only as a last resort + except StyleError: + raise + except _Refused as refused: + raise StyleError(_UNTRUSTED_SOURCE if not trusted else "'(style)'", [str(refused)]) from None + except RecursionError: # only as a last resort: the depth is bounded before it comes to this raise StyleError(_UNTRUSTED_SOURCE if not trusted else "'(style)'", ["the style is nested too deeply"]) from None + except Exception: + if trusted: + raise + # Whatever else a style from elsewhere makes the loader raise (a bug, or + # a case nobody thought of) must not reach the caller as a traceback + # whose message quotes the style: it is the style that is refused. + raise StyleError(_UNTRUSTED_SOURCE, ["the style could not be read"]) from None def _load(style: Any, overrides: Mapping[str, Any] | None, trusted: bool) -> Style: @@ -300,9 +376,9 @@ def _untrusted_yaml_problem(error: yaml.YAMLError) -> str: def _merge(base: Mapping, override: Mapping) -> dict: """*override* deep-merged onto *base*; mappings merge, anything else (lists included) replaces.""" - merged = {str(key): value for key, value in base.items()} + merged = {_key_text(key): value for key, value in base.items()} for key, value in override.items(): - key = str(key) + key = _key_text(key) if isinstance(value, Mapping) and isinstance(merged.get(key), Mapping): merged[key] = _merge(merged[key], value) else: @@ -317,7 +393,7 @@ def _expand_dotted(overrides: Mapping[str, Any], trusted: bool = True) -> dict: expanded: dict = {} for key, value in overrides.items(): target = expanded - *parents, last = str(key).split(".") + *parents, last = _key_text(key).split(".") for part in parents: target = target.setdefault(part, {}) if not isinstance(target, dict): @@ -347,13 +423,21 @@ def _refuse_meta_overrides(expanded: Mapping) -> None: raise StyleError("overrides", problems) +def _key_text(key: Any) -> str: + """A mapping key as the text it is read as: ``str(key)``, except that an + enormous number (``str`` would raise, or take quadratic time) is refused.""" + if isinstance(key, int) and not isinstance(key, bool) and key.bit_length() > MAX_NUMBER_BITS: + raise _Refused("a key is a number too large to be a setting") + return str(key) + + def _copy(value: Any, depth: int = 0) -> Any: """A deep copy of *value* with every mapping made a plain dict. A value - nested more than :data:`MAX_STYLE_DEPTH` levels raises :class:`_TooDeep`.""" + nested more than :data:`MAX_STYLE_DEPTH` levels raises :class:`_Refused`.""" if depth > MAX_STYLE_DEPTH: - raise _TooDeep + raise _Refused("the style is nested too deeply") if isinstance(value, Mapping): - return {str(key): _copy(item, depth + 1) for key, item in value.items()} + return {_key_text(key): _copy(item, depth + 1) for key, item in value.items()} if isinstance(value, list): return [_copy(item, depth + 1) for item in value] if isinstance(value, tuple): @@ -441,11 +525,35 @@ def _build(data: Mapping, source: str, trusted: bool = True) -> Style: problems: list[str] = [] style = _convert(data, Style, "", problems, trusted=trusted) problems.extend(_template_problems(style, trusted)) + problems.extend(_value_problems(style, trusted)) if problems: + if len(problems) > MAX_PROBLEMS: + problems = [*problems[:MAX_PROBLEMS], f"and {len(problems) - MAX_PROBLEMS} more"] raise StyleError(source, problems) return style +def _value_problems(style: Style, trusted: bool) -> list[str]: + """What is wrong with the settings that only mean something to a library: + ``values.date``, which is a name or a CLDR pattern for Babel, and which + Babel refuses at render time when it is no pattern (``GGGGGGG``) — so + every render of a document with a date would raise. An untrusted style + names one of the four formats; any other pattern is tried here, on a fixed + date, so that no render raises.""" + date = style.values.date + if date in DATE_FORMATS: + return [] + if not trusted: + return [f"values.date: must be one of {', '.join(DATE_FORMATS)} in an untrusted style"] + try: + for day in (datetime.date(2001, 2, 3), datetime.date(2026, 12, 31), datetime.date.min): + for locale in ("en", "cs", "de"): + format_date(day, format=date, locale=locale) + except Exception: # Babel raises KeyError, ValueError, AttributeError, ... for a bad pattern + return ["values.date: not a date format (short, medium, long, full) or a valid CLDR date pattern"] + return [] + + def _template_problems(style: Style, trusted: bool = True) -> list[str]: """What is wrong with the ``{field}`` names in the style's text templates (:func:`~.model.template_settings`): an identifier-shaped field the @@ -476,7 +584,11 @@ def _template_problems(style: Style, trusted: bool = True) -> list[str]: def _got(value: Any, trusted: bool) -> str: """`` (got 3)`` for a message about a value — nothing for an untrusted style, whose text a message does not quote.""" - return f" (got {value!r})" if trusted else "" + if not trusted: + return "" + if isinstance(value, int) and not isinstance(value, bool) and value.bit_length() > MAX_NUMBER_BITS: + return " (got a very large number)" # repr would raise, or take quadratic time + return f" (got {value!r})" def _convert(value: Any, hint: Any, path: str, problems: list[str], *, trusted: bool = True, @@ -517,14 +629,16 @@ def _convert(value: Any, hint: Any, path: str, problems: list[str], *, trusted: converted = {} for key, item in value.items(): if key_hint is int: + named = _key_text(key) try: key = int(key) - except (TypeError, ValueError): - problems.append(f"{path}.{key}: key must be a number" if trusted + except (TypeError, ValueError, OverflowError): + problems.append(f"{path}.{named}: key must be a number" if trusted else f"{path}: a key must be a number") continue if not 1 <= key <= 5: - problems.append(f"{path}.{key}: heading levels are 1 to 5 (§4.1)") + problems.append(f"{path}.{named}: heading levels are 1 to 5 (§4.1)" if trusted + else f"{path}: heading levels are 1 to 5 (§4.1)") continue converted[key] = _convert(item, item_hint, f"{path}.{key}", problems, trusted=trusted) return converted @@ -540,6 +654,9 @@ def _convert(value: Any, hint: Any, path: str, problems: list[str], *, trusted: return value if hint is str: # YAML reads 700 or 1.5 as numbers; they are fine as text here. + if isinstance(value, int) and not isinstance(value, bool) and value.bit_length() > MAX_NUMBER_BITS: + problems.append(f"{label}: must be text, not an enormous number") # str() would raise + return "" if isinstance(value, (int, float)) and not isinstance(value, bool): value = str(value) elif not isinstance(value, str): @@ -548,10 +665,23 @@ def _convert(value: Any, hint: Any, path: str, problems: list[str], *, trusted: if css is not None and (problem := css_problem(value, css, trusted=trusted)): problems.append(f"{label}: {problem}") return "" + if not trusted and (problem := _text_problem(value)): + problems.append(f"{label}: {problem}") # the value stays, so the template checks do not add to it return value raise TypeError(f"unsupported style field type {hint!r} at {label}") # a bug in model.py +def _text_problem(value: str) -> str | None: + """What is wrong with a free-text setting of an untrusted style: it is + repeated wherever the output shows it, so it is short, and a character that + no output encoding can write (NUL, a lone surrogate) would crash the writer.""" + if len(value) > MAX_UNTRUSTED_TEXT: + return f"is longer than {MAX_UNTRUSTED_TEXT} characters" + if any(char == "\x00" or unicodedata.category(char) == "Cs" for char in value): + return "must not contain NUL or a lone surrogate" + return None + + def _convert_dataclass(value: Any, cls: type, path: str, problems: list[str], *, trusted: bool = True) -> Any: label = path or "(root)" if isinstance(value, cls): diff --git a/src/legaldown_render/style/model.py b/src/legaldown_render/style/model.py index 43b5052..05ab2e2 100644 --- a/src/legaldown_render/style/model.py +++ b/src/legaldown_render/style/model.py @@ -271,10 +271,12 @@ def css_problem(value: str, kind: str, *, trusted: bool) -> str | None: """What is wrong with *value* for a setting of *kind* written into the stylesheet, or None. It never quotes the value. - Always: no comment delimiter, `` str | None: for token in CSS_FORBIDDEN: if token in value: return f"must not contain '{token}' (it would break out of the stylesheet)" - if any(unicodedata.category(char) in ("Cc", "Zl", "Zp") for char in value): - return "must not contain a control character or a line break" + if any(unicodedata.category(char) in ("Cc", "Zl", "Zp", "Cs") for char in value): + return "must not contain a control character, a line break or a lone surrogate" if not trusted: css = CSS_KINDS[kind] if not css.grammar.fullmatch(value): diff --git a/src/legaldown_render/writers/html.py b/src/legaldown_render/writers/html.py index cc64e53..f505343 100644 --- a/src/legaldown_render/writers/html.py +++ b/src/legaldown_render/writers/html.py @@ -462,7 +462,9 @@ def stylesheet(style: Style) -> str: lines.append(f"@page {{ size: {_css(style.page.size)}; margin: {_css(style.page.margin)}; }}") css = _BASE_CSS + "\n".join(lines) + "\n" if style.html.extra_css: - css += style.html.extra_css.rstrip() + "\n" + # The loader refuses `` None: from legaldown_render.writers.html import stylesheet assert "max-width: calc(100% / 3);" in stylesheet(Style(html=Html(max_width="calc(100% / 3)"))) + + +def test_the_stylesheet_cannot_end_its_element_through_raw_css() -> None: + """The loader refuses ``\n"))) + assert "", "�", "\x85
"]) +def test_a_hostile_string_in_each_place_is_harmless(hostile: str) -> None: + quoted = yaml.safe_dump(hostile, allow_unicode=True).rstrip("\n").removesuffix("\n...").rstrip("\n") + source = (f"---\nlegaldown: '0.2'\ntitle: {quoted}\nsubtitle: {quoted}\n---\n\n# {hostile}\n\n{hostile} " + f"[{hostile}]()\n\n```\n{hostile}\n```\n\n| {hostile} |\n|---|\n| {hostile} |\n") + overrides = {key: hostile + "{page}" for key in FREE_TEXT_SETTINGS} + assert_well_formed(render(source, format="docx", overrides=overrides).binary or b"") + + +def test_nothing_from_the_document_is_left_as_a_control_character() -> None: + source = "---\nlegaldown: '0.2'\ntitle: 'a\\x01b'\n---\n\n# H\x01\x08\x0b\x0c\x1f\x7f\x85x\n\nbody\x01\n" + contents = parts(render(source, format="docx").binary or b"") + for name, content in contents.items(): + text = content.decode("utf-8") + assert not any(ord(c) < 0x20 and c not in "\t\n\r" or 0x7F <= ord(c) <= 0x9F for c in text), name + assert "\ud800" not in text, name + + +def test_a_header_template_with_a_control_character_is_sanitised_not_refused() -> None: + contents = parts(render("---\ntitle: T\n---\n# A\n", format="docx", + overrides={"page.footer.center": "a\x00b\x1bc{page}"}).binary or b"") + footer = ET.fromstring(contents["word/footer1.xml"]) + assert "abc" in "".join(node.text or "" for node in footer.iter() if node.tag.endswith("}t")) diff --git a/tests/test_docx_package.py b/tests/test_docx_package.py new file mode 100644 index 0000000..c1988c7 --- /dev/null +++ b/tests/test_docx_package.py @@ -0,0 +1,635 @@ +"""The OOXML package the DOCX writer produces: its parts, relationships and content types, +the settings, styles, headers and footers, properties and page setup generated from the +style template, determinism, and the builders later work reuses. Schema validity is in +test_docx_validation.py, LibreOffice in test_docx_libreoffice.py; this module needs neither.""" +from __future__ import annotations + +import io +import sys +import xml.etree.ElementTree as ET +import zipfile +from pathlib import Path + +import pytest +from conftest import DOCUMENTS, FEATURES, FRONT, run_cli +from docx_support import W_NS, parts, texts + +from legaldown_render import RenderOptions, render, render_file +from legaldown_render.cli import main +from legaldown_render.errors import InternalError +from legaldown_render.writers import FORMATS, DocxWriter, format_for_path +from legaldown_render.writers.docx import ooxml, package, styles +from legaldown_render.writers.docx.headers import template_runs + +W = f"{{{W_NS}}}" +SOURCE = ("---\nlegaldown: '0.2'\ntitle: Services Agreement\nsubtitle: A subtitle\nlanguage: cs\nversion: '2.1'\n" + "effective_date: 2026-03-04\n---\n\nPreamble text.\n\n# First {#first}\n\nBody *one*.\n\n## Nested\n\n" + "- item a\n- item b\n\n> quoted\n\n```\ncode 1\ncode 2\n```\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\n---\n") + + +def docx(source: str = SOURCE, **settings: object) -> dict[str, bytes]: + result = render(source, format="docx", **settings) + assert result.binary is not None + return parts(result.binary) + + +def xml(part: bytes) -> ET.Element: + return ET.fromstring(part) + + +# -- The package --------------------------------------------------------------------------- + + +def test_a_package_has_the_parts_a_word_document_needs_in_a_fixed_order() -> None: + assert list(docx()) == [ + "[Content_Types].xml", "_rels/.rels", "docProps/core.xml", "docProps/app.xml", "word/document.xml", + "word/_rels/document.xml.rels", "word/styles.xml", "word/numbering.xml", "word/settings.xml", + "word/footer1.xml"] + + +def test_content_types_name_every_part() -> None: + contents = docx() + root = xml(contents["[Content_Types].xml"]) + ns = "{http://schemas.openxmlformats.org/package/2006/content-types}" + defaults = {entry.get("Extension"): entry.get("ContentType") for entry in root.iter(f"{ns}Default")} + overrides = {entry.get("PartName"): entry.get("ContentType") for entry in root.iter(f"{ns}Override")} + assert defaults == {"rels": "application/vnd.openxmlformats-package.relationships+xml", + "xml": "application/xml"} + assert overrides == { + "/docProps/core.xml": "application/vnd.openxmlformats-package.core-properties+xml", + "/docProps/app.xml": "application/vnd.openxmlformats-officedocument.extended-properties+xml", + "/word/document.xml": "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml", + "/word/styles.xml": "application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml", + "/word/numbering.xml": "application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml", + "/word/settings.xml": "application/vnd.openxmlformats-officedocument.wordprocessingml.settings+xml", + "/word/footer1.xml": "application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml", + } + + +def test_the_relationships_point_at_the_parts() -> None: + contents = docx() + ns = "{http://schemas.openxmlformats.org/package/2006/relationships}" + package_rels = [(r.get("Id"), r.get("Type").rsplit("/", 1)[1], r.get("Target")) + for r in xml(contents["_rels/.rels"]).iter(f"{ns}Relationship")] + assert package_rels == [("rId1", "officeDocument", "word/document.xml"), + ("rId2", "core-properties", "docProps/core.xml"), + ("rId3", "extended-properties", "docProps/app.xml")] + document_rels = [(r.get("Id"), r.get("Type").rsplit("/", 1)[1], r.get("Target")) + for r in xml(contents["word/_rels/document.xml.rels"]).iter(f"{ns}Relationship")] + assert document_rels == [("rId1", "styles", "styles.xml"), ("rId2", "numbering", "numbering.xml"), + ("rId3", "settings", "settings.xml"), ("rId4", "footer", "footer1.xml")] + + +def test_every_part_is_utf8_xml_with_a_declaration() -> None: + for name, content in docx().items(): + assert content.startswith(b'\n'), name + xml(content) # well-formed + + +def test_the_zip_is_deterministic_and_has_no_timestamps_or_platform_bits() -> None: + first = render(SOURCE, format="docx").binary + second = render(SOURCE, format="docx").binary + assert first == second + assert first is not None + with zipfile.ZipFile(io.BytesIO(first)) as archive: + assert archive.testzip() is None + for info in archive.infolist(): + assert info.date_time == (1980, 1, 1, 0, 0, 0) + assert info.compress_type == zipfile.ZIP_DEFLATED + assert (info.create_system, info.external_attr >> 16, info.extra, info.comment) == (3, 0o644, b"", b"") + + +def test_a_different_document_is_a_different_file() -> None: + assert render(SOURCE, format="docx").binary != render(SOURCE.replace("First", "Second"), format="docx").binary + + +# -- settings.xml ------------------------------------------------------------------------ + + +def test_settings_have_compatibility_mode_15_and_no_revision_ids() -> None: + settings = xml(docx()["word/settings.xml"]) + modes = [(c.get(f"{W}name"), c.get(f"{W}val")) for c in settings.iter(f"{W}compatSetting")] + assert modes == [("compatibilityMode", "15")] + assert b"rsid" not in docx()["word/settings.xml"] + + +def test_update_fields_is_in_schema_order_after_the_character_spacing_and_before_compat() -> None: + settings = xml(docx()["word/settings.xml"]) + assert [child.tag.removeprefix(W) for child in settings] == [ + "defaultTabStop", "characterSpacingControl", "updateFields", "compat"] + assert settings.find(f"{W}updateFields").get(f"{W}val") == "true" + + +def test_update_fields_on_open_can_be_turned_off() -> None: + settings = xml(docx(overrides={"docx.update_fields_on_open": False})["word/settings.xml"]) + assert settings.find(f"{W}updateFields") is None + assert settings.find(f"{W}compat") is not None + + +def test_the_numbering_part_is_an_empty_stub_that_later_work_fills() -> None: + assert list(xml(docx()["word/numbering.xml"])) == [] + from legaldown_render.writers.docx.settings import build_numbering + + filled = xml(build_numbering([ooxml.el("abstractNum", {"abstractNumId": 0})], [ooxml.el("num", {"numId": 1})])) + assert [child.tag.removeprefix(W) for child in filled] == ["abstractNum", "num"] + + +# -- styles.xml ---------------------------------------------------------------------------- + + +def style_map(contents: dict[str, bytes]) -> dict[str, ET.Element]: + return {s.get(f"{W}styleId"): s for s in xml(contents["word/styles.xml"]).iter(f"{W}style")} + + +def test_the_styles_the_body_and_word_need_exist_under_the_names_word_knows() -> None: + by_id = style_map(docx()) + names = {style_id: s.find(f"{W}name").get(f"{W}val") for style_id, s in by_id.items()} + assert names == { + "Normal": "Normal", "DefaultParagraphFont": "Default Paragraph Font", "TableNormal": "Normal Table", + "NoList": "No List", "Heading1": "heading 1", "Heading2": "heading 2", "Heading3": "heading 3", + "Heading4": "heading 4", "Heading5": "heading 5", "Title": "Title", "Subtitle": "Subtitle", + "Quote": "Quote", "ContentsHeading": "Contents Heading", "DraftingNote": "Drafting Note", "Code": "Code", + "TableGrid": "Table Grid", "Header": "header", "Footer": "footer", "TOC1": "toc 1", "TOC2": "toc 2", + "TOC3": "toc 3", "TOC4": "toc 4", "TOC5": "toc 5", "Hyperlink": "Hyperlink", "InlineCode": "Inline Code", + "DefinedTerm": "Defined Term", "TermReference": "Term Reference", "Blank": "Blank", "Choice": "Choice", + "FailureMarker": "Failure Marker", "ConditionLabel": "Condition Label", + "DraftingNoteLabel": "Drafting Note Label"} + assert by_id["Normal"].get(f"{W}default") == "1" + assert by_id["TableGrid"].get(f"{W}type") == "table" + assert by_id["Hyperlink"].get(f"{W}type") == "character" + assert by_id["ContentsHeading"].get(f"{W}customStyle") == "1" + assert by_id["Heading1"].get(f"{W}customStyle") is None + + +def test_headings_are_outline_levels_zero_to_four_and_not_numbered_yet() -> None: + by_id = style_map(docx()) + for level in range(1, 6): + heading = by_id[f"Heading{level}"] + assert heading.find(f"{W}pPr/{W}outlineLvl").get(f"{W}val") == str(level - 1) + assert heading.find(f"{W}basedOn").get(f"{W}val") == "Normal" + assert heading.find(f"{W}pPr/{W}numPr") is None + assert by_id["ContentsHeading"].find(f"{W}pPr/{W}outlineLvl") is None # in no table of contents + + +def test_headings_link_to_a_numbering_definition_when_given_one() -> None: + from legaldown_render.style import load_style + from legaldown_render.style.docx import docx_settings + + style = load_style("default") + root = xml(styles.build_styles(style, docx_settings(style), "en", heading_num_id=7)) + for style_element in root.iter(f"{W}style"): + style_id = style_element.get(f"{W}styleId") + num = style_element.find(f"{W}pPr/{W}numPr") + if style_id.startswith("Heading"): + level = int(style_id.removeprefix("Heading")) - 1 + assert (num.find(f"{W}ilvl").get(f"{W}val"), num.find(f"{W}numId").get(f"{W}val")) == (str(level), "7") + else: + assert num is None + + +def test_headings_follow_the_style_template() -> None: + by_id = style_map(docx(style="continental")) + heading = by_id["Heading1"] + assert heading.find(f"{W}pPr/{W}jc").get(f"{W}val") == "center" + assert heading.find(f"{W}rPr/{W}caps") is None # continental: transform none + assert heading.find(f"{W}rPr/{W}sz").get(f"{W}val") == "24" # 1.1em of 11pt, rounded to half-points + default = style_map(docx())["Heading1"] + assert default.find(f"{W}rPr/{W}caps") is not None and default.find(f"{W}rPr/{W}b") is not None + + +def test_normal_carries_size_colour_justification_and_line_height() -> None: + normal = style_map(docx(overrides={"typography.justify": True, "typography.font_size": "12pt", + "typography.line_height": "1.25", "typography.color": "#112233"}))["Normal"] + assert normal.find(f"{W}rPr/{W}sz").get(f"{W}val") == "24" + assert normal.find(f"{W}rPr/{W}color").get(f"{W}val") == "112233" + assert normal.find(f"{W}pPr/{W}jc").get(f"{W}val") == "both" + spacing = normal.find(f"{W}pPr/{W}spacing") + assert (spacing.get(f"{W}line"), spacing.get(f"{W}lineRule")) == ("300", "auto") + + +def test_doc_defaults_have_the_language_and_the_first_font_family() -> None: + root = xml(docx()["word/styles.xml"]) + run = root.find(f"{W}docDefaults/{W}rPrDefault/{W}rPr") + fonts = run.find(f"{W}rFonts") + assert {key.removeprefix(W): value for key, value in fonts.attrib.items()} == { + "ascii": "Georgia", "hAnsi": "Georgia", "cs": "Georgia", "eastAsia": "Georgia"} + assert run.find(f"{W}lang").get(f"{W}val") == "cs" # the document's language + english = xml(docx(FRONT + "# A\n")["word/styles.xml"]).find(f"{W}docDefaults/{W}rPrDefault/{W}rPr/{W}lang") + assert english.get(f"{W}val") == "en" + + +def test_a_language_that_is_not_a_tag_falls_back_to_the_locale_then_en_us() -> None: + assert styles.language_tag("cs-CZ", "en") == "cs-CZ" + assert styles.language_tag("not a tag", "de-DE") == "de-DE" + assert styles.language_tag("", None) == "en-US" + assert styles.language_tag(None, "") == "en-US" + + +def test_term_styles_follow_the_definitions_settings() -> None: + by_id = style_map(docx(overrides={"definitions.style": "italic", "definitions.term_style": "underline"})) + assert by_id["DefinedTerm"].find(f"{W}rPr/{W}i") is not None + assert by_id["TermReference"].find(f"{W}rPr/{W}u").get(f"{W}val") == "single" + plain = style_map(docx(overrides={"definitions.style": "plain"}))["DefinedTerm"] + assert plain.find(f"{W}rPr") is None + + +def test_the_hyperlink_and_toc_styles_use_the_link_colour_and_the_text_width() -> None: + by_id = style_map(docx(overrides={"typography.link_color": "#ff0000", "page.margin": "20mm"})) + assert by_id["Hyperlink"].find(f"{W}rPr/{W}color").get(f"{W}val") == "FF0000" + tab = by_id["TOC2"].find(f"{W}pPr/{W}tabs/{W}tab") + assert (tab.get(f"{W}val"), tab.get(f"{W}leader"), tab.get(f"{W}pos")) == ("right", "dot", str(11906 - 2 * 1134)) + assert by_id["TOC3"].find(f"{W}pPr/{W}ind").get(f"{W}left") == "480" + + +# -- Page setup, header and footer --------------------------------------------------------- + + +def section(contents: dict[str, bytes]) -> ET.Element: + return xml(contents["word/document.xml"]).find(f"{W}body/{W}sectPr") + + +def test_the_page_setup_is_in_twips() -> None: + sect = section(docx(overrides={"page.size": "letter", "page.margin": "1in 2in"})) + assert {k.removeprefix(W): v for k, v in sect.find(f"{W}pgSz").attrib.items()} == {"w": "12240", "h": "15840"} + margins = {k.removeprefix(W): v for k, v in sect.find(f"{W}pgMar").attrib.items()} + assert margins == {"top": "1440", "right": "2880", "bottom": "1440", "left": "2880", "header": "720", + "footer": "720", "gutter": "0"} + + +def test_landscape_has_the_long_edge_as_width_and_says_so() -> None: + size = section(docx(overrides={"page.size": "A4 landscape"})).find(f"{W}pgSz") + assert {k.removeprefix(W): v for k, v in size.attrib.items()} == {"w": "16838", "h": "11906", "orient": "landscape"} + + +def test_the_section_references_the_footer_and_there_is_no_header_by_default() -> None: + contents = docx() + sect = section(contents) + assert sect.find(f"{W}headerReference") is None + reference = sect.find(f"{W}footerReference") + assert reference.get(f"{W}type") == "default" + assert reference.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id") == "rId4" + assert [child.tag.removeprefix(W) for child in sect] == ["footerReference", "pgSz", "pgMar"] + + +def test_a_document_without_header_and_footer_has_no_such_parts() -> None: + contents = docx(overrides={"page.footer": {"center": ""}}) + assert not [name for name in contents if "header" in name or "footer" in name] + assert section(contents).find(f"{W}footerReference") is None + + +def test_the_default_footer_is_a_page_and_pages_field_centred() -> None: + footer = xml(docx()["word/footer1.xml"]) + (paragraph,) = footer.iter(f"{W}p") + assert paragraph.find(f"{W}pPr/{W}pStyle").get(f"{W}val") == "Footer" + assert paragraph.find(f"{W}pPr/{W}jc").get(f"{W}val") == "center" + instructions = [t.text for t in paragraph.iter(f"{W}instrText")] + assert instructions == [" PAGE ", " NUMPAGES "] + kinds = [c.get(f"{W}fldCharType") for c in paragraph.iter(f"{W}fldChar")] + assert kinds == ["begin", "separate", "end"] * 2 + assert texts(footer) == ["1 / 1"] # the cached results and the literal between them + + +def header_footer(**templates: str) -> dict[str, bytes]: + return docx(overrides={f"page.{key.replace('_', '.')}": value for key, value in templates.items()}) + + +def test_three_templates_are_set_with_tab_stops_at_the_middle_and_right_margin() -> None: + contents = header_footer(header_left="{title}", header_center="{date}", header_right="v{version}, {page}") + header = xml(contents["word/header1.xml"]) + (paragraph,) = header.iter(f"{W}p") + assert texts(header) == ["Services Agreement\t4. března 2026\tv2.1, 1"] + width = 11906 - 2 * 1417 + tabs = [(t.get(f"{W}val"), t.get(f"{W}pos")) for t in paragraph.iter(f"{W}tab") if t.get(f"{W}pos")] + assert tabs == [("center", str(width // 2)), ("right", str(width))] + assert paragraph.find(f"{W}pPr/{W}pStyle").get(f"{W}val") == "Header" + assert section(contents).find(f"{W}headerReference").get( + "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id") == "rId4" + + +@pytest.mark.parametrize(("templates", "line", "alignment", "stops"), [ + ({"header_left": "L"}, "L", None, []), + ({"header_center": "C"}, "C", "center", []), + ({"header_right": "R"}, "R", "right", []), + ({"header_left": "L", "header_center": "C"}, "L\tC", None, ["center"]), + ({"header_left": "L", "header_right": "R"}, "L\tR", None, ["right"]), + ({"header_center": "C", "header_right": "R"}, "\tC\tR", None, ["center", "right"]), + ({"header_left": "L", "header_center": "C", "header_right": "R"}, "L\tC\tR", None, ["center", "right"]), +]) +def test_how_the_templates_are_laid_out(templates: dict[str, str], line: str, alignment: str | None, + stops: list[str]) -> None: + header = xml(header_footer(**templates)["word/header1.xml"]) + assert texts(header) == [line] + jc = header.find(f".//{W}pPr/{W}jc") + assert (jc.get(f"{W}val") if jc is not None else None) == alignment + assert [t.get(f"{W}val") for t in header.iter(f"{W}tab") if t.get(f"{W}pos")] == stops + + +def test_title_version_and_date_are_text_and_a_missing_one_is_empty() -> None: + footer = xml(header_footer(footer_center="{title}|{version}|{date}")["word/footer1.xml"]) + assert texts(footer) == ["Services Agreement|2.1|4. března 2026"] + bare = xml(docx(FRONT + "# A\n", overrides={"page.footer.center": "[{version}][{date}]"})["word/footer1.xml"]) + assert texts(bare) == ["[][]"] + + +def test_the_date_follows_the_style_and_the_locale() -> None: + footer = xml(docx(overrides={"page.footer.center": "{date}", "values.date": "short", "locale": "cs-CZ"}) + ["word/footer1.xml"]) + assert texts(footer) == ["04.03.26"] + + +def test_a_version_or_date_that_holds_directives_is_left_out() -> None: + source = "---\ntitle: T\nversion: '{{placeholder: v}}'\n---\n# A\n" + footer = xml(docx(source, overrides={"page.footer.center": "[{version}]"})["word/footer1.xml"]) + assert texts(footer) == ["[]"] + + +def test_a_field_name_the_writer_does_not_know_stays_as_written() -> None: + runs = template_runs("{page} of {pages}: {nope}", {"title": "x"}) + root = ET.Element("p") + root.extend(runs) + assert "".join(t.text for t in root.iter(f"{W}t")) == "1 of 1: {nope}" + + +# -- Properties -------------------------------------------------------------------------- + + +def test_core_properties_have_the_title_and_language_and_no_timestamps() -> None: + core = xml(docx()["docProps/core.xml"]) + assert [(child.tag.rsplit("}", 1)[1], child.text) for child in core] == [ + ("title", "Services Agreement"), ("language", "cs")] + raw = docx()["docProps/core.xml"].decode("utf-8") + assert "created" not in raw and "modified" not in raw and "creator" not in raw + + +def test_core_properties_leave_out_an_empty_title() -> None: + core = xml(docx("---\ntitle: ''\n---\n# A\n")["docProps/core.xml"]) + assert [child.tag.rsplit("}", 1)[1] for child in core] == ["language"] + + +def test_app_properties_name_the_program() -> None: + app = xml(docx()["docProps/app.xml"]) + assert app.tag == "{http://schemas.openxmlformats.org/officeDocument/2006/extended-properties}Properties" + assert app[0].text.startswith("legaldown-render ") + + +# -- The interim body ---------------------------------------------------------------------- + + +def test_the_interim_body_has_the_title_headings_and_text_and_no_numbers() -> None: + contents = docx() + body = xml(contents["word/document.xml"]).find(f"{W}body") + styles_used = [(p.find(f"{W}pPr/{W}pStyle").get(f"{W}val") if p.find(f"{W}pPr/{W}pStyle") is not None else None, + texts(p)[0]) for p in body.iter(f"{W}p")] + assert styles_used[:5] == [("Title", "Services Agreement"), ("Subtitle", "A subtitle"), (None, "Preamble text."), + ("Heading1", "First"), (None, "Body one.")] + assert ("Heading2", "Nested") in styles_used + assert (None, "item a") in styles_used and ("Quote", "quoted") in styles_used + assert [text for style, text in styles_used if style == "Code"] == ["code 1", "code 2"] + assert (None, "A\tB") in styles_used and (None, "1\t2") in styles_used + assert all(not text.startswith(("1.", "(a)")) for _, text in styles_used) + + +def test_a_heading_level_is_clamped_to_the_five_word_styles() -> None: + source = FRONT + "# 1\n\n## 2\n\n### 3\n\n#### 4\n\n##### 5\n" + used = [p.find(f"{W}pPr/{W}pStyle").get(f"{W}val") + for p in xml(docx(source)["word/document.xml"]).iter(f"{W}p") if p.find(f"{W}pPr/{W}pStyle") is not None] + assert used == ["Title", "Heading1", "Heading2", "Heading3", "Heading4", "Heading5"] + + +def test_a_code_block_is_one_paragraph_per_line_and_a_hard_break_is_a_break() -> None: + document = xml(docx(FRONT + "# A\n\nline one\\\nline two\n")["word/document.xml"]) + assert "line one\nline two" in texts(document) + + +# -- Sanitising text ----------------------------------------------------------------------- + + +def test_the_sanitiser_drops_what_xml_cannot_carry() -> None: + assert ooxml._t("a\x00b\x01c\x08d\x0be\x0cf\x1fg\x7fh\x85i\x9fj\ufffek\uffffl\ud800m\udfffn") == "abcdefghijklmn" + assert ooxml._t("keep ž € \U0001f600 
 ") == "keep ž € \U0001f600 
 " + assert ooxml._t("a\tb\nc\rd") == "a b c d" + assert ooxml._t("a\tb\nc\rd", breaks=True) == "a\tb\nc\rd" + + +def test_text_runs_turn_tab_and_line_breaks_into_elements() -> None: + run = ooxml.text_run("a\tb\nc\r\nd\re f") + assert [(child.tag.removeprefix(W), child.text) for child in run] == [ + ("t", "a"), ("tab", None), ("t", "b"), ("br", None), ("t", "c"), ("br", None), ("t", "d"), ("br", None), + ("t", "e f")] + assert all(t.get("{http://www.w3.org/XML/1998/namespace}space") == "preserve" for t in run.iter(f"{W}t")) + assert list(ooxml.text_run("")) == [] + assert [child.tag.removeprefix(W) for child in ooxml.text_run("\x00\x01")] == [] + + +def test_every_attribute_and_text_goes_through_the_sanitiser() -> None: + element = ooxml.el("bookmarkStart", {"name": "a\x01b", "id": 3, "flag": True, "off": False, "none": None}, + text="x\x02y") + assert element.attrib == {f"{W}name": "ab", f"{W}id": "3", f"{W}flag": "1", f"{W}off": "0"} + assert element.text == "xy" + assert ET.tostring(ooxml.el("t", text="a\ud800b"), encoding="utf-8") # encodable: no lone surrogate survives + + +def test_serialisation_is_valid_utf8_for_any_text() -> None: + hostile = "".join(chr(code) for code in (*range(0x20), 0x7F, 0x85, 0xD800, 0xDFFF, 0xFFFE, 0xFFFF)) + "<&>\"'" + data = ooxml.serialize(ooxml.el("p", {"x": hostile}, ooxml.text_run(hostile), ooxml.el("instrText", text=hostile))) + ET.fromstring(data.decode("utf-8")) + + +def test_properties_are_sorted_into_schema_order() -> None: + properties = ooxml.ppr(ooxml.val("jc", "left"), ooxml.val("outlineLvl", 0), ooxml.val("pStyle", "X"), + ooxml.el("spacing"), ooxml.el("keepNext"), ooxml.el("numPr"), ooxml.el("ind")) + assert [child.tag.removeprefix(W) for child in properties] == [ + "pStyle", "keepNext", "numPr", "spacing", "ind", "jc", "outlineLvl"] + run = ooxml.rpr(ooxml.val("sz", 2), ooxml.el("b"), ooxml.val("rStyle", "S"), ooxml.val("color", "000000"), + ooxml.el("i")) + assert [child.tag.removeprefix(W) for child in run] == ["rStyle", "b", "i", "color", "sz"] + assert ooxml.ppr(None, [None, False]) is None + with pytest.raises(InternalError): + ooxml.ppr(ooxml.el("notAChild")) + + +def test_a_paragraph_puts_its_properties_before_its_content() -> None: + paragraph = ooxml.p(ooxml.text_run("x"), style="Quote", props=[ooxml.el("keepNext")]) + assert [child.tag.removeprefix(W) for child in paragraph] == ["pPr", "r"] + assert [child.tag.removeprefix(W) for child in paragraph[0]] == ["pStyle", "keepNext"] + + +# -- Fields, bookmarks, hyperlinks --------------------------------------------------------- + + +def test_a_complex_field_is_begin_instruction_separate_result_end() -> None: + runs = ooxml.complex_field("REF ld_x \\n \\h", [ooxml.text_run("2.1")], dirty=True) + root = ET.Element("p") + root.extend(runs) + chars = [(c.get(f"{W}fldCharType"), c.get(f"{W}dirty")) for c in root.iter(f"{W}fldChar")] + assert chars == [("begin", "1"), ("separate", None), ("end", None)] + assert [t.text for t in root.iter(f"{W}instrText")] == [" REF ld_x \\n \\h "] + assert [child[0].tag.removeprefix(W) if len(child) else None for child in runs] == [ + "fldChar", "instrText", "fldChar", "t", "fldChar"] + empty = ooxml.complex_field("PAGEREF x") + assert len(empty) == 4 # no cached result + + +def test_a_field_instruction_is_sanitised_and_kept_on_one_line() -> None: + runs = ooxml.complex_field("REF a\x01b\nc") + assert runs[1].find(f"{W}instrText").text == " REF ab c " + + +def test_bookmark_ids_are_unique_across_the_package_and_names_are_checked() -> None: + bookmarks = ooxml.Bookmarks() + first_start, first_end = bookmarks.pair("ld_first") + second = bookmarks.around("_Ld2", ooxml.text_run("x")) + assert first_start.get(f"{W}id") == first_end.get(f"{W}id") == "1" + assert [c.tag.removeprefix(W) for c in second] == ["bookmarkStart", "r", "bookmarkEnd"] + assert second[0].get(f"{W}id") == second[2].get(f"{W}id") == "2" + for bad in ("", "1abc", "a-b", "a b", "x" * 41, "ž"): + with pytest.raises(InternalError): + bookmarks.pair(bad) + with pytest.raises(InternalError): + bookmarks.pair("LD_FIRST") # Word compares names without regard to case + assert bookmarks.pair("x" * 40) + + +def test_hyperlinks_are_internal_by_anchor_or_external_by_relationship() -> None: + relationships = package.Relationships() + first = relationships.add_external("https://example.com/a b?q=ž") + again = relationships.add_external("https://example.com/a b?q=ž") + other = relationships.add_external("mailto:x@example.com") + assert (first, again, other) == ("rId1", "rId1", "rId2") + external = ooxml.hyperlink_external(first, ooxml.text_run("here")) + assert external.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id") == "rId1" + internal = ooxml.hyperlink_internal("ld_x", ooxml.text_run("there"), tooltip="t") + assert (internal.get(f"{W}anchor"), internal.get(f"{W}tooltip")) == ("ld_x", "t") + listed = ET.fromstring(relationships.to_xml()) + assert [(r.get("Id"), r.get("Target"), r.get("TargetMode")) for r in listed] == [ + ("rId1", "https://example.com/a%20b?q=%C5%BE", "External"), ("rId2", "mailto:x@example.com", "External")] + + +@pytest.mark.parametrize(("url", "expected"), [ + ("https://example.com/a?b=c&d=e#f", "https://example.com/a?b=c&d=e#f"), + ("https://example.com/a b", "https://example.com/a%20b"), + ("https://example.com/%20already", "https://example.com/%20already"), + ("https://example.com/ž", "https://example.com/%C5%BE"), + ('https://example.com/"<>\\^`{|}', "https://example.com/%22%3C%3E%5C%5E%60%7B%7C%7D"), + (" https://example.com ", "https://example.com"), + ("", None), (" ", None), ("https://e/\x01", None), ("https://e/\x7f", None), ("https://e/a\nb", None), + ("https://e/\ud800", None), +]) +def test_a_relationship_target_is_a_valid_uri_or_none(url: str, expected: str | None) -> None: + assert package.external_target(url) == expected + assert package.Relationships().add_external(url) == (None if expected is None else "rId1") + + +def test_a_relationship_value_is_escaped_in_the_part() -> None: + relationships = package.Relationships() + relationships.add("http://x/\"&<", "a&b\"c", external=True) + (entry,) = ET.fromstring(relationships.to_xml()) + assert (entry.get("Type"), entry.get("Target")) == ("http://x/\"&<", "a&b\"c") + + +def test_a_part_added_twice_is_a_bug() -> None: + bundle = package.Package() + bundle.add("word/x.xml", package.CT_XML, b"") + with pytest.raises(InternalError): + bundle.add("word/x.xml", package.CT_XML, b"") + + +# -- The API and the command line ------------------------------------------------------------ + + +def test_docx_is_a_format_selected_by_its_extension() -> None: + assert FORMATS["docx"] == ("docx",) + assert format_for_path("contract.docx") == "docx" + assert format_for_path("CONTRACT.DOCX") == "docx" + + +def test_the_result_carries_the_bytes_and_an_empty_output() -> None: + result = render(SOURCE, RenderOptions(format="docx")) + assert result.format == "docx" + assert result.output == "" + assert isinstance(result.binary, bytes) and result.binary.startswith(b"PK") + assert result.ok + assert render(SOURCE, format="html").binary is None and render(SOURCE, format="text").binary is None + + +def test_render_file_renders_docx(tmp_path: Path) -> None: + source = tmp_path / "contract.lgd" + source.write_text(SOURCE, encoding="utf-8") + assert render_file(source, format="docx").binary == render(SOURCE, format="docx").binary + + +def test_diagnostics_are_the_same_as_for_the_other_formats() -> None: + document = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + assert render(document, format="docx").diagnostics == render(document, format="text").diagnostics + + +def test_the_docx_writer_can_be_used_directly() -> None: + from legaldown_render.style import load_style + + result = render(SOURCE, format="text") + data = DocxWriter(load_style("default"), generator="test 1").write(result.tree) + assert parts(data)["docProps/app.xml"].count(b"test 1") == 1 + + +def test_the_cli_writes_a_docx_file(tmp_path: Path) -> None: + target = tmp_path / "out.docx" + assert main([FEATURES, "-q", "-o", str(target)]) == 0 + assert target.read_bytes() == render((DOCUMENTS / "features.lgd").read_text(encoding="utf-8"), + format="docx").binary + other = tmp_path / "other.bin" + assert main([FEATURES, "-q", "-f", "docx", "-o", str(other)]) == 0 + assert other.read_bytes() == target.read_bytes() + + +def test_an_uppercase_extension_selects_docx(tmp_path: Path) -> None: + target = tmp_path / "OUT.DOCX" + assert main([FEATURES, "-q", "-o", str(target)]) == 0 + assert target.read_bytes().startswith(b"PK") + + +def test_docx_to_standard_output_needs_dash_and_writes_bytes() -> None: + done = run_cli(FEATURES, "-q", "-f", "docx", "-o", "-") + assert done.returncode == 0 and done.stderr == b"" + assert done.stdout.startswith(b"PK") + assert done.stdout == render((DOCUMENTS / "features.lgd").read_text(encoding="utf-8"), format="docx").binary + from_extension = run_cli(FEATURES, "-q", "-f", "docx", "-o", "-") + assert from_extension.stdout == done.stdout + + +def test_docx_without_output_is_a_usage_error_not_a_stream_of_bytes() -> None: + done = run_cli(FEATURES, "-q", "-f", "docx") + assert done.returncode == 2 and done.stdout == b"" + assert b"-o FILE.docx" in done.stderr and b"-o -" in done.stderr + + +def test_docx_is_not_written_to_a_terminal(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(sys.stdout, "isatty", lambda: True) + with pytest.raises(SystemExit) as exit_info: + main([FEATURES, "-q", "-f", "docx", "-o", "-"]) + assert exit_info.value.code == 2 + assert "refusing to write a DOCX file to a terminal" in capsys.readouterr().err + + +def test_a_style_word_cannot_express_exits_with_status_2_and_writes_no_file(tmp_path: Path) -> None: + target = tmp_path / "out.docx" + done = run_cli(FEATURES, "-q", "-o", str(target), "--set", "page.margin=5%") + assert done.returncode == 2 + assert b"page.margin: uses '%'" in done.stderr + assert not target.exists() + assert run_cli(FEATURES, "-q", "-o", str(tmp_path / "out.html"), "--set", "page.margin=5%").returncode == 0 + + +def test_the_cli_cannot_write_a_docx_into_a_missing_folder(tmp_path: Path) -> None: + done = run_cli(FEATURES, "-q", "-o", str(tmp_path / "no" / "out.docx")) + assert done.returncode == 1 and b"cannot write" in done.stderr + + +def test_text_and_html_to_standard_output_with_dash_still_work_and_name_no_file(tmp_path: Path) -> None: + done = run_cli(FEATURES, "-q", "-f", "text", "-o", "-", cwd=tmp_path) + assert done.returncode == 0 and done.stdout.startswith(b"Services Agreement") + assert list(tmp_path.iterdir()) == [] + + +def test_an_untrusted_render_makes_a_docx_too(tmp_path: Path) -> None: + result = render(SOURCE, format="docx", untrusted=True, document_root=tmp_path) + assert result.binary is not None and result.ok diff --git a/tests/test_errors.py b/tests/test_errors.py index 1968390..f446398 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -34,9 +34,9 @@ def test_unknown_format_raises_options_error() -> None: with pytest.raises(OptionsError, match="unknown output format 'pdf'"): render("# x\n", RenderOptions(format="pdf")) with pytest.raises(OptionsError): - render("# x\n", format="docx") + render("# x\n", format="odt") with pytest.raises(RenderError): - render("# x\n", format="docx") + render("# x\n", format="pdf") def test_invalid_style_through_the_api_is_a_render_error() -> None: From 0b9f73a77b5ec9c34225fe66e2b998572430dd57 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:21:38 +0000 Subject: [PATCH 069/118] Verify DOCX against the OOXML schemas and LibreOffice Every part of the generated documents is validated against the ISO/IEC 29500-4 transitional schemas (lxml, a dev dependency), and the package is checked for content types and relationship targets. The schemas are fetched by tests/schemas/fetch.py with pinned checksums, not stored: no licence that allows redistribution could be established (tests/schemas/README.md). tests/lo/uno_export.py, run by the system Python that has uno, opens documents in headless LibreOffice, updates fields and indexes, and writes the text, the header, the footer and the page count; the libreoffice tests compare them with what was meant. Both kinds of test skip when their tool is missing and fail instead when LEGALDOWN_REQUIRE_DOCX_TOOLS is set. A docx CI job installs libreoffice-writer and python3-uno, fetches the schemas and runs them. --- .github/workflows/ci.yml | 23 ++++++ .gitignore | 3 + pyproject.toml | 7 +- tests/lo/uno_export.py | 110 +++++++++++++++++++++++++++ tests/schemas/README.md | 45 +++++++++++ tests/schemas/fetch.py | 73 ++++++++++++++++++ tests/test_docx_libreoffice.py | 134 +++++++++++++++++++++++++++++++++ tests/test_docx_validation.py | 97 ++++++++++++++++++++++++ 8 files changed, 490 insertions(+), 2 deletions(-) create mode 100644 tests/lo/uno_export.py create mode 100644 tests/schemas/README.md create mode 100644 tests/schemas/fetch.py create mode 100644 tests/test_docx_libreoffice.py create mode 100644 tests/test_docx_validation.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55c2ff4..174c6e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,29 @@ jobs: env: LEGALDOWN_SPEC_DIR: .legaldown-spec + docx: + # The DOCX writer is checked against the ISO/IEC 29500 schemas (fetched, not stored: tests/schemas/README.md) + # and against LibreOffice, which opens each document, updates its fields and says what it shows + # (tests/lo/uno_export.py runs under the system Python, which python3-uno provides, not the virtual + # environment's). A missing tool fails the job instead of skipping the tests. + name: DOCX (schemas and LibreOffice) + runs-on: ubuntu-latest + env: + LEGALDOWN_REQUIRE_DOCX_TOOLS: "1" + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + cache: pip + - run: | + sudo apt-get update + sudo apt-get install -y libreoffice-writer python3-uno + - run: pip install -e ".[dev]" + - run: python tests/schemas/fetch.py + - run: pytest -q -m ooxml_schema + - run: pytest -q -m libreoffice + validator-main: # Surfaces changes in legaldown-validator's model before they are released # (docs/decisions/0001). Informational: a failure here is a heads-up. diff --git a/.gitignore b/.gitignore index c2225b2..fc9e61f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ build/ dist/ .pytest_cache/ .ruff_cache/ + +# OOXML schemas fetched for the DOCX tests (tests/schemas/README.md) +tests/schemas/ooxml/ diff --git a/pyproject.toml b/pyproject.toml index eed7a93..b1573f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" name = "legaldown-render" # Single-sourced from src/legaldown_render/__init__.py. dynamic = ["version"] -description = "Reference renderer for the LegalDown legal document format: HTML and plain text, DOCX and PDF planned" +description = "Reference renderer for the LegalDown legal document format: HTML and plain text, DOCX in progress, PDF planned" readme = "README.md" requires-python = ">=3.11" license = "MIT" @@ -40,7 +40,8 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=9.2", "ruff>=0.17", "build>=1.6"] +# lxml validates the DOCX parts against the OOXML schemas in the tests (tests/schemas/README.md). +dev = ["pytest>=9.2", "ruff>=0.17", "build>=1.6", "lxml>=6.1"] [project.scripts] legaldown-render = "legaldown_render.cli:main" @@ -74,6 +75,8 @@ addopts = "-ra --strict-markers" pythonpath = ["src"] markers = [ "conformance: renders the LegalDown specification's examples and fixtures (needs a checkout of the specification)", + "ooxml_schema: validates DOCX parts against the ISO/IEC 29500 schemas (needs lxml and the schemas: python tests/schemas/fetch.py)", + "libreoffice: opens DOCX files in LibreOffice and compares what it shows (needs soffice and python3-uno)", ] # A deprecated legaldown-validator API fails the tests: the next minor # release removes it, and the dependency range admits only one minor version. diff --git a/tests/lo/uno_export.py b/tests/lo/uno_export.py new file mode 100644 index 0000000..cae746b --- /dev/null +++ b/tests/lo/uno_export.py @@ -0,0 +1,110 @@ +#!/usr/bin/python3 +"""Open DOCX files in headless LibreOffice through UNO, update every field and index, +and write what the document says: one ``NAME.txt`` per file (the body as plain text, +UTF-8, as LibreOffice's text export writes it) and one ``NAME.json`` with the page +count and the text of the running header and footer. + +Run it with the system Python that has the ``uno`` module (``/usr/bin/python3`` once +``python3-uno`` is installed), **not** with the project's virtual environment: + + /usr/bin/python3 tests/lo/uno_export.py OUTDIR FILE.docx [FILE.docx ...] + +The same LibreOffice instance opens every file, so a test module pays the start-up +once. Exit status 0 when every file was exported. +""" +from __future__ import annotations + +import contextlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import uuid + +import uno # type: ignore[import-not-found] # noqa: I001 - provided by python3-uno, not by pip +from com.sun.star.beans import PropertyValue # type: ignore[import-not-found] + + +def _value(name: str, value: object) -> PropertyValue: + prop = PropertyValue() + prop.Name, prop.Value = name, value + return prop + + +def _connect(pipe: str, process: subprocess.Popen[bytes]) -> object: + local = uno.getComponentContext() + resolver = local.ServiceManager.createInstanceWithContext("com.sun.star.bridge.UnoUrlResolver", local) + for _ in range(120): + if process.poll() is not None: + raise SystemExit("soffice exited before it accepted a connection") + try: + return resolver.resolve(f"uno:pipe,name={pipe};urp;StarOffice.ComponentContext") + except Exception: # not listening yet + time.sleep(0.5) + raise SystemExit("soffice did not accept a connection in time") + + +def _update(document: object) -> None: + """Update every index and field, twice: page numbers settle after the first layout.""" + for _ in range(2): + indexes = document.getDocumentIndexes() + for position in range(indexes.getCount()): + indexes.getByIndex(position).update() + document.getTextFields().refresh() + document.refresh() + + +def _page_texts(document: object) -> tuple[str, str]: + """The text of the first page style's header and footer ("" when it has none).""" + styles = document.getStyleFamilies().getByName("PageStyles") + page = styles.getByName(document.getCurrentController().getViewCursor().PageStyleName) + header = page.HeaderText.getString() if page.HeaderIsOn else "" + footer = page.FooterText.getString() if page.FooterIsOn else "" + return header, footer + + +def export(out_dir: str, files: list[str]) -> None: + profile = tempfile.mkdtemp(prefix="lo-profile-") + pipe = f"legaldown_{uuid.uuid4().hex}" + process = subprocess.Popen( + ["soffice", "--headless", "--invisible", "--norestore", "--nologo", "--nodefault", + f"--accept=pipe,name={pipe};urp;", f"-env:UserInstallation=file://{profile}"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + desktop = None + try: + context = _connect(pipe, process) + desktop = context.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", context) + for path in files: + path = os.path.abspath(path) + stem = os.path.splitext(os.path.basename(path))[0] + document = desktop.loadComponentFromURL(uno.systemPathToFileUrl(path), "_blank", 0, + (_value("Hidden", True),)) + if document is None: + raise SystemExit(f"LibreOffice could not open {path}") + try: + _update(document) + header, footer = _page_texts(document) + document.storeToURL(uno.systemPathToFileUrl(os.path.join(out_dir, f"{stem}.txt")), + (_value("FilterName", "Text (encoded)"), _value("FilterOptions", "UTF8"))) + pages = document.getCurrentController().PageCount + with open(os.path.join(out_dir, f"{stem}.json"), "w", encoding="utf-8") as handle: + json.dump({"header": header, "footer": footer, "pages": pages}, handle) + finally: + document.close(True) + finally: + if desktop is not None: + with contextlib.suppress(Exception): # already gone + desktop.terminate() + time.sleep(0.5) + process.kill() + process.wait() + shutil.rmtree(profile, ignore_errors=True) + + +if __name__ == "__main__": + if len(sys.argv) < 3: + raise SystemExit(__doc__) + export(sys.argv[1], sys.argv[2:]) diff --git a/tests/schemas/README.md b/tests/schemas/README.md new file mode 100644 index 0000000..83dc158 --- /dev/null +++ b/tests/schemas/README.md @@ -0,0 +1,45 @@ +# OOXML schemas for the DOCX tests + +`tests/test_docx_validation.py` validates every part of the generated documents against the ISO/IEC +29500-4 (transitional) schemas. The schemas are **not stored in this repository**. Fetch them with + +``` +python tests/schemas/fetch.py # writes tests/schemas/ooxml/, which git ignores +``` + +(standard library only; `LEGALDOWN_OOXML_SCHEMAS=DIR` points the tests at another copy). The tests +skip when they are missing, and fail instead when `LEGALDOWN_REQUIRE_DOCX_TOOLS` is set, as the `docx` +job of `.github/workflows/ci.yml` does after fetching them. + +## What is fetched, and from where + +| Files | Source | Used for | +|---|---|---| +| `wml.xsd` and the schemas it imports (`dml-*.xsd`, `shared-*.xsd`, ...) | `OfficeOpenXML-XMLSchema-Transitional.zip` in [ECMA-376 5th edition, Part 4](https://ecma-international.org/wp-content/uploads/ECMA-376-4_5th_edition_december_2016.zip) (the transitional schemas of ISO/IEC 29500-4:2016) | `document.xml`, `styles.xml`, `numbering.xml`, `settings.xml`, headers and footers; `app.xml` (`shared-documentPropertiesExtended.xsd`) | +| `opc-contentTypes.xsd`, `opc-relationships.xsd` | `OpenPackagingConventions-XMLSchema.zip` in [ECMA-376 5th edition, Part 2](https://ecma-international.org/wp-content/uploads/ECMA-376-2_5th_edition_december_2021.zip) | `[Content_Types].xml` and every `.rels` | +| `xml.xsd` | [W3C](https://www.w3.org/2001/xml.xsd) | `xml:space` and `xml:lang`, which `wml.xsd` imports without saying where they are | +| `ld-wml.xsd` | written by `fetch.py` (ours, a few lines) | a wrapper that loads `xml.xsd` before `wml.xsd`, as libxml2 needs | + +Each download is checked against the SHA-256 recorded in `fetch.py`; a changed file stops the script. + +**Not validated:** `docProps/core.xml`. Its schema (`opc-coreProperties.xsd`) imports the Dublin Core +schemas from `dublincore.org` over the network. The tests check that part's shape directly +(tests/test_docx_package.py). Everything else is validated; the package itself (content types for +every part, every relationship target present, `[Content_Types].xml` first) is checked in +`tests/docx_support.py`. + +## Why they are fetched and not vendored + +Committing them would need a licence that allows redistribution. None is clear, so they are fetched: + +* The XSD files carry no licence or copyright header, and the standards' own text (the Part 4 and Part 2 PDFs + in the same downloads) contains no redistribution grant that could be extracted from it. +* The schemas are published, free of charge, by Ecma International (ECMA-376) and, as ISO/IEC 29500, by + ISO/IEC, whose webstore terms apply to that copy. Both are copyrighted standards texts. Ecma's general + practice is to allow copying with the notice retained, but that was not confirmed here for these files, and + the ISO/IEC terms differ. +* A copy of the same schemas was available in this environment under `/mnt/skills/public/docx/`. It is part of + a skill whose licence forbids copying and distributing it, so it was neither used nor copied. + +If the maintainers clear the licence (for example by asking Ecma), the files can be committed here and +`fetch.py` dropped; nothing else depends on where they live. diff --git a/tests/schemas/fetch.py b/tests/schemas/fetch.py new file mode 100644 index 0000000..f9240ca --- /dev/null +++ b/tests/schemas/fetch.py @@ -0,0 +1,73 @@ +"""Download the OOXML schemas the DOCX tests validate against. + +Usage: ``python tests/schemas/fetch.py [DIRECTORY]`` (default: ``tests/schemas/ooxml``, +which is ignored by git). Standard library only. See tests/schemas/README.md for what +is fetched and why it is not stored in the repository. + +Every download is checked against the SHA-256 it had when the schemas were last +reviewed; a different file stops the script, so a changed upstream is noticed rather +than trusted. +""" +from __future__ import annotations + +import hashlib +import io +import sys +import time +import urllib.request +import zipfile +from pathlib import Path + +#: (url, sha256 of the download, the inner zip to open, the files to take from it) +SOURCES = ( + ("https://ecma-international.org/wp-content/uploads/ECMA-376-4_5th_edition_december_2016.zip", + "bd25da1109f73762356596918bf5ff8b74a1331642dba5f1c1d1dfc6bed34ecd", + "OfficeOpenXML-XMLSchema-Transitional.zip", None), + ("https://ecma-international.org/wp-content/uploads/ECMA-376-2_5th_edition_december_2021.zip", + "1d489dc491168ea1f9e9a59063acc8dd5f02b4ad1d21aa7ec19ba9a58d020c70", + "OpenPackagingConventions-XMLSchema.zip", ("opc-contentTypes.xsd", "opc-relationships.xsd")), +) +XML_XSD = ("https://www.w3.org/2001/xml.xsd", "61960fb3131e38022caad5360e2f33a3382578ab3c80cd58bd74320ede61b20c") + +#: wml.xsd imports the XML namespace without saying where it is; libxml2 needs it loaded first. +WRAPPER = """ + + + + +""" + + +def download(url: str, sha256: str) -> bytes: + last: Exception | None = None + for attempt in range(4): + try: + with urllib.request.urlopen(url, timeout=120) as response: # noqa: S310 - fixed https URLs + data = response.read() + break + except OSError as error: + last = error + time.sleep(2 * (attempt + 1)) + else: + raise SystemExit(f"cannot download {url}: {last}") + found = hashlib.sha256(data).hexdigest() + if found != sha256: + raise SystemExit(f"{url}: expected SHA-256 {sha256}, got {found}; review the change before updating this script") + return data + + +def fetch(target: Path) -> None: + target.mkdir(parents=True, exist_ok=True) + for url, sha256, inner, wanted in SOURCES: + with zipfile.ZipFile(io.BytesIO(download(url, sha256))) as outer, \ + zipfile.ZipFile(io.BytesIO(outer.read(inner))) as schemas: + for name in schemas.namelist(): + if name.endswith(".xsd") and (wanted is None or name in wanted): + (target / Path(name).name).write_bytes(schemas.read(name)) + (target / "xml.xsd").write_bytes(download(*XML_XSD)) + (target / "ld-wml.xsd").write_text(WRAPPER, encoding="utf-8") + print(f"schemas written to {target}") + + +if __name__ == "__main__": + fetch(Path(sys.argv[1]) if len(sys.argv) > 1 else Path(__file__).parent / "ooxml") diff --git a/tests/test_docx_libreoffice.py b/tests/test_docx_libreoffice.py new file mode 100644 index 0000000..7176f40 --- /dev/null +++ b/tests/test_docx_libreoffice.py @@ -0,0 +1,134 @@ +"""LibreOffice as the oracle: each document is opened headless through UNO (tests/lo/uno_export.py, +run by the system Python that has ``uno``), its fields and indexes are updated, and what it shows is +compared with what was meant: the words in order, the running header and footer, the page count. +Skipped when ``soffice`` or ``python3-uno`` is missing (it is not on Windows, and not in the test +jobs that do not install it); a failure instead when ``LEGALDOWN_REQUIRE_DOCX_TOOLS`` is set, as in +the CI job that installs them (``pytest -m libreoffice``).""" +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET + +import pytest +from conftest import DOCUMENTS +from docx_support import Exported, libreoffice_export, parts, require_libreoffice, texts + +from legaldown_render import render + +pytestmark = pytest.mark.libreoffice + +ROUND_TRIP = """--- +legaldown: '0.2' +title: Round Trip +subtitle: Sub +language: cs +version: '1.2' +effective_date: 2026-03-04 +--- + +Preamble ž€. + +# First + +Body with *emphasis*, **strong** and `code`.\\ +Second line. + +## Second + +- item one +- item two + +> A quote + +``` +code line 1 + code line 2 +``` + +| H1 | H2 | +|----|----| +| a | b | +""" + +LONG = "---\ntitle: Long\n---\n\n" + "".join(f"# Heading {n}\n\nParagraph {n} " + "word " * 120 + "\n\n" for n in range(40)) + +HOSTILE = "---\nlegaldown: '0.2'\ntitle: \"a\\x01b\"\n---\n\n# H\x01\x08\x0b\x0c\x1f\x7f\x85x\n\nbody\x01 <&> ]]>\n" + + +def docx(source: str, **settings: object) -> bytes: + result = render(source, format="docx", **settings) + assert result.binary is not None + return result.binary + + +def expected_lines(data: bytes) -> list[str]: + """What the paragraphs of the document part say, one line per line, without blank ones.""" + lines = [] + for paragraph in texts(ET.fromstring(parts(data)["word/document.xml"])): + lines += paragraph.split("\n") + return [line for line in lines if line] + + +@pytest.fixture(scope="module") +def shown() -> dict[str, Exported]: + require_libreoffice() + documents = { + "round_trip": docx(ROUND_TRIP, overrides={"page.header.left": "{title}", "page.header.center": "{date}", + "page.header.right": "v{version}, p{page}"}), + "features": docx((DOCUMENTS / "features.lgd").read_text(encoding="utf-8")), + "template": docx((DOCUMENTS / "template.lgd").read_text(encoding="utf-8"), style="continental"), + "long": docx(LONG), + "hostile": docx(HOSTILE), + "no_footer": docx(ROUND_TRIP, overrides={"page.footer": {"center": ""}}), + "landscape": docx(ROUND_TRIP, overrides={"page.size": "A4 landscape", "page.footer.right": "{page}"}), + } + return libreoffice_export(documents) + + +def test_the_words_come_back_in_order(shown: dict[str, Exported]) -> None: + assert shown["round_trip"].lines == [ + "Round Trip", "Sub", "Preamble ž€.", "First", "Body with emphasis, strong and code.", "Second line.", + "Second", "item one", "item two", "A quote", "code line 1", " code line 2", "H1\tH2", "a\tb"] + + +@pytest.mark.parametrize("name", ["features", "template"]) +def test_the_test_documents_read_as_their_paragraphs(shown: dict[str, Exported], name: str) -> None: + data = docx((DOCUMENTS / f"{name}.lgd").read_text(encoding="utf-8"), **({"style": "continental"} + if name == "template" else {})) + assert [line for line in shown[name].lines if line] == expected_lines(data) + + +def test_the_header_shows_the_templates_and_the_page_field(shown: dict[str, Exported]) -> None: + assert shown["round_trip"].header == "Round Trip\t4. března 2026\tv1.2, p1" + + +def test_the_default_footer_counts_the_pages(shown: dict[str, Exported]) -> None: + footer = shown["round_trip"].footer + assert re.fullmatch(r"1 / 1", footer), footer + assert shown["round_trip"].pages == 1 + assert shown["round_trip"].header.count("\t") == 2 + + +def test_the_page_count_field_follows_the_layout(shown: dict[str, Exported]) -> None: + long = shown["long"] + assert long.pages > 2 + match = re.fullmatch(r"(\d+) / (\d+)", long.footer) + assert match is not None, long.footer + assert int(match.group(2)) == long.pages + assert 1 <= int(match.group(1)) <= long.pages + + +def test_no_footer_means_none(shown: dict[str, Exported]) -> None: + assert shown["no_footer"].footer == "" + + +def test_a_landscape_page_with_a_right_aligned_page_number_opens(shown: dict[str, Exported]) -> None: + assert shown["landscape"].footer == "\t1 / 1\t1" + assert shown["landscape"].lines == shown["round_trip"].lines + + +def test_hostile_text_opens_with_the_unwritable_characters_dropped(shown: dict[str, Exported]) -> None: + lines = shown["hostile"].lines + assert lines[0] == "ab" + assert lines[1] == "Hx" + assert lines[2] == "body <&> ]]>" diff --git a/tests/test_docx_validation.py b/tests/test_docx_validation.py new file mode 100644 index 0000000..55b7268 --- /dev/null +++ b/tests/test_docx_validation.py @@ -0,0 +1,97 @@ +"""Every part of every generated document validates against the ISO/IEC 29500-4 transitional +schemas (WordprocessingML, content types, relationships, extended properties), and the package +is sound: each relationship targets a part, each part has a content type. The schemas are +fetched, not stored (tests/schemas/README.md); the tests skip without them, and fail instead when +``LEGALDOWN_REQUIRE_DOCX_TOOLS`` is set, as in CI.""" +from __future__ import annotations + +import random + +import pytest +from conftest import DOCUMENTS +from docx_support import parts, require_schemas, schema_for, validation_errors +from test_docx_escaping import FREE_TEXT_SETTINGS, document + +from legaldown_render import render + +pytestmark = pytest.mark.ooxml_schema + +DOCUMENT_NAMES = ("features", "template") +STYLES = ("default", "continental", "outline") + + +def make(source: str, **settings: object) -> bytes: + result = render(source, format="docx", **settings) + assert result.binary is not None + return result.binary + + +@pytest.fixture(autouse=True) +def _schemas() -> None: + require_schemas() + + +@pytest.mark.parametrize("style", STYLES) +@pytest.mark.parametrize("name", DOCUMENT_NAMES) +def test_the_test_documents_validate_under_every_built_in_style(name: str, style: str) -> None: + source = (DOCUMENTS / f"{name}.lgd").read_text(encoding="utf-8") + assert validation_errors(make(source, style=style)) == [] + + +def test_every_xml_part_has_a_schema_except_the_core_properties() -> None: + names = parts(make("---\ntitle: T\n---\n# A\n")) + assert {name: schema_for(name) for name in names} == { + "[Content_Types].xml": "types", "_rels/.rels": "rels", "docProps/core.xml": None, "docProps/app.xml": "app", + "word/document.xml": "wml", "word/_rels/document.xml.rels": "rels", "word/styles.xml": "wml", + "word/numbering.xml": "wml", "word/settings.xml": "wml", "word/footer1.xml": "wml"} + + +@pytest.mark.parametrize("overrides", [ + {}, + {"page.footer": {"center": ""}}, + {"page.header.left": "{title}", "page.header.center": "{date}", "page.header.right": "{page}/{pages} {version}"}, + {"page.header.center": "C", "page.footer": {"left": "L", "center": "", "right": "R"}}, + {"page.header.right": "R", "page.footer.center": ""}, + {"docx.update_fields_on_open": False}, + {"page.size": "A4 landscape", "page.margin": "1in 2in 3in 4in"}, + {"page.size": "letter", "page.margin": "0"}, + {"page.size": "10cm 20cm", "typography.font_size": "9pt", "typography.line_height": "14pt"}, + {"typography.justify": True, "typography.color": "rgb(10, 20, 30)", "typography.link_color": "hsl(200, 50%, 40%)"}, + {"definitions.style": "small-caps", "definitions.term_style": "underline"}, + {"headings": {1: {"transform": "small-caps", "align": "center", "italic": True, "weight": "400"}}}, + {"typography.font_family": "'Open Sans', sans-serif"}, +]) +def test_the_settings_that_shape_the_parts_keep_them_valid(overrides: dict) -> None: + source = ("---\nlegaldown: '0.2'\ntitle: Agreement\nversion: '1'\neffective_date: 2026-01-02\n---\n\n" + "# One\n\ntext\n\n## Two\n\ntext\n") + assert validation_errors(make(source, overrides=overrides)) == [] + + +def test_an_empty_document_validates() -> None: + assert validation_errors(make("---\ntitle: T\n---\n")) == [] + + +def test_the_schema_check_catches_a_wrong_order_in_the_settings() -> None: + """The validation is not vacuous: ``updateFields`` after ``compat`` (the order both prototypes + had) is rejected.""" + import io + import zipfile + + original = parts(make("---\ntitle: T\n---\n# A\n")) + settings = original["word/settings.xml"].decode("utf-8") + start = settings.index("", start) + 2 + element = settings[start:end] + broken = settings.replace(element, "").replace("", element + "") + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for name, content in original.items(): + archive.writestr(name, broken.encode("utf-8") if name == "word/settings.xml" else content) + assert any("updateFields" in problem for problem in validation_errors(buffer.getvalue())) + + +@pytest.mark.parametrize("seed", range(15)) +def test_documents_of_any_text_validate(seed: int) -> None: + rnd = random.Random(1000 + seed) + overrides = {key: "".join(rnd.choice("ab \x01\x00\ud800<&ž") for _ in range(10)) + "{page}" for key in FREE_TEXT_SETTINGS} + assert validation_errors(make(document(rnd), overrides=overrides)) == [] From db466d3bda14c5848f12a005af06ce1c7f29100c Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:22:06 +0000 Subject: [PATCH 070/118] Pin and document the numbering structure's edge cases A list's first number is items[0].number (List.start): sibling lists nested in one item go on counting, which is now documented and tested, including that alternatives never join across lists. Document that List.depth only selects the format and that lists in quotes and drafting notes are built on the section, that paragraph alternatives are not separated by a list or table, and that sections share by number. _Counter.next now returns the first holder of a shared number itself. The invariant test also runs with every label written as {path}, checks that an alternative stands where its first does, and that numbers count on without gaps; more tests cover three-way alternatives, subsections only the later alternative has, a sixth heading level and mixed list families. --- docs/architecture.md | 7 +- src/legaldown_render/resolve/resolver.py | 47 ++--- src/legaldown_render/tree.py | 52 ++++-- tests/test_numbering_structure.py | 217 +++++++++++++++++++++-- 4 files changed, 261 insertions(+), 62 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index a5f9352..d754af8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -132,7 +132,8 @@ Guessing at lost structure was tried, and it traded each gap for new bugs. a skipped heading level counts as 1. The style only formats them, the n-th part of a number with the n-th level format. For paragraphs and list items, **alternatives** follow the validator's rule, applied the same way: an alternative directly follows its - sibling, has the same identifier and a valid condition, and its full presence (its own condition + sibling among the units numbered together (a list or table between two top-level paragraphs does + not separate them), has the same identifier and a valid condition, and its full presence (its own condition and every enclosing one) excludes that of every unit already holding the number (§15.4, the validator's `exclusive()`). It then labels list items by depth and top-level paragraphs, and allocates output anchors. Every anchor @@ -181,10 +182,10 @@ carries the structure behind every number, in terms that name no output format: | `RenderTree.formats` | `NumberingFormats` | The formats the labels were made with, after the style is resolved: `scheme`; `headings`, one `LevelFormat` (counter, `label` and `ref` templates) per heading level (empty under `none`); `enumerated`, the formats by list depth (cycling); `renumber`, the one format of ordered lists (None under `enumeration.ordered: enumerate`); `paragraphs` (None unless numbered); `reference`, `references.format` | | `Section`, `ListItem`, `Paragraph` | `unit` | An id, unique in the document and equal on every render: `s` for the k-th section, `u` for the k-th item or top-level paragraph of a section in document order, the preamble first. Not the anchor. Every section, every item (in a quote too) and every top-level paragraph of a section has one, numbered or not, since an unnumbered one can still be the target of a reference; a paragraph inside an item, or in the preamble, is not a unit (None) | | same | `number` | The unit's own counter value (the last part of "2.1" is 1; the item's position in its list; the paragraph's in its section). None when the unit is not numbered: any section under `none`, an item of a list that is not enumerated, a paragraph when the style does not number them | -| same | `parent_unit` | The unit its designation is built on. For a section, the nearest section above it with fewer number parts (a skipped heading level counts as 1 and is no unit: "1.2.1.1" under "1.2" has a gap of one level, which `level` shows); for an item of a list not nested in an item, and for a paragraph, its section (None in the preamble); for a nested item, the item. A list in a quote inside an item is built on the section, as its designation is | +| same | `parent_unit` | The unit its designation is built on. For a section, the nearest section above it with fewer number parts (a skipped heading level counts as 1 and is no unit: "1.2.1.1" under "1.2" has a gap of one level, which `level` shows); for an item of a list not nested in an item, and for a paragraph, its section (None in the preamble); for a nested item, the item. A list in a quote or drafting note, even inside an item, is built on the section, as its designation is; to see where a list nests, follow `parent_unit` (an item or a section), never `depth` | | same | `shares_number_with` | For a later alternative that shares its number (§15.8, §15.4), the unit of the **first**; None otherwise, and always None on an unnumbered unit. Every alternative has a unit of its own. Sections share by number, as the validator numbers them: the subsections of a later alternative section repeat the numbers of the earlier one's ("1.1" again), and each repeat shares the number of the first section that had it | | `List` | `family` | `renumber` (ordered, counted 1., 2., … whatever the source says), `enumerated` (counted in the format of the nesting depth) or `bullet` (no numbers; `enumerated` is False) | -| `List` | `depth`, `format` | The nesting depth the format was chosen at (a list in a quote in an item is one deeper) and the `LevelFormat` itself, so the entry of `formats` is never selected again; None for `bullet` | +| `List` | `depth`, `format`, `start` | The nesting depth the format was chosen at (a list in a quote in an item is one deeper), which selects the format and nothing else; the `LevelFormat` itself, so the entry of `formats` is never selected again (None for `bullet`); and `start`, the number of the first item (`items[0].number`). A list is not always counted from 1: lists nested directly in one item share a counter per way of writing a designation (counter style and `ref`), so a second list after a paragraph, or a renumbered list beside an enumerated one with the same designations, continues the first's numbers (a, b, then c) and no designation repeats; top-level lists and lists in quotes start at 1. A list has no id of its own: name it by `items[0].unit` | | `CrossRef` | `target_unit`, `section_unit` | The unit the identifier is on, and the section that qualifies the designation (for a section target, itself). With alternatives sharing the identifier, `target_unit` is the first, as the anchor is. For an unnumbered item or paragraph the unit is still the target and `text` is the section's designation. Under `none` both are set as usual and the section's `number` is None. A reference that does not resolve is a `FailureMarker`, and so is a `{{ref:}}` to an attachment id (§5.6); an attachment is reached with `{{attach:}}`, a `Value` whose `href` is `#` and the `AttachmentPart.anchor`. Both are None for a copy that no longer links (`unlinked`, as in the table of contents) | A designation is the parent's designation followed by the unit's `ref` template filled in diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index a9333f5..ce2ec86 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -771,8 +771,9 @@ def _structure_section(self, section: Section, presence: Presence) -> tuple[Bloc if isinstance(block, Paragraph) and block.top_level: # Alternative paragraphs share a number, as sections do (§15.8). own = self._own_presence(block.condition, presence) - number = paragraphs.next(block.anchor_id, own) - block = self._number_paragraph(block, section, paragraphs, number, + unit = self._unit() + number, shares = paragraphs.next(block.anchor_id, own, unit) + block = self._number_paragraph(block, section, paragraphs, number, unit, shares, presence if own is None else own) out.append(self._structure(block, section=section, depth=0, presence=presence)) return tuple(out) @@ -784,10 +785,9 @@ def _own_presence(self, condition: str, enclosing: Presence) -> Presence | None: return enclosing | own if own else None def _number_paragraph(self, block: Paragraph, section: Section, counter: _Counter, number: int, - presence: Presence) -> Paragraph: + unit: str, shares: str | None, presence: Presence) -> Paragraph: numbering = self.style.paragraphs base = section.designation or "" - unit = self._unit() if numbering.numbered: fmt = numbering.format n = format_counter(number, fmt.counter) @@ -800,7 +800,6 @@ def _number_paragraph(self, block: Paragraph, section: Section, counter: _Counte else: label = None target = _Target(base, None, enumerated=False, section_index=self._section_index, unit=unit) - shares = counter.claim(unit) anchor = self._anchor(block.anchor_id) if block.anchor_id: self._register(block.anchor_id, replace(target, anchor=anchor)) @@ -831,7 +830,8 @@ def _structure(self, block: Block, *, section: Section | None, depth: int, prese return block def _structure_list(self, block: List, *, section: Section | None, depth: int, parent: str, - parent_unit: str | None, presence: Presence, counters: dict[tuple[str, str] | None, _Counter] | None = None, + parent_unit: str | None, presence: Presence, + counters: dict[tuple[str, str] | None, _Counter] | None = None, quoted: bool = False, parts: tuple[tuple[str, str], ...] = ()) -> List: """*block* numbered under *parent*. Sibling lists nested in the same item share *counters*, one per way of writing a designation (counter @@ -858,9 +858,8 @@ def _structure_list(self, block: List, *, section: Section | None, depth: int, p counter = counters.setdefault(key, _Counter(self._exclusive)) counter.new_list() for item in block.items: - index = counter.next(item.anchor_id, self._own_presence(item.condition, presence)) unit = self._unit() - shares = counter.claim(unit) + index, shares = counter.next(item.anchor_id, self._own_presence(item.condition, presence), unit) # An item's nested blocks are present only when the item is. inner = presence | self._presence(item.condition) if fmt is not None: @@ -1628,17 +1627,17 @@ class _Counter: """Numbers sibling units in order (§13.2, §15.8). A unit joins the previous unit's number when it is an alternative to it: the same identifier, and a presence that excludes the presence of every unit - already holding that number (§15.4). Only adjacent units are compared, - as the validator does for sections.""" + already holding that number (§15.4). Only consecutive units *of this + counter* are compared, as the validator does for sections: for the + top-level paragraphs of a section, a list or table between two of them + does not separate them.""" def __init__(self, exclusive: Callable[[Presence, Presence], bool]) -> None: self.exclusive = exclusive self.count = 0 self.identifier = "" self.holders: list[Presence] = [] - # Whether the last unit joined the number before it, and the unit that - # first held the current number (see claim). - self.joined = False + # The unit that first held the current number (see next). self.first_unit: str | None = None def new_list(self) -> None: @@ -1647,28 +1646,20 @@ def new_list(self) -> None: self.identifier = "" self.holders = [] - def next(self, identifier: str, presence: Presence | None) -> int: - """The number for the next unit. *presence* is None for a unit with - no valid condition of its own, which is never an alternative.""" - self.joined = False + def next(self, identifier: str, presence: Presence | None, unit: str) -> tuple[int, str | None]: + """The number for the next unit, *unit*, and the unit that first held + it when *unit* is a later alternative, else None. *presence* is None + for a unit with no valid condition of its own, which is never an + alternative.""" if (presence is not None and identifier and identifier == self.identifier and all(self.exclusive(holder, presence) for holder in self.holders)): self.holders.append(presence) - self.joined = True - return self.count + return self.count, self.first_unit self.count += 1 self.identifier = identifier if presence is not None else "" self.holders = [presence] if presence is not None else [] - return self.count - - def claim(self, unit: str) -> str | None: - """Record *unit* as the holder of the number :meth:`next` just gave: - None if it is the first to hold it, else the unit that was (the first - alternative, §15.8).""" - if self.joined: - return self.first_unit self.first_unit = unit - return None + return self.count, None class _Slot(NamedTuple): diff --git a/src/legaldown_render/tree.py b/src/legaldown_render/tree.py index 3b7ad33..cc0bd57 100644 --- a/src/legaldown_render/tree.py +++ b/src/legaldown_render/tree.py @@ -118,8 +118,8 @@ class CrossRef: the section's number is ``None`` and its title is what the designation starts from. - Both are ``None`` where there is no unit to name: for a copy that no longer - links (:func:`unlinked`, as in the table of contents). A reference that does + Both are ``None`` in a copy that no longer links (:func:`unlinked`, as in + the table of contents), which names no unit. A reference that does not resolve is a :class:`FailureMarker`, not a ``CrossRef``, and so is a ``{{ref:}}`` to an attachment id, which is a broken reference (§5.6); ``{{attach:}}`` is a :class:`Value` of kind ``attach``, whose ``href`` is @@ -219,7 +219,9 @@ class Paragraph: #: The numbering of a top-level paragraph of a section (see Section): #: its unit id, counter value (None when the style does not number #: paragraphs), the section's unit, and the first alternative it shares - #: its number with. Other paragraphs are not units: all four are None. + #: its number with. Alternatives are the section's top-level paragraphs + #: only: a list or table between two of them does not separate them. + #: Other paragraphs are not units: all four are None. unit: str | None = None number: int | None = None parent_unit: str | None = None @@ -244,7 +246,8 @@ class ListItem: #: a list that is not enumerated), the unit its designation is built on (the #: section for an item of a list that is not nested in an item; the enclosing #: item for a nested one; None in the preamble), and the first alternative - #: it shares its number with. + #: it shares its number with. The counter value does not always start at 1 + #: in a list: see :class:`List`. unit: str | None = None number: int | None = None parent_unit: str | None = None @@ -269,9 +272,12 @@ class List: #: Set by the resolve stage: which numbering the list is under. It is #: ``bullet`` exactly when ``enumerated`` is False. family: ListFamily = "bullet" - #: Set by the resolve stage: how deep the list is nested, from 0 for a list - #: that is not inside an item (a list in a quote in an item counts as one - #: level deeper, as its format does). + #: Set by the resolve stage: the nesting depth the list's format was + #: chosen at, from 0 for a list that is not inside an item. It selects the + #: format and nothing else: a list in a quote or drafting note inside an + #: item is one level deeper than the item, yet its items' ``parent_unit`` + #: is the section. Where a list nests, follow ``parent_unit`` (an item, or + #: a section), not ``depth``. depth: int = 0 #: Set by the resolve stage: the format its items are numbered in (the #: counter, ``label`` and ``ref`` templates of §13.2), None for ``bullet``. @@ -279,6 +285,19 @@ class List: #: and depth select, so that nobody has to select it again. format: LevelFormat | None = None + @property + def start(self) -> int | None: + """The number of the first item (``items[0].number``), None when the + list is not enumerated. It is 1 unless the list goes on from a sibling: + lists nested directly in one item share a counter per way of writing a + designation (counter style and ``ref`` template), so that a second + list after a paragraph, or a list of another family with the same + designations, continues the first's numbers ("(c)" after "(a)", "(b)") + and no designation is repeated. Top-level lists, and lists in quotes, + always start at 1. A list has no id of its own: name it by + ``items[0].unit``.""" + return self.items[0].number if self.items else None + @dataclass(frozen=True, slots=True) class Quote: @@ -351,13 +370,20 @@ class Section: when the style does not number paragraphs. * ``parent_unit``: the unit whose designation this one's is built on: for a section the nearest section above it with a smaller ``level`` (a skipped - level counts as 1 and is not a unit); for an item of a list that is not - inside an item, and for a paragraph, the section (None in the preamble); - for a nested item, the item it is nested in. + level counts as 1 and is not a unit, yet that 1 is part of every + designation below it, and so of the ``{section}`` of an item or paragraph + there); for an item of a list that is not inside an item, and for a + paragraph, the section (None in the preamble); for a nested item, the + item it is nested in. A list in a quote or drafting note, even inside an + item, is built on the section. * ``shares_number_with``: for a later alternative that shares its number - (§15.8; same identifier and exclusive conditions, §15.4) the unit of the - **first** alternative, else None. It is set only on a numbered unit; the - first alternative is a unit like any other, and each alternative has a + (§15.8, §15.4) the unit of the **first** unit that has that number, else + None. Items and paragraphs share when they have the same identifier and a + presence that excludes that of every unit already holding the number; + sections share by number (the validator's): the subsections of a later + alternative section repeat the numbers of the earlier one's, whatever + their identifiers. It is set only on a numbered unit; the first + alternative is a unit like any other, and each alternative has a ``unit`` of its own. """ diff --git a/tests/test_numbering_structure.py b/tests/test_numbering_structure.py index 8677595..e465d6d 100644 --- a/tests/test_numbering_structure.py +++ b/tests/test_numbering_structure.py @@ -15,6 +15,7 @@ from legaldown_render import DocumentError, render from legaldown_render.resolve.numbering import extend, fill, format_counter +from legaldown_render.style.model import LevelFormat from legaldown_render.tree import ( CrossRef, DraftingNote, @@ -30,6 +31,8 @@ ) NONE = {"numbering.scheme": "none"} +THREE_WAY = FRONT.replace("title: T", ( + "title: T\nquestions:\n q:\n type: choice\n choices:\n a: A\n b: B\n c: C")) ALTERNATIVES = FRONT.replace("title: T", ( "title: T\nquestions:\n forum:\n type: choice\n choices:\n courts: Courts\n arb: Arbitration")) ATTACHED = FRONT.replace("title: T", "title: T\nattachments:\n - id: att\n title: Schedule\n file: att.lgd") @@ -231,6 +234,32 @@ def test_alternative_sections_share_a_number_and_name_the_first() -> None: ("s6", "2", 2, None, None)] +def test_three_alternative_sections_all_name_the_first() -> None: + body = ("# D {#d when=q:a}\n\n## s\n\n# D {#d when=q:b}\n\n## s2\n\n## s3\n\n" + "# D {#d when=q:c}\n\n## s4\n\n## s5\n\n## s6\n") + tree = tree_of(body, THREE_WAY) + assert [(s.unit, s.designation, s.parent_unit, s.shares_number_with) for s in tree.sections] == [ + ("s1", "1", None, None), ("s2", "1.1", "s1", None), + ("s3", "1", None, "s1"), ("s4", "1.1", "s3", "s2"), ("s5", "1.2", "s3", None), + ("s6", "1", None, "s1"), ("s7", "1.1", "s6", "s2"), ("s8", "1.2", "s6", "s5"), ("s9", "1.3", "s6", None)] + + +def test_a_later_alternative_may_have_more_subsections_and_the_extra_ones_are_first_holders() -> None: + body = ("# D {#d when=forum:courts}\n\n## One\n\n# D {#d when=forum:arb}\n\n## One alt\n\n## Extra\n\n" + "### Deep\n\n# After\n") + tree = tree_of(body, ALTERNATIVES) + assert [(s.designation, s.shares_number_with) for s in tree.sections] == [ + ("1", None), ("1.1", None), ("1", "s1"), ("1.1", "s2"), ("1.2", None), ("1.2.1", None), ("2", None)] + assert [s.parent_unit for s in tree.sections] == [None, "s1", None, "s3", "s3", "s5", None] + + +def test_a_heading_below_level_five_is_numbered_as_level_five() -> None: + tree = tree_of("# A\n\n## B\n\n### C\n\n#### D\n\n##### E\n\n###### F\n\n## G\n") + assert [(s.level, s.number, s.parent_unit, s.designation) for s in tree.sections] == [ + (1, 1, None, "1"), (2, 1, "s1", "1.1"), (3, 1, "s2", "1.1.1"), (4, 1, "s3", "1.1.1.1"), + (5, 1, "s4", "1.1.1.1.1"), (5, 2, "s4", "1.1.1.1.2"), (2, 2, "s1", "1.2")] + + def test_alternative_sections_under_the_none_scheme_share_nothing() -> None: body = "# Disputes {#d when=forum:courts}\n\n# Disputes {#d when=forum:arb}\n" tree = tree_of(body, ALTERNATIVES, overrides=NONE) @@ -277,6 +306,20 @@ def test_nested_enumerated_items_carry_number_parent_and_family() -> None: assert [f.format for f in lists(tree)] == [*formats.enumerated, formats.renumber] +def test_families_mix_by_depth_and_the_enumerated_formats_cycle() -> None: + body = "# A\n\n1. r1\n - e1\n 1. r2\n - e2\n - e3\n - e4\n 1. r3\n" + tree = tree_of(body) + formats = tree.formats + assert [(f.family, f.depth) for f in lists(tree)] == [ + ("renumber", 0), ("enumerated", 1), ("renumber", 2), ("enumerated", 3), ("enumerated", 4), + ("enumerated", 5), ("renumber", 6)] + assert [f.format for f in lists(tree)] == [ + formats.renumber, formats.enumerated[1], formats.renumber, formats.enumerated[0], formats.enumerated[1], + formats.enumerated[2], formats.renumber] + assert [i.label for i in items(tree)] == ["1.", "(i)", "1.", "(a)", "(i)", "(A)", "1."] + assert [i.parent_unit for i in items(tree)] == ["s1", "u1", "u2", "u3", "u4", "u5", "u6"] + + def test_a_list_nested_deeper_than_the_formats_goes_back_to_the_first() -> None: body = "# A\n\n- 1\n - 2\n - 3\n - 4\n - 5\n" tree = tree_of(body) @@ -351,9 +394,7 @@ def test_alternative_items_share_a_number() -> None: def test_three_alternatives_all_name_the_first() -> None: - front = FRONT.replace("title: T", "title: T\nquestions:\n q:\n type: choice\n choices:\n" - " a: A\n b: B\n c: C") - tree = tree_of("# A\n\n- x\n- one {#i when=q:a}\n- two {#i when=q:b}\n- three {#i when=q:c}\n- y\n", front) + tree = tree_of("# A\n\n- x\n- one {#i when=q:a}\n- two {#i when=q:b}\n- three {#i when=q:c}\n- y\n", THREE_WAY) assert [(i.number, i.shares_number_with) for i in items(tree)] == [ (1, None), (2, None), (2, "u2"), (2, "u2"), (3, None)] @@ -368,6 +409,55 @@ def test_only_adjacent_alternatives_share_a_number_and_a_list_never_shares_with_ assert [(i.number, i.shares_number_with) for i in items(tree_of(body, ALTERNATIVES))] == [(1, None), (1, None)] +def test_sibling_lists_in_one_item_go_on_counting() -> None: + """Lists nested directly in one item share a counter per way of writing a designation, so no + designation repeats: a list's first number is items[0].number (List.start), not always 1.""" + tree = tree_of("# A\n\n- x\n - a\n - b\n\n text\n\n - c\n - d\n") + assert [(i.label, i.number, i.parent_unit) for i in items(tree)[1:]] == [ + ("(i)", 1, "u1"), ("(ii)", 2, "u1"), ("(iii)", 3, "u1"), ("(iv)", 4, "u1")] + assert [(f.depth, f.start) for f in lists(tree)] == [(0, 1), (1, 1), (1, 3)] + + +def test_a_sibling_list_of_another_family_with_the_same_designations_goes_on_too() -> None: + # A renumbered list is "(n)" in a reference, so is an enumerated one whose ref is "({n})" in decimal. + levels = [{"counter": "decimal", "label": "{n}.", "ref": "({n})"}] * 2 + tree = tree_of("# A\n\n- x\n - a\n\n text\n\n 1. p\n 1. q\n", overrides={"enumeration.levels": levels}) + assert [(f.family, f.start) for f in lists(tree)] == [("enumerated", 1), ("enumerated", 1), ("renumber", 2)] + assert [i.label for i in items(tree)[1:]] == ["1.", "2.", "3."] + # With other designations the lists are independent. + other = tree_of("# A\n\n- x\n - a\n\n text\n\n 1. p\n 1. q\n") + assert [(f.family, f.start) for f in lists(other)] == [("enumerated", 1), ("enumerated", 1), ("renumber", 1)] + + +def test_top_level_lists_and_lists_in_quotes_start_at_one() -> None: + tree = tree_of("# A\n\n- x\n\ntext\n\n- y\n\n> - q\n\n> text\n\n> - r\n") + assert [f.start for f in lists(tree)] == [1, 1, 1, 1] + + +def test_alternatives_never_join_across_sibling_lists() -> None: + body = "# A\n\n- x\n - a {#i when=forum:courts}\n\n text\n\n - b {#i when=forum:arb}\n" + tree = tree_of(body, ALTERNATIVES) + assert [(i.number, i.shares_number_with) for i in items(tree)[1:]] == [(1, None), (2, None)] + # Within one list they do. + body = "# A\n\n- x\n - a {#i when=forum:courts}\n - b {#i when=forum:arb}\n" + assert [(i.number, i.shares_number_with) for i in items(tree_of(body, ALTERNATIVES))[1:]] == [(1, None), (1, "u2")] + + +def test_a_list_in_a_quote_or_drafting_note_inside_an_item_is_built_on_the_section() -> None: + body = ("# A\n\n- x\n > [!DRAFTING]\n > - n1\n > - n2\n\n > - q1\n\n - nested\n\n" + "> [!DRAFTING]\n> - top\n") + tree = tree_of(body) + assert [(i.unit, i.label, i.parent_unit) for i in items(tree)] == [ + ("u1", "(a)", "s1"), + ("u2", "(i)", "s1"), ("u3", "(ii)", "s1"), # depth 1 gives the format; the parent is the section + ("u4", "(i)", "s1"), + ("u5", "(i)", "u1"), # a list nested in the item itself is built on it + ("u6", "(a)", "s1")] + assert [(f.depth, f.start, f.items[0].parent_unit) for f in lists(tree)] == [ + (0, 1, "s1"), (1, 1, "s1"), (1, 1, "s1"), (1, 1, "u1"), (0, 1, "s1")] + assert [f.items[0].unit for f in lists(tree)] == ["u1", "u2", "u4", "u5", "u6"] # a list has no id: its first item + + # -- Paragraphs -------------------------------------------------------------------- @@ -399,6 +489,24 @@ def test_alternative_paragraphs_share_a_number() -> None: ("u1", "1.1", 1, None), ("u2", "1.2", 2, None), ("u3", "1.2", 2, "u2"), ("u4", "1.3", 3, None)] +def test_alternative_paragraphs_need_not_be_adjacent_blocks() -> None: + """Only the top-level paragraphs are counted together: a list or a table between two of them does not + keep them from sharing a number, and a paragraph does.""" + body = ("# A\n\nCourts. {#p when=forum:courts}\n\n- between\n\n| a |\n|---|\n| b |\n\n" + "Arbitration. {#p when=forum:arb}\n\nMid.\n\nCourts again. {#q when=forum:courts}\n\n" + "Mid.\n\nArbitration again. {#q when=forum:arb}\n") + tree = tree_of(body, ALTERNATIVES, overrides={"paragraphs.numbered": True}) + assert [(p.unit, p.number, p.shares_number_with) for p in paragraphs(tree)] == [ + ("u1", 1, None), ("u3", 1, "u1"), ("u4", 2, None), ("u5", 3, None), ("u6", 4, None), ("u7", 5, None)] + + +def test_three_alternative_paragraphs_all_name_the_first() -> None: + tree = tree_of("# A\n\nx {#p when=q:a}\n\n- l\n\ny {#p when=q:b}\n\nz {#p when=q:c}\n\nlast\n", + THREE_WAY, overrides={"paragraphs.numbered": True}) + assert [(p.number, p.shares_number_with) for p in paragraphs(tree)] == [ + (1, None), (1, "u1"), (1, "u1"), (2, None)] + + def test_alternative_paragraphs_share_nothing_where_none_are_numbered() -> None: body = "# A\n\nCourts. {#p when=forum:courts}\n\nArbitration. {#p when=forum:arb}\n" assert [p.shares_number_with for p in paragraphs(tree_of(body, ALTERNATIVES))] == [None, None] @@ -662,6 +770,10 @@ def check(tree: RenderTree) -> None: first = recompute.found[node.shares_number_with].node assert position[node.shares_number_with] < position[unit] assert node.number is not None and first.number == node.number and first.shares_number_with is None + # An alternative stands where its first does: same kind, same parent, same label. + assert type(first) is type(node) and first.label == node.label, unit + if not isinstance(node, Section): + assert first.parent_unit == node.parent_unit, unit if isinstance(node, Section): assert node.label == recompute.label(unit), unit assert node.designation == recompute.designation(unit), unit @@ -692,6 +804,65 @@ def check(tree: RenderTree) -> None: assert section == (target_found.node.unit if isinstance(target_found.node, Section) else target_found.section.unit) assert ref.text == recompute.reference(ref), (ref.text, ref.target) + check_counting(tree, recompute.found) + + +def _counts(group: list, found: dict[str, Found], first: int, what: str) -> None: + """Consecutive numbered units of *group* count up by one, or share the number of an earlier one.""" + previous = first - 1 + for node in group: + if node.number is None: + continue + shared = node.shares_number_with + assert node.number == previous + 1 or (shared is not None and node.number == found[shared].node.number), ( + what, node.unit, node.number, previous) + previous = node.number + + +def check_counting(tree: RenderTree, found: dict[str, Found]) -> None: + """Numbers start where they should and go on without a gap: sections of one parent, paragraphs of one + section, and the items of a list, which goes on from an earlier sibling list in the same item when + the two write their designations alike.""" + by_parent: dict[str | None, list] = {} + for section in tree.sections: + by_parent.setdefault(section.parent_unit, []).append(section) + for parent, group in by_parent.items(): + _counts(group, found, 1, f"sections under {parent}") + in_section: dict[str, list] = {} + for paragraph in paragraphs(tree): + in_section.setdefault(paragraph.parent_unit, []).append(paragraph) + for parent, group in in_section.items(): + _counts(group, found, 1, f"paragraphs of {parent}") + last: dict[tuple, int] = {} # (item, counter, ref) -> the last number of the sibling lists so far + for block in lists(tree): + if block.format is None: + continue + parent = block.items[0].parent_unit + under_item = parent is not None and isinstance(found[parent].node, ListItem) + key = (parent, block.format.counter, block.format.ref) + first = last.get(key, 0) + 1 if under_item else 1 + _counts(block.items, found, first, f"items of the list at {block.items[0].unit}") + assert block.start == block.items[0].number + last[key] = block.items[-1].number + + +def path_labelled(style: str, overrides: dict) -> dict: + """*overrides* with the label of every list level and of paragraphs written as ``{path}``, so that + each label is the unit's whole designation and comparing labels compares designations.""" + formats = tree_of("# A\n", style=style, overrides=overrides).formats + out = dict(overrides) + if formats.enumerated: + out["enumeration.levels"] = [{"counter": f.counter, "label": "{path}", "ref": f.ref} + for f in formats.enumerated] + paragraph = formats.paragraphs or LevelFormat("decimal", "{section}.{n}", ".{n}") + out["paragraphs.numbered"] = True + out["paragraphs.format"] = {"counter": paragraph.counter, "label": "{path}", "ref": paragraph.ref} + return out + + +def variants(style: str, overrides: dict) -> list[dict]: + """The settings to check a style under: as they are, and with designations as labels.""" + return [overrides, path_labelled(style, overrides)] STYLES = [("default", {}), ("outline", {}), ("continental", {}), @@ -705,8 +876,9 @@ def check(tree: RenderTree) -> None: @pytest.mark.parametrize(("style", "overrides"), STYLES, ids=STYLE_IDS) def test_the_corpus_of_the_tests_is_numbered_as_its_structure_says(style: str, overrides: dict) -> None: for name in ("features.lgd", "template.lgd"): - check(render((DOCUMENTS / name).read_text(encoding="utf-8"), format="text", style=style, - overrides=overrides).tree) + for variant in variants(style, overrides): + check(render((DOCUMENTS / name).read_text(encoding="utf-8"), format="text", style=style, + overrides=variant).tree) @pytest.mark.parametrize(("body", "overrides"), [ @@ -715,18 +887,26 @@ def test_the_corpus_of_the_tests_is_numbered_as_its_structure_says(style: str, o ("# A\n\n### B\n\n## C\n\n#### D\n\n### E\n\n- x {#x}\n\nSee {{ref: x}}.\n", {"numbering.scheme": "mixed"}), ("# A\n\n> - q\n\n- x\n > - y\n", {}), ("- p\n\n# A\n", {}), -], ids=["nested", "refs", "refs-paragraphs", "refs-none", "skips", "skips-mixed", "quotes", "preamble"]) + ("# A\n\n- x\n - a\n - b\n\n text\n\n 1. c\n 1. d\n\n - e\n\n> [!DRAFTING]\n> - n\n", {}), + ("# A\n\n1. r\n - e1\n 1. r2\n - e2\n - e3\n - e4\n", {}), +], ids=["nested", "refs", "refs-paragraphs", "refs-none", "skips", "skips-mixed", "quotes", "preamble", + "sibling-lists", "mixed-families"]) @pytest.mark.parametrize("style", ["default", "outline", "continental"]) def test_small_documents_are_numbered_as_their_structure_says(body: str, overrides: dict, style: str) -> None: - check(tree_of(body, style=style, overrides=overrides)) + for variant in variants(style, overrides): + check(tree_of(body, style=style, overrides=variant)) def test_alternatives_are_numbered_as_their_structure_says() -> None: body = ("# Disputes {#d when=forum:courts}\n\n## Sub one\n\n- a {#i when=forum:courts}\n- b {#i when=forum:arb}\n\n" "# Disputes {#d when=forum:arb}\n\n## Sub alt {#sub-alt}\n\nOne. {#p when=forum:arb}\n\n" "Two. {#p when=forum:courts}\n\nSee {{ref: d}}, {{ref: i}}, {{ref: p}}, {{ref: sub-alt}}.\n") + siblings = ("# A\n\n- x\n - a {#i when=forum:courts}\n\n text\n\n - b {#i when=forum:arb}\n - c {#i when=forum:courts}\n" + " - d {#i when=forum:arb}\n") for style, overrides in STYLES: - check(tree_of(body, ALTERNATIVES, style=style, overrides={**overrides, "paragraphs.numbered": True})) + for variant in variants(style, {**overrides, "paragraphs.numbered": True}): + check(tree_of(body, ALTERNATIVES, style=style, overrides=variant)) + check(tree_of(siblings, ALTERNATIVES, style=style, overrides=variant)) # -- The specification's corpus ---------------------------------------------------- @@ -742,13 +922,14 @@ def test_the_specifications_corpus_is_numbered_as_its_structure_says(style: str, assert CORPUS, "no documents found in the specification checkout" rendered = 0 for path in CORPUS: - try: - result = render(path.read_text(encoding="utf-8"), format="text", style=style, overrides=overrides) - except DocumentError: - continue - try: - check(result.tree) - except AssertionError as error: # name the document - raise AssertionError(f"{path.relative_to(SPEC)}: {error}") from error - rendered += 1 - assert rendered > 100 + for variant in variants(style, overrides): + try: + result = render(path.read_text(encoding="utf-8"), format="text", style=style, overrides=variant) + except DocumentError: + continue + try: + check(result.tree) + except AssertionError as error: # name the document + raise AssertionError(f"{path.relative_to(SPEC)}: {error}") from error + rendered += 1 + assert rendered > 200 From dcd7095fee493f31ff9d37c4c05e93ac5ad48142 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:52:37 +0000 Subject: [PATCH 071/118] Bound the lengths and colours the DOCX settings accept An enormous length (page.size, page.margin, font and heading sizes, line height) crashed with OverflowError or wrote a value Word and the schema refuse. Numbers of more than 32 digits and lengths over 10 million twips are refused, then Word's limits are checked on the converted values: page edges of 144 to 31680 twips, margins of 0 to 31680 that leave at least 720 for text each way (so no text width or tab stop is negative), line of at most 31680, a body size small enough for the 1.6 times title. Non-finite rgb() and hsl() components are refused. Units are case-insensitive. Messages never quote the value. --- src/legaldown_render/style/docx.py | 69 +++++++++++++++++++++++------- tests/test_docx_style.py | 67 ++++++++++++++++++++++++++++- 2 files changed, 118 insertions(+), 18 deletions(-) diff --git a/src/legaldown_render/style/docx.py b/src/legaldown_render/style/docx.py index b801295..93bdbcb 100644 --- a/src/legaldown_render/style/docx.py +++ b/src/legaldown_render/style/docx.py @@ -27,6 +27,7 @@ from __future__ import annotations import colorsys +import math import re from dataclasses import dataclass @@ -70,6 +71,17 @@ #: The largest size Word accepts for a font (``ST_HpsMeasure``), in points. _MAX_FONT_POINTS = 1638 +#: The largest body text: the title is 1.6 times it and must stay within :data:`_MAX_FONT_POINTS`. +_MAX_BODY_POINTS = 1023 +#: Word's limit for a page edge, a margin or a line (22 inches), in twips; and the least page edge. +_MAX_TWIPS = 31680 +_MIN_PAGE_TWIPS = 144 +#: The least width and height of the text area, so that no tab stop or text width is negative or absurd. +_MIN_TEXT_TWIPS = 720 +#: A number of more characters than this is no length (and is not converted: ``float`` of a +#: few hundred digits is infinity); a length over this many twips is refused before it is used. +_MAX_NUMBER_CHARS = 32 +_MAX_LENGTH_TWIPS = 10_000_000 _BASIC_COLORS = { "black": "000000", "silver": "C0C0C0", "gray": "808080", "grey": "808080", "white": "FFFFFF", @@ -165,7 +177,8 @@ def docx_settings(style: Style, *, trusted: bool = True) -> DocxSettings: problems = _Problems() typography = style.typography - font_points = _font_points(typography.font_size, "typography.font_size", BASE_FONT_POINTS, problems) + font_points = _font_points(typography.font_size, "typography.font_size", BASE_FONT_POINTS, problems, + _MAX_BODY_POINTS) font_points = font_points if font_points is not None else 11.0 half_points = round(font_points * 2) @@ -200,10 +213,13 @@ def docx_settings(style: Style, *, trusted: bool = True) -> DocxSettings: def _length(value: str, key: str, em_points: float, problems: _Problems) -> int | None: """*value*, a CSS length, in twips; None (with a problem) when Word cannot have it. ``em`` and ``rem`` are *em_points* points.""" - match = _LENGTH_RE.match(value.strip()) + match = _LENGTH_RE.match(value.strip().lower()) if match is None: problems.add(key, "is not a length the DOCX writer can read (a number and a unit such as 25mm, 11pt or 1.2em)") return None + if len(match.group(1)) > _MAX_NUMBER_CHARS: + problems.add(key, "is a number too large for a Word document") + return None number, unit = float(match.group(1)), match.group(2) if unit is None: if number != 0: @@ -214,18 +230,21 @@ def _length(value: str, key: str, em_points: float, problems: _Problems) -> int problems.add(key, f"uses '{unit}', which a Word document cannot express; use pt, mm, cm, in, px, em or rem " f"(HTML accepts it)") return None - if unit in _RELATIVE: - return round(number * em_points * _TWIPS_PER_POINT) - return round(number * _ABSOLUTE[unit]) + twips = number * (em_points * _TWIPS_PER_POINT if unit in _RELATIVE else _ABSOLUTE[unit]) + if not math.isfinite(twips) or abs(twips) > _MAX_LENGTH_TWIPS: + problems.add(key, "is a length too large for a Word document") + return None + return round(twips) -def _font_points(value: str, key: str, em_points: float, problems: _Problems) -> float | None: +def _font_points(value: str, key: str, em_points: float, problems: _Problems, + maximum: int = _MAX_FONT_POINTS) -> float | None: twips = _length(value, key, em_points, problems) if twips is None: return None points = twips / _TWIPS_PER_POINT - if not 1 <= points <= _MAX_FONT_POINTS: - problems.add(key, f"must be between 1pt and {_MAX_FONT_POINTS}pt in a Word document") + if not 1 <= points <= maximum: + problems.add(key, f"must be between 1pt and {maximum}pt in a Word document") return None return points @@ -239,23 +258,36 @@ def _line_height(value: str, font_points: float, problems: _Problems) -> tuple[i return 240, "auto" if _NUMBER_RE.match(text): factor = float(text) - if not 0 < factor <= 100: + if not 0 < factor <= 100: # also false for infinity problems.add("typography.line_height", "a number must be greater than 0 and at most 100") return 240, "auto" return round(factor * 240), "auto" twips = _length(text, "typography.line_height", font_points, problems) - if twips is None or twips <= 0: + if twips is None or not 0 < twips <= _MAX_TWIPS: if twips is not None: - problems.add("typography.line_height", "must be greater than 0") + problems.add("typography.line_height", f"must be greater than 0 and at most {_MAX_TWIPS // 20}pt") return 240, "auto" return twips, "atLeast" def _page(size: str, margin: str, font_points: float, problems: _Problems) -> PageGeometry: + """The page in twips. Word's limits hold: an edge of 144 to 31680, a margin of 0 to + 31680, and margins that leave a text area of at least 720 in each direction, so a + tab stop at the text width can never be negative.""" + before = len(problems.items) width, height = _page_size(size, font_points, problems) top, right, bottom, left = _margins(margin, font_points, problems) + if len(problems.items) == before: # a value that is already wrong has said so + if not all(_MIN_PAGE_TWIPS <= edge <= _MAX_TWIPS for edge in (width, height)): + problems.add("page.size", f"each edge must be between {_MIN_PAGE_TWIPS / 20:g}pt and " + f"{_MAX_TWIPS // 20}pt in a Word document") + elif not all(0 <= edge <= _MAX_TWIPS for edge in (top, right, bottom, left)): + problems.add("page.margin", f"each margin must be between 0 and {_MAX_TWIPS // 20}pt in a Word document") + elif width - left - right < _MIN_TEXT_TWIPS or height - top - bottom < _MIN_TEXT_TWIPS: + problems.add("page.margin", "the margins leave no room for text on the page (page.size)") return PageGeometry(width=width, height=height, top=top, right=right, bottom=bottom, left=left, - header=min(720, top // 2), footer=min(720, bottom // 2), landscape=width > height) + header=max(0, min(720, top // 2)), footer=max(0, min(720, bottom // 2)), + landscape=width > height) def _page_size(value: str, font_points: float, problems: _Problems) -> tuple[int, int]: @@ -347,18 +379,23 @@ def _channel(part: str) -> int | None: number = float(part[:-1]) * 2.55 if part.endswith("%") else float(part) except ValueError: return None - return round(min(255.0, max(0.0, number))) + return round(min(255.0, max(0.0, number))) if math.isfinite(number) else None def _hsl(parts: list[str]) -> str | None: try: - hue = float(parts[0].removesuffix("deg")) % 360 / 360 + hue_degrees = float(parts[0].removesuffix("deg")) if not (parts[1].endswith("%") and parts[2].endswith("%")): return None - saturation = min(100.0, max(0.0, float(parts[1][:-1]))) / 100 - lightness = min(100.0, max(0.0, float(parts[2][:-1]))) / 100 + saturation = float(parts[1][:-1]) + lightness = float(parts[2][:-1]) except ValueError: return None + if not all(math.isfinite(number) for number in (hue_degrees, saturation, lightness)): + return None + hue = hue_degrees % 360 / 360 + saturation = min(100.0, max(0.0, saturation)) / 100 + lightness = min(100.0, max(0.0, lightness)) / 100 red, green, blue = colorsys.hls_to_rgb(hue, lightness, saturation) return f"{round(red * 255):02X}{round(green * 255):02X}{round(blue * 255):02X}" diff --git a/tests/test_docx_style.py b/tests/test_docx_style.py index 30c318a..5c8d139 100644 --- a/tests/test_docx_style.py +++ b/tests/test_docx_style.py @@ -218,8 +218,9 @@ def test_the_render_api_refuses_before_rendering_and_html_still_renders() -> Non ("page.size", "A4 A3", "not a length"), ("page.size", "210mm landscape", "orientation goes with a paper size"), ("page.size", "0 10mm", "greater than 0"), - ("typography.font_size", "0", "between 1pt and 1638pt"), - ("typography.font_size", "2000pt", "between 1pt and 1638pt"), + ("typography.font_size", "0", "between 1pt and 1023pt"), + ("typography.font_size", "2000pt", "between 1pt and 1023pt"), + ("headings.1.size", "2000pt", "between 1pt and 1638pt"), ("typography.line_height", "0", "greater than 0"), ("typography.line_height", "calc(1 + 1)", "not a length"), ("typography.color", "hsl(", "cannot read"), @@ -245,3 +246,65 @@ def test_every_problem_is_reported_together_and_an_untrusted_style_is_not_quoted def test_the_built_in_styles_convert() -> None: for name in ("default", "outline", "continental"): docx_settings(load_style(name)) + + +# -- Numeric extremes: a StyleError, never a crash and never a value Word or the schema refuses --------------- + +DIGITS = ["1" + "0" * 40, "9" * 30, "9" * 33, "9" * 100, "9" * 400, "9" * 5000, "1" + "0" * 40 + "." + "9" * 40] +LENGTH_KEYS = ["page.size", "page.margin", "typography.font_size", "typography.line_height", "headings.1.size"] + + +@pytest.mark.parametrize("unit", ["mm", "pt", "in", "em", "px"]) +@pytest.mark.parametrize("digits", DIGITS) +@pytest.mark.parametrize("key", LENGTH_KEYS) +def test_an_enormous_length_is_a_style_error_that_does_not_quote_it(key: str, digits: str, unit: str) -> None: + for trusted in (True, False) if len(digits) < 150 else (True,): # an untrusted text is at most 200 characters + style = load_style("default", overrides={key: digits + unit}, trusted=trusted) + with pytest.raises(StyleError) as caught: + docx_settings(style, trusted=trusted) + assert digits[:12] not in str(caught.value) + with pytest.raises(StyleError): + render(FRONT + "# A\n", format="docx", overrides={key: digits + unit}) + + +@pytest.mark.parametrize("value", ["inf", "nan", "1e999", "9" * 400, "-" + "9" * 400]) +@pytest.mark.parametrize("function", ["rgb({0}, 0, 0)", "rgb(0 {0} 0)", "rgba(0,0,{0},1)", "hsl({0} 50% 50%)", + "hsl(10 {0}% 50%)", "hsl(10 50% {0}%)"]) +def test_a_colour_component_that_is_not_a_finite_number_is_a_style_error(function: str, value: str) -> None: + with pytest.raises(StyleError, match="typography.color: is a colour the DOCX writer cannot read"): + docx_settings(load_style("default", overrides={"typography.color": function.format(value)})) + + +@pytest.mark.parametrize(("key", "value", "fragment"), [ + ("page.size", "7pt", "each edge must be between 7.2pt and 1584pt"), + ("page.size", "1600pt 1600pt", "each edge must be between"), + ("page.size", "5pt 20in", "each edge must be between"), + ("page.margin", "1600pt", "each margin must be between 0 and 1584pt"), + ("page.margin", "50mm 100mm", "leave no room for text"), + ("page.margin", "150mm 0", "leave no room for text"), + ("page.margin", "105mm", "leave no room for text"), + ("typography.line_height", "1600pt", "at most 1584pt"), + ("typography.font_size", "1024pt", "between 1pt and 1023pt"), +]) +def test_values_outside_word_s_limits_are_named(key: str, value: str, fragment: str) -> None: + with pytest.raises(StyleError) as caught: + docx_settings(load_style("default", overrides={key: value})) + assert any(problem.startswith(f"{key}: ") and fragment in problem for problem in caught.value.problems), \ + caught.value.problems + + +def test_the_largest_values_that_fit_are_accepted_and_nothing_is_negative_or_overflows() -> None: + word = docx_settings(load_style("default", overrides={ + "page.size": "22in 22in", "page.margin": "0", "typography.font_size": "1023pt", + "typography.line_height": "1584pt", "headings.1.size": "1638pt"})) + assert (word.page.width, word.page.height, word.page.text_width) == (31680, 31680, 31680) + assert word.half_points == 2046 and round(word.half_points * 1.6) <= 3276 + narrow = docx_settings(load_style("default", overrides={"page.size": "1.5in 3in", "page.margin": "0.5in"})) + assert narrow.page.text_width == 720 and narrow.page.header >= 0 + + +@pytest.mark.parametrize("value", ["25MM", "25Mm", "1IN", "72PT", "6PC", "96PX", "1EM", "1.5REM"]) +def test_units_are_not_case_sensitive(value: str) -> None: + assert docx_settings(load_style("default", overrides={"page.margin": value})).page.top > 0 + with pytest.raises(StyleError, match="uses '%'"): + docx_settings(load_style("default", overrides={"page.margin": "5%"})) From fe47efa0b34bd6a6a64b2b5d9ff6790f803a40c9 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:52:37 +0000 Subject: [PATCH 072/118] Take {version} and {date} from the tree, and warn that the DOCX body is interim RenderTree gains version and effective_date as plain text, resolved exactly as the title block shows them (directives included, a valid date formatted by the style), whether or not the title block shows them. The DOCX writer reads them from the tree; its keyword arguments and api._page_fields are gone. The title is one text for the body, the properties and the header. Every DOCX render reports the Warning render-docx-interim, from api.py, until the body is complete. --- src/legaldown_render/api.py | 37 ++++++++----------- src/legaldown_render/resolve/resolver.py | 35 +++++++++++++----- src/legaldown_render/tree.py | 6 +++ src/legaldown_render/writers/docx/__init__.py | 21 ++++------- src/legaldown_render/writers/docx/body.py | 10 ++++- 5 files changed, 64 insertions(+), 45 deletions(-) diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index 840dea0..e552af2 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -20,7 +20,6 @@ import yaml from legaldown import AssemblyError, Diagnostic, Document, ValidationResult, parse, parse_template, validate -from legaldown.grammar import is_valid_iso_date from .build import build_tree, normalize_source from .diagnostics import by_line, key_line, repeats @@ -121,6 +120,17 @@ def ok(self) -> bool: return not any(diagnostic.level == "error" for diagnostic in self.diagnostics) +def _interim_warning(filename: str) -> Diagnostic: + """The Warning that DOCX output is not complete yet: the writer's body is interim (no numbers, + references, contents, title block, attachments or signature blocks). It goes when the body is + complete.""" + return Diagnostic( + rule="render-docx-interim", level="warning", file=filename, + message="DOCX output is incomplete in this version: headings and list items carry no numbers, and " + "cross-references, the table of contents, the title block, attachments and signature blocks are " + "not written yet. Do not use the file as the finished document.") + + def render(source: str, options: RenderOptions | None = None, /, **settings: Any) -> RenderResult: """Render LegalDown *source*. @@ -176,8 +186,10 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any if options.strict and any(d.level == "error" for d in diagnostics): raise RenderRefused(diagnostics) + if options.format == "docx": + diagnostics.append(_interim_warning(document.filename)) assert_resolved(tree) - written = _writer(options, style, docx, document, tree).write(tree) + written = _writer(options, style, docx).write(tree) if isinstance(written, bytes): return RenderResult("", diagnostics, options.format, tree, style, binary=written) return RenderResult(written, diagnostics, options.format, tree, style) @@ -300,8 +312,7 @@ def _resolve(document: Document, result: ValidationResult, style: Style, return resolved, diagnostics + found -def _writer(options: RenderOptions, style: Style, docx: DocxSettings | None, document: Document, - tree: RenderTree) -> HtmlWriter | TextWriter | DocxWriter: +def _writer(options: RenderOptions, style: Style, docx: DocxSettings | None) -> HtmlWriter | TextWriter | DocxWriter: if options.format == "text": return TextWriter(style.attachments.separator) from . import __version__ @@ -309,21 +320,5 @@ def _writer(options: RenderOptions, style: Style, docx: DocxSettings | None, doc generator = f"legaldown-render {__version__}" if options.format == "docx": assert docx is not None - version, date = _page_fields(document, tree, style) - return DocxWriter(style, generator=generator, version=version, date=date, settings=docx) + return DocxWriter(style, generator=generator, settings=docx) return HtmlWriter(style, standalone=options.standalone, generator=generator) - - -def _page_fields(document: Document, tree: RenderTree, style: Style) -> tuple[str, str]: - """The document's ``version`` and ``effective_date`` as a running header or - footer shows them (``{version}``, ``{date}``): the date formatted as the style's - ``values.date`` says, for the formatting locale the resolver used. A value that - holds directives (``{{...}}``) is left out: they are resolved where the document - shows them, not here, and a running text is no place for a blank to fill in.""" - metadata = document.metadata - version = metadata.version or "" - date = metadata.effective_date or "" - locale = parse_locale(tree.locale) - if date and locale is not None and is_valid_iso_date(date): - date = Formatter(locale, style.values).date(date) - return ("" if "{{" in version else version), ("" if "{{" in date else date) diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index ce2ec86..d237660 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -357,6 +357,8 @@ def run(self) -> RenderTree: title = resolve(self.tree.title) with self._at(self.key_line("subtitle")): subtitle = resolve(self.tree.subtitle) + effective_date = self._effective_date() + version = self._version() resolved = replace( self.tree, title=title, @@ -364,7 +366,9 @@ def run(self) -> RenderTree: locale=self.formatter.tag, preamble=preamble, sections=sections, - header=self._header(), + header=self._header(effective_date, version), + version=plain_text(version), + effective_date=plain_text(effective_date), sides=self._sides() if self.style.title_block.parties else (), attachments=attachments, attachments_label=self.labels.attachments or "", @@ -1389,15 +1393,28 @@ def _plain_value(self, text: str, *key: Any) -> str: def _frontmatter(self, text: str) -> tuple[Inline, ...]: return self._resolve_inlines(plain_inlines(text)) - def _header(self) -> tuple[tuple[str, tuple[Inline, ...]], ...]: + def _effective_date(self) -> tuple[Inline, ...]: + """The effective date as the title block shows it (a valid date formatted, any + other text resolved), or nothing when the document has none.""" + if not self.metadata.effective_date: + return () + with self._at(self.key_line("effective_date")): + return self._date_or_text(self.metadata.effective_date) + + def _version(self) -> tuple[Inline, ...]: + """The version as the title block shows it, or nothing when the document has none.""" + if not self.metadata.version: + return () + with self._at(self.key_line("version")): + return self._frontmatter(self.metadata.version) + + def _header(self, effective_date: tuple[Inline, ...], + version: tuple[Inline, ...]) -> tuple[tuple[str, tuple[Inline, ...]], ...]: rows: list[tuple[str, tuple[Inline, ...]]] = [] - metadata = self.metadata - if self.style.title_block.effective_date and metadata.effective_date: - with self._at(self.key_line("effective_date")): - rows.append((self.labels.effective_date or "", self._date_or_text(metadata.effective_date))) - if self.style.title_block.version and metadata.version: - with self._at(self.key_line("version")): - rows.append((self.labels.version or "", self._frontmatter(metadata.version))) + if self.style.title_block.effective_date and effective_date: + rows.append((self.labels.effective_date or "", effective_date)) + if self.style.title_block.version and version: + rows.append((self.labels.version or "", version)) return tuple(rows) def _date_or_text(self, value: str) -> tuple[Inline, ...]: diff --git a/src/legaldown_render/tree.py b/src/legaldown_render/tree.py index cc0bd57..0a255c9 100644 --- a/src/legaldown_render/tree.py +++ b/src/legaldown_render/tree.py @@ -520,6 +520,12 @@ class RenderTree: colon: str = ": " #: The numbering formats the labels and designations were made with. formats: NumberingFormats = NumberingFormats() + #: The frontmatter's ``version`` and ``effective_date`` as plain text, exactly as + #: the title block shows them (directives resolved, a valid date formatted by the + #: style for the locale), whether or not the style shows them there: the + #: running header and footer of a paged format use them. Empty when absent. + version: str = "" + effective_date: str = "" # --------------------------------------------------------------------------- diff --git a/src/legaldown_render/writers/docx/__init__.py b/src/legaldown_render/writers/docx/__init__.py index 117ff4c..d68ab26 100644 --- a/src/legaldown_render/writers/docx/__init__.py +++ b/src/legaldown_render/writers/docx/__init__.py @@ -19,9 +19,9 @@ from ...style import Style from ...style.docx import DocxSettings, docx_settings -from ...tree import RenderTree, plain_text +from ...tree import RenderTree from . import package as pkg -from .body import interim_body +from .body import document_title, interim_body from .headers import build_footer, build_header from .ooxml import el, serialize from .properties import build_app, build_core @@ -37,29 +37,24 @@ class DocxWriter: *style* is the style template (the DOCX settings are converted from it, see :func:`~legaldown_render.style.docx.docx_settings`, which raises :class:`~legaldown_render.errors.StyleError` for one Word cannot express); - *generator* names the program in the document's properties. *version* and *date* - are the document's own, as the running header and footer show them - (``{version}``, ``{date}``: the frontmatter's ``version``, and its - ``effective_date`` as the style formats dates); the title is the tree's. Both are - empty for a document without one. + *generator* names the program in the document's properties. The running header and + footer take ``{title}``, ``{version}`` and ``{date}`` from the tree (its title, and the + ``version`` and ``effective_date`` the resolver formatted as the title block shows them). """ extension = "docx" media_type = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" - def __init__(self, style: Style, *, generator: str = "", version: str = "", date: str = "", - settings: DocxSettings | None = None) -> None: + def __init__(self, style: Style, *, generator: str = "", settings: DocxSettings | None = None) -> None: self.style = style self.generator = generator or "legaldown-render" - self.version = version - self.date = date self.settings = settings if settings is not None else docx_settings(style) def write(self, tree: RenderTree) -> bytes: settings = self.settings language = language_tag(tree.language, self.style.locale) - title = plain_text(tree.title) - values: Mapping[str, str] = {"title": title, "version": self.version, "date": self.date} + title = document_title(tree) + values: Mapping[str, str] = {"title": title, "version": tree.version, "date": tree.effective_date} text_width = settings.page.text_width package = pkg.Package() diff --git a/src/legaldown_render/writers/docx/body.py b/src/legaldown_render/writers/docx/body.py index adf5ade..d8a8ddd 100644 --- a/src/legaldown_render/writers/docx/body.py +++ b/src/legaldown_render/writers/docx/body.py @@ -16,7 +16,7 @@ from collections.abc import Iterator from ...style.docx import PageGeometry -from ...tree import Block, CodeBlock, DraftingNote, List, Paragraph, Quote, RenderTree, Rule, Table +from ...tree import Block, CodeBlock, DraftingNote, List, Paragraph, Quote, RenderTree, Rule, Table, plain_text from ..text import inline_text from .ooxml import el, p, text_run from .styles import CODE, DRAFTING_NOTE, DRAFTING_NOTE_LABEL, QUOTE, SUBTITLE, TITLE, heading_style @@ -37,11 +37,17 @@ def section_properties(page: PageGeometry, *, header_id: str | None = None, foot ) +def document_title(tree: RenderTree) -> str: + """The document's title as plain text, trimmed: the one text the title paragraph, the + ``{title}`` of a header or footer and the document properties all show.""" + return plain_text(tree.title).strip() + + def interim_body(tree: RenderTree, page: PageGeometry, *, header_id: str | None = None, footer_id: str | None = None) -> ET.Element: """``w:body`` for *tree*: see the module docstring. Interim.""" content: list[ET.Element] = [] - title = inline_text(tree.title).strip() + title = document_title(tree) if title: content.append(p(text_run(title), style=TITLE)) subtitle = inline_text(tree.subtitle).strip() From 4460f5c081ad3468208d31ec878f51d9a170c4cd Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:52:37 +0000 Subject: [PATCH 073/118] Harden the DOCX package after review Serialise with the writer's own prefix table instead of xml.etree's global registry, and sanitise in the serialiser too; pin the uncompressed parts by SHA-256. ZIP entries carry a regular-file mode. Hyperlink relationships are made only for link schemes (no relative, //host or backslash targets), and relationships are found in constant time. Bookmark names and languages must match in full; text passed as a child is a bug, not a recursion error; docDefaults set only the Latin font and the language; code is left-aligned. The command writes a DOCX without -o to a standard output that is not a terminal, and refuses --fragment for it. Tests: numeric extremes, the specification corpus validated in the schema run, more negative controls, missing lxml fails under LEGALDOWN_REQUIRE_DOCX_TOOLS. CI caches the schemas and checks out the specification. Docs say what changed in behaviour. --- .github/workflows/ci.yml | 17 +- CONFORMANCE.md | 2 +- README.md | 7 +- docs/architecture.md | 1 + docs/decisions/0008-docx-word-native.md | 10 +- docs/roadmap.md | 5 + docs/style-templates.md | 16 +- src/legaldown_render/cli.py | 17 +- src/legaldown_render/writers/docx/ooxml.py | 74 +++++- src/legaldown_render/writers/docx/package.py | 25 ++- src/legaldown_render/writers/docx/styles.py | 12 +- tests/docx_support.py | 27 ++- tests/schemas/README.md | 5 +- tests/test_docx_package.py | 225 +++++++++++++++++-- tests/test_docx_validation.py | 109 +++++++-- 15 files changed, 464 insertions(+), 88 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 174c6e0..16122fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,12 @@ jobs: LEGALDOWN_REQUIRE_DOCX_TOOLS: "1" steps: - uses: actions/checkout@v7 + - name: Check out the LegalDown specification (its documents are validated too) + uses: actions/checkout@v7 + with: + repository: ForLegalAI/LegalDown + ref: ${{ env.LEGALDOWN_SPEC_REF }} + path: .legaldown-spec - uses: actions/setup-python@v7 with: python-version: "3.13" @@ -94,8 +100,17 @@ jobs: sudo apt-get update sudo apt-get install -y libreoffice-writer python3-uno - run: pip install -e ".[dev]" - - run: python tests/schemas/fetch.py + - name: Cache the OOXML schemas + id: schemas + uses: actions/cache@v4 + with: + path: tests/schemas/ooxml + key: ooxml-schemas-${{ hashFiles('tests/schemas/fetch.py') }} + - if: steps.schemas.outputs.cache-hit != 'true' + run: python tests/schemas/fetch.py - run: pytest -q -m ooxml_schema + env: + LEGALDOWN_SPEC_DIR: .legaldown-spec - run: pytest -q -m libreoffice validator-main: diff --git a/CONFORMANCE.md b/CONFORMANCE.md index fae90b2..2814a9d 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -20,7 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | -| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. The **body is interim**: the title, headings as `heading N` paragraphs and every other block as plain text, with no numbers, cross-references, table of contents, title block, attachments or signature blocks. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)); none of the numbering is built yet. HTML is the format this renderer claims §13.6 with | +| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. The **body is interim**: the title, headings as `heading N` paragraphs and every other block as plain text, with no numbers, cross-references, table of contents, title block, attachments or signature blocks. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)); none of the numbering is built yet. Every DOCX render reports the Warning `render-docx-interim` ("DOCX output is incomplete in this version") until the body is complete; it is the renderer's own rule, not the specification's, and goes away then. HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | diff --git a/README.md b/README.md index 0410241..348fa70 100644 --- a/README.md +++ b/README.md @@ -131,8 +131,8 @@ may not contain `version` or `extends`, so `overrides=legaldown_render.style.sty ## DOCX (in progress) `-o contract.docx` (or `--format docx`, or `render(text, format="docx")`) writes a Word document. The -file is bytes: the command writes it to a file or, with `-o -`, to standard output, and refuses to write -it to a terminal; `render()` returns it as `result.binary` (`result.output` is empty). The writer needs no +file is bytes: the command writes it to a file or, with `-o -` (or no `-o` when standard output is not a +terminal), to standard output, and refuses to write it to a terminal; `--fragment` is a usage error for it; `render()` returns it as `result.binary` (`result.output` is empty). The writer needs no dependency beyond the base package. The design is that **in the Word file every number, cross-reference, the table of contents and the page @@ -148,7 +148,8 @@ and margins, a running header and footer from `page.header` and `page.footer` (t as live page fields), the document's title and language, `w:updateFields` so Word refreshes fields when it opens the file (`docx.update_fields_on_open`), and deterministic output. **The body is interim:** the title, the headings as Word headings, and every other block as plain text, with **no numbers, references, contents, -title block, attachments or signature blocks yet**. Do not send such a file as a contract. Settings Word +title block, attachments or signature blocks yet**, and every render reports the Warning +`render-docx-interim` to say so. Do not send such a file as a contract. Settings Word cannot express (a `%` length, say) are an error for DOCX only. See [Styles and preferences](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/style-templates.md#docx) and the [roadmap](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/roadmap.md). diff --git a/docs/architecture.md b/docs/architecture.md index 12a227f..a9f8f61 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -179,6 +179,7 @@ carries the structure behind every number, in terms that name no output format: | Where | Field | Holds | |---|---|---| +| `RenderTree.version`, `RenderTree.effective_date` | `str` | The frontmatter's version and effective date as plain text, exactly as the title block shows them (directives resolved, a valid date formatted by the style for the locale), **whether or not** `title_block` shows them: a running header or footer uses them (`{version}`, `{date}`). Empty when absent | | `RenderTree.formats` | `NumberingFormats` | The formats the labels were made with, after the style is resolved: `scheme`; `headings`, one `LevelFormat` (counter, `label` and `ref` templates) per heading level (empty under `none`); `enumerated`, the formats by list depth (cycling); `renumber`, the one format of ordered lists (None under `enumeration.ordered: enumerate`); `paragraphs` (None unless numbered); `reference`, `references.format` | | `Section`, `ListItem`, `Paragraph` | `unit` | An id, unique in the document and equal on every render: `s` for the k-th section, `u` for the k-th item or top-level paragraph of a section in document order, the preamble first. Not the anchor. Every section, every item (in a quote too) and every top-level paragraph of a section has one, numbered or not, since an unnumbered one can still be the target of a reference; a paragraph inside an item, or in the preamble, is not a unit (None) | | same | `number` | The unit's own counter value (the last part of "2.1" is 1; the item's position in its list; the paragraph's in its section). None when the unit is not numbered: any section under `none`, an item of a list that is not enumerated, a paragraph when the style does not number them | diff --git a/docs/decisions/0008-docx-word-native.md b/docs/decisions/0008-docx-word-native.md index 44530b9..8f432d5 100644 --- a/docs/decisions/0008-docx-word-native.md +++ b/docs/decisions/0008-docx-word-native.md @@ -68,9 +68,15 @@ Also decided here, because every later part rests on them: - **Safety.** One sanitiser is applied to every text, attribute, field instruction, bookmark name, relationship and property value. It drops the characters XML cannot carry. Without it `xml.etree` writes a control character raw and a lone surrogate as a character reference, and Word - reports the file as unreadable. The untrusted mode and its style caps apply unchanged. + reports the file as unreadable. The untrusted mode and its style caps apply unchanged. A hyperlink + relationship is made only for the schemes links may use (no relative target, no `//host`, no + backslash), because Word follows what a relationship says. - **Deterministic output.** Fixed part order, relationship ids by order of use, fixed ZIP times, - attributes and compression, no timestamps in the properties. + attributes and compression, no timestamps in the properties, and prefixes from the writer's own table, + not from `xml.etree`'s global registry (which another library may change). The ZIP is + deterministic **for a given zlib**: a different zlib may compress the same parts to different, + equivalent bytes. The parts themselves, uncompressed, are the same everywhere, and the tests pin + them by SHA-256. ### Cross-references diff --git a/docs/roadmap.md b/docs/roadmap.md index 20974fb..b65ab13 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -102,6 +102,11 @@ values appear only as the cached results of fields. Decided in mode 15, `w:updateFields`), page setup, running headers and footers with `PAGE`/`NUMPAGES` fields, document properties, deterministic output. `RenderResult.binary`, `-o x.docx`, `-o -`. The body is an interim one: the title, `heading N` paragraphs and plain-text paragraphs, with no numbers yet +- **Behaviour changes this brings.** `-o -` now writes to standard output for every format (it used to + create a file named `-`); `.docx` is a writable extension, and a DOCX with no `-o` goes to standard output + when that is not a terminal; `--fragment` with DOCX is a usage error; `RenderTree` gains `version` and + `effective_date` (plain text), filled whether or not the title block shows them; `.docx` renders add + the Warning `render-docx-interim` until the body is complete - **Style settings (done).** `page.header` and `page.footer` (the HTML writer ignores them for now), `docx.update_fields_on_open`; settings Word cannot express are a `StyleError` for DOCX - **Verification (done).** Every part validated against the ISO/IEC 29500 schemas (fetched in CI, not diff --git a/docs/style-templates.md b/docs/style-templates.md index 0cb626c..7106b2c 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -216,12 +216,13 @@ may use these fields, and any other `{name}` in braces is refused when the style | `{version}` | the frontmatter's `version`; empty when the document has none | | `{date}` | the frontmatter's `effective_date`, formatted as `values.date` says for the formatting locale; empty when the document has none | -A `version` or `effective_date` that holds a directive (`{{placeholder: ...}}`) is empty in a header or -footer: directives are resolved where the document shows them. A text that is empty everywhere (the -header by default) writes no header at all. +`{version}` and `{date}` are the same text the title block shows for them (directives in them resolved +the same way, a valid date formatted as `values.date` says), whether or not `title_block` shows them +there. A text that is empty everywhere (the header by default) writes no header at all. **Where it applies.** The **DOCX** writer writes them: `{page}` and `{pages}` are `PAGE` and `NUMPAGES` -fields that Word and LibreOffice count while laying out the pages, and the other three fields are text. +fields that Word and LibreOffice count while laying out the pages (a viewer that lays out no pages shows +the cached "1", so the default footer reads "1 / 1" there), and the other three fields are text. A single text is aligned; two or three are separated by tab stops at the middle and the right margin. The **HTML** and plain-text writers do not read them yet; the PDF writer will (v0.6). @@ -243,7 +244,7 @@ keeps accepting the same style. | `page.size` | the paper size of the page: `A3` to `A5`, `B4`, `B5`, `letter`, `legal`, `ledger`, with `landscape` or `portrait`; or one or two lengths; `auto` and an orientation alone are A4 | | `page.margin` | one to four lengths (top, right, bottom, left, as in CSS); the header and footer sit half a margin from the edge, at most 1.27 cm | | `typography.font_family` | the **first** family: `Georgia` for `'Georgia, "Times New Roman", serif'`; `serif`, `sans-serif` and `monospace` alone are Times New Roman, Arial and Courier New. Word falls back to its own substitute when the font is not installed | -| `typography.font_size`, `headings.N.size` | half-points, between 1 pt and 1638 pt | +| `typography.font_size`, `headings.N.size` | half-points (Word's unit for a font size, so a size is rounded to the nearest half-point: `11.3pt` is 11.5 pt); `headings.N.size` between 1 pt and 1638 pt, and `typography.font_size` between 1 pt and 1023 pt, because the title is 1.6 times the body | | `typography.line_height` | a number is that many lines (`1.5`), `normal` is one line, a length is the least height of a line | | `typography.color`, `typography.link_color` | `#rgb`, `#rrggbb` (an alpha part is dropped), `rgb()`, `hsl()`, or one of `black silver gray white maroon red purple fuchsia green lime olive yellow navy blue teal aqua orange`; any other colour name is an error, so write it as `#rrggbb` | | `headings.N.weight` | bold from 600 up (`bold`, `bolder`), otherwise regular | @@ -252,7 +253,10 @@ Lengths: `mm`, `cm`, `in`, `pt`, `pc` and `px` (96 to the inch) are absolute. ** relative to `typography.font_size`, which is itself relative to 12 pt when it is written in them, as in a browser. **`%`, `ch` and `ex`** depend on a layout or a font that Word does not give the renderer, so each is an error for DOCX: `page.margin: 5%` raises "page.margin: uses '%', which a Word document cannot -express". `html.max_width` and `html.extra_css` are never read. +express". Units are not case-sensitive (`25MM`). Word's limits are enforced too, as style errors that never +quote the value: a page edge of 7.2 pt to 1584 pt (22 inches), a margin of 0 to 1584 pt, margins that leave +at least 36 pt (0.5 inch) of text in each direction, a line height of at most 1584 pt (or 100 lines), and a +number of more than 32 digits is no length at all. `html.max_width` and `html.extra_css` are never read. The writer generates Word styles from the template (`Normal`, `heading 1` to `heading 5`, `Title`, `Subtitle`, `Quote`, `toc 1` to `toc 5`, `Hyperlink`, `Table Grid`, and styles of its own for drafting diff --git a/src/legaldown_render/cli.py b/src/legaldown_render/cli.py index 5853dc4..806f690 100644 --- a/src/legaldown_render/cli.py +++ b/src/legaldown_render/cli.py @@ -9,7 +9,7 @@ Documents and output are always UTF-8 (the HTML says ````), never the console's or the locale's encoding, and output has LF line endings on every platform. A DOCX file is bytes: it is written to a file, or to standard output's -binary interface with ``-o -``, never to a terminal. +binary interface (``-o -``, or no ``-o`` when standard output is not a terminal), never to a terminal. """ from __future__ import annotations @@ -82,7 +82,7 @@ def _parser() -> argparse.ArgumentParser: ) parser.add_argument("input", nargs="?", help="the .lgd file to render, or - for standard input") parser.add_argument("-o", "--output", help="output file (default: standard output, except for DOCX, which " - "is binary and needs a file or -o -). Without --format, .html/.htm write HTML, " + "is binary and is refused on a terminal). Without --format, .html/.htm write HTML, " ".txt/.text plain text and .docx a Word document; any other name, with or without an " "extension, writes HTML, except the extension of a format not written (.doc, " ".odt, .rtf, .pdf, .md, .markdown, .epub, .tex), which needs --format. -o - writes " @@ -187,10 +187,9 @@ def main(argv: list[str] | None = None) -> int: output_format = output_format or "html" to_stdout = not args.output or args.output == _STDOUT if output_format == "docx": - if not args.output: - parser.error("a DOCX file is binary: name it with -o FILE.docx, or write it to standard output " - "with -o -") - if to_stdout and sys.stdout.isatty(): + if args.fragment: + parser.error("--fragment applies to HTML only: a DOCX file is always the whole document") + if to_stdout and getattr(sys.stdout, "isatty", lambda: False)(): parser.error("refusing to write a DOCX file to a terminal; redirect standard output or name a file " "with -o FILE.docx") name = "" if args.input == "-" else args.input @@ -253,9 +252,9 @@ def main(argv: list[str] | None = None) -> int: def _write_binary(data: bytes, output: str) -> int: - """Write *data*, a rendered binary file, to the file *output*, or with ``-`` to standard - output's bytes (never through its text encoding).""" - if output == _STDOUT: + """Write *data*, a rendered binary file, to the file *output*, or (with ``-``, or no + output at all) to standard output's bytes, never through its text encoding.""" + if not output or output == _STDOUT: buffer = getattr(sys.stdout, "buffer", None) if buffer is None: _complain("legaldown-render: cannot write to standard output: it has no binary interface") diff --git a/src/legaldown_render/writers/docx/ooxml.py b/src/legaldown_render/writers/docx/ooxml.py index 83740c7..b185a48 100644 --- a/src/legaldown_render/writers/docx/ooxml.py +++ b/src/legaldown_render/writers/docx/ooxml.py @@ -34,9 +34,10 @@ CONTENT_TYPES = "http://schemas.openxmlformats.org/package/2006/content-types" EXTENDED_PROPERTIES = "http://schemas.openxmlformats.org/officeDocument/2006/extended-properties" +#: The prefix of each namespace in the parts written here. :func:`serialize` uses this table, not +#: ``xml.etree``'s global registry, which any other library may change (``register_namespace``). _PREFIXES = {"w": W, "r": R, "cp": CP, "dc": DC, "dcterms": DCTERMS, "dcmitype": DCMITYPE, "xsi": XSI} -for _prefix, _uri in _PREFIXES.items(): - ET.register_namespace(_prefix, _uri) +_PREFIX_OF = {uri: prefix for prefix, uri in {**_PREFIXES, "xml": XML}.items()} #: Characters XML 1.0 cannot carry (C0 but TAB, LF and CR), and the ones Word #: refuses or reads as something else: DEL and C1 (NEL included), the @@ -99,6 +100,8 @@ def _flatten(children: Iterable[Any]) -> list[ET.Element]: continue if isinstance(child, ET.Element): flat.append(child) + elif isinstance(child, str | bytes): + raise InternalError("text was passed where an element belongs (a bug in the DOCX writer)") else: flat.extend(_flatten(child)) return flat @@ -113,11 +116,62 @@ def val(name: str, value: Any = None) -> ET.Element: _DECLARATION = b'\n' +def _name(tag: str) -> tuple[str, str]: + """The namespace and the local name of an ElementTree *tag* (``""`` for none).""" + if tag.startswith("{"): + namespace, _, local = tag[1:].partition("}") + return namespace, local + return "", tag + + +def _attribute(value: str) -> str: + """*value* as the text of a double-quoted attribute: sanitised, and escaped so that + nothing, white space included, changes when a parser normalises it.""" + return _t(value).replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """) + + def serialize(root: ET.Element) -> bytes: - """*root* as a UTF-8 XML part with the declaration Word writes. Attributes keep - the order they were set in, and namespace declarations are sorted by prefix, - so the same tree is always the same bytes.""" - return _DECLARATION + ET.tostring(root, encoding="utf-8") + """*root* as a UTF-8 XML part with the declaration Word writes. Prefixes come from this + module's own table, never from ``xml.etree``'s global registry, so what another library + registers cannot change the bytes. Attributes keep the order they were set in, the + namespaces are declared on the root sorted by prefix, and the same tree is always the + same bytes. Every text and attribute is sanitised here as well (:func:`_t`), whoever + built the element. Only ``text`` is written, as no part here has mixed content.""" + namespaces: set[str] = set() + for element in root.iter(): + namespaces.add(_name(element.tag)[0]) + namespaces.update(_name(key)[0] for key in element.attrib) + namespaces.discard("") + unknown = namespaces - _PREFIX_OF.keys() + if unknown: + raise InternalError("an element is in a namespace the DOCX writer has no prefix for (a bug in the writer)") + declarations = "".join(f' xmlns:{prefix}="{uri}"' + for prefix, uri in sorted((_PREFIX_OF[uri], uri) for uri in namespaces if uri != XML)) + + def qualified(tag: str) -> str: + namespace, local = _name(tag) + return f"{_PREFIX_OF[namespace]}:{local}" if namespace else local + + out: list[str] = [] + stack: list[tuple[ET.Element, bool]] = [(root, False)] + while stack: + element, closing = stack.pop() + name = qualified(element.tag) + if closing: + out.append(f"") + continue + attributes = "".join(f' {qualified(key)}="{_attribute(value)}"' for key, value in element.attrib.items()) + extra = declarations if element is root else "" + text = element.text + if text is None and len(element) == 0: + out.append(f"<{name}{extra}{attributes}/>") + continue + out.append(f"<{name}{extra}{attributes}>") + if text: + out.append(_t(text, breaks=True).replace("&", "&").replace("<", "<").replace(">", ">")) + stack.append((element, True)) + stack.extend((child, False) for child in reversed(element)) + return _DECLARATION + "".join(out).encode("utf-8") def flat_part(namespace: str, root: str, children: Iterable[tuple[str, Mapping[str, Any], str | None]]) -> bytes: @@ -226,13 +280,13 @@ def tr_pr(*children: Any) -> ET.Element | None: def p(*content: Any, style: str | None = None, props: Iterable[Any] = ()) -> ET.Element: """A paragraph: *style* (a style id) and *props* (``w:pPr`` children, any order) first, then *content* (runs, fields, bookmarks, hyperlinks).""" - return el("p", None, ppr(style and val("pStyle", style), props), content) + return el("p", None, ppr(val("pStyle", style) if style else None, props), content) def r(*content: Any, style: str | None = None, props: Iterable[Any] = ()) -> ET.Element: """A run: *style* (a character style id) and *props* (``w:rPr`` children, any order) first, then *content* (``w:t``, ``w:tab``, ``w:br``, field characters).""" - return el("r", None, rpr(style and val("rStyle", style), props), content) + return el("r", None, rpr(val("rStyle", style) if style else None, props), content) def text_run(text: str, *, style: str | None = None, props: Iterable[Any] = ()) -> ET.Element: @@ -282,7 +336,7 @@ def hyperlink_internal(bookmark: str, *runs: ET.Element, tooltip: str | None = N #: A bookmark name: a letter (or ``_`` for a hidden one), then letters, digits and ``_``, #: at most 40 characters (Word's limit). -_BOOKMARK_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,39}$") +_BOOKMARK_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,39}") class Bookmarks: @@ -296,7 +350,7 @@ def __init__(self) -> None: def pair(self, name: str) -> tuple[ET.Element, ET.Element]: """The ``w:bookmarkStart`` and ``w:bookmarkEnd`` of a new bookmark *name*.""" - if not _BOOKMARK_RE.match(name): + if not _BOOKMARK_RE.fullmatch(name): raise InternalError("a bookmark name is not valid (a bug in the DOCX writer)") if name.lower() in self._names: raise InternalError("a bookmark name is used twice (a bug in the DOCX writer)") diff --git a/src/legaldown_render/writers/docx/package.py b/src/legaldown_render/writers/docx/package.py index 1124442..073475e 100644 --- a/src/legaldown_render/writers/docx/package.py +++ b/src/legaldown_render/writers/docx/package.py @@ -14,6 +14,7 @@ from urllib.parse import quote from ...errors import InternalError +from ...urls import LINK_SCHEMES, is_link_target, url_scheme from .ooxml import CONTENT_TYPES, PACKAGE_RELATIONSHIPS, _t, flat_part #: Relationship types (ISO/IEC 29500-1, 11.3 and 15.2). @@ -62,20 +63,32 @@ class Relationships: def __init__(self) -> None: self._items: list[tuple[str, str, bool]] = [] + self._ids: dict[tuple[str, str, bool], str] = {} def add(self, relationship_type: str, target: str, *, external: bool = False) -> str: """The id of the relationship to *target* (a part name relative to the owning part, or with *external* an address, see :func:`external_target`), added if it is not there yet.""" key = (relationship_type, target, external) - if key not in self._items: + if key not in self._ids: self._items.append(key) - return f"rId{self._items.index(key) + 1}" + self._ids[key] = f"rId{len(self._items)}" + return self._ids[key] def add_external(self, url: str) -> str | None: - """The id of a hyperlink relationship to *url*, or None when *url* cannot be - one (:func:`external_target`): the caller then shows the link's text alone.""" - target = external_target(url) + """The id of a hyperlink relationship to *url*, or None when *url* is not one the + document may link to: the caller then shows the link's text alone. It is the + HTML writer's rule (:func:`~legaldown_render.urls.is_link_target`) and more, as + Word, unlike a browser, follows what a relationship says without asking: the URL + needs a scheme from :data:`~legaldown_render.urls.LINK_SCHEMES` (no relative path, + and no ``//host``), has no backslash, and can be written as a URI + (:func:`external_target`).""" + stripped = url.strip() + scheme = url_scheme(stripped) + if scheme is None or scheme not in LINK_SCHEMES or not is_link_target(stripped) \ + or stripped.startswith("//") or "\\" in stripped: + return None + target = external_target(stripped) return None if target is None else self.add(REL_HYPERLINK, target, external=True) def __bool__(self) -> bool: @@ -132,6 +145,6 @@ def to_bytes(self) -> bytes: info = zipfile.ZipInfo(_t(name), date_time=(1980, 1, 1, 0, 0, 0)) info.compress_type = zipfile.ZIP_DEFLATED info.create_system = 3 - info.external_attr = 0o644 << 16 + info.external_attr = 0o100644 << 16 archive.writestr(info, data, compresslevel=6) return buffer.getvalue() diff --git a/src/legaldown_render/writers/docx/styles.py b/src/legaldown_render/writers/docx/styles.py index 7e41a3f..ad4a5a5 100644 --- a/src/legaldown_render/writers/docx/styles.py +++ b/src/legaldown_render/writers/docx/styles.py @@ -59,17 +59,19 @@ def toc_style(level: int) -> str: _NOTE_FILL, _NOTE_RULE = "FFF8E1", "D9A400" _CODE_FILL = "F5F5F5" -_LANGUAGE_RE = re.compile(r"^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{1,8}){0,3}$") +_LANGUAGE_RE = re.compile(r"[A-Za-z]{2,3}(?:-[A-Za-z0-9]{1,8}){0,3}") def language_tag(*candidates: str | None) -> str: """The first of *candidates* that is a language tag (``en``, ``cs-CZ``), else ``en-US``: the document's ``language`` is free text to the renderer, and ``w:lang`` is read by Word's proofing.""" - return next((tag for tag in candidates if tag and _LANGUAGE_RE.match(tag)), "en-US") + return next((tag for tag in candidates if tag and _LANGUAGE_RE.fullmatch(tag)), "en-US") def _fonts(name: str) -> ET.Element: - return el("rFonts", {"ascii": name, "hAnsi": name, "cs": name, "eastAsia": name}) + """The Latin font. The complex-script and East Asian fonts stay unset, for Word to choose by + script: they are not the Latin font's to give.""" + return el("rFonts", {"ascii": name, "hAnsi": name}) def _size(half_points: int) -> list[ET.Element]: @@ -136,7 +138,7 @@ def half_points(factor: float) -> int: "docDefaults", None, el("rPrDefault", None, rpr(_fonts(settings.font), _size(settings.half_points), - el("lang", {"val": language, "eastAsia": language, "bidi": language}))), + el("lang", {"val": language}))), el("pPrDefault"), ) @@ -193,7 +195,7 @@ def half_points(factor: float) -> int: el("spacing", {"before": em(0.4), "after": em(0.4)}), el("ind", {"left": 227, "right": 113})]), _style(CODE, "Code", based_on=NORMAL, priority=31, custom=True, props=[_shade(_CODE_FILL), el("spacing", {"before": 0, "after": 0, "line": 240, "lineRule": "auto"}), - el("ind", {"left": 113, "right": 113})], + el("ind", {"left": 113, "right": 113}), el("jc", {"val": "left"})], run=[_fonts(_MONOSPACE), _size(half_points(0.9))]), _style(TABLE_GRID, "Table Grid", kind="table", based_on="TableNormal", priority=39, table=el("tblPr", None, el("tblBorders", None, *(_border(side, _RULE, 4) diff --git a/tests/docx_support.py b/tests/docx_support.py index f290901..cc09f67 100644 --- a/tests/docx_support.py +++ b/tests/docx_support.py @@ -16,6 +16,7 @@ import tempfile import xml.etree.ElementTree as ET import zipfile +from collections.abc import Callable from functools import cache from pathlib import Path @@ -28,8 +29,9 @@ W = f"{{{W_NS}}}" -def _missing(reason: str) -> None: - """Skip, or fail when the environment promises the tools (CI).""" +def tool_missing(reason: str) -> None: + """Skip, or fail when the environment promises the tools (CI): a job that sets + ``LEGALDOWN_REQUIRE_DOCX_TOOLS`` must not pass by skipping what it exists to run.""" if os.environ.get("LEGALDOWN_REQUIRE_DOCX_TOOLS"): pytest.fail(f"required tool missing: {reason}") pytest.skip(reason) @@ -84,9 +86,12 @@ def _schemas() -> dict[str, object]: def require_schemas() -> None: """Skip (or fail in CI) unless lxml and the schemas are there.""" - pytest.importorskip("lxml", reason="lxml is a test dependency: pip install -e '.[dev]'") + try: + import lxml # noqa: F401 + except ImportError: + tool_missing("lxml is not installed (a test dependency: pip install -e '.[dev]')") if schema_directory() is None: - _missing("the OOXML schemas are not fetched (python tests/schemas/fetch.py)") + tool_missing("the OOXML schemas are not fetched (python tests/schemas/fetch.py)") def schema_for(name: str) -> str | None: @@ -183,10 +188,10 @@ def system_python() -> str | None: def require_libreoffice() -> str: """The Python to run the UNO script with, or skip (fail in CI) when LibreOffice or its Python bindings are missing.""" if shutil.which("soffice") is None: - _missing("LibreOffice (soffice) is not installed") + tool_missing("LibreOffice (soffice) is not installed") python = system_python() if python is None: - _missing("no Python with the uno module (python3-uno) was found") + tool_missing("no Python with the uno module (python3-uno) was found") assert python is not None return python @@ -205,6 +210,16 @@ def lines(self) -> list[str]: return self.text.splitlines() +def repackage(data: bytes, name: str, change: Callable[[str], str]) -> bytes: + """The package *data* with the text of the part *name* changed by *change*: for a test that + the checks catch what they are meant to.""" + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for part, content in parts(data).items(): + archive.writestr(part, change(content.decode("utf-8")).encode("utf-8") if part == name else content) + return buffer.getvalue() + + def libreoffice_export(documents: dict[str, bytes]) -> dict[str, Exported]: """Open each document (name to DOCX bytes) in one LibreOffice instance, update its fields and indexes, and return its text, header, footer and page count.""" diff --git a/tests/schemas/README.md b/tests/schemas/README.md index 83dc158..727a77b 100644 --- a/tests/schemas/README.md +++ b/tests/schemas/README.md @@ -38,8 +38,9 @@ Committing them would need a licence that allows redistribution. None is clear, ISO/IEC, whose webstore terms apply to that copy. Both are copyrighted standards texts. Ecma's general practice is to allow copying with the notice retained, but that was not confirmed here for these files, and the ISO/IEC terms differ. -* A copy of the same schemas was available in this environment under `/mnt/skills/public/docx/`. It is part of - a skill whose licence forbids copying and distributing it, so it was neither used nor copied. +* Another copy of the same schemas, bundled with a third-party tool whose licence forbids copying and + distributing it, was available to the author. It was neither used nor copied: only the files fetched from + the publishers are used. If the maintainers clear the licence (for example by asking Ecma), the files can be committed here and `fetch.py` dropped; nothing else depends on where they live. diff --git a/tests/test_docx_package.py b/tests/test_docx_package.py index c1988c7..912c872 100644 --- a/tests/test_docx_package.py +++ b/tests/test_docx_package.py @@ -4,6 +4,7 @@ test_docx_validation.py, LibreOffice in test_docx_libreoffice.py; this module needs neither.""" from __future__ import annotations +import hashlib import io import sys import xml.etree.ElementTree as ET @@ -17,6 +18,7 @@ from legaldown_render import RenderOptions, render, render_file from legaldown_render.cli import main from legaldown_render.errors import InternalError +from legaldown_render.tree import plain_text from legaldown_render.writers import FORMATS, DocxWriter, format_for_path from legaldown_render.writers.docx import ooxml, package, styles from legaldown_render.writers.docx.headers import template_runs @@ -96,7 +98,7 @@ def test_the_zip_is_deterministic_and_has_no_timestamps_or_platform_bits() -> No for info in archive.infolist(): assert info.date_time == (1980, 1, 1, 0, 0, 0) assert info.compress_type == zipfile.ZIP_DEFLATED - assert (info.create_system, info.external_attr >> 16, info.extra, info.comment) == (3, 0o644, b"", b"") + assert (info.create_system, info.external_attr >> 16, info.extra, info.comment) == (3, 0o100644, b"", b"") def test_a_different_document_is_a_different_file() -> None: @@ -211,9 +213,10 @@ def test_doc_defaults_have_the_language_and_the_first_font_family() -> None: root = xml(docx()["word/styles.xml"]) run = root.find(f"{W}docDefaults/{W}rPrDefault/{W}rPr") fonts = run.find(f"{W}rFonts") - assert {key.removeprefix(W): value for key, value in fonts.attrib.items()} == { - "ascii": "Georgia", "hAnsi": "Georgia", "cs": "Georgia", "eastAsia": "Georgia"} - assert run.find(f"{W}lang").get(f"{W}val") == "cs" # the document's language + # Only the Latin font: complex-script and East Asian fonts are not the Latin font's to give. + assert {key.removeprefix(W): value for key, value in fonts.attrib.items()} == {"ascii": "Georgia", "hAnsi": "Georgia"} + lang = run.find(f"{W}lang") + assert {key.removeprefix(W): value for key, value in lang.attrib.items()} == {"val": "cs"} # the document's language english = xml(docx(FRONT + "# A\n")["word/styles.xml"]).find(f"{W}docDefaults/{W}rPrDefault/{W}rPr/{W}lang") assert english.get(f"{W}val") == "en" @@ -337,10 +340,31 @@ def test_the_date_follows_the_style_and_the_locale() -> None: assert texts(footer) == ["04.03.26"] -def test_a_version_or_date_that_holds_directives_is_left_out() -> None: - source = "---\ntitle: T\nversion: '{{placeholder: v}}'\n---\n# A\n" - footer = xml(docx(source, overrides={"page.footer.center": "[{version}]"})["word/footer1.xml"]) - assert texts(footer) == ["[]"] +def test_a_version_or_date_that_holds_directives_reads_as_the_title_block_shows_it() -> None: + source = "---\ntitle: T\nversion: 'v {{date: 2026-03-04}} {{placeholder: v}}'\neffective_date: '{{date: 2026-01-02}}'\n---\n# A\n" + overrides = {"page.footer.center": "[{version}|{date}]", "title_block.version": True} + result = render(source, format="docx", overrides=overrides) + footer = xml(parts(result.binary)["word/footer1.xml"]) + assert texts(footer) == ["[v March 4, 2026 [_____]|January 2, 2026]"] + shown = {label: plain_text(value) for label, value in result.tree.header} + assert shown["Version"] == "v March 4, 2026 [_____]" # the same text the title block has + + +def test_the_tree_carries_version_and_date_whether_or_not_the_title_block_shows_them() -> None: + tree = render(SOURCE, format="text", overrides={"title_block.version": False, + "title_block.effective_date": False}).tree + assert (tree.version, tree.effective_date, tree.header) == ("2.1", "4. března 2026", ()) + bare = render(FRONT + "# A\n", format="text").tree + assert (bare.version, bare.effective_date) == ("", "") + + +def test_the_title_is_the_same_text_in_the_body_the_properties_and_the_footer() -> None: + source = "---\ntitle: ' A {{date: 2026-03-04}} B '\n---\n# A\n" + contents = docx(source, overrides={"page.footer.center": "{title}"}) + expected = "A March 4, 2026 B" + assert texts(xml(contents["word/document.xml"]))[0] == expected + assert texts(xml(contents["word/footer1.xml"])) == [expected] + assert xml(contents["docProps/core.xml"])[0].text == expected def test_a_field_name_the_writer_does_not_know_stays_as_written() -> None: @@ -510,14 +534,41 @@ def test_hyperlinks_are_internal_by_anchor_or_external_by_relationship() -> None ("https://example.com/a b", "https://example.com/a%20b"), ("https://example.com/%20already", "https://example.com/%20already"), ("https://example.com/ž", "https://example.com/%C5%BE"), - ('https://example.com/"<>\\^`{|}', "https://example.com/%22%3C%3E%5C%5E%60%7B%7C%7D"), + ('https://example.com/"<>^`{|}', "https://example.com/%22%3C%3E%5E%60%7B%7C%7D"), + ("https://e/a\\b", "https://e/a%5Cb"), (" https://example.com ", "https://example.com"), ("", None), (" ", None), ("https://e/\x01", None), ("https://e/\x7f", None), ("https://e/a\nb", None), ("https://e/\ud800", None), ]) def test_a_relationship_target_is_a_valid_uri_or_none(url: str, expected: str | None) -> None: assert package.external_target(url) == expected - assert package.Relationships().add_external(url) == (None if expected is None else "rId1") + if "\\" not in url: + assert package.Relationships().add_external(url) == (None if expected is None else "rId1") + + +@pytest.mark.parametrize("url", [ + "javascript:alert(1)", "JaVaScRiPt:alert(1)", "java\tscript:alert(1)", "file:///etc/passwd", "ms-msdt:/id PCWDiagnostic", + "ms-word:ofe|u|https://x/a.docx", "data:text/html,x", "vbscript:x", "//evil.example/x", "///x", "\\\\host\\share", + "\\host", "https://e/a\\b", "relative/path", "../up", "#fragment", "?q=1", "C:\\x", "localhost:8080", "", +]) +def test_a_hyperlink_relationship_is_refused_for_what_is_not_a_link(url: str) -> None: + """The HTML writer's rule (is_link_target), and more: no relative target and no //host, + since Word follows a relationship without a base to read it against.""" + assert package.Relationships().add_external(url) is None + + +@pytest.mark.parametrize("url", ["https://example.com", "http://e/x", "mailto:a@example.com", "tel:+420123456789", + "ftp://e/f", "urn:isbn:0451450523", "sms:+1555"]) +def test_a_hyperlink_relationship_is_made_for_a_link_scheme(url: str) -> None: + assert package.Relationships().add_external(url) == "rId1" + + +def test_relationships_are_found_in_constant_time_and_keep_their_order() -> None: + relationships = package.Relationships() + ids = [relationships.add_external(f"https://example.com/{n}") for n in range(2000)] + assert ids == [f"rId{n + 1}" for n in range(2000)] + assert relationships.add_external("https://example.com/1999") == "rId2000" + assert relationships.add(package.REL_STYLES, "styles.xml") == "rId2001" def test_a_relationship_value_is_escaped_in_the_part() -> None: @@ -558,9 +609,21 @@ def test_render_file_renders_docx(tmp_path: Path) -> None: assert render_file(source, format="docx").binary == render(SOURCE, format="docx").binary -def test_diagnostics_are_the_same_as_for_the_other_formats() -> None: +def test_diagnostics_are_the_other_formats_plus_the_interim_warning() -> None: document = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") - assert render(document, format="docx").diagnostics == render(document, format="text").diagnostics + docx_result = render(document, format="docx").diagnostics + assert docx_result[:-1] == render(document, format="text").diagnostics + warning = docx_result[-1] + assert (warning.rule, warning.level, warning.file) == ("render-docx-interim", "warning", "") + assert warning.message.startswith("DOCX output is incomplete in this version:") + + +def test_the_interim_warning_is_one_warning_that_does_not_refuse_under_strict() -> None: + result = render(FRONT + "# A\n", format="docx", strict=True, filename="d.lgd") + assert [(d.rule, d.file) for d in result.diagnostics if d.rule.startswith("render-")] == [ + ("render-docx-interim", "d.lgd")] + assert result.ok + assert not [d for d in render(FRONT + "# A\n", format="html").diagnostics if d.rule == "render-docx-interim"] def test_the_docx_writer_can_be_used_directly() -> None: @@ -587,19 +650,49 @@ def test_an_uppercase_extension_selects_docx(tmp_path: Path) -> None: assert target.read_bytes().startswith(b"PK") -def test_docx_to_standard_output_needs_dash_and_writes_bytes() -> None: +def test_docx_to_standard_output_with_dash_writes_bytes() -> None: + expected = render((DOCUMENTS / "features.lgd").read_text(encoding="utf-8"), format="docx").binary done = run_cli(FEATURES, "-q", "-f", "docx", "-o", "-") assert done.returncode == 0 and done.stderr == b"" + assert done.stdout.startswith(b"PK") and done.stdout == expected + + +def test_docx_without_output_goes_to_standard_output_when_that_is_not_a_terminal() -> None: + done = run_cli(FEATURES, "-q", "-f", "docx") # the test's standard output is a pipe + assert done.returncode == 0 and done.stderr == b"" assert done.stdout.startswith(b"PK") - assert done.stdout == render((DOCUMENTS / "features.lgd").read_text(encoding="utf-8"), format="docx").binary - from_extension = run_cli(FEATURES, "-q", "-f", "docx", "-o", "-") - assert from_extension.stdout == done.stdout + assert done.stdout == run_cli(FEATURES, "-q", "-f", "docx", "-o", "-").stdout + + +def test_fragment_is_a_usage_error_for_docx(tmp_path: Path) -> None: + done = run_cli(FEATURES, "-q", "-f", "docx", "--fragment", "-o", str(tmp_path / "x.docx")) + assert done.returncode == 2 and b"--fragment applies to HTML only" in done.stderr + assert list(tmp_path.iterdir()) == [] + + +def test_docx_without_output_is_refused_on_a_terminal(monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(sys.stdout, "isatty", lambda: True) + with pytest.raises(SystemExit) as exit_info: + main([FEATURES, "-q", "-f", "docx"]) + assert exit_info.value.code == 2 + assert "refusing to write a DOCX file to a terminal" in capsys.readouterr().err -def test_docx_without_output_is_a_usage_error_not_a_stream_of_bytes() -> None: - done = run_cli(FEATURES, "-q", "-f", "docx") - assert done.returncode == 2 and done.stdout == b"" - assert b"-o FILE.docx" in done.stderr and b"-o -" in done.stderr +def test_a_standard_output_without_isatty_is_not_a_terminal(monkeypatch: pytest.MonkeyPatch, + capsysbinary: pytest.CaptureFixture[bytes]) -> None: + class Plain: # no isatty, no reconfigure: some embedders replace the stream so + buffer = io.BytesIO() + + def write(self, text: str) -> int: + return len(text) + + def flush(self) -> None: + pass + + monkeypatch.setattr(sys, "stdout", Plain()) + assert main([FEATURES, "-q", "-f", "docx"]) == 0 + assert Plain.buffer.getvalue().startswith(b"PK") def test_docx_is_not_written_to_a_terminal(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: @@ -633,3 +726,95 @@ def test_text_and_html_to_standard_output_with_dash_still_work_and_name_no_file( def test_an_untrusted_render_makes_a_docx_too(tmp_path: Path) -> None: result = render(SOURCE, format="docx", untrusted=True, document_root=tmp_path) assert result.binary is not None and result.ok + + +# -- Output that does not depend on anything outside the writer ------------------------------------- + + +def test_the_output_does_not_depend_on_the_global_namespace_registry() -> None: + expected = render(SOURCE, format="docx").binary + saved = dict(ET._namespace_map) # type: ignore[attr-defined] + try: + for uri in (W_NS, "http://schemas.openxmlformats.org/officeDocument/2006/relationships", + "http://schemas.openxmlformats.org/package/2006/metadata/core-properties", + "http://purl.org/dc/elements/1.1/", "http://www.w3.org/XML/1998/namespace"): + ET.register_namespace("host", uri) # another library's registration + assert render(SOURCE, format="docx").binary == expected + finally: + ET._namespace_map.clear() # type: ignore[attr-defined] + ET._namespace_map.update(saved) # type: ignore[attr-defined] + + +def test_importing_the_writer_registers_no_namespace() -> None: + code = ("import xml.etree.ElementTree as ET\nbefore = dict(ET._namespace_map)\n" + "import legaldown_render.writers.docx\nassert ET._namespace_map == before\n") + assert run_python(code).returncode == 0 + + +def run_python(code: str): # noqa: ANN201 - a test helper + import os + import subprocess + + env = dict(os.environ, PYTHONPATH=str(Path(__file__).resolve().parents[1] / "src")) + return subprocess.run([sys.executable, "-c", code], capture_output=True, env=env, check=False) + + +def test_the_serialiser_sanitises_whoever_built_the_element() -> None: + root = ET.Element(f"{W}t", {f"{W}val": "a\x01b\n\"<&>", "plain": "x"}) + root.text = "a\x00b\ud800c&<>\t" + data = ooxml.serialize(root).decode("utf-8") + assert data.endswith('abc&<>\t') + with pytest.raises(InternalError): + ooxml.serialize(ET.Element("{urn:other}x")) + + +def test_a_text_passed_as_a_child_is_a_bug_not_a_recursion_error() -> None: + for bad in ("text", b"bytes", ["ok", ooxml.el("x")]): + with pytest.raises(InternalError): + ooxml.el("p", None, bad) + assert [c.tag for c in ooxml.el("p", None, [None, False, [ooxml.el("x")]])] == [f"{W}x"] + assert ooxml.p(style="").find(f"{W}pPr") is None + + +def test_names_and_languages_must_match_in_full() -> None: + bookmarks = ooxml.Bookmarks() + with pytest.raises(InternalError): + bookmarks.pair("ld_x\n") + assert styles.language_tag("cs\n", "en") == "en" + assert styles.language_tag("cs-CZ\n", None) == "en-US" + + +def test_code_is_left_aligned_even_when_the_body_is_justified() -> None: + by_id = style_map(docx(overrides={"typography.justify": True})) + assert by_id["Normal"].find(f"{W}pPr/{W}jc").get(f"{W}val") == "both" + assert by_id["Code"].find(f"{W}pPr/{W}jc").get(f"{W}val") == "left" + + +#: SHA-256 of each part of the fixed document below, uncompressed. The ZIP's own bytes depend on the +#: zlib that compressed them (a different zlib may write different, equivalent, bytes); these do not. +#: When a part changes on purpose, run this test, review the change, and replace the digest. +GOLDEN = { + '[Content_Types].xml': '0edc3b54ddf02325a53d17a23256a46b50f782562ff0319d5c48b73ceb50d29b', + '_rels/.rels': 'd6f21829c0081670bdb417d3e3f644b81134a121e074fb153c7686faaa8b167b', + 'docProps/core.xml': '159ed0e537106ab29739fd357a5f6d89a2c0b0154daefd4c13d878bd16dff47e', + 'docProps/app.xml': 'b285204d159fd58da80b6a0242cef523448f9c3a598b357b15faf4627160dcc3', + 'word/document.xml': 'd84c7bc463bf0a89b6022fb8b39ca15601d8c722f951e8f8fb1c023da173172a', + 'word/_rels/document.xml.rels': 'ada136c2e7d05a317579c9503c1f0dd8dfaa8f0cce687df86fe59e36d5fa5d57', + 'word/styles.xml': '83adf799de91eac3e9bf1e658b8c2f53586c700be35df179e1b2f1765f189dfc', + 'word/numbering.xml': '8d1f7769c35af16db98942ba09882646b45ba404e83686d4ce7a376c6a911f0f', + 'word/settings.xml': 'b3c72bc5720862c1a9c1449d89903673777f30df6ff38ceb23da73e21cad22e1', + 'word/footer1.xml': 'e693781f92e5a7ccf747fd2ed3facb627e448f389b021882cac589d746aa709b', +} + + +def part_digests() -> dict[str, str]: + from legaldown_render.style import load_style + + tree = render(SOURCE, format="text").tree + data = DocxWriter(load_style("default"), generator="golden").write(tree) + return {name: hashlib.sha256(content).hexdigest() for name, content in parts(data).items()} + + +def test_the_parts_of_a_fixed_document_are_what_they_were() -> None: + digests = part_digests() + assert digests == GOLDEN, "\n".join(f" {name!r}: {digest!r}," for name, digest in digests.items()) diff --git a/tests/test_docx_validation.py b/tests/test_docx_validation.py index 55b7268..78a7d93 100644 --- a/tests/test_docx_validation.py +++ b/tests/test_docx_validation.py @@ -8,11 +8,11 @@ import random import pytest -from conftest import DOCUMENTS -from docx_support import parts, require_schemas, schema_for, validation_errors +from conftest import DOCUMENTS, spec_root +from docx_support import parts, repackage, require_schemas, schema_for, tool_missing, validation_errors from test_docx_escaping import FREE_TEXT_SETTINGS, document -from legaldown_render import render +from legaldown_render import DocumentError, render pytestmark = pytest.mark.ooxml_schema @@ -60,6 +60,10 @@ def test_every_xml_part_has_a_schema_except_the_core_properties() -> None: {"definitions.style": "small-caps", "definitions.term_style": "underline"}, {"headings": {1: {"transform": "small-caps", "align": "center", "italic": True, "weight": "400"}}}, {"typography.font_family": "'Open Sans', sans-serif"}, + {"page.size": "22in 22in", "page.margin": "0", "typography.font_size": "1023pt", "typography.line_height": "1584pt", + "headings.1.size": "1638pt"}, + {"page.size": "1.5in 3in", "page.margin": "0.5in", "typography.font_size": "1pt"}, + {"page.margin": "25MM 1IN", "typography.font_size": "12PT"}, ]) def test_the_settings_that_shape_the_parts_keep_them_valid(overrides: dict) -> None: source = ("---\nlegaldown: '0.2'\ntitle: Agreement\nversion: '1'\neffective_date: 2026-01-02\n---\n\n" @@ -71,23 +75,42 @@ def test_an_empty_document_validates() -> None: assert validation_errors(make("---\ntitle: T\n---\n")) == [] +def moved(text: str, first: str, second: str) -> str: + """*text* with the element that starts with *first* moved after the one that starts with *second* + (both self-closing, *first* the earlier).""" + start = text.index(first) + end = text.index("/>", start) + 2 + element = text[start:end] + after = text.index("/>", text.index(second)) + 2 + return text[:start] + text[end:after] + element + text[after:] + + def test_the_schema_check_catches_a_wrong_order_in_the_settings() -> None: """The validation is not vacuous: ``updateFields`` after ``compat`` (the order both prototypes had) is rejected.""" - import io - import zipfile - - original = parts(make("---\ntitle: T\n---\n# A\n")) - settings = original["word/settings.xml"].decode("utf-8") - start = settings.index("", start) + 2 - element = settings[start:end] - broken = settings.replace(element, "").replace("", element + "") - buffer = io.BytesIO() - with zipfile.ZipFile(buffer, "w") as archive: - for name, content in original.items(): - archive.writestr(name, broken.encode("utf-8") if name == "word/settings.xml" else content) - assert any("updateFields" in problem for problem in validation_errors(buffer.getvalue())) + original = make("---\ntitle: T\n---\n# A\n") + settings = parts(original)["word/settings.xml"].decode("utf-8") + element = '' + assert element in settings + broken = repackage(original, "word/settings.xml", + lambda text: text.replace(element, "").replace("", element + "")) + assert any("updateFields" in problem for problem in validation_errors(broken)) + + +def test_the_schema_check_catches_an_element_out_of_order_in_a_style_and_in_the_section() -> None: + original = make("---\ntitle: T\n---\n# A\n") + assert validation_errors(original) == [] + style = repackage(original, "word/styles.xml", + lambda text: text.replace('', + '', 1)) + assert any("styles.xml" in problem for problem in validation_errors(style)) + section = repackage(original, "word/document.xml", lambda text: moved(text, " None: rnd = random.Random(1000 + seed) overrides = {key: "".join(rnd.choice("ab \x01\x00\ud800<&ž") for _ in range(10)) + "{page}" for key in FREE_TEXT_SETTINGS} assert validation_errors(make(document(rnd), overrides=overrides)) == [] + + +UNITS = ["mm", "cm", "in", "pt", "pc", "px", "em", "rem", "%", "ch", "ex", "", "MM"] + + +def random_length(rnd: random.Random) -> str: + """A number as small, as large or as odd as a style might write it.""" + whole = rnd.choice(["0", str(rnd.randint(0, 40)), str(rnd.randint(0, 100000)), "9" * rnd.randint(1, 45), + "1" + "0" * rnd.randint(1, 400)]) + fraction = rnd.choice(["", "", "." + "".join(rnd.choice("0123456789") for _ in range(rnd.randint(1, 40)))]) + return f"{whole}{fraction}{rnd.choice(UNITS)}" + + +@pytest.mark.parametrize("seed", range(60)) +def test_random_lengths_are_a_style_error_or_a_valid_document(seed: int) -> None: + from legaldown_render import StyleError + + rnd = random.Random(5000 + seed) + keys = ["page.size", "page.margin", "typography.font_size", "typography.line_height", "headings.1.size", + "headings.3.size"] + overrides = {key: " ".join(random_length(rnd) for _ in range(rnd.randint(1, 4))) + for key in rnd.sample(keys, rnd.randint(1, len(keys)))} + try: + data = make("---\ntitle: T\n---\n# A\n\ntext\n", overrides=overrides) + except StyleError as error: + assert not any(value.split()[0][:12] in str(error) for value in overrides.values() if len(value.split()[0]) > 12) + return + assert validation_errors(data) == [] + + +# -- The specification's own documents --------------------------------------------------------------- + +SPEC = spec_root() +SPEC_FILES = sorted(path for folder in ("examples", "fixtures") for path in (SPEC / folder).rglob("*.lgd")) if SPEC else [] + + +def test_the_specification_corpus_is_available() -> None: + """A run that promises its tools (CI) must have the corpus it validates.""" + if SPEC is None: + tool_missing("no LegalDown specification checkout (set LEGALDOWN_SPEC_DIR)") + assert SPEC_FILES + + +@pytest.mark.parametrize("style", ["default", "outline"]) +@pytest.mark.parametrize("path", SPEC_FILES, ids=lambda path: str(path.relative_to(SPEC)) if SPEC else str(path)) +def test_every_specification_document_validates(path: object, style: str) -> None: + source = path.read_text(encoding="utf-8") # type: ignore[attr-defined] + try: + data = make(source, style=style) + except DocumentError: + pytest.skip("not a document (invalid frontmatter)") + assert validation_errors(data) == [] From c1c42da9564eac9dfad71be0f7a957dd32360693 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:01:24 +0000 Subject: [PATCH 074/118] Fix five small DOCX and style issues: cache v6, CR escaping, tail refusal, diagnostics doc, untrusted style name --- .github/workflows/ci.yml | 2 +- docs/architecture.md | 2 +- src/legaldown_render/style/loader.py | 2 +- src/legaldown_render/writers/docx/ooxml.py | 9 +++++++-- tests/test_docx_package.py | 15 +++++++++++++++ tests/test_untrusted.py | 5 +++++ 6 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16122fb..cd2fbec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,7 +102,7 @@ jobs: - run: pip install -e ".[dev]" - name: Cache the OOXML schemas id: schemas - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: tests/schemas/ooxml key: ooxml-schemas-${{ hashFiles('tests/schemas/fetch.py') }} diff --git a/docs/architecture.md b/docs/architecture.md index a9f8f61..2232608 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -315,7 +315,7 @@ Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can ev `ref-not-enumerated`, `path-not-relative` and `path-outside-root` (Errors for a file reference that is not relative, or that leaves the document root, §2.3; reported here because validator 0.4 checks neither), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id). The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and `render-docx-interim` (a Warning on every DOCX render, while the DOCX body is incomplete). The validator's diagnostics carry their line (§16.9), and so do the renderer's own, and the CLI prints it as `file:line:`. diff --git a/src/legaldown_render/style/loader.py b/src/legaldown_render/style/loader.py index ce2eda9..f4bbb57 100644 --- a/src/legaldown_render/style/loader.py +++ b/src/legaldown_render/style/loader.py @@ -209,7 +209,7 @@ def _load(style: Any, overrides: Mapping[str, Any] | None, trusted: bool) -> Sty expanded = _expand_dotted(overrides, trusted) _refuse_meta_overrides(expanded) data = _merge(data, expanded) - return _build(data, source, trusted) + return _build(data, source if trusted else _UNTRUSTED_SOURCE, trusted) def style_to_dict(style: Style) -> dict[str, Any]: diff --git a/src/legaldown_render/writers/docx/ooxml.py b/src/legaldown_render/writers/docx/ooxml.py index b185a48..4f9bc9a 100644 --- a/src/legaldown_render/writers/docx/ooxml.py +++ b/src/legaldown_render/writers/docx/ooxml.py @@ -136,9 +136,13 @@ def serialize(root: ET.Element) -> bytes: registers cannot change the bytes. Attributes keep the order they were set in, the namespaces are declared on the root sorted by prefix, and the same tree is always the same bytes. Every text and attribute is sanitised here as well (:func:`_t`), whoever - built the element. Only ``text`` is written, as no part here has mixed content.""" + built the element. Only ``text`` is written, as no part here has mixed content, so a + tail is refused rather than dropped. A carriage return in text is written as `` ``, + which a parser reads back as CR, where a raw CR would read as LF.""" namespaces: set[str] = set() for element in root.iter(): + if element.tail: + raise InternalError("mixed content is not written (a bug in the DOCX writer)") namespaces.add(_name(element.tag)[0]) namespaces.update(_name(key)[0] for key in element.attrib) namespaces.discard("") @@ -168,7 +172,8 @@ def qualified(tag: str) -> str: continue out.append(f"<{name}{extra}{attributes}>") if text: - out.append(_t(text, breaks=True).replace("&", "&").replace("<", "<").replace(">", ">")) + out.append(_t(text, breaks=True).replace("&", "&").replace("<", "<").replace(">", ">") + .replace("\r", " ")) stack.append((element, True)) stack.extend((child, False) for child in reversed(element)) return _DECLARATION + "".join(out).encode("utf-8") diff --git a/tests/test_docx_package.py b/tests/test_docx_package.py index 912c872..b6c5a82 100644 --- a/tests/test_docx_package.py +++ b/tests/test_docx_package.py @@ -768,6 +768,21 @@ def test_the_serialiser_sanitises_whoever_built_the_element() -> None: ooxml.serialize(ET.Element("{urn:other}x")) +def test_a_carriage_return_in_text_is_written_as_a_character_reference() -> None: + root = ET.Element(f"{W}t") + root.text = "a\rb" + data = ooxml.serialize(root).decode("utf-8") + assert " " in data and "\r" not in data + assert ET.fromstring(data).text == "a\rb" + + +def test_an_element_with_a_tail_is_refused_rather_than_dropped() -> None: + root = ET.Element(f"{W}p") + ET.SubElement(root, f"{W}r").tail = "text" + with pytest.raises(InternalError, match="mixed content is not written"): + ooxml.serialize(root) + + def test_a_text_passed_as_a_child_is_a_bug_not_a_recursion_error() -> None: for bad in ("text", b"bytes", ["ok", ooxml.el("x")]): with pytest.raises(InternalError): diff --git a/tests/test_untrusted.py b/tests/test_untrusted.py index 9ae0ce7..c7d5f30 100644 --- a/tests/test_untrusted.py +++ b/tests/test_untrusted.py @@ -387,6 +387,11 @@ def test_an_override_that_cannot_hold_keys_is_not_quoted() -> None: assert SECRET in str(caught.value) +def test_an_untrusted_default_style_is_named_like_any_other_untrusted_style() -> None: + assert refused("default", {"page.margin": "9" * 400 + "mm"}).source == "'untrusted style'" + assert refused(None, {"page.margin": "9" * 400 + "mm"}).source == "'untrusted style'" + + def test_the_style_object_name_is_not_quoted() -> None: error = refused(Style(name=SECRET, typography=Typography(font_size=SECRET))) assert SECRET not in str(error) From a7fe71f68e9067053a713605e359f5f3a8e26d1c Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:05:58 +0000 Subject: [PATCH 075/118] Plan the numbering Word will draw for a resolved tree plan_docx maps the numbering structure of the render tree to Word's: the abstractNums (one multilevel list for headings, enumerated, renumbered, paragraph and bullet lists), the nums with their restarts, and the numId and level of every section, list item and numbered paragraph. A list nested in an item of the same kind goes on a deeper level of its parent's num; a sibling list that the resolver continues shares the num; any other nested list, and a list in a quote, has a num of its own. Word's counters are simulated (docx_numbering) and every placement is checked against the number the resolver gave the unit: letters past 26, Roman numerals from 4000, a clamped depth, a skipped heading level, labels that need the section's number and any other difference are reported as render-docx-* diagnostics. docx_style_errors names the labels Word's level text cannot express, for the DOCX settings step to refuse before rendering (ADR 0006). --- .../resolve/docx_numbering.py | 299 +++++++++ src/legaldown_render/resolve/docx_plan.py | 613 ++++++++++++++++++ tests/test_docx_simulator.py | 215 ++++++ 3 files changed, 1127 insertions(+) create mode 100644 src/legaldown_render/resolve/docx_numbering.py create mode 100644 src/legaldown_render/resolve/docx_plan.py create mode 100644 tests/test_docx_simulator.py diff --git a/src/legaldown_render/resolve/docx_numbering.py b/src/legaldown_render/resolve/docx_numbering.py new file mode 100644 index 0000000..3ce794a --- /dev/null +++ b/src/legaldown_render/resolve/docx_numbering.py @@ -0,0 +1,299 @@ +"""Word's list numbering, as the DOCX planner needs it (docs/decisions/0008, §13.1, §13.2). + +Two things live here, both pure: + +* **Token strings.** The renderer's labels and designations, and Word's expansion of an ``lvlText``, are written as + lists of :class:`Lit` (text that is not a counter) and :class:`Count` (the counter of one owner, as formatted). The + functions :func:`fill_tokens` and :func:`extend_tokens` are :func:`.numbering.fill` and :func:`.numbering.extend` on + tokens, so that a template filled here reads exactly as the resolver filled it, with the owner of every counter kept. + :func:`level_text` derives the ``lvlText`` of a level from a label template. +* **The simulator** (:class:`WordNumbering`): what Word's numbering gives each paragraph of a list, from the + abstractNum and num definitions alone. The planner uses it to place every unit and to check each one against the + number the resolver gave it, and its results are the cached results of the fields Word will recompute. + +The simulator models Word's list semantics (docs/architecture.md, "The DOCX planner"): counters belong to the +abstractNum and are shared by all its ``w:num``; a ``startOverride`` restarts the named level at the first use of that +num; a paragraph restarts every deeper level; ``numId 0`` paragraphs count nothing (the planner never steps them). +""" +from __future__ import annotations + +import re +from collections.abc import Sequence +from typing import TYPE_CHECKING, Literal, NamedTuple + +from ..errors import InternalError +from .numbering import _FIELD_RE + +if TYPE_CHECKING: + from .docx_plan import LevelDef, Num + +# --------------------------------------------------------------------------- +# Counters +# --------------------------------------------------------------------------- + +#: The renderer's counters (``style.model.Counter``) and the ``numFmt`` that writes each in Word. +NUM_FMT: dict[str, str] = { + "decimal": "decimal", + "lower-alpha": "lowerLetter", + "upper-alpha": "upperLetter", + "lower-roman": "lowerRoman", + "upper-roman": "upperRoman", +} + +_ROMAN = ( + (900, "cm"), (500, "d"), (400, "cd"), (100, "c"), (90, "xc"), (50, "l"), (40, "xl"), + (10, "x"), (9, "ix"), (5, "v"), (4, "iv"), (1, "i"), +) + + +def word_counter(value: int, num_fmt: str) -> str: + """*value* as Word writes it in *num_fmt*. + + Letters repeat after 26 (``aa``, ``bb``, ... ``zz``, ``aaa``), as in ECMA-376 and Word's documentation; the + renderer counts ``aa``, ``ab`` (§13.2), so the two differ from 28 on. Roman numerals are the usual form below + 1000 and ``m`` repeated above, so that 4000 reads ``mmmm`` where the renderer writes digits. A value below 1 is + ``0`` for ``decimal`` and empty otherwise.""" + if value < 1: + return "0" if num_fmt == "decimal" else "" + match num_fmt: + case "lowerLetter" | "upperLetter": + letters = chr(ord("a") + (value - 1) % 26) * ((value - 1) // 26 + 1) + return letters.upper() if num_fmt == "upperLetter" else letters + case "lowerRoman" | "upperRoman": + numeral, rest = "m" * (value // 1000), value % 1000 + for amount, symbol in _ROMAN: + while rest >= amount: + numeral += symbol + rest -= amount + return numeral.upper() if num_fmt == "upperRoman" else numeral + case _: + return str(value) + + +#: How wide a counter of each ``numFmt`` is taken to be, to size a hanging indent. +SAMPLE_COUNTER: dict[str, str] = { + "decimal": "88", "lowerLetter": "mm", "upperLetter": "MM", "lowerRoman": "viii", "upperRoman": "VIII", +} + +# --------------------------------------------------------------------------- +# Token strings +# --------------------------------------------------------------------------- + + +class Lit(NamedTuple): + """Text that is not a counter: delimiters, or words of a label.""" + + text: str + + +class Count(NamedTuple): + """The counter of *owner* (a unit id, a :class:`Ghost`, or a level number in a template), as formatted.""" + + owner: object + text: str + + +class Ghost(NamedTuple): + """The counter of a skipped heading level: *level* (from 0) was never used before *unit* needed it.""" + + unit: str + level: int + + +Token = Lit | Count + + +def fill_tokens(template: str, *, n: Sequence[Token], path: Sequence[Token] = (), + section: Sequence[Token] | None = None) -> list[Token]: + """:func:`.numbering.fill` on tokens: *template* with ``{n}``, ``{path}`` and ``{section}`` replaced by token + lists, other text (and other braces) kept as :class:`Lit`. *section* given and empty (a preamble item) drops + ``{section}`` and ONE ``.`` after it; None means "not given" (headings), and ``{section}`` is then empty.""" + if section is not None and not section: + template = re.sub(r"\{section\}\.", "", template) + values: dict[str, Sequence[Token]] = {"n": n, "path": path, "section": section or ()} + out: list[Token] = [] + position = 0 + for match in _FIELD_RE.finditer(template): + if match.start() > position: + out.append(Lit(template[position:match.start()])) + out.extend(values[match.group(1)]) + position = match.end() + if position < len(template): + out.append(Lit(template[position:])) + return out + + +def extend_tokens(designation: Sequence[Token], piece: Sequence[Token], *, textual: bool) -> list[Token]: + """:func:`.numbering.extend` on tokens: *designation* followed by *piece*; the piece loses leading dots when + nothing precedes it or under the ``none`` scheme, where a space goes before it.""" + piece = list(piece) + if (not designation or textual) and piece and isinstance(piece[0], Lit): + piece[0] = Lit(piece[0].text.lstrip(".")) + if not designation: + return [token for token in piece if token != Lit("")] + if textual: + return [*designation, Lit(" "), *(token for token in piece if token != Lit(""))] + return [*designation, *piece] + + +def text_of(tokens: Sequence[Token]) -> str: + return "".join(token.text for token in tokens) + + +# --------------------------------------------------------------------------- +# lvlText +# --------------------------------------------------------------------------- + +#: A level text as parts: text, and the number of the level (from 0) a ``%k`` stands for. +LevelParts = tuple[str | int, ...] + +_PLACEHOLDER = re.compile(r"%([1-9])") + + +def level_text(label: str, refs: Sequence[str], level: int, *, section: Sequence[Token] | None, + textual: bool = False) -> LevelParts: + """The ``lvlText`` of level *level* of a list, from its ``label`` template, as parts. + + Only counters of the list's own levels can be shown (Word's ``%k`` names a level of the same abstractNum), so + ``{n}`` is this level's counter and ``{path}`` the designation over levels ``0..level``, each with its own ``ref`` + template (*refs*, one per level). *section* is ``None`` for headings (``{section}`` is empty) and ``()`` for the + foreign-free form of a list item or paragraph, the one a preamble item has: ``{section}`` and the dot after it + are dropped, which is the documented fallback where a label needs a section number Word cannot show there.""" + designation: list[Token] = [] + for j in range(level + 1): + piece = fill_tokens(refs[j], n=[Count(j, "")], section=section) + designation = extend_tokens(designation, piece, textual=textual) + tokens = fill_tokens(label, n=[Count(level, "")], path=designation, section=section) + parts: list[str | int] = [] + for token in tokens: + if isinstance(token, Count): + parts.append(token.owner) # type: ignore[arg-type] + elif parts and isinstance(parts[-1], str): + parts[-1] += token.text + else: + parts.append(token.text) + return tuple(parts) + + +def lvl_text(parts: LevelParts) -> str: + """*parts* in OOXML form: ``%k`` for level ``k-1``. A literal ``%`` before a digit, or before a counter, cannot + be told from a placeholder in ``lvlText``; style validation (:func:`.docx_plan.docx_style_errors`) refuses it before + rendering, so one reaching here is a bug.""" + text = "".join(f"%{part + 1}" if isinstance(part, int) else part for part in parts) + before_counter = any(isinstance(after, int) and isinstance(part, str) and part.endswith("%") + for part, after in zip(parts, parts[1:], strict=False)) + if before_counter or len(_PLACEHOLDER.findall(text)) != sum(isinstance(part, int) for part in parts): + raise InternalError("a label with a literal '%' before a digit reached the DOCX planner. This is a bug; " + "style validation refuses it.") + return text + + +def split_level_text(text: str) -> list[str | int]: + """An ``lvlText`` as literal strings and level numbers (from 0), as :func:`lvl_text` writes it.""" + parts: list[str | int] = [] + position = 0 + for match in _PLACEHOLDER.finditer(text): + if match.start() > position: + parts.append(text[position:match.start()]) + parts.append(int(match.group(1)) - 1) + position = match.end() + if position < len(text): + parts.append(text[position:]) + return parts + + +# --------------------------------------------------------------------------- +# The simulator +# --------------------------------------------------------------------------- + + +class Step(NamedTuple): + """What Word gives one numbered paragraph.""" + + #: The counter of the paragraph's own level. + count: int + #: The level's own counter as written (``a``, ``IV``). + own: str + #: The full ``lvlText`` expansion, trailing period kept (``1.``, ``(a)``). + label: str + #: What ``REF \n`` returns: the label minus its trailing periods. + word_ref: str + #: The label as tokens, each counter with the unit (or ghost) that holds it. + tokens: tuple[Token, ...] + + +class WordNumbering: + """Word's counters for a set of abstractNums and nums. Register each with :meth:`add_abstract` and + :meth:`add_num` (an abstractNum's levels are read when a paragraph is stepped, so a level may be defined + after the abstractNum is added), then :meth:`step` paragraphs in document order. + + *skipped* is the model of a heading level that was never used: ``"lo"`` shows its start value and counts on + (what the renderer does and LibreOffice shows); ``"word"`` shows ``start - 1`` and does not advance it, which is + what Word is expected to do (the documents that skip a level have a Core Error, ``heading-skip``).""" + + def __init__(self, *, skipped: Literal["lo", "word"] = "lo") -> None: + self.skipped = skipped + self._levels: dict[int, Sequence[LevelDef]] = {} + self._nums: dict[int, Num] = {} + self._count: dict[int, list[int]] = {} + self._holder: dict[int, list[object | None]] = {} + self._used: set[tuple[int, int]] = set() + + def add_abstract(self, abstract_id: int, levels: Sequence[LevelDef]) -> None: + self._levels[abstract_id] = levels + self._count[abstract_id] = [0] * 9 + self._holder[abstract_id] = [None] * 9 + + def add_num(self, num: Num) -> None: + self._nums[num.id] = num + + def _overrides(self, num_id: int) -> dict[int, int]: + return dict(self._nums[num_id].start_overrides) + + def peek(self, num_id: int, ilvl: int) -> int: + """The number the next paragraph at (*num_id*, *ilvl*) would get.""" + num = self._nums[num_id] + overrides = self._overrides(num_id) + if ilvl in overrides and (num_id, ilvl) not in self._used: + return overrides[ilvl] + return self._count[num.abstract_id][ilvl] + 1 + + def current(self, num_id: int, ilvl: int) -> tuple[int, object | None]: + """The counter of the level now, and the unit that last counted it (None if unused).""" + abstract_id = self._nums[num_id].abstract_id + return self._count[abstract_id][ilvl], self._holder[abstract_id][ilvl] + + def step(self, num_id: int, ilvl: int, owner: object) -> Step: + """Number the next paragraph at (*num_id*, *ilvl*), which *owner* (a unit id) is.""" + num = self._nums[num_id] + levels = self._levels[num.abstract_id] + count, holder = self._count[num.abstract_id], self._holder[num.abstract_id] + overrides = self._overrides(num_id) + if ilvl in overrides and (num_id, ilvl) not in self._used: + self._used.add((num_id, ilvl)) + count[ilvl] = overrides[ilvl] - 1 + count[ilvl] += 1 + holder[ilvl] = owner + for deeper in range(ilvl + 1, 9): + count[deeper], holder[deeper] = 0, None + for above in range(ilvl): + if count[above] == 0: + holder[above] = Ghost(str(owner), above) + if self.skipped == "lo": + count[above] = levels[above].start + else: + count[above] = levels[above].start - 1 + tokens: list[Token] = [] + for part in split_level_text(levels[ilvl].lvl_text): + if isinstance(part, int): + tokens.append(Count(holder[part], word_counter(count[part], levels[part].num_fmt))) + else: + tokens.append(Lit(part)) + if self.skipped == "word": + # Never-used levels show start - 1 and stay unused: the next heading of that level starts at `start`. + for above in range(ilvl): + if isinstance(holder[above], Ghost): + count[above], holder[above] = 0, None + label = text_of(tokens) + return Step(count[ilvl], word_counter(count[ilvl], levels[ilvl].num_fmt), label, + label.rstrip("."), tuple(tokens)) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py new file mode 100644 index 0000000..0e51b05 --- /dev/null +++ b/src/legaldown_render/resolve/docx_plan.py @@ -0,0 +1,613 @@ +"""The DOCX planner: from the render tree to the numbering Word will draw (docs/decisions/0008). + +``plan_docx(tree, style)`` is the one place that knows how LegalDown numbering becomes Word numbering; the DOCX +writer serialises its :class:`DocxPlan` and decides nothing (docs/decisions/0003). It is a pure function of the +resolved tree: the numbering structure the resolver put on every section, list item and top-level paragraph +(``unit``, ``number``, ``parent_unit``, ``List.family``, ``RenderTree.formats``; §13.1, §13.2) is mapped to +abstractNums, nums and, for each unit, the (numId, ilvl) of its paragraph, and every placement is checked against the +number the resolver gave the unit, by simulating Word's counters (:mod:`.docx_numbering`). + +What a plan holds so far: the numbering definitions and the numbering of every unit. The reference fields, +bookmarks and table of contents are further parts of the plan that are empty here. + +**Data model.** Frozen data only; the planner assigns bookmark *names*, never ids (the writer allocates those); +names are valid Word bookmark names of at most 40 characters and unique ignoring case. All sequences are tuples, all +mappings are plain dicts built once in document order and never changed afterwards. +""" +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Literal, NamedTuple + +from legaldown import Diagnostic + +from ..diagnostics import by_line, record +from ..errors import InternalError, StyleError +from ..style.model import LevelFormat, Style +from ..tree import ( + DraftingNote, + Inline, + List, + ListItem, + NumberingFormats, + Paragraph, + Quote, + RenderTree, + Section, + plain_text, +) +from ..urls import printable +from .docx_numbering import ( + NUM_FMT, + SAMPLE_COUNTER, + LevelParts, + Step, + WordNumbering, + level_text, + lvl_text, + split_level_text, + word_counter, +) +from .numbering import RENUMBERED, format_counter, heading_levels + +# --------------------------------------------------------------------------- +# The plan (frozen data) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class LevelDef: + """One ``w:lvl`` of an abstractNum.""" + + ilvl: int + #: ``decimal``, ``lowerLetter``, ``upperLetter``, ``lowerRoman``, ``upperRoman`` or ``bullet``. + num_fmt: str + #: In OOXML form (``%1.%2``, ``(%3)``, ``Article %1``, ``•``): ``%k`` is the counter of level ``k-1``. + lvl_text: str + start: int = 1 + #: ``Heading1``..``Heading5``, only on levels 0-4 of the heading abstractNum. + p_style: str | None = None + suff: Literal["tab", "space"] = "tab" + #: The level's own ``w:ind`` in twips. On the heading abstractNum this serves the heading; a unit that sits on + #: the level at another nesting depth carries its own indent (:attr:`UnitPlan.ind`). + left: int = 0 + hanging: int = 0 + + +@dataclass(frozen=True, slots=True) +class AbstractNum: + id: int # 0.., in order of first use (headings first) + kind: Literal["headings", "enumerated", "renumber", "paragraphs", "bullet"] + #: Eight hex digits, deterministic (``0x4C440000 + id``). Equal in every LegalDown document, so Word may + #: merge lists on paste between two of them (cosmetic). + nsid: str + multi_level: Literal["multilevel", "hybridMultilevel"] + #: Always nine levels. + levels: tuple[LevelDef, ...] + + +@dataclass(frozen=True, slots=True) +class Num: + """One ``w:num``. ``nums[0]`` is the heading num (numId 1) only when headings are numbered.""" + + id: int # from 1 (numId 0 means "no numbering" and is never a Num) + abstract_id: int + #: ``(ilvl, value)``: the first number of the list at that level; deeper levels restart at 1. Overrides that + #: are never reached by a paragraph are harmless. + start_overrides: tuple[tuple[int, int], ...] = () + + +@dataclass(frozen=True, slots=True) +class Literal_: + """Text of a reference or number that is not a counter: delimiters and the words of ``references.format``.""" + + text: str + + +@dataclass(frozen=True, slots=True) +class FieldSeg: + """A ``REF`` field with its cached result. Instruction: ``REF `` + ``\\n`` if *number* + ``\\t`` if + *suppress_text* + ``\\h`` if *hyperlink*. Without *number* it returns the bookmarked text (heading text under + the ``none`` scheme).""" + + bookmark: str + number: bool + suppress_text: bool + hyperlink: bool + #: What Word computes (the simulator's value). + cached: str + #: What LibreOffice computes (it ignores ``\t``); equal to *cached* unless *suppress_text*. For tests only. + cached_lo: str + + +Segment = Literal_ | FieldSeg + + +@dataclass(frozen=True, slots=True) +class AltNumber: + """The number of a later alternative that shares a number (§15.8): its paragraph has ``numId 0`` and starts + with these segments (the label as a cover of the holder's number: ``REF holder \\n \\h`` and the stripped + period), then a tab or space. Its indent is :attr:`UnitPlan.ind`.""" + + segments: tuple[Segment, ...] + suff: Literal["tab", "space"] + + +@dataclass(frozen=True, slots=True) +class UnitPlan: + """What the paragraph of one tree unit (section, list item, top-level paragraph) needs, key: ``unit``.""" + + unit: str + kind: Literal["section", "item", "paragraph"] + #: None: the writer writes no ``w:numPr`` (a heading takes its style's; nothing else is unnumbered). + num_id: int | None + ilvl: int | None + #: True: the writer MUST write ``w:numPr`` (``numId`` 0 for an alternative). + explicit: bool + #: ``(left, hanging)`` in twips, written as ``w:ind`` on the paragraph, for a unit whose nesting depth differs + #: from its level's natural depth (a nested list on the heading abstractNum, an alternative); else None. + ind: tuple[int, int] | None + alt: AltNumber | None + #: Word's number text for the unit (the full ``lvlText`` expansion, trailing period kept: ``1.``, ``(a)``). + label: str + #: What ``REF \n`` returns (trailing period dropped): the cached text of the unit's own field. + word_ref: str + #: Word's counter value (None for an alternative, a bullet and an unnumbered unit). + count: int | None + #: Why the unit is not on the plain path: ``resync``, ``alternative``, ``reserved-level``, ``restart-family``, + #: ``restart-section``, ``restart-alternative``, ``label-fallback``, ``clamped``, ``skipped-level``. + cause: tuple[str, ...] + #: Names to wrap around the paragraph's title/first paragraph content: the anchor bookmark first (when the + #: unit has an anchor), then the helper bookmark a cover needs. + bookmarks: tuple[str, ...] + #: ``_Toc`` name of a heading listed in the contents. + toc_bookmark: str | None + + +@dataclass(frozen=True, slots=True) +class RefPlan: + """The fields of every ``{{ref:}}`` to one unit (``CrossRef.target_unit``), wherever it stands.""" + + #: ``references.format`` literals around the cover of the designation. + segments: tuple[Segment, ...] + #: ``concat(cached) == CrossRef.text``. + exact: bool + + +@dataclass(frozen=True, slots=True) +class TocEntryPlan: + #: The ``ContentsEntry`` it is made from (``ContentsEntry.unit``: the section's unit, None for attachments). + unit: str | None + level: int # 1..5, the TOCn style + #: Word's number text of the heading incl. the trailing period; ``""`` for attachments. + number: str + sep: Literal["\t", " ", ""] + title: tuple[Inline, ...] # ContentsEntry.title, already unlinked + condition_label: str | None + bookmark: str # the ``_Toc`` name the entry's hyperlink and PAGEREF point to + cached_page: str = "" # always empty + + +@dataclass(frozen=True, slots=True) +class DocxPlan: + abstract_nums: tuple[AbstractNum, ...] + nums: tuple[Num, ...] + #: The num the heading styles' ``numPr`` points to; None under the ``none`` scheme. + heading_num_id: int | None + units: Mapping[str, UnitPlan] = field(default_factory=dict) + #: Every output anchor of the tree (units, ``DefinedTerm.anchor``, ``AttachmentPart.anchor``, + #: ``attachments_anchor``) -> bookmark name. + anchors: Mapping[str, str] = field(default_factory=dict) + #: ``CrossRef.target_unit`` -> its fields. + refs: Mapping[str, RefPlan] = field(default_factory=dict) + toc: tuple[TocEntryPlan, ...] = () + + +# --------------------------------------------------------------------------- +# Style checks that belong before rendering (ADR 0006) +# --------------------------------------------------------------------------- + +#: The longest ``lvlText`` Word accepts (taken from Word's limit of 255 characters on a level's text). +MAX_LVL_TEXT = 255 + +#: A literal ``%`` that ``lvlText`` could not tell from a placeholder once the label is filled in: before a digit, +#: or before a field that becomes ``%k``. +_AMBIGUOUS_PERCENT = re.compile(r"%(?=[0-9{])") + + +def _formats_of(style: Style) -> NumberingFormats: + """The numbering formats *style* gives (as ``Resolver._formats`` writes them into the tree).""" + enumeration = style.enumeration + return NumberingFormats( + scheme=style.numbering.scheme, + headings=() if style.numbering.scheme == "none" else heading_levels(style.numbering), + enumerated=tuple(enumeration.levels) if enumeration.enabled or enumeration.ordered == "enumerate" else (), + renumber=RENUMBERED if enumeration.ordered == "renumber" else None, + paragraphs=style.paragraphs.format if style.paragraphs.numbered else None, + reference=style.references.format, + ) + + +def docx_style_errors(style: Style) -> list[str]: + """What in *style* Word's numbering cannot express, one message per problem naming the setting; empty when + the style is fine for DOCX. + + Word's ``lvlText`` has no way to write a ``%`` before a digit, and a level's text is at most + :data:`MAX_LVL_TEXT` characters. These are problems of configuration, so a DOCX render calls this before + rendering and raises :class:`~legaldown_render.errors.StyleError` (ADR 0006); :func:`plan_docx` checks again.""" + problems: list[str] = [] + formats = _formats_of(style) + templates: list[tuple[str, LevelFormat]] = [] + if formats.headings: + templates += [(f"numbering.levels[{i}]", f) for i, f in enumerate(style.numbering.levels[:5])] + if formats.enumerated: + templates += [(f"enumeration.levels[{i}]", f) for i, f in enumerate(style.enumeration.levels)] + if formats.paragraphs is not None: + templates.append(("paragraphs.format", formats.paragraphs)) + for path, level in templates: + for name, template in (("label", level.label), ("ref", level.ref)): + if _AMBIGUOUS_PERCENT.search(template): + problems.append( + f"{path}.{name}: a '%' before a digit or a field cannot be written in a Word numbering level " + f"(DOCX); write it without the '%'") + if problems: + return problems + for kind, levels in _definitions(formats, set()).items(): + for level in levels: + if len(level.lvl_text) > MAX_LVL_TEXT: + problems.append(f"the {kind} numbering level {level.ilvl + 1} is longer than {MAX_LVL_TEXT} " + f"characters in Word (DOCX): shorten its label or the refs it shows") + return problems + + +# --------------------------------------------------------------------------- +# Level definitions +# --------------------------------------------------------------------------- + +_BULLETS = ("•", "–", "◦") +#: Twips: one centimetre, the step of the indent of items. +_STEP = 567 +#: Twips per character of a heading's widest label. +_CHARACTER = 113 + + +def _num_fmt(counter: str, unmapped: set[str]) -> str: + """The ``numFmt`` of *counter*; a counter Word has no format for is written as ``decimal`` and noted.""" + found = NUM_FMT.get(counter) + if found is None: + unmapped.add(counter) + return "decimal" + return found + + +def _level(ilvl: int, num_fmt: str, parts: LevelParts, *, p_style: str | None = None, + widths: Mapping[int, str] | None = None) -> LevelDef: + """One level. Headings (*widths*: the ``numFmt`` of each level a ``%k`` names) get an indent wide enough for + their widest label; items step one centimetre per level. A label with a word in it (``Article {n}``) is + followed by a space, any other by a tab.""" + words = any(isinstance(part, str) and any(c.isalpha() for c in part) for part in parts) + if widths is None: + left, hanging = _STEP * (ilvl + 1), _STEP + else: + sample = sum(len(part) if isinstance(part, str) else len(SAMPLE_COUNTER[widths[part]]) for part in parts) + left = hanging = max(_STEP, _CHARACTER * (sample + 2)) + return LevelDef(ilvl, num_fmt, lvl_text(parts), p_style=p_style, suff="space" if words else "tab", + left=left, hanging=hanging) + + +def _neutral(ilvl: int) -> LevelDef: + """A level nothing uses: its own counter, decimal.""" + return LevelDef(ilvl, "decimal", f"%{ilvl + 1}", left=_STEP * (ilvl + 1), hanging=_STEP) + + +def _definitions(formats: NumberingFormats, unmapped: set[str]) -> dict[str, list[LevelDef]]: + """The nine levels of every abstractNum *formats* needs, by kind. A label that needs a number of the section + (``{section}``, or ``{path}`` of an item or paragraph in one) is written without it, as for a preamble item: + Word shows only counters of the list itself in a level.""" + textual = formats.scheme == "none" + out: dict[str, list[LevelDef]] = {} + if formats.headings: + refs = [f.ref for f in formats.headings] + fmts = {j: _num_fmt(f.counter, unmapped) for j, f in enumerate(formats.headings)} + levels = [_level(k, fmts[k], level_text(f.label, refs, k, section=None), p_style=f"Heading{k + 1}", + widths=fmts) for k, f in enumerate(formats.headings)] + out["headings"] = [*levels, *(_neutral(k) for k in range(5, 9))] + if formats.enumerated: + size = len(formats.enumerated) + refs = [formats.enumerated[j % size].ref for j in range(9)] + out["enumerated"] = [ + _level(d, _num_fmt(formats.enumerated[d % size].counter, unmapped), + level_text(formats.enumerated[d % size].label, refs, d, section=(), textual=textual)) + for d in range(9)] + if formats.renumber is not None: + f = formats.renumber + out["renumber"] = [_level(d, _num_fmt(f.counter, unmapped), + level_text(f.label, [f.ref] * 9, d, section=(), textual=textual)) + for d in range(9)] + if formats.paragraphs is not None: + f = formats.paragraphs + out["paragraphs"] = [ + _level(0, _num_fmt(f.counter, unmapped), level_text(f.label, [f.ref], 0, section=(), textual=textual)), + *(_neutral(k) for k in range(1, 9))] + out["bullet"] = [LevelDef(d, "bullet", _BULLETS[d % 3], left=_STEP * (d + 1), hanging=_STEP) for d in range(9)] + return out + + +def _needs_section(level: LevelFormat) -> bool: + """True if *level*'s label shows a number of the section: ``{section}``, or ``{path}`` (which holds it) of an + item or paragraph in a section.""" + return "{section}" in level.label or "{path}" in level.label or "{section}" in level.ref + + +def _reads(level: LevelDef, levels: Sequence[LevelDef]) -> str: + """*level* as it reads with every counter at 1, for a message.""" + return "".join(word_counter(1, levels[part].num_fmt) if isinstance(part, int) else part + for part in split_level_text(level.lvl_text)) + + +# --------------------------------------------------------------------------- +# The planner +# --------------------------------------------------------------------------- + +_Unit = Section | ListItem | Paragraph + + +class _Abstract: + """An abstractNum being built: the simulator reads its levels while the plan grows.""" + + def __init__(self, abstract_id: int, kind: str, levels: list[LevelDef]) -> None: + self.id, self.kind, self.levels = abstract_id, kind, levels + + +class _Placed(NamedTuple): + """Where a list item is numbered, for the lists nested in it.""" + + num: Num + abstract: _Abstract + + +class _Planner: + def __init__(self, tree: RenderTree, file: str | None) -> None: + self.tree, self.file = tree, file or "" + self.formats = tree.formats + self.textual = tree.formats.scheme == "none" + self.diagnostics: list[Diagnostic] = [] + self.sim = WordNumbering() + self.abstracts: list[_Abstract] = [] + self.nums: list[Num] = [] + self.units: dict[str, UnitPlan] = {} + self.parent: dict[str, str | None] = {} + self.section_level: dict[str, int] = {} + self.placed: dict[str, _Placed] = {} + self.paragraph_nums: dict[str, Num] = {} + self.flagged: set[str] = set() + self.reported: set[tuple[str, object]] = set() + self.heading_num: Num | None = None + self.depth_reported = False + self.unmapped: set[str] = set() + self.definitions = _definitions(self.formats, self.unmapped) + self.by_kind: dict[str, _Abstract] = {} + + # -- registry ------------------------------------------------------------------- + + def abstract(self, kind: str) -> _Abstract: + """The abstractNum of *kind*, made on first use (so ids follow the order of use).""" + found = self.by_kind.get(kind) + if found is None: + found = _Abstract(len(self.abstracts), kind, self.definitions[kind]) + self.abstracts.append(found) + self.by_kind[kind] = found + self.sim.add_abstract(found.id, found.levels) + return found + + def new_num(self, abstract: _Abstract, ilvl: int | None = None, first: int = 1) -> Num: + """A num on *abstract*; with *ilvl*, its list starts there at *first* and deeper levels at 1.""" + overrides = () if ilvl is None else tuple((i, first if i == ilvl else 1) for i in range(ilvl, 9)) + num = Num(len(self.nums) + 1, abstract.id, overrides) + self.nums.append(num) + self.sim.add_num(num) + return num + + def report(self, rule: str, level: str, message: str, line: int | None) -> None: + record(Diagnostic(rule=rule, level=level, message=message, line=line, file=self.file), self.diagnostics) + + # -- the pass ------------------------------------------------------------------- + + def run(self) -> tuple[DocxPlan, list[Diagnostic]]: + if self.formats.headings: + self.heading_num = self.new_num(self.abstract("headings")) + for block in self.tree.preamble: + self.block(block, None) + for section in self.tree.sections: + self.section(section) + for block in section.blocks: + self.block(block, section) + for counter in sorted(self.unmapped): + self.report("render-docx-counter", "warning", + f"The counter '{printable(counter)}' has no Word number format; it is written as decimal.", + None) + abstracts = tuple( + AbstractNum(a.id, a.kind, f"{0x4C440000 + a.id:08X}", # type: ignore[arg-type] + "multilevel" if a.kind == "headings" else "hybridMultilevel", tuple(a.levels)) + for a in self.abstracts) + heading_id = self.heading_num.id if self.heading_num else None + return DocxPlan(abstracts, tuple(self.nums), heading_id, units=self.units), by_line(self.diagnostics) + + def block(self, block: object, section: Section | None) -> None: + match block: + case Paragraph(unit=str()) if section is not None: + self.paragraph(block, section) + case List(): + self.list_(block, section) + case Quote(blocks=blocks) | DraftingNote(blocks=blocks): + for child in blocks: + self.block(child, section) + + # -- what a unit gets ----------------------------------------------------------- + + def add(self, node: _Unit, kind: Literal["section", "item", "paragraph"], num: Num | None, ilvl: int | None, *, + explicit: bool, cause: tuple[str, ...] = (), step: Step | None = None) -> None: + assert node.unit is not None + self.units[node.unit] = UnitPlan( + node.unit, kind, num.id if num else None, ilvl, explicit, None, None, + step.label if step else "", step.word_ref if step else "", step.count if step else None, + cause, (), None) + + def ancestor_flagged(self, unit: str) -> bool: + parent = self.parent.get(unit) + while parent is not None: + if parent in self.flagged: + return True + parent = self.parent.get(parent) + return False + + def check(self, node: _Unit, kind: str, step: Step, counter: str, group: object, cause: tuple[str, ...]) -> None: + """Compare what Word gives *node* with what the resolver gave it, and report the first difference of a + chain (a unit with a flagged ancestor is not reported again). Word's count must be the resolver's; the + counter must read the same (letters past 26 and Roman numerals from 4000 do not); and the whole label must + read the same, unless a diagnostic already says why it cannot (an unsupported label, a clamped depth).""" + assert node.unit is not None and node.number is not None + fresh = not self.ancestor_flagged(node.unit) + expected = format_counter(node.number, counter) + word = kind[0].upper() + kind[1:] + if step.count == node.number and "clamped" in cause: + return # a level 9 in the format of level 9: the clamp is the reason, and it is reported + if step.count == node.number and step.own == expected: + if step.label == node.label or "label-fallback" in cause: + return + rule = "render-docx-number-mismatch" + message = f"Word will number the {kind} '{printable(node.label or '')}' as '{printable(step.label)}'." + elif step.count != node.number: + rule = "render-docx-number-mismatch" + message = f"Word will number the {kind} '{printable(node.label or '')}' as '{printable(step.label)}'." + elif counter.endswith("roman") and node.number >= 4000: + rule = "render-docx-roman-overflow" + message = (f"{word} {node.number} is a Roman numeral of 4000 or more: the renderer shows digits " + f"('{printable(expected)}'), Word shows '{printable(step.own)}'.") + elif counter.endswith("alpha"): + rule = "render-docx-letter-overflow" + message = (f"{word} {node.number} ('{printable(expected)}') will show as '{printable(step.own)}' in " + f"Word: Word repeats the letter after 26 ('aa', 'bb'), the renderer counts 'aa', 'ab'. " + f"References to it and to its sub-items read the same way.") + else: + rule = "render-docx-number-mismatch" + message = f"Word will write the counter of the {kind} '{printable(expected)}' as '{printable(step.own)}'." + self.flagged.add(node.unit) + # An overflow is reported once for the list, heading level or section it starts in. + once = rule.endswith("overflow") + if fresh and not (once and (rule, group) in self.reported): + self.reported.add((rule, group)) + self.report(rule, "warning", message, node.line) + + def fallback_label(self, level: LevelFormat, family: str, kind: str, line: int | None) -> None: + """Report a label that needs the section's number where Word can show only the list's own counters.""" + levels = self.definitions[family] + self.report("render-docx-section-label", "warning", + f"The label '{printable(level.label)}' of {kind}s in a section shows the section's number, " + f"which Word can show only for headings: in Word it reads '{printable(_reads(levels[0], levels))}'.", + line) + + # -- units ---------------------------------------------------------------------- + + def section(self, section: Section) -> None: + unit = section.unit + assert unit is not None + self.parent[unit] = section.parent_unit + self.section_level[unit] = section.level + if self.heading_num is None or section.number is None: + self.add(section, "section", None, None, explicit=False) + return + ilvl = section.level - 1 + parent = section.parent_unit + cause: tuple[str, ...] = () + if (parent is None and section.level > 1) or (parent is not None + and self.section_level[parent] + 1 < section.level): + cause = ("skipped-level",) + self.report("render-docx-heading-skip", "warning", + f"Heading level skipped before '{printable(plain_text(section.title))}': the number shows " + f"the skipped level as 1. Word may number the headings after it differently.", section.line) + step = self.sim.step(self.heading_num.id, ilvl, unit) + self.add(section, "section", self.heading_num, ilvl, explicit=False, cause=cause, step=step) + self.check(section, "heading", step, self.formats.headings[ilvl].counter, ("heading", section.level), cause) + + def paragraph(self, paragraph: Paragraph, section: Section) -> None: + assert paragraph.unit is not None and section.unit is not None + self.parent[paragraph.unit] = paragraph.parent_unit + fmt = self.formats.paragraphs + if paragraph.number is None or fmt is None: + self.add(paragraph, "paragraph", None, None, explicit=False) + return + num = self.paragraph_nums.get(section.unit) + if num is None: + num = self.paragraph_nums[section.unit] = self.new_num(self.abstract("paragraphs"), 0) + cause: tuple[str, ...] = () + if _needs_section(fmt): + cause = ("label-fallback",) + self.fallback_label(fmt, "paragraphs", "paragraph", paragraph.line) + step = self.sim.step(num.id, 0, paragraph.unit) + self.add(paragraph, "paragraph", num, 0, explicit=True, cause=cause, step=step) + self.check(paragraph, "paragraph", step, fmt.counter, ("paragraphs", section.unit), cause) + + def list_(self, lst: List, section: Section | None) -> None: + first = lst.items[0] + abstract = self.abstract(lst.family) + ilvl = min(lst.depth, 8) + parent = first.parent_unit + placed = self.placed.get(parent) if parent is not None else None + cause: tuple[str, ...] = () + # Nested in an item: that item's num, if it is of the same kind and Word counts the list from where the + # resolver does (a sibling list continues, a list in a new item restarts); otherwise a num of its own. + if (placed is not None and placed.abstract is abstract and lst.depth <= 8 + and (lst.family == "bullet" or self.sim.peek(placed.num.id, ilvl) == first.number)): + num = placed.num + else: + num = self.new_num(abstract, ilvl, first.number or 1) + if lst.depth > 8: + cause = ("clamped",) + if not self.depth_reported: + self.depth_reported = True + self.report("render-docx-list-depth", "warning", + f"A list is nested {lst.depth} levels deep: Word has 9. Items from level 10 on are " + f"numbered in the format of level 9 and restart by fixed values.", first.line) + elif placed is not None: + kinds = placed.abstract is not abstract + cause = ("restart-family",) if kinds else ("resync",) + self.report("render-docx-static-restart", "info", + "A list nested in a list of another kind restarts by a fixed setting (Restart at 1), " + "not by structure: an item added above it in Word will not restart it." if kinds else + "A list that Word would continue from another list in the same item restarts by a fixed " + "setting (Restart at 1), not by structure.", first.line) + fmt = lst.format + qualified = fmt is not None and section is not None and _needs_section(fmt) + if fmt is not None and qualified: + self.fallback_label(fmt, lst.family, "list item", first.line) + for index, item in enumerate(lst.items): + assert item.unit is not None + self.parent[item.unit] = item.parent_unit + item_cause = (cause if index == 0 else ()) + (("label-fallback",) if qualified else ()) + if fmt is None: + self.units[item.unit] = UnitPlan(item.unit, "item", num.id, ilvl, True, None, None, "", "", None, + item_cause, (), None) + else: + step = self.sim.step(num.id, ilvl, item.unit) + self.add(item, "item", num, ilvl, explicit=True, cause=item_cause, step=step) + self.check(item, "list item", step, fmt.counter, ("list", first.unit), item_cause) + self.placed[item.unit] = _Placed(num, abstract) + for child in item.blocks: + self.block(child, section) + + +def plan_docx(tree: RenderTree, style: Style, *, file: str | None = None) -> tuple[DocxPlan, list[Diagnostic]]: + """The numbering plan of *tree* for Word, and the diagnostics only planning can produce (all + ``render-docx-*``, in the order of their lines, naming *file*). Pure: the same tree and style give an equal plan. + *style* is the one the tree was resolved under. Raises :class:`~legaldown_render.errors.StyleError` for a style + Word's numbering cannot express (:func:`docx_style_errors`; a render checks that first, ADR 0006).""" + problems = docx_style_errors(style) + if problems: + raise StyleError(f"'{style.name}'", problems) + if tree.formats != _formats_of(style): + raise InternalError("the tree and the style given to the DOCX planner do not belong together. This is a bug; " + "please report it with the document that triggers it.") + return _Planner(tree, file).run() diff --git a/tests/test_docx_simulator.py b/tests/test_docx_simulator.py new file mode 100644 index 0000000..e71cbcb --- /dev/null +++ b/tests/test_docx_simulator.py @@ -0,0 +1,215 @@ +"""Word's list numbering as the DOCX planner models it (docs/architecture.md, "The DOCX planner"): counters as Word +writes them, label templates as tokens (the same text the resolver fills in, with the owner of each counter kept), +the ``lvlText`` of a level, and the simulator that gives each paragraph of a list its number.""" +from __future__ import annotations + +import pytest + +from legaldown_render.resolve.docx_numbering import ( + Count, + Ghost, + Lit, + WordNumbering, + extend_tokens, + fill_tokens, + level_text, + lvl_text, + split_level_text, + text_of, + word_counter, +) +from legaldown_render.resolve.docx_plan import AbstractNum, LevelDef, Num +from legaldown_render.resolve.numbering import extend, fill, format_counter + +# -- Counters --------------------------------------------------------------------- + + +@pytest.mark.parametrize(("value", "num_fmt", "expected"), [ + (1, "decimal", "1"), (4000, "decimal", "4000"), + (1, "lowerLetter", "a"), (26, "lowerLetter", "z"), (27, "lowerLetter", "aa"), (28, "lowerLetter", "bb"), + (52, "lowerLetter", "zz"), (53, "lowerLetter", "aaa"), (54, "upperLetter", "BBB"), + (4, "lowerRoman", "iv"), (14, "upperRoman", "XIV"), (3999, "upperRoman", "MMMCMXCIX"), + (4000, "lowerRoman", "mmmm"), (4999, "lowerRoman", "mmmmcmxcix"), + (0, "decimal", "0"), (0, "lowerLetter", ""), +]) +def test_word_counters(value: int, num_fmt: str, expected: str) -> None: + assert word_counter(value, num_fmt) == expected + + +@pytest.mark.parametrize(("counter", "num_fmt"), [ + ("decimal", "decimal"), ("lower-alpha", "lowerLetter"), ("upper-alpha", "upperLetter"), + ("lower-roman", "lowerRoman"), ("upper-roman", "upperRoman")]) +def test_word_and_renderer_counters_agree_where_word_defines_them(counter: str, num_fmt: str) -> None: + limit = 4000 if counter.endswith("roman") else 27 if counter.endswith("alpha") else 5000 + for value in range(1, limit): + assert word_counter(value, num_fmt) == format_counter(value, counter) + + +def test_word_repeats_letters_where_the_renderer_counts_on() -> None: + assert [word_counter(v, "lowerLetter") for v in (27, 28, 29)] == ["aa", "bb", "cc"] + assert [format_counter(v, "lower-alpha") for v in (27, 28, 29)] == ["aa", "ab", "ac"] + assert word_counter(4000, "upperRoman") == "MMMM" and format_counter(4000, "upper-roman") == "4000" + + +# -- Token strings ---------------------------------------------------------------- + + +@pytest.mark.parametrize(("template", "section"), [ + ("{n}.", None), ("({n})", None), ("{section}.{n}", "5"), ("{section}.{n}", ""), ("{section}..{n}", ""), + ("({section}){n}", ""), ("{path}.{n}", "2"), ("a {other} {n}", "1"), ("{n}{n}", None)]) +def test_filling_tokens_reads_as_filling_the_template(template: str, section: str | None) -> None: + tokens = fill_tokens(template, n=[Count("n", "b")], path=[Lit("2.1")], + section=None if section is None else [Lit(section)] if section else []) + kwargs = {} if section is None else {"section": section} + assert text_of(tokens) == fill(template, n="b", path="2.1", **kwargs) + + +@pytest.mark.parametrize(("designation", "piece", "textual"), [ + ("", ".1", False), ("2.1", "(b)", False), ("Termination", "(a)", True), ("", ".1", True), ("A", ".1", True)]) +def test_extending_tokens_reads_as_extending_the_designation(designation: str, piece: str, textual: bool) -> None: + tokens = extend_tokens([Lit(designation)] if designation else [], [Lit(piece)], textual=textual) + assert text_of(tokens) == extend(designation, piece, textual=textual) + + +def test_tokens_keep_the_owner_of_each_counter() -> None: + tokens = fill_tokens("{path}({n})", n=[Count("u2", "b")], path=[Count("s1", "1"), Lit(".")]) + assert tokens == [Count("s1", "1"), Lit("."), Lit("("), Count("u2", "b"), Lit(")")] + + +# -- lvlText ---------------------------------------------------------------------- + + +def test_a_level_shows_only_counters_of_its_own_list() -> None: + refs = ["{n}", ".{n}", ".{n}"] + assert lvl_text(level_text("{path}.", refs, 0, section=None)) == "%1." + assert lvl_text(level_text("{path}", refs, 2, section=None)) == "%1.%2.%3" + assert lvl_text(level_text("({n})", ["({n})"] * 3, 1, section=())) == "(%2)" + assert lvl_text(level_text("Article {n}", ["{n}"], 0, section=None)) == "Article %1" + + +def test_a_label_that_needs_the_section_is_written_as_in_the_preamble() -> None: + # Word cannot show the section's counter in a list level: the label loses the field and the dot after it. + assert lvl_text(level_text("{section}.{n}", [".{n}"], 0, section=())) == "%1" + assert lvl_text(level_text("{path}", [".{n}", "({n})"], 1, section=())) == "%1(%2)" + + +def test_literal_percent_before_a_digit_cannot_be_written() -> None: + assert lvl_text(level_text("100% {n}", ["{n}"], 0, section=None)) == "100% %1" + from legaldown_render.errors import InternalError + with pytest.raises(InternalError): + lvl_text(level_text("%1 {n}", ["{n}"], 0, section=None)) + with pytest.raises(InternalError): + lvl_text(level_text("%{n}", ["{n}"], 0, section=None)) + + +def test_split_level_text() -> None: + assert split_level_text("%1.%2(%10)x") == [0, ".", 1, "(", 0, "0)x"] + assert split_level_text("plain") == ["plain"] + + +# -- The simulator ---------------------------------------------------------------- + + +def _levels(*specs: tuple[str, str]) -> list[LevelDef]: + levels = [LevelDef(i, fmt, text) for i, (fmt, text) in enumerate(specs)] + return levels + [LevelDef(i, "decimal", f"%{i + 1}") for i in range(len(levels), 9)] + + +HEADINGS = _levels(("decimal", "%1."), ("decimal", "%1.%2"), ("decimal", "%1.%2.%3")) +ITEMS = _levels(("lowerLetter", "(%1)"), ("lowerRoman", "(%2)"), ("upperLetter", "(%3)")) + + +def numbering(*nums: Num, skipped: str = "lo") -> WordNumbering: + sim = WordNumbering(skipped=skipped) # type: ignore[arg-type] + sim.add_abstract(0, HEADINGS) + sim.add_abstract(1, ITEMS) + for num in nums: + sim.add_num(num) + return sim + + +def test_counters_count_and_deeper_levels_restart() -> None: + sim = numbering(Num(1, 0)) + labels = [sim.step(1, i, f"u{k}").label for k, i in enumerate([0, 1, 1, 2, 0, 1])] + assert labels == ["1.", "1.1", "1.2", "1.2.1", "2.", "2.1"] + + +def test_a_step_says_what_ref_gives_and_which_unit_holds_each_counter() -> None: + sim = numbering(Num(1, 0)) + sim.step(1, 0, "s1") + step = sim.step(1, 1, "s2") + assert (step.count, step.own, step.label, step.word_ref) == (1, "1", "1.1", "1.1") + assert step.tokens == (Count("s1", "1"), Lit("."), Count("s2", "1")) + assert sim.step(1, 0, "s3").word_ref == "2" and sim.step(1, 0, "s4").label == "3." + + +def test_a_start_override_restarts_a_level_at_its_first_use_only() -> None: + sim = numbering(Num(1, 1), Num(2, 1, ((0, 1), (1, 1))), Num(3, 1, ((0, 3), (1, 1)))) + assert [sim.step(1, 0, "a").label, sim.step(1, 0, "b").label] == ["(a)", "(b)"] + assert sim.peek(2, 0) == 1 + assert [sim.step(2, 0, "c").label, sim.step(2, 0, "d").label] == ["(a)", "(b)"] + assert sim.peek(3, 0) == 3 + assert [sim.step(3, 0, "e").label, sim.step(3, 0, "f").label] == ["(c)", "(d)"] + + +def test_counters_are_shared_by_every_num_of_an_abstract_num() -> None: + # A num without an override goes on from the counter of the abstractNum; one with an override restarts the level + # while the levels above keep their values. + sim = numbering(Num(1, 0), Num(2, 0, ((1, 1), (2, 1)))) + sim.step(1, 0, "s1") + sim.step(1, 1, "s2") + sim.step(1, 1, "s3") + assert sim.step(2, 1, "s4").label == "1.1" + assert sim.step(1, 1, "s5").label == "1.2" + assert sim.step(2, 0, "s6").label == "2." + + +def test_peek_and_current_do_not_count() -> None: + sim = numbering(Num(1, 1)) + assert (sim.peek(1, 0), sim.current(1, 0)) == (1, (0, None)) + sim.step(1, 0, "u1") + assert (sim.peek(1, 0), sim.current(1, 0)) == (2, (1, "u1")) + assert sim.peek(1, 1) == 1 + + +def test_a_skipped_level_reads_as_its_start_value_and_counts_on() -> None: + sim = numbering(Num(1, 0)) + sim.step(1, 0, "s1") + deep = sim.step(1, 2, "s2") + assert deep.label == "1.1.1" + assert deep.tokens[2] == Count(Ghost("s2", 1), "1") + assert sim.step(1, 1, "s3").label == "1.2" + + +def test_in_the_word_model_a_skipped_level_reads_zero_and_is_not_advanced() -> None: + sim = numbering(Num(1, 0), skipped="word") + sim.step(1, 0, "s1") + assert sim.step(1, 2, "s2").label == "1.0.1" + assert sim.step(1, 1, "s3").label == "1.1" + + +def test_the_first_paragraph_of_a_list_deeper_than_the_level_it_starts_at_fills_the_levels_above() -> None: + sim = numbering(Num(1, 1, ((1, 1), (2, 1)))) + assert sim.step(1, 1, "u1").label == "(i)" + + +def test_letters_past_twenty_six_repeat_in_the_word_model() -> None: + sim = numbering(Num(1, 1)) + labels = [sim.step(1, 0, f"u{i}").own for i in range(1, 30)] + assert labels[25:] == ["z", "aa", "bb", "cc"] + + +def test_an_abstract_num_may_gain_its_levels_after_it_is_added() -> None: + levels = _levels(("decimal", "%1.")) + sim = WordNumbering() + sim.add_abstract(0, levels) + sim.add_num(Num(1, 0)) + levels[1] = LevelDef(1, "lowerLetter", "%2)") + sim.step(1, 0, "a") + assert sim.step(1, 1, "b").label == "a)" + + +def test_abstract_num_is_frozen_data() -> None: + abstract = AbstractNum(0, "headings", "4C440000", "multilevel", tuple(HEADINGS)) + with pytest.raises(AttributeError): + abstract.id = 1 # type: ignore[misc] From a2edc774dc719c5fce07da3441b27f1ce9971b06 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:05:58 +0000 Subject: [PATCH 076/118] Test the DOCX numbering plan against the resolver, and describe the planner Unit tests for each placement rule and diagnostic, and a property over the tests' documents, the specification's corpus and random documents under every built-in style and list setting: Word's simulated label for each numbered unit is the resolver's, or a diagnostic of the plan explains the difference. The architecture document gets a section on the planner and on the list semantics it relies on. --- docs/architecture.md | 45 +++ tests/test_docx_plan.py | 685 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 730 insertions(+) create mode 100644 tests/test_docx_plan.py diff --git a/docs/architecture.md b/docs/architecture.md index d754af8..9adff09 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -241,6 +241,47 @@ piece after it is kept), and `render-designation-limit` says so for that heading cycle or past a cap, or one whose terms reach into one, may read differently from the designation references show; the Warning says so. Every other heading reads exactly as its designation. +### The DOCX planner (`resolve/docx_plan.py`) + +The DOCX writer will contain no numbering rules either (ADR 0003). Between the resolved tree and that writer sits a +pure function, `plan_docx(tree, style, *, file=None) -> (DocxPlan, diagnostics)`, which maps the numbering structure +above to what Word needs: the abstractNums and nums, and for every section, list item and numbered paragraph the +`(numId, ilvl)` its paragraph takes. It is a planning step, not a stage of the pipeline: nothing calls it yet. It +reads `tree.formats` and the units' `number`, `parent_unit` and `List.family`/`depth`/`format`, and checks every +placement against the number the resolver gave the unit by simulating Word's counters +(`resolve/docx_numbering.py`), so that what Word draws is what the resolver numbered, or a diagnostic says why not. + +**Word's list semantics**, stated once (the planner and its diagnostics rely on them): + +- Counters belong to the abstractNum and are shared by every `w:num` of it. A num with a `startOverride` restarts + the named level at the first use of that num, and the levels above keep their values: this is Word's own + "Restart at 1". +- A paragraph restarts every deeper level. A paragraph with `numId 0` counts nothing. +- A list nested in an item of the same kind goes on a deeper level of its parent's num, so it restarts by + structure when a parent item is added. A list of another kind, a list in a quote, and a list Word would continue + where the resolver restarts have a num of their own that restarts by a fixed setting: an item added above one in + Word does not restart it. A sibling list under the same item that the resolver continues (its first item is 3) + shares the num, so that Word continues it too. +- Paragraph numbering has one num per section, and a section added in Word does not restart it by itself. +- `REF \n` returns the paragraph's level text with its counters, minus trailing periods. + +**What is planned.** Heading numbering (one multilevel abstractNum linked to the heading styles, the label templates +turned into `lvlText`, the counters into `numFmt`), list families (enumerated, renumbered, bullets) with their nesting +and restarts, depth beyond Word's nine levels (clamped), and numbered paragraphs. A label may show only counters of +its own list, so a label that needs the section's number (`{section}`, or the `{path}` of an item or paragraph in a +section: the continental style's `5.1`) is written without it, as for a preamble item, and reported +(`render-docx-section-label`). `docx_style_errors(style)` names what Word's numbering cannot express at all (a `%` +before a digit, a level text over 255 characters); a DOCX render calls it before rendering and raises `StyleError` +(ADR 0006), and `plan_docx` checks again. + +**Diagnostics** (all `render-docx-*`, with the unit's line): `heading-skip`, `section-label`, `list-depth`, +`letter-overflow` (Word repeats letters after 26, the renderer counts on: `aa`, `ab`), `roman-overflow` (4000 and +beyond), `number-mismatch` (any other difference, which must not occur without a reason: alternatives that share a +number are not yet planned and are reported this way), `counter` (a counter Word has no format for), and, as +information, `static-restart` (a list that restarts by a fixed setting). The same finding is reported once. + +References, bookmarks and the table of contents are further parts of the plan, empty so far. + ### 5. Write (`writers/`) > **A writer contains no LegalDown rules.** `tree.assert_resolved()` rejects a tree that still @@ -266,6 +307,8 @@ src/legaldown_render/ ├── resolve/ │ ├── resolver.py stage 4 │ ├── numbering.py counters, the §13.1 scheme presets, label templates +│ ├── docx_plan.py the DOCX plan: Word numbering for the resolved tree (planning step, not yet wired) +│ ├── docx_numbering.py Word's counters, label templates as tokens, lvlText, the numbering simulator │ └── values.py Babel formatting of dates, money, durations ├── style/ │ ├── model.py the style dataclasses (every field has a default) @@ -459,6 +502,8 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_values.py` | Money, dates and durations in a locale: exact digits, plural of the value shown, currency display, CLDR fallback to English | | `tests/test_numbering.py` | Counter formats, label templates and the preamble's `{section}` | | `tests/test_numbering_structure.py` | The numbering structure in the tree (`formats`, unit ids, numbers, parents, alternatives, list families, cross-reference units): headings in each scheme and with skipped levels, nested lists, section-qualified items, numbered paragraphs, alternatives, the `none` scheme and its second pass; and that every label, designation and reference can be written again from the structure alone, over the tests' and the specification's documents | +| `tests/test_docx_plan.py` | The DOCX numbering plan: abstractNums per style and scheme, heading levels, top-level, nested, sibling, cross-family, quoted and clamped lists, numbered paragraphs, labels that need the section, letter and Roman overflow, diagnostics (lines, file, order, repeats), style errors, determinism; and that every numbered unit's simulated Word label is the resolver's or a diagnostic says why not, over the tests' and the specification's documents under each style and list setting, and over random documents | +| `tests/test_docx_simulator.py` | Word's counters, label templates as tokens (equal to filling the template as text), `lvlText`, and the simulator: restarts, start overrides, shared counters, skipped levels, `peek`/`current` | | `tests/test_build.py` | Building the tree: comment-only items and quotes, emphasis next to a directive, spacing where an inline comment was dropped | | `tests/test_terms.py` | What `{{term:}}` shows, references inside terms, terms that name themselves or each other, the cap on a term text | | `tests/test_none_designations.py` | Designations under the `none` scheme: references and terms in headings, items and paragraphs under them, ambiguity on the final designations, cycles, the depth, length and nesting limits, the lines of the Warnings, work bounds, invariants on random documents | diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py new file mode 100644 index 0000000..584e7a3 --- /dev/null +++ b/tests/test_docx_plan.py @@ -0,0 +1,685 @@ +"""The DOCX numbering plan (docs/architecture.md, "The DOCX planner"): which abstractNums and nums a document needs, +which (numId, ilvl) each section, list item and numbered paragraph takes, and that Word, simulated, numbers every unit +as the resolver did, or a diagnostic says why not. Everything is checked on the plan's data; no file is written.""" +from __future__ import annotations + +import random +from collections.abc import Iterator +from pathlib import Path + +import pytest +from conftest import DOCUMENTS, FRONT, spec_root + +from legaldown_render import DocumentError, render +from legaldown_render.errors import StyleError +from legaldown_render.resolve.docx_numbering import NUM_FMT +from legaldown_render.resolve.docx_plan import ( + DocxPlan, + _formats_of, + docx_style_errors, + plan_docx, +) +from legaldown_render.style import load_style +from legaldown_render.tree import DraftingNote, List, ListItem, Paragraph, Quote, RenderTree, Section + +NONE = {"numbering.scheme": "none"} +PARAGRAPHS = {"paragraphs.numbered": True} +ALTERNATIVES = FRONT.replace("title: T", ( + "title: T\nquestions:\n forum:\n type: choice\n choices:\n courts: Courts\n arb: Arbitration")) + +#: The styles every document is planned under: the built-in ones, and the overrides that change which lists exist +#: and which labels need the section. +VARIANTS: dict[str, dict] = { + "default": {}, + "outline": {"style": "outline"}, + "continental": {"style": "continental"}, + "mixed": {"overrides": {"numbering.scheme": "mixed"}}, + "none": {"overrides": NONE}, + "paragraphs": {"overrides": PARAGRAPHS}, + "continental-none": {"style": "continental", "overrides": NONE}, + "enumerate-ordered": {"overrides": {"enumeration.ordered": "enumerate"}}, + "bullets": {"overrides": {"enumeration.enabled": False}}, +} + + +def planned(body: str, front: str = FRONT, **settings: object): # noqa: ANN201 + result = render(front + body, format="text", **settings) + plan, diagnostics = plan_docx(result.tree, result.style) + return result.tree, plan, diagnostics + + +def rules(diagnostics: list) -> set[str]: # noqa: ANN001 + return {d.rule for d in diagnostics} + + +# -- A walk of the units ---------------------------------------------------------- + + +def _blocks(blocks: tuple) -> Iterator[Section | ListItem | Paragraph]: + for block in blocks: + match block: + case Paragraph() if block.unit is not None: + yield block + case List(): + for item in block.items: + yield item + yield from _blocks(item.blocks) + case Quote(blocks=inner) | DraftingNote(blocks=inner): + yield from _blocks(inner) + + +def units(tree: RenderTree) -> list[Section | ListItem | Paragraph]: + found = list(_blocks(tree.preamble)) + for section in tree.sections: + found.append(section) + found.extend(_blocks(section.blocks)) + return found + + +def lists(tree: RenderTree) -> list[List]: + found: list[List] = [] + + def walk(blocks: tuple) -> None: + for block in blocks: + match block: + case List(): + found.append(block) + for item in block.items: + walk(item.blocks) + case Quote(blocks=inner) | DraftingNote(blocks=inner): + walk(inner) + + walk(tree.preamble) + for section in tree.sections: + walk(section.blocks) + return found + + +def items(tree: RenderTree) -> list[ListItem]: + """Every list item, in document order (unit ids also count paragraphs, so tests ask for items by position).""" + return [u for u in units(tree) if isinstance(u, ListItem)] + + +def shown(plan: DocxPlan, *nodes: ListItem) -> list[tuple]: + """(numId, ilvl, label) of each node.""" + return [(plan.units[n.unit].num_id, plan.units[n.unit].ilvl, plan.units[n.unit].label) for n in nodes] + + +# -- The abstractNums --------------------------------------------------------------- + + +def test_the_default_style_needs_a_heading_a_list_and_a_renumber_abstract_num() -> None: + _, plan, diagnostics = planned("# A\n\n- x\n\n1. y\n") + assert [(a.id, a.kind, a.nsid, a.multi_level) for a in plan.abstract_nums] == [ + (0, "headings", "4C440000", "multilevel"), + (1, "enumerated", "4C440001", "hybridMultilevel"), + (2, "renumber", "4C440002", "hybridMultilevel")] + assert all(len(a.levels) == 9 for a in plan.abstract_nums) + assert plan.heading_num_id == 1 and plan.nums[0].id == 1 and plan.nums[0].start_overrides == () + assert diagnostics == [] + + +def test_abstract_nums_are_made_in_the_order_of_use_and_only_when_used() -> None: + _, plan, _ = planned("# A\n\n1. y\n\n- x\n") + assert [a.kind for a in plan.abstract_nums] == ["headings", "renumber", "enumerated"] + _, plan, _ = planned("Text.\n") + assert [a.kind for a in plan.abstract_nums] == ["headings"] + + +def test_heading_levels_follow_the_scheme() -> None: + def texts(**settings: object) -> list[tuple[str, str]]: + _, plan, _ = planned("# A\n", **settings) + return [(level.num_fmt, level.lvl_text) for level in plan.abstract_nums[0].levels[:5]] + + assert texts() == [("decimal", "%1."), ("decimal", "%1.%2"), ("decimal", "%1.%2.%3"), + ("decimal", "%1.%2.%3.%4"), ("decimal", "%1.%2.%3.%4.%5")] + assert texts(style="outline") == [("upperRoman", "%1."), ("upperLetter", "%2."), ("decimal", "%3."), + ("lowerLetter", "%4."), ("lowerRoman", "(%5)")] + assert texts(overrides={"numbering.scheme": "mixed"}) == [ + ("decimal", "%1."), ("lowerLetter", "(%2)"), ("lowerRoman", "(%3)"), ("upperLetter", "(%4)"), + ("decimal", "(%5)")] + assert texts(style="continental")[:2] == [("decimal", "%1."), ("decimal", "%1.%2")] + + +def test_heading_levels_are_linked_to_the_heading_styles_and_unused_levels_are_neutral() -> None: + _, plan, _ = planned("# A\n") + levels = plan.abstract_nums[0].levels + assert [level.p_style for level in levels] == [f"Heading{k}" for k in range(1, 6)] + [None] * 4 + assert [(level.num_fmt, level.lvl_text) for level in levels[5:]] == [("decimal", f"%{k}") for k in range(6, 10)] + + +def test_a_custom_heading_label_is_a_level_text_and_a_word_label_is_followed_by_a_space() -> None: + levels = [{"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}] + _, plan, _ = planned("# A\n\n## B\n", overrides={"numbering.levels": levels}) + first, second = plan.abstract_nums[0].levels[:2] + assert (first.num_fmt, first.lvl_text, first.suff) == ("upperRoman", "Article %1", "space") + assert (second.lvl_text, second.suff) == ("%1.%2", "tab") + assert plan.units["s1"].label == "Article I" + + +def test_level_indents_step_by_level_and_headings_are_as_wide_as_their_widest_label() -> None: + _, plan, _ = planned("# A\n\n- x\n") + headings, items = plan.abstract_nums[0].levels, plan.abstract_nums[1].levels + assert [(level.left, level.hanging) for level in items[:3]] == [(567, 567), (1134, 567), (1701, 567)] + assert headings[0].left == headings[0].hanging == 567 + assert headings[4].left == headings[4].hanging > headings[1].left > headings[0].left + + +def test_the_none_scheme_has_no_heading_abstract_num_and_sections_are_unnumbered() -> None: + tree, plan, _ = planned("# A\n\n- x\n", overrides=NONE) + assert plan.heading_num_id is None + assert [a.kind for a in plan.abstract_nums] == ["enumerated"] + assert plan.units["s1"].num_id is None and plan.units["s1"].explicit is False and plan.units["s1"].count is None + assert plan.nums[0].abstract_id == 0 + + +def test_a_counter_without_a_word_format_is_written_as_decimal(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delitem(NUM_FMT, "lower-alpha") + tree, plan, diagnostics = planned("# A\n\n- x\n- y\n") + assert plan.abstract_nums[1].levels[0].num_fmt == "decimal" + assert [plan.units[i.unit].label for i in items(tree)] == ["(1)", "(2)"] + assert [(d.rule, d.level) for d in diagnostics if d.rule == "render-docx-counter"] == [ + ("render-docx-counter", "warning")] + + +# -- Headings ------------------------------------------------------------------------ + + +def test_headings_take_the_style_num_at_their_level() -> None: + tree, plan, _ = planned("# A\n\n## B\n\n### C\n\n# D\n") + assert [(u.num_id, u.ilvl, u.explicit, u.label, u.word_ref, u.count) for u in map(plan.units.get, ("s1", "s2", "s3", "s4"))] == [ + (1, 0, False, "1.", "1", 1), (1, 1, False, "1.1", "1.1", 1), (1, 2, False, "1.1.1", "1.1.1", 1), + (1, 0, False, "2.", "2", 2)] + + +def test_a_skipped_heading_level_is_numbered_as_the_resolver_does_and_reported() -> None: + tree, plan, diagnostics = planned("# A\n\n### B\n\n## C\n") + assert [plan.units[s].label for s in ("s1", "s2", "s3")] == ["1.", "1.1.1", "1.2"] + assert plan.units["s2"].cause == ("skipped-level",) and plan.units["s3"].cause == () + [found] = [d for d in diagnostics if d.rule == "render-docx-heading-skip"] + assert found.level == "warning" and found.line == tree.sections[1].line and "'B'" in found.message + assert rules(diagnostics) == {"render-docx-heading-skip"} + + +def test_a_document_that_starts_below_level_one_skips_nothing() -> None: + _, plan, diagnostics = planned("## A\n\n## B\n") + assert [plan.units[s].label for s in ("s1", "s2")] == ["1.", "2."] and diagnostics == [] + + +# -- Lists --------------------------------------------------------------------------- + +NESTED = """# A + +- one + - nested a + - nested b + - deep +- two +- three + +1. first +1. second +""" + + +def test_every_top_level_list_has_a_num_that_starts_it_at_one() -> None: + tree, plan, _ = planned(NESTED) + items = [u for u in units(tree) if isinstance(u, ListItem)] + first_list, ordered = plan.units[items[0].unit], plan.units[items[-2].unit] + assert (first_list.num_id, first_list.ilvl, first_list.explicit, first_list.label) == (2, 0, True, "(a)") + assert (ordered.num_id, ordered.ilvl, ordered.label) == (3, 0, "1.") + assert plan.nums[1].start_overrides == tuple((i, 1) for i in range(9)) + assert plan.nums[2].abstract_id == 2 + + +def test_two_lists_in_a_section_restart_each_at_its_own_num() -> None: + tree, plan, _ = planned("# A\n\n- x\n- y\n\nText.\n\n- z\n") + assert [(n, label) for n, _, label in shown(plan, *items(tree))] == [(2, "(a)"), (2, "(b)"), (3, "(a)")] + + +def test_a_nested_list_of_the_same_kind_uses_a_deeper_level_of_the_parents_num() -> None: + tree, plan, diagnostics = planned(NESTED) + assert shown(plan, *items(tree)[:6]) == [ + (2, 0, "(a)"), (2, 1, "(i)"), (2, 1, "(ii)"), (2, 2, "(A)"), (2, 0, "(b)"), (2, 0, "(c)")] + assert len(plan.nums) == 3 and diagnostics == [] + + +def test_a_list_in_a_new_item_restarts_by_structure_and_needs_no_override() -> None: + tree, plan, _ = planned("# A\n\n- a\n - x\n- b\n - y\n") + _, x, _, y = items(tree) + assert shown(plan, x, y) == [(2, 1, "(i)"), (2, 1, "(i)")] and plan.units[y.unit].cause == () + + +def test_sibling_lists_in_one_item_share_the_num_so_that_the_second_continues() -> None: + body = "# A\n\n- a\n - x\n - y\n\n Between.\n\n - z\n" + tree, plan, diagnostics = planned(body) + nested = items(tree)[1:] + assert [i.number for i in nested] == [1, 2, 3] # the resolver goes on with the second list + assert shown(plan, *nested) == [(2, 1, "(i)"), (2, 1, "(ii)"), (2, 1, "(iii)")] + assert len(plan.nums) == 2 and diagnostics == [] + + +def test_a_sibling_list_of_another_kind_that_continues_gets_a_num_starting_at_its_number() -> None: + # The nested ordered list and the second nested ordered list share one counter in the resolver: the second + # list's first item is 3. Another kind of list in between puts them on different nums. + body = "# A\n\n- a\n 1. x\n 1. y\n\n Between.\n\n - w\n\n 1. z\n" + tree, plan, diagnostics = planned(body) + ordered = [u for u in units(tree) if isinstance(u, ListItem) and u.label and u.label.endswith(".")] + assert [item.number for item in ordered] == [1, 2, 3] + assert [plan.units[item.unit].label for item in ordered] == ["1.", "2.", "3."] + # Another kind of list in between: the second ordered list cannot go on from the first by structure, so its num + # starts at its number, 3. Static, and reported as information only. + last = plan.units[ordered[-1].unit] + assert last.num_id != plan.units[ordered[0].unit].num_id and last.cause == ("restart-family",) + assert plan.nums[last.num_id - 1].start_overrides[0] == (1, 3) + assert {d.level for d in diagnostics} == {"info"} + + +def test_a_nested_list_of_another_kind_has_a_num_of_its_own_and_is_reported_as_information() -> None: + tree, plan, diagnostics = planned("# A\n\n- a\n 1. x\n 1. y\n- b\n 1. z\n") + _, x, y, _, z = items(tree) + assert shown(plan, x, y, z) == [(3, 1, "1."), (3, 1, "2."), (4, 1, "1.")] + assert plan.units[x.unit].cause == ("restart-family",) + assert plan.nums[2].start_overrides[0] == (1, 1) and plan.nums[2].abstract_id == 2 + assert [(d.rule, d.level) for d in diagnostics] == [("render-docx-static-restart", "info")] + assert diagnostics[0].line == tree.sections[0].blocks[0].items[0].blocks[1].items[0].line + + +def test_a_list_in_a_quote_inside_an_item_is_numbered_from_the_section_with_a_num_of_its_own() -> None: + body = "# A\n\n- a\n - x\n\n > - quoted\n > - quoted two\n\n - y\n- b\n" + tree, plan, diagnostics = planned(body) + _, x, quoted, quoted_too, y, _ = items(tree) + assert [quoted.parent_unit, quoted_too.parent_unit] == ["s1", "s1"] # built on the section, not on the item + # a num of its own, at the level its depth gives: fresh counter, as the resolver has + assert shown(plan, quoted, quoted_too) == [(3, 1, "(i)"), (3, 1, "(ii)")] + # The list after the quote goes on from the first nested list (the resolver counts them as siblings), but + # Word's counter of that level was used by the quote's list: it starts again where the resolver is, by a + # fixed setting. + assert shown(plan, x, y) == [(2, 1, "(i)"), (4, 1, "(ii)")] + assert plan.nums[3].start_overrides[0] == (1, 2) and plan.units[y.unit].cause == ("resync",) + assert [(d.rule, d.level) for d in diagnostics] == [("render-docx-static-restart", "info")] + + +def test_bullets_are_numbered_paragraphs_with_a_bullet_of_their_own() -> None: + tree, plan, diagnostics = planned("# A\n\n- a\n - b\n", overrides={"enumeration.enabled": False}) + assert [a.kind for a in plan.abstract_nums] == ["headings", "bullet"] + first, second = items(tree) + assert shown(plan, first, second) == [(2, 0, ""), (2, 1, "")] and plan.units[first.unit].count is None + assert [level.lvl_text for level in plan.abstract_nums[1].levels[:4]] == ["•", "–", "◦", "•"] + assert diagnostics == [] + + +def test_ordered_lists_that_are_enumerated_take_the_enumeration_levels() -> None: + _, plan, _ = planned("# A\n\n1. x\n 1. y\n", overrides={"enumeration.ordered": "enumerate"}) + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated"] + assert [plan.units[u].label for u in ("u1", "u2")] == ["(a)", "(i)"] + + +def test_an_item_in_the_preamble_is_numbered_without_a_section() -> None: + _, plan, _ = planned("- a\n- b\n\n# A\n", style="continental") + assert [(plan.units[u].label, plan.units[u].num_id) for u in ("u1", "u2")] == [("1", 2), ("2", 2)] + + +def test_items_in_a_quote_or_a_drafting_note_are_numbered_like_any_item() -> None: + tree, plan, _ = planned("# A\n\n> - quoted\n\n> [!DRAFTING]\n> - noted\n") + assert [(n, label) for n, _, label in shown(plan, *items(tree))] == [(2, "(a)"), (3, "(a)")] + + +def test_a_list_deeper_than_word_has_levels_is_clamped_and_reported_once() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(11)) + tree, plan, diagnostics = planned(body) + deep = items(tree) + assert [plan.units[i.unit].ilvl for i in deep] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8] + assert [plan.units[i.unit].cause for i in deep[8:]] == [(), ("clamped",), ("clamped",)] + # each clamped list restarts on a num of its own: Word's level 9 would count on otherwise + assert len({plan.units[i.unit].num_id for i in deep[8:]}) == 3 + assert all(plan.units[i.unit].count == 1 for i in deep[8:]) + [found] = [d for d in diagnostics if d.rule == "render-docx-list-depth"] + assert found.level == "warning" and found.line == deep[9].line and "9" in found.message + assert rules(diagnostics) == {"render-docx-list-depth"} + + +# -- Paragraphs ---------------------------------------------------------------------- + + +def test_numbered_paragraphs_have_a_num_per_section_that_starts_at_one() -> None: + body = "# A\n\nOne.\n\nTwo.\n\n# B\n\nThree.\n" + tree, plan, diagnostics = planned(body, style="continental") + paragraphs = [u for u in units(tree) if isinstance(u, Paragraph)] + assert [(plan.units[p.unit].num_id, plan.units[p.unit].ilvl, plan.units[p.unit].label) for p in paragraphs] == [ + (2, 0, "(1)"), (2, 0, "(2)"), (3, 0, "(1)")] + assert plan.abstract_nums[1].kind == "paragraphs" and plan.abstract_nums[1].levels[0].lvl_text == "(%1)" + assert diagnostics == [] + + +def test_paragraphs_are_not_numbered_unless_the_style_numbers_them() -> None: + _, plan, _ = planned("# A\n\nOne.\n") + assert plan.units["u1"].num_id is None and plan.units["u1"].count is None and plan.units["u1"].label == "" + + +# -- Labels that need the section ----------------------------------------------------- + + +def test_a_label_that_needs_the_sections_number_is_written_without_it_and_reported() -> None: + body = "# A\n\n- x\n- y\n\n## B\n\n- z\n" + tree, plan, diagnostics = planned(body, style="continental") + assert [plan.units[i.unit].label for i in items(tree)] == ["1", "2", "1"] # the resolver wrote 1.1, 1.2, 1.1.1 + assert [u.label for u in units(tree) if isinstance(u, ListItem)] == ["1.1", "1.2", "1.1.1"] + assert plan.units[items(tree)[0].unit].cause == ("label-fallback",) + [found] = [d for d in diagnostics if d.rule == "render-docx-section-label"] + assert found.level == "warning" and "{section}.{n}" in found.message and "'1'" in found.message + assert found.line == tree.sections[0].blocks[0].items[0].line + + +def test_paragraph_numbers_that_need_the_section_are_reported_too() -> None: + tree, plan, diagnostics = planned("# A\n\nOne.\n", overrides=PARAGRAPHS) + [paragraph] = [u for u in units(tree) if isinstance(u, Paragraph)] + assert plan.units[paragraph.unit].label == "1" and plan.units[paragraph.unit].cause == ("label-fallback",) + assert rules(diagnostics) == {"render-docx-section-label"} + + +def test_a_preamble_item_needs_no_section_and_is_not_reported() -> None: + _, plan, diagnostics = planned("- x\n\n# A\n", style="continental") + assert plan.units["u1"].label == "1" and diagnostics == [] + + +# -- Counters Word writes differently -------------------------------------------------- + + +def _items(n: int) -> str: + return "# A\n\n" + "".join(f"- item {i}\n" for i in range(1, n + 1)) + + +def test_letters_past_twenty_six_are_reported_once_for_the_list() -> None: + tree, plan, diagnostics = planned(_items(60)) + items = [u for u in units(tree) if isinstance(u, ListItem)] + assert plan.units[items[26].unit].label == "(aa)" and plan.units[items[27].unit].label == "(bb)" + assert items[27].label == "(ab)" + [found] = diagnostics + assert (found.rule, found.level, found.line) == ("render-docx-letter-overflow", "warning", items[27].line) + assert "'ab'" in found.message and "'bb'" in found.message + assert plan.units[items[27].unit].count == 28 + + +def test_an_overflow_that_reads_the_same_in_two_lists_is_one_finding_at_the_first() -> None: + tree, _, diagnostics = planned(_items(30) + "\nText.\n\n" + _items(30)[5:]) + [found] = diagnostics + assert found.rule == "render-docx-letter-overflow" and found.line == items(tree)[27].line + + +def test_roman_numerals_from_4000_are_reported() -> None: + levels = [{"counter": "upper-roman", "label": "{n}.", "ref": "{n}"}] + body = "".join(f"# H{i}\n\n" for i in range(1, 4)) + tree, plan, diagnostics = planned(body, overrides={"numbering.levels": levels}) + assert diagnostics == [] + levels = [{"counter": "lower-roman", "label": "({n})", "ref": "({n})"}] * 3 + many = "".join(f"- i{i}\n" for i in range(1, 4002)) + tree, plan, diagnostics = planned("# A\n\n" + many, overrides={"enumeration.levels": levels}) + items = [u for u in units(tree) if isinstance(u, ListItem)] + assert (items[3998].label, plan.units[items[3998].unit].label) == ("(mmmcmxcix)", "(mmmcmxcix)") + assert (items[3999].label, plan.units[items[3999].unit].label) == ("(4000)", "(mmmm)") + [found] = [d for d in diagnostics if d.rule == "render-docx-roman-overflow"] + assert found.line == items[3999].line and "'4000'" in found.message and "'mmmm'" in found.message + + +# -- Alternatives: planned in a later step; today Word's numbers differ and the finding says so --------- + + +def test_an_alternative_that_shares_a_number_is_reported_while_it_is_numbered_in_sequence() -> None: + body = "# Disputes {#d when=forum:courts}\n\n# Disputes {#d when=forum:arb}\n\n# Last\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES) + assert tree.sections[1].shares_number_with == "s1" + assert [plan.units[s].label for s in ("s1", "s2", "s3")] == ["1.", "2.", "3."] + assert [(d.rule, d.line) for d in diagnostics] == [ + ("render-docx-number-mismatch", tree.sections[1].line), ("render-docx-number-mismatch", tree.sections[2].line)] + assert "'1.'" in diagnostics[0].message and "'2.'" in diagnostics[0].message + + +# -- Diagnostics: lines, files, order, repeats -------------------------------------------- + + +def test_diagnostics_name_the_file_and_the_line_and_come_in_line_order() -> None: + result = render(FRONT + "# A\n\n### B\n\n- x\n 1. y\n", format="text") + plan, diagnostics = plan_docx(result.tree, result.style, file="deal.lgd") + assert [(d.rule, d.file) for d in diagnostics] == [ + ("render-docx-heading-skip", "deal.lgd"), ("render-docx-static-restart", "deal.lgd")] + lines = [d.line for d in diagnostics] + assert lines == sorted(lines) and None not in lines + _, anonymous = plan_docx(result.tree, result.style) + assert {d.file for d in anonymous} == {""} + + +def test_the_same_finding_is_reported_once() -> None: + _, _, diagnostics = planned("# A\n\n- a\n 1. x\n- b\n 1. y\n- c\n 1. z\n") + assert [d.rule for d in diagnostics] == ["render-docx-static-restart"] + + +def test_untrusted_text_in_a_message_is_escaped() -> None: + _, _, diagnostics = planned("# A\n\n### B\u0007x\n") + [found] = diagnostics + assert "\u0007" not in found.message + + +# -- Determinism and the plan's data ------------------------------------------------------- + + +def test_the_same_tree_gives_an_equal_plan_and_the_plan_is_frozen() -> None: + result = render(FRONT + NESTED, format="text", style="continental") + first = plan_docx(result.tree, result.style) + again = plan_docx(result.tree, result.style) + assert first[0] == again[0] and first[1] == again[1] + with pytest.raises(AttributeError): + first[0].heading_num_id = 5 # type: ignore[misc] + assert list(first[0].units) == [u.unit for u in units(result.tree)] + assert first[0].refs == {} and first[0].anchors == {} and first[0].toc == () + + +def test_unit_plans_carry_the_whole_plan_for_a_unit() -> None: + _, plan, _ = planned("# A\n\n- x\n") + unit = plan.units["u1"] + assert (unit.kind, unit.ind, unit.alt, unit.bookmarks, unit.toc_bookmark) == ("item", None, None, (), None) + assert plan.units["s1"].kind == "section" + + +def test_a_tree_and_a_style_that_do_not_belong_together_are_refused() -> None: + from legaldown_render.errors import InternalError + result = render(FRONT + "# A\n", format="text") + with pytest.raises(InternalError): + plan_docx(result.tree, load_style("continental")) + + +# -- Style checks (ADR 0006) ---------------------------------------------------------------- + + +def test_formats_of_a_style_are_the_formats_in_the_tree() -> None: + for settings in VARIANTS.values(): + result = render(FRONT + "# A\n", format="text", **settings) + assert _formats_of(result.style) == result.tree.formats + + +def test_the_built_in_styles_are_fine_for_docx() -> None: + for name in ("default", "outline", "continental"): + assert docx_style_errors(load_style(name)) == [] + + +@pytest.mark.parametrize(("key", "value"), [ + ("numbering.levels", [{"counter": "decimal", "label": "%1 {n}", "ref": "{n}"}]), + ("numbering.levels", [{"counter": "decimal", "label": "%{n}", "ref": "{n}"}]), + ("numbering.levels", [{"counter": "decimal", "label": "{n}", "ref": "%2{n}"}]), + ("enumeration.levels", [{"counter": "decimal", "label": "{n}%9", "ref": "{n}"}]), + ("paragraphs.format", {"counter": "decimal", "label": "%3{n}", "ref": "{n}"}), +]) +def test_a_percent_that_lvl_text_cannot_write_is_a_style_error(key: str, value: object) -> None: + overrides = {key: value, "paragraphs.numbered": True} + style = load_style(overrides=overrides) + problems = docx_style_errors(style) + assert len(problems) == 1 and key.split(".")[0] in problems[0] and "'%'" in problems[0] + with pytest.raises(StyleError): + plan_docx(render(FRONT + "# A\n", format="text", overrides=overrides).tree, style) + + +def test_a_percent_that_cannot_be_mistaken_is_fine() -> None: + levels = [{"counter": "decimal", "label": "{n}% of", "ref": "{n}"}] + assert docx_style_errors(load_style(overrides={"numbering.levels": levels})) == [] + tree, plan, _ = planned("# A\n", overrides={"numbering.levels": levels}) + assert plan.abstract_nums[0].levels[0].lvl_text == "%1% of" + + +def test_settings_that_are_off_are_not_checked() -> None: + overrides = {"enumeration.enabled": False, "enumeration.levels": [ + {"counter": "decimal", "label": "%1{n}", "ref": "{n}"}]} + assert docx_style_errors(load_style(overrides=overrides)) == [] + heads = {"numbering.scheme": "none"} + assert docx_style_errors(load_style(overrides=heads)) == [] + + +def test_a_level_text_longer_than_word_allows_is_a_style_error() -> None: + long = "x" * 260 + style = load_style(overrides={"numbering.levels": [{"counter": "decimal", "label": long + "{n}", "ref": "{n}"}]}) + [problem] = docx_style_errors(style) + assert "headings" in problem and "255" in problem + + +# -- Word numbers every unit as the resolver did, or a diagnostic says why not --------------------- + +EXPLAINING = { + "label-fallback": "render-docx-section-label", + "clamped": "render-docx-list-depth", + "skipped-level": "render-docx-heading-skip", +} +MISMATCH = {"render-docx-number-mismatch", "render-docx-letter-overflow", "render-docx-roman-overflow"} + + +def assert_planned_as_resolved(result) -> None: # noqa: ANN001 + """The property: for every numbered unit, Word's label and count are the resolver's, or a diagnostic of the plan + explains the exception; and nothing is reported that did not happen.""" + plan, diagnostics = plan_docx(result.tree, result.style) + found = rules(diagnostics) + differs = False + for unit in units(result.tree): + mine = plan.units[unit.unit] + if unit.number is None: + assert mine.count is None and mine.label == "" or isinstance(unit, Section) + continue + for cause, rule in EXPLAINING.items(): + if cause in mine.cause: + assert rule in found, (unit.unit, cause) + if mine.count != unit.number or (mine.label != unit.label and not {"label-fallback", "clamped"} & set(mine.cause)): + differs = True + if mine.cause == () and mine.count == unit.number: + assert mine.label == unit.label or found & MISMATCH, (unit.unit, mine.label, unit.label) + assert (not differs) or found & MISMATCH, "a difference without a diagnostic" + for rule in found & MISMATCH: + assert differs, f"{rule} without a difference" + assert all(d.rule.startswith("render-docx-") for d in diagnostics) + assert {u.unit for u in units(result.tree)} == set(plan.units) + uses = {n.id for n in plan.nums} + for mine in plan.units.values(): + assert mine.num_id is None or mine.num_id in uses + # Every num with overrides is used, and the first unit on it is on a level its overrides name. + first: dict[int, str] = {} + for unit in units(result.tree): + mine = plan.units[unit.unit] + if mine.num_id is not None: + first.setdefault(mine.num_id, unit.unit) + for num in plan.nums: + if num.start_overrides: + assert num.id in first, f"num {num.id} is never used" + assert plan.units[first[num.id]].ilvl in {i for i, _ in num.start_overrides} + + +@pytest.mark.parametrize("variant", VARIANTS) +@pytest.mark.parametrize("name", ["features.lgd", "template.lgd"]) +def test_the_test_documents_are_planned_as_resolved(name: str, variant: str) -> None: + source = (DOCUMENTS / name).read_text(encoding="utf-8") + assert_planned_as_resolved(render(source, format="text", **VARIANTS[variant])) + + +SPEC = spec_root() + + +def _spec_documents() -> list[Path]: + if SPEC is None: + return [] + return sorted((SPEC / "examples").rglob("*.lgd")) + sorted((SPEC / "fixtures" / "valid").rglob("*.lgd")) + + +@pytest.mark.conformance +@pytest.mark.skipif(SPEC is None, reason="LegalDown specification checkout not found") +@pytest.mark.parametrize("variant", VARIANTS) +def test_the_specifications_documents_are_planned_as_resolved(variant: str) -> None: + for path in _spec_documents(): + try: + result = render(path.read_text(encoding="utf-8"), format="text", **VARIANTS[variant]) + except DocumentError: + continue + try: + assert_planned_as_resolved(result) + except AssertionError as error: + raise AssertionError(f"{path.relative_to(SPEC)}: {error}") from error + + +# -- Random documents --------------------------------------------------------------------------- + + +def _random_list(rng: random.Random, depth: int, indent: str) -> list[str]: + ordered = rng.random() < 0.3 + marker = "1. " if ordered else "- " + lines: list[str] = [] + for index in range(rng.choice([1, 2, 2, 3, 5, 30]) if depth == 0 else rng.randint(1, 3)): + lines.append(f"{indent}{marker}item {depth}.{index}") + inner = indent + " " * len(marker) + if depth < 4 and rng.random() < 0.45: + lines.extend(_random_list(rng, depth + 1, inner)) + if rng.random() < 0.3: # a sibling list after text: it goes on from the first + lines += ["", f"{inner}Between.", ""] + lines.extend(_random_list(rng, depth + 1, inner)) + if depth < 3 and rng.random() < 0.12: + lines += ["", f"{inner}> - quoted", f"{inner}> - quoted too", ""] + return lines + [""] + + +def _random_document(rng: random.Random) -> str: + out: list[str] = [] + level = 1 + for _ in range(rng.randint(1, 7)): + move = rng.random() + level = max(1, min(5, level + (1 if move < 0.4 else -1 if move < 0.55 else 2 if move < 0.6 else 0))) + out.append("#" * level + " Heading\n") + for _ in range(rng.randint(0, 3)): + kind = rng.random() + if kind < 0.45: + out.extend(_random_list(rng, 0, "")) + elif kind < 0.8: + out.append("A paragraph.\n") + else: + out.append("> quoted\n") + if rng.random() < 0.3: + out.insert(0, "- preamble item\n - nested\n") + return "\n".join(out) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_random_documents_are_planned_as_resolved(variant: str) -> None: + rng = random.Random(20261028) + for _ in range(40): + source = FRONT + _random_document(rng) + try: + assert_planned_as_resolved(render(source, format="text", **VARIANTS[variant])) + except AssertionError as error: + raise AssertionError(f"{error}\n--- document ---\n{source}") from error + + +def test_the_random_documents_reach_every_case_the_property_is_about() -> None: + seen_rules: set[str] = set() + seen_causes: set[str] = set() + for settings in VARIANTS.values(): + rng = random.Random(20261028) + for _ in range(40): + result = render(FRONT + _random_document(rng), format="text", **settings) + plan, diagnostics = plan_docx(result.tree, result.style) + seen_rules |= rules(diagnostics) + seen_causes |= {cause for unit in plan.units.values() for cause in unit.cause} + assert {"render-docx-letter-overflow", "render-docx-section-label", "render-docx-heading-skip", + "render-docx-static-restart"} <= seen_rules + assert {"restart-family", "resync", "label-fallback", "skipped-level"} <= seen_causes From f4804aa94bcacbb6587b074827d7eac8c3bf0a36 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:21:23 +0000 Subject: [PATCH 077/118] Add the DOCX plan data model The planner hands the DOCX writer frozen data: numbering definitions, which paragraph carries which number, bookmark names, reference fields with cached results and the cached table of contents. The writer lays it out and decides nothing (ADR 0003). --- src/legaldown_render/resolve/docx_plan.py | 165 ++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 src/legaldown_render/resolve/docx_plan.py diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py new file mode 100644 index 0000000..f78163f --- /dev/null +++ b/src/legaldown_render/resolve/docx_plan.py @@ -0,0 +1,165 @@ +"""The DOCX plan: the data the planner (`resolve/docx_plan.py`, `plan_docx`) hands to the writer. + +Frozen data only, no logic. The planner decides every Word-specific matter (abstractNums, nums, which paragraph +carries which number, bookmark names, reference fields and their cached results, the cached table of contents); +the writer serialises these decisions and decides nothing (docs/decisions/0003). Bookmark *ids* are not here: the +writer allocates them (`writers/docx/ooxml.Bookmarks`), the plan only names bookmarks. Names are valid for +`ooxml._BOOKMARK_RE` (at most 40 characters) and unique ignoring case. + +All sequences are tuples, all mappings are plain dicts built once in document order and never changed afterwards. +""" +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Literal + +from ..tree import Inline # in the implementation: from ...tree import Inline + + +@dataclass(frozen=True, slots=True) +class LevelDef: + """One ``w:lvl`` of an abstractNum.""" + + ilvl: int + #: ``decimal``, ``lowerLetter``, ``upperLetter``, ``lowerRoman``, ``upperRoman`` or ``bullet``. + num_fmt: str + #: In OOXML form (``%1.%2``, ``(%3)``, ``Article %1``, ``•``): ``%k`` is the counter of level ``k-1``. + lvl_text: str + start: int = 1 + #: ``Heading1``..``Heading5``, only on levels 0-4 of the heading abstractNum. + p_style: str | None = None + suff: Literal["tab", "space"] = "tab" + #: The level's own ``w:ind`` in twips. On the heading abstractNum this serves the heading; a unit that sits on + #: the level at another nesting depth carries its own indent (:attr:`UnitPlan.ind`). + left: int = 0 + hanging: int = 0 + + +@dataclass(frozen=True, slots=True) +class AbstractNum: + id: int # 0.., in order of first use (headings first) + kind: Literal["headings", "enumerated", "renumber", "paragraphs", "bullet"] + #: Eight hex digits, deterministic (``0x4C440000 + id``). Equal in every LegalDown document, so Word may + #: merge lists on paste between two of them (cosmetic). + nsid: str + multi_level: Literal["multilevel", "hybridMultilevel"] + #: Always nine levels. + levels: tuple[LevelDef, ...] + + +@dataclass(frozen=True, slots=True) +class Num: + """One ``w:num``. ``nums[0]`` is the heading num (numId 1) only when headings are numbered.""" + + id: int # from 1 (numId 0 means "no numbering" and is never a Num) + abstract_id: int + #: ``(ilvl, value)``: the first number of the list at that level; deeper levels restart at 1. Overrides that + #: are never reached by a paragraph are harmless. + start_overrides: tuple[tuple[int, int], ...] = () + + +@dataclass(frozen=True, slots=True) +class Literal_: + """Text of a reference or number that is not a counter: delimiters and the words of ``references.format``.""" + + text: str + + +@dataclass(frozen=True, slots=True) +class FieldSeg: + """A ``REF`` field with its cached result. Instruction: ``REF `` + ``\\n`` if *number* + ``\\t`` if + *suppress_text* + ``\\h`` if *hyperlink*. Without *number* it returns the bookmarked text (heading text under + the ``none`` scheme).""" + + bookmark: str + number: bool + suppress_text: bool + hyperlink: bool + #: What Word computes (the simulator's value). + cached: str + #: What LibreOffice computes (it ignores ``\t``); equal to *cached* unless *suppress_text*. For tests only. + cached_lo: str + + +Segment = Literal_ | FieldSeg + + +@dataclass(frozen=True, slots=True) +class AltNumber: + """The number of a later alternative that shares a number (§15.8): its paragraph has ``numId 0`` and starts + with these segments (the label as a cover of the holder's number: ``REF holder \\n \\h`` and the stripped + period), then a tab or space. Its indent is :attr:`UnitPlan.ind`.""" + + segments: tuple[Segment, ...] + suff: Literal["tab", "space"] + + +@dataclass(frozen=True, slots=True) +class UnitPlan: + """What the paragraph of one tree unit (section, list item, top-level paragraph) needs, key: ``unit``.""" + + unit: str + kind: Literal["section", "item", "paragraph"] + #: None: the writer writes no ``w:numPr`` (a heading takes its style's; nothing else is unnumbered). + num_id: int | None + ilvl: int | None + #: True: the writer MUST write ``w:numPr`` (``numId`` 0 for an alternative). + explicit: bool + #: ``(left, hanging)`` in twips, written as ``w:ind`` on the paragraph, for a unit whose nesting depth differs + #: from its level's natural depth (a nested list on the heading abstractNum, an alternative); else None. + ind: tuple[int, int] | None + alt: AltNumber | None + #: Word's number text for the unit (the full ``lvlText`` expansion, trailing period kept: ``1.``, ``(a)``). + label: str + #: What ``REF \n`` returns (trailing period dropped): the cached text of the unit's own field. + word_ref: str + #: Word's counter value (None for an alternative, a bullet and an unnumbered unit). + count: int | None + #: Why the unit is not on the plain path: ``resync``, ``alternative``, ``reserved-level``, ``restart-family``, + #: ``restart-section``, ``restart-alternative``, ``label-fallback``, ``clamped``, ``skipped-level``. + cause: tuple[str, ...] + #: Names to wrap around the paragraph's title/first paragraph content: the anchor bookmark first (when the + #: unit has an anchor), then the helper bookmark a cover needs. + bookmarks: tuple[str, ...] + #: ``_Toc`` name of a heading listed in the contents. + toc_bookmark: str | None + + +@dataclass(frozen=True, slots=True) +class RefPlan: + """The fields of every ``{{ref:}}`` to one unit (``CrossRef.target_unit``), wherever it stands.""" + + #: ``references.format`` literals around the cover of the designation. + segments: tuple[Segment, ...] + #: ``concat(cached) == CrossRef.text``. + exact: bool + + +@dataclass(frozen=True, slots=True) +class TocEntryPlan: + #: The ``ContentsEntry`` it is made from (``ContentsEntry.unit``: the section's unit, None for attachments). + unit: str | None + level: int # 1..5, the TOCn style + #: Word's number text of the heading incl. the trailing period; ``""`` for attachments. + number: str + sep: Literal["\t", " ", ""] + title: tuple[Inline, ...] # ContentsEntry.title, already unlinked + condition_label: str | None + bookmark: str # the ``_Toc`` name the entry's hyperlink and PAGEREF point to + cached_page: str = "" # always empty + + +@dataclass(frozen=True, slots=True) +class DocxPlan: + abstract_nums: tuple[AbstractNum, ...] + nums: tuple[Num, ...] + #: The num the heading styles' ``numPr`` points to; None under the ``none`` scheme. + heading_num_id: int | None + units: Mapping[str, UnitPlan] = field(default_factory=dict) + #: Every output anchor of the tree (units, ``DefinedTerm.anchor``, ``AttachmentPart.anchor``, + #: ``attachments_anchor``) -> bookmark name. + anchors: Mapping[str, str] = field(default_factory=dict) + #: ``CrossRef.target_unit`` -> its fields. + refs: Mapping[str, RefPlan] = field(default_factory=dict) + toc: tuple[TocEntryPlan, ...] = () From 9cbe181d5167be8e44b9074c86d6b1340fc2b81c Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:21:23 +0000 Subject: [PATCH 078/118] Write the DOCX body from a plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit numbering.xml comes from the plan's abstract numberings and nums. Each section, list item and top-level paragraph gets w:numPr, w:ind and bookmarks as its unit plan says, a later alternative has numId 0 and its number as REF fields (§15.8), and an item that opens with another block is an empty numbered paragraph. A cross-reference is the literal text and REF fields of its plan, a term reference an internal hyperlink, a defined term a bookmark, an attachment value a REF or a link. Every block (lists with continuation blocks, quotes, drafting notes, code, tables with a repeated header row, rules) and inline has its Word form. Until the planner is wired in the writer is given a plan with no numbering, so output stays valid and still carries the interim warning. Part digests are re-pinned: only document.xml changes, as tables are now tables. --- src/legaldown_render/writers/docx/__init__.py | 30 +- src/legaldown_render/writers/docx/body.py | 361 +++++-- .../writers/docx/interim_plan.py | 47 + .../writers/docx/numbering.py | 48 + src/legaldown_render/writers/docx/runs.py | 223 +++++ src/legaldown_render/writers/docx/styles.py | 12 +- tests/test_docx_body.py | 937 ++++++++++++++++++ tests/test_docx_libreoffice.py | 2 +- tests/test_docx_package.py | 10 +- 9 files changed, 1595 insertions(+), 75 deletions(-) create mode 100644 src/legaldown_render/writers/docx/interim_plan.py create mode 100644 src/legaldown_render/writers/docx/numbering.py create mode 100644 src/legaldown_render/writers/docx/runs.py create mode 100644 tests/test_docx_body.py diff --git a/src/legaldown_render/writers/docx/__init__.py b/src/legaldown_render/writers/docx/__init__.py index d68ab26..c4b560c 100644 --- a/src/legaldown_render/writers/docx/__init__.py +++ b/src/legaldown_render/writers/docx/__init__.py @@ -11,21 +11,27 @@ * :mod:`.ooxml` -- namespaces, the sanitiser, element, run, field, bookmark and hyperlink builders; * :mod:`.package` -- parts, relationships, content types and the deterministic ZIP; * :mod:`.styles`, :mod:`.settings`, :mod:`.headers`, :mod:`.properties` -- the parts that come from the style template; -* :mod:`.body` -- the body of the document (interim, see there). +* :mod:`.numbering` -- ``numbering.xml``, from the plan's numbering definitions; +* :mod:`.body` and :mod:`.runs` -- the body of the document and its inlines, laid out as the plan says (interim + for the title block, contents, attachments and signatures, see there); +* :mod:`.interim_plan` -- the plan with no numbering that stands in until the planner is wired. """ from __future__ import annotations from collections.abc import Mapping +from ...resolve.docx_plan import DocxPlan from ...style import Style from ...style.docx import DocxSettings, docx_settings from ...tree import RenderTree from . import package as pkg -from .body import document_title, interim_body +from .body import BodyWriter, document_title from .headers import build_footer, build_header -from .ooxml import el, serialize +from .interim_plan import _unnumbered_plan +from .numbering import numbering_part +from .ooxml import Bookmarks, el, serialize from .properties import build_app, build_core -from .settings import build_numbering, build_settings +from .settings import build_settings from .styles import build_styles, language_tag __all__ = ["DocxWriter"] @@ -50,7 +56,12 @@ def __init__(self, style: Style, *, generator: str = "", settings: DocxSettings self.generator = generator or "legaldown-render" self.settings = settings if settings is not None else docx_settings(style) - def write(self, tree: RenderTree) -> bytes: + def write(self, tree: RenderTree, plan: DocxPlan | None = None) -> bytes: + """The ``.docx`` file for *tree*, numbered, bookmarked and cross-referenced as *plan* says. Without + one the document has no numbering (:func:`~.interim_plan._unnumbered_plan`): temporary, until the + planner is wired in.""" + if plan is None: + plan = _unnumbered_plan(tree) settings = self.settings language = language_tag(tree.language, self.style.locale) title = document_title(tree) @@ -73,12 +84,15 @@ def write(self, tree: RenderTree) -> bytes: if page.footer: footer_id = rels.add(pkg.REL_FOOTER, "footer1.xml") - document = el("document", None, interim_body(tree, settings.page, header_id=header_id, footer_id=footer_id)) + body = BodyWriter(tree, plan, Bookmarks(), rels, text_width).body(settings.page, header_id=header_id, + footer_id=footer_id) + document = el("document", None, body) package.add("docProps/core.xml", pkg.CT_CORE_PROPERTIES, build_core(title, language)) package.add("docProps/app.xml", pkg.CT_EXTENDED_PROPERTIES, build_app(self.generator)) package.add("word/document.xml", pkg.CT_DOCUMENT, serialize(document), rels) - package.add("word/styles.xml", pkg.CT_STYLES, build_styles(self.style, settings, language)) - package.add("word/numbering.xml", pkg.CT_NUMBERING, build_numbering()) + package.add("word/styles.xml", pkg.CT_STYLES, build_styles(self.style, settings, language, + heading_num_id=plan.heading_num_id)) + package.add("word/numbering.xml", pkg.CT_NUMBERING, numbering_part(plan)) package.add("word/settings.xml", pkg.CT_SETTINGS, build_settings(update_fields=settings.update_fields_on_open)) if header_id: diff --git a/src/legaldown_render/writers/docx/body.py b/src/legaldown_render/writers/docx/body.py index d8a8ddd..40fdaff 100644 --- a/src/legaldown_render/writers/docx/body.py +++ b/src/legaldown_render/writers/docx/body.py @@ -1,25 +1,75 @@ """The body of the document: ``w:body`` and its closing ``w:sectPr``. -**Interim.** :func:`interim_body` writes the title, the headings as ``heading N`` -paragraphs, and every other block as paragraphs of plain text (through the -text writer's inline text), so that a document round-trips through Word and -LibreOffice with its words in the right order. It writes no numbers: headings -are unnumbered until the numbering definitions exist, and a list item or a -numbered paragraph shows its text alone. The full body writer (numbering, -cross-references, the table of contents, the title block, attachments and -signature blocks) replaces it; :func:`section_properties` and the builders in -:mod:`.ooxml` are what it keeps. +:class:`BodyWriter` lays out the resolved tree as the plan says (docs/decisions/0008). It decides +nothing: which paragraph carries which number, which bookmarks wrap it and what a cross-reference +is made of are the plan's (:mod:`~legaldown_render.resolve.docx_plan`), looked up by the unit id +the tree gives each section, list item and top-level paragraph: + +* a unit's paragraph gets ``w:numPr`` when its :class:`~legaldown_render.resolve.docx_plan.UnitPlan` + says it is ``explicit`` (a heading otherwise takes the numbering of its style), ``numId`` 0 and + the segments of its :class:`~legaldown_render.resolve.docx_plan.AltNumber` for a later + alternative that shares a number (§15.8), and ``w:ind`` where the plan sets one; +* its bookmarks wrap the title or first paragraph's content, not the condition label or the + alternative's number; +* a list item whose first block is not a paragraph is an empty numbered paragraph, so that the + number and the bookmarks have a paragraph to stand on, followed by its blocks; +* the other blocks of an item, and a quote's or a drafting note's, are plain paragraphs at the indent + of what they belong to. + +**Interim.** The title block (the header rows and parties), the table of contents, the attachments +and the signature blocks are not written yet: only the title and the subtitle are. The +:func:`section_properties` and the builders in :mod:`.ooxml` are what the rest will keep. """ from __future__ import annotations import xml.etree.ElementTree as ET -from collections.abc import Iterator +from collections.abc import Iterable, Iterator, Sequence +from dataclasses import dataclass, replace +from ...errors import InternalError +from ...resolve.docx_plan import AltNumber, DocxPlan, UnitPlan from ...style.docx import PageGeometry -from ...tree import Block, CodeBlock, DraftingNote, List, Paragraph, Quote, RenderTree, Rule, Table, plain_text -from ..text import inline_text -from .ooxml import el, p, text_run -from .styles import CODE, DRAFTING_NOTE, DRAFTING_NOTE_LABEL, QUOTE, SUBTITLE, TITLE, heading_style +from ...tree import ( + Block, + CodeBlock, + DefinedTerm, + DraftingNote, + Inline, + List, + ListItem, + Paragraph, + Quote, + RenderTree, + Rule, + Section, + Table, + iter_located_body_inlines, + plain_text, +) +from .ooxml import Bookmarks, el, p, tbl_pr, tc_pr, text_run, tr_pr, val +from .package import Relationships +from .runs import InlineWriter +from .styles import ( + CODE, + CODE_INDENT, + CONDITION_LABEL, + DRAFTING_NOTE, + DRAFTING_NOTE_LABEL, + NOTE_INDENT, + QUOTE, + QUOTE_INDENT, + SUBTITLE, + TABLE_GRID, + TITLE, + heading_style, +) + +#: The left indent the paragraph styles set, in twips: what a paragraph in that style has without an indent of its own. +_STYLE_LEFT = {QUOTE: QUOTE_INDENT, DRAFTING_NOTE: NOTE_INDENT} + +_RULE_COLOR = "C8C8C8" +#: The space above and below the text of a table cell, in twips. +_CELL_PADDING = 40 def section_properties(page: PageGeometry, *, header_id: str | None = None, footer_id: str | None = None) -> ET.Element: @@ -43,48 +93,241 @@ def document_title(tree: RenderTree) -> str: return plain_text(tree.title).strip() -def interim_body(tree: RenderTree, page: PageGeometry, *, header_id: str | None = None, - footer_id: str | None = None) -> ET.Element: - """``w:body`` for *tree*: see the module docstring. Interim.""" - content: list[ET.Element] = [] - title = document_title(tree) - if title: - content.append(p(text_run(title), style=TITLE)) - subtitle = inline_text(tree.subtitle).strip() - if subtitle: - content.append(p(text_run(subtitle), style=SUBTITLE)) - for block in tree.preamble: - content += _block(block) - for section in tree.sections: - content.append(p(text_run(inline_text(section.title)), style=heading_style(min(max(section.level, 1), 5)))) - for block in section.blocks: - content += _block(block) - return el("body", None, content, section_properties(page, header_id=header_id, footer_id=footer_id)) - - -def _block(block: Block, style: str | None = None) -> Iterator[ET.Element]: - """The paragraphs of *block*, in *style* where it has none of its own.""" - match block: - case Paragraph(inlines=inlines): - yield p(text_run(inline_text(inlines)), style=style) - case List(items=items): - for item in items: - for child in item.blocks: - yield from _block(child, style) - case Quote(blocks=blocks): - for child in blocks: - yield from _block(child, QUOTE) - case DraftingNote(blocks=blocks, label=label): - if label: - yield p(text_run(label, style=DRAFTING_NOTE_LABEL), style=DRAFTING_NOTE) - for child in blocks: - yield from _block(child, DRAFTING_NOTE) - case CodeBlock(text=text): - for line in text.rstrip("\n").split("\n"): - yield p(text_run(line), style=CODE) - case Table(header=header, rows=rows): - for row in (header, *rows): - yield p(text_run("\t".join(inline_text(cell) for cell in row)), style=style) - case Rule(): - yield p(style=style, props=[el("pBdr", None, el("bottom", {"val": "single", "sz": 4, "space": 1, - "color": "C8C8C8"}))]) +@dataclass(frozen=True, slots=True) +class Context: + """Where a block is written: the paragraph style of a quote or drafting note it is in, how far + its paragraphs are indented, and how much the quotes and notes around it add to a numbered unit.""" + + #: The style of the plain paragraphs here (``Quote``, ``DraftingNote``), None at the top. + style: str | None = None + #: Where the text of a plain paragraph starts, in twips from the margin: the text of the list item + #: it continues, or what the enclosing quotes and notes indent. + left: int = 0 + #: What the quotes and notes around a list item add to its own indent (§13.2). + extra: int = 0 + + +class BodyWriter: + """Writes the body of one document: see the module docstring. + + *bookmarks* and *relationships* are the document's (the bookmark ids are unique across the package; + the relationships receive the external links); *text_width* is the width between the margins in + twips, which tables take.""" + + def __init__(self, tree: RenderTree, plan: DocxPlan, bookmarks: Bookmarks, relationships: Relationships, + text_width: int) -> None: + self.tree = tree + self.plan = plan + self.bookmarks = bookmarks + self.text_width = text_width + abstracts = {abstract.id: abstract for abstract in plan.abstract_nums} + self._levels = {num.id: abstracts[num.abstract_id].levels for num in plan.nums if num.abstract_id in abstracts} + attachment_titles = {a.anchor: plain_text(a.title) for a in tree.attachments if a.anchor} + self.inlines = InlineWriter(plan, relationships, bookmarks, self._placed(), attachment_titles) + + def _placed(self) -> set[str]: + """The names of the bookmarks the body writes: the plan's for every unit, and the definitions' + (a defined term's bookmark is placed where the term is). A field or link to any other name would + point at nothing.""" + names = {name for unit in self.plan.units.values() for name in (*unit.bookmarks, unit.toc_bookmark) if name} + for inline, _ in iter_located_body_inlines(self.tree): + if isinstance(inline, DefinedTerm) and inline.defining and inline.anchor: + name = self.plan.anchors.get(inline.anchor) + if name: + names.add(name) + return names + + # -- the document --------------------------------------------------------------------- + + def body(self, page: PageGeometry, *, header_id: str | None = None, footer_id: str | None = None) -> ET.Element: + """``w:body`` for the tree, closed by the section properties of *page*.""" + tree = self.tree + content: list[ET.Element] = [] + title = document_title(tree) + if title: + content.append(p(text_run(title), style=TITLE)) + subtitle = plain_text(tree.subtitle).strip() + if subtitle: + content.append(p(text_run(subtitle), style=SUBTITLE)) + content += self.blocks(tree.preamble, Context()) + for section in tree.sections: + content += self.section(section) + if content and content[-1].tag.endswith("}tbl"): + content.append(p()) # a table cannot be the last thing before the section properties + return el("body", None, content, section_properties(page, header_id=header_id, footer_id=footer_id)) + + def section(self, section: Section) -> list[ET.Element]: + """The heading paragraph of *section* (``heading N``: the resolver has clamped the level to + 1-5) and its blocks. The condition label (§15.8) is a run inside the heading, after the title, + so that Word's table of contents keeps it.""" + level = min(max(section.level, 1), 5) + heading = self._unit(section.unit, self.inlines.runs(section.title), style=heading_style(level), + context=Context(), conditions=(section.condition_label,), after=True) + return [heading, *self.blocks(section.blocks, Context())] + + # -- blocks --------------------------------------------------------------------------- + + def blocks(self, blocks: Iterable[Block], context: Context) -> list[ET.Element]: + """The elements of *blocks*. Two tables in a row get an empty paragraph between them, which + Word needs to keep them apart.""" + content: list[ET.Element] = [] + for block in blocks: + for element in self.block(block, context): + if element.tag.endswith("}tbl") and content and content[-1].tag.endswith("}tbl"): + content.append(p()) + content.append(element) + return content + + def block(self, block: Block, context: Context) -> Iterator[ET.Element]: + match block: + case Paragraph(): + yield self._paragraph(block, context) + case List(items=items): + for item in items: + yield from self._item(item, context) + case Quote(blocks=blocks): + yield from self.blocks(blocks, replace(context, style=QUOTE, left=context.left + QUOTE_INDENT, + extra=context.extra + QUOTE_INDENT)) + case DraftingNote(blocks=blocks, label=label): + inner = replace(context, style=DRAFTING_NOTE, left=context.left + NOTE_INDENT, + extra=context.extra + NOTE_INDENT) + if label: + yield p(text_run(label, style=DRAFTING_NOTE_LABEL), style=DRAFTING_NOTE, + props=[self._plain_indent(inner)]) + yield from self.blocks(blocks, inner) + case CodeBlock(text=text): + # One paragraph per line: Word shades and keeps them together as one block. + for line in text.rstrip("\n").split("\n"): + yield p(text_run(line), style=CODE, + props=[el("ind", {"left": context.left + CODE_INDENT}) if context.left else None]) + case Table(): + yield from self._table(block, context) + case Rule(): + yield p(style=context.style, props=[ + el("pBdr", None, el("bottom", {"val": "single", "sz": 4, "space": 1, "color": _RULE_COLOR})), + self._plain_indent(context)]) + + def _plain_indent(self, context: Context) -> ET.Element | None: + """``w:ind`` for a plain paragraph in *context*, unless its style already indents so far.""" + return el("ind", {"left": context.left}) if context.left != _STYLE_LEFT.get(context.style or "", 0) else None + + def _paragraph(self, block: Paragraph, context: Context) -> ET.Element: + content = self.inlines.runs(block.inlines) + if block.unit is not None: + return self._unit(block.unit, content, style=context.style, context=context, + conditions=(block.condition_label,)) + return p(self._conditions((block.condition_label,), content), content, style=context.style, + props=[self._plain_indent(context)]) + + def _item(self, item: ListItem, context: Context) -> Iterator[ET.Element]: + """A list item: its first paragraph is the numbered paragraph, or an empty numbered paragraph + stands for it when the item opens with another block (or has none).""" + first = item.blocks[0] if item.blocks and isinstance(item.blocks[0], Paragraph) else None + rest = item.blocks[1:] if first is not None else item.blocks + conditions = (item.condition_label, first.condition_label if first is not None else None) + content = self.inlines.runs(first.inlines) if first is not None else [] + yield self._unit(item.unit, content, style=context.style, context=context, conditions=conditions) + inner = replace(context, left=self._text_left(self.plan.units.get(item.unit or ""), context)) + yield from self.blocks(rest, inner) + + # -- numbered paragraphs -------------------------------------------------------------- + + def _indent(self, unit: UnitPlan) -> tuple[int, int] | None: + """``(left, hanging)`` of the unit's paragraph: its own indent from the plan, else the indent of its + level in the numbering definition; None when the paragraph has neither.""" + if unit.ind is not None: + return unit.ind + levels = self._levels.get(unit.num_id) if unit.num_id else None + if levels is None or unit.ilvl is None: + return None + return levels[unit.ilvl].left, levels[unit.ilvl].hanging + + def _text_left(self, unit: UnitPlan | None, context: Context) -> int: + """Where the text of a unit's paragraph starts: what its continuation blocks are indented to.""" + indent = self._indent(unit) if unit is not None else None + return context.left if indent is None else indent[0] + context.extra + + def _unit(self, unit_id: str | None, content: Sequence[ET.Element], *, style: str | None, context: Context, + conditions: Sequence[str | None], after: bool = False) -> ET.Element: + """The paragraph of the unit *unit_id* (None, or a unit the plan does not know, has no numbering) with + *content*, the title or first paragraph's runs. The condition labels lead the paragraph, or follow the + content with *after* (a heading); the unit's bookmarks wrap the content alone.""" + unit = self.plan.units.get(unit_id) if unit_id else None + props: list[ET.Element | None] = [] + number: list[ET.Element] = [] + names: list[str] = [] + if unit is not None: + if unit.explicit: + props.append(self._numbering(unit)) + indent = self._indent(unit) + if indent is not None and (unit.ind is not None or context.extra): + left, hanging = indent + props.append(el("ind", {"left": left + context.extra, "hanging": hanging or None})) + if unit.alt is not None: + number = self._alternative(unit.alt) + names = [*unit.bookmarks, *([unit.toc_bookmark] if unit.toc_bookmark else [])] + else: + props.append(self._plain_indent(context)) + wrapped = self.inlines.wrap(names, content) if names else list(content) + labels = self._conditions(conditions, content, after=after) + return p(number, wrapped + labels if after else [*labels, *wrapped], style=style, props=props) + + def _numbering(self, unit: UnitPlan) -> ET.Element: + """``w:numPr`` of an explicit unit: its num and level, ``numId`` 0 (no numbering) for an alternative.""" + num_id = 0 if unit.alt is not None else unit.num_id + if num_id is None: + raise InternalError(f"the unit {unit.unit} must carry numbering but the plan gives no num " + "(a bug in the DOCX plan)") + return el("numPr", None, unit.ilvl is not None and val("ilvl", unit.ilvl), val("numId", num_id)) + + def _alternative(self, alt: AltNumber) -> list[ET.Element]: + """The number of a later alternative as text and fields (§15.8), then its tab or space.""" + separator = el("r", None, el("tab")) if alt.suff == "tab" else text_run(" ") + return [*self.inlines.segments(alt.segments), separator] + + def _conditions(self, labels: Sequence[str | None], content: Sequence[ET.Element], *, + after: bool = False) -> list[ET.Element]: + """The condition labels (§15.8) as ``ConditionLabel`` runs, each one apart from the text by a space + (none after an empty paragraph's labels).""" + runs: list[ET.Element] = [] + for label in filter(None, labels): + run = text_run(label, style=CONDITION_LABEL) + if after: + runs += [text_run(" "), run] + else: + runs += [run, text_run(" ")] if content else [run] + return runs + + # -- tables --------------------------------------------------------------------------- + + def _table(self, table: Table, context: Context) -> Iterator[ET.Element]: + """A table: ``TableGrid``, the header row repeated on every page (``w:tblHeader``), columns aligned as + the table says, and indented with the list item or quote it is in. A row shorter than the header is + completed with empty cells.""" + columns = max(len(table.header), *(len(row) for row in table.rows), 0) + if not columns: + return + width = max(self.text_width - context.left, columns) + grid = [width // columns] * columns + grid[-1] += width - sum(grid) + align = (*table.align, *[""] * columns)[:columns] + + def row(cells: tuple[tuple[Inline, ...], ...], *, header: bool = False) -> ET.Element: + padded = (*cells, *[()] * (columns - len(cells))) + return el("tr", None, tr_pr(header and el("tblHeader")), [ + el("tc", None, tc_pr(el("tcW", {"w": size, "type": "dxa"})), + p(self.inlines.runs(cell), props=[ + el("spacing", {"before": 0, "after": 0}), el("jc", {"val": alignment or "left"})])) + for cell, size, alignment in zip(padded, grid, align, strict=True)]) + + yield el( + "tbl", None, + tbl_pr(val("tblStyle", TABLE_GRID), el("tblW", {"w": width, "type": "dxa"}), + context.left > 0 and el("tblInd", {"w": context.left, "type": "dxa"}), + el("tblCellMar", None, el("top", {"w": _CELL_PADDING, "type": "dxa"}), + el("bottom", {"w": _CELL_PADDING, "type": "dxa"})), + el("tblLook", {"val": "0420", "firstRow": 1, "lastRow": 0, "firstColumn": 0, "lastColumn": 0, + "noHBand": 1, "noVBand": 1})), + el("tblGrid", None, [el("gridCol", {"w": size}) for size in grid]), + row(table.header, header=True) if table.header else None, + [row(cells) for cells in table.rows]) diff --git a/src/legaldown_render/writers/docx/interim_plan.py b/src/legaldown_render/writers/docx/interim_plan.py new file mode 100644 index 0000000..f8b127d --- /dev/null +++ b/src/legaldown_render/writers/docx/interim_plan.py @@ -0,0 +1,47 @@ +"""A plan with no numbering, for as long as the DOCX planner is not wired to the writer. + +**Temporary.** :func:`_unnumbered_plan` builds a valid :class:`~legaldown_render.resolve.docx_plan.DocxPlan` +that has no numbering definitions, no bookmarks and no reference fields: every unit is unnumbered and +every cross-reference shows its text, so the document reads in the right order but carries no numbers +(the ``render-docx-interim`` warning says so). A list item is indented by its depth, so that the +structure of nested lists shows. The planner (``resolve/docx_plan.py``) +replaces it: it is the only place with the Word numbering rules (docs/decisions/0003). +""" +from __future__ import annotations + +from collections.abc import Iterable, Iterator +from typing import Literal + +from ...resolve.docx_plan import DocxPlan, UnitPlan +from ...tree import Block, DraftingNote, List, Paragraph, Quote, RenderTree + +#: The left indent of one level of nesting, in twips (1 cm). +_STEP = 567 + + +def _unit(unit: str, kind: Literal["section", "item", "paragraph"], ind: tuple[int, int] | None = None) -> UnitPlan: + return UnitPlan(unit=unit, kind=kind, num_id=None, ilvl=None, explicit=False, ind=ind, alt=None, label="", word_ref="", count=None, cause=(), bookmarks=(), toc_bookmark=None) + + +def _units(blocks: Iterable[Block]) -> Iterator[UnitPlan]: + for block in blocks: + match block: + case Paragraph(unit=unit) if unit: + yield _unit(unit, "paragraph") + case List(items=items, depth=depth): + for item in items: + if item.unit: + yield _unit(item.unit, "item", ((depth + 1) * _STEP, 0)) + yield from _units(item.blocks) + case Quote(blocks=inner) | DraftingNote(blocks=inner): + yield from _units(inner) + + +def _unnumbered_plan(tree: RenderTree) -> DocxPlan: + """A :class:`DocxPlan` for *tree* with no numbering: see the module docstring.""" + units = [*_units(tree.preamble)] + for section in tree.sections: + if section.unit: + units.append(_unit(section.unit, "section")) + units += _units(section.blocks) + return DocxPlan(abstract_nums=(), nums=(), heading_num_id=None, units={unit.unit: unit for unit in units}) diff --git a/src/legaldown_render/writers/docx/numbering.py b/src/legaldown_render/writers/docx/numbering.py new file mode 100644 index 0000000..dcd7127 --- /dev/null +++ b/src/legaldown_render/writers/docx/numbering.py @@ -0,0 +1,48 @@ +"""``word/numbering.xml`` from the plan: the abstract numbering definitions and their instances. + +The planner decides every one of them (:class:`~legaldown_render.resolve.docx_plan.AbstractNum`, +:class:`~legaldown_render.resolve.docx_plan.Num`); this module lays them out, in the order +CT_Numbering, CT_AbstractNum and CT_Lvl require (ISO/IEC 29500-1, 17.9), and decides nothing. +""" +from __future__ import annotations + +import xml.etree.ElementTree as ET + +from ...resolve.docx_plan import AbstractNum, DocxPlan, LevelDef, Num +from .ooxml import el, val +from .settings import build_numbering + + +def level(definition: LevelDef) -> ET.Element: + """One ``w:lvl``: start, number format, paragraph style, suffix, text, justification and indent, in schema order.""" + return el( + "lvl", {"ilvl": definition.ilvl}, + val("start", definition.start), + val("numFmt", definition.num_fmt), + definition.p_style is not None and val("pStyle", definition.p_style), + definition.suff == "space" and val("suff", "space"), + val("lvlText", definition.lvl_text), + val("lvlJc", "left"), + (definition.left > 0 or definition.hanging > 0) and el( + "pPr", None, el("ind", {"left": definition.left, "hanging": definition.hanging or None})), + ) + + +def abstract_num(abstract: AbstractNum) -> ET.Element: + """One ``w:abstractNum``: its list identifier, its kind of list and its levels.""" + return el("abstractNum", {"abstractNumId": abstract.id}, val("nsid", abstract.nsid.upper()), + val("multiLevelType", abstract.multi_level), [level(definition) for definition in abstract.levels]) + + +def num(instance: Num) -> ET.Element: + """One ``w:num``: the definition it uses and, for each level that starts elsewhere than at the + level's own start, a ``w:startOverride``.""" + return el("num", {"numId": instance.id}, val("abstractNumId", instance.abstract_id), + [el("lvlOverride", {"ilvl": ilvl}, val("startOverride", value)) + for ilvl, value in instance.start_overrides]) + + +def numbering_part(plan: DocxPlan) -> bytes: + """``word/numbering.xml`` for *plan*: its abstract numberings, then its nums, in the plan's order.""" + return build_numbering([abstract_num(abstract) for abstract in plan.abstract_nums], + [num(instance) for instance in plan.nums]) diff --git a/src/legaldown_render/writers/docx/runs.py b/src/legaldown_render/writers/docx/runs.py new file mode 100644 index 0000000..6f14895 --- /dev/null +++ b/src/legaldown_render/writers/docx/runs.py @@ -0,0 +1,223 @@ +"""Inlines as runs: text, formatting, fields, hyperlinks and bookmarks (§13.3 to §13.5, §8). + +:class:`InlineWriter` turns the resolved inlines of the tree into the content of a paragraph, and +the segments of the plan into runs. The plan says what a cross-reference is made of (literal text +and ``REF`` fields with their cached results) and which bookmark every anchor is; the writer +lays that out and decides nothing (docs/decisions/0003). It writes the cached result of a field +exactly as the plan gives it: Word computes the live value when it updates the field. +""" +from __future__ import annotations + +import xml.etree.ElementTree as ET +from collections.abc import Collection, Iterable, Mapping, Sequence +from dataclasses import dataclass, replace +from typing import Any + +from ...errors import InternalError +from ...resolve.docx_plan import DocxPlan, FieldSeg, Literal_, Segment +from ...tree import ( + Blank, + Choice, + Code, + CrossRef, + DefinedTerm, + Emphasis, + FailureMarker, + HardBreak, + Image, + Inline, + Link, + SoftBreak, + Strong, + TermRef, + Text, + Value, + has_visible_content, + plain_text, +) +from ...urls import displayed_url +from .ooxml import Bookmarks, complex_field, el, hyperlink_external, hyperlink_internal, r, text_run, val +from .package import Relationships +from .styles import ( + BLANK, + CHOICE, + DEFINED_TERM, + FAILURE_MARKER, + HYPERLINK, + INLINE_CODE, + TERM_REFERENCE, +) + + +@dataclass(frozen=True, slots=True) +class Format: + """The character formatting of the runs being written: a character style and the emphasis + that enclosing ``*``/``**`` add.""" + + style: str | None = None + bold: bool = False + italic: bool = False + + @property + def props(self) -> list[Any]: + """The run properties (``w:rPr`` children, in any order, :func:`~.ooxml.rpr` sorts them).""" + return [val("rStyle", self.style) if self.style else None, self.bold and [el("b"), el("bCs")], + self.italic and [el("i"), el("iCs")]] + + def styled(self, style: str) -> Format: + return replace(self, style=style) + + +#: No character formatting. +PLAIN = Format() + + +def reference_instruction(segment: FieldSeg) -> str: + """The instruction of *segment*: ``REF bookmark``, then ``\\n`` (the paragraph number), ``\\t`` (without + the text around the counter) and ``\\h`` (a link to the bookmark), as the segment says.""" + return " ".join(["REF", segment.bookmark, *(switch for switch, on in ( + ("\\n", segment.number), ("\\t", segment.suppress_text), ("\\h", segment.hyperlink)) if on)]) + + +class InlineWriter: + """Writes inlines for one document. + + *plan* names the bookmarks and the reference fields; *relationships* receives the + external hyperlinks (:meth:`~.package.Relationships.add_external`: a link the document may not + make stays text); *bookmarks* allocates the ids of the bookmarks written here; *placed* holds the + names of the bookmarks the document writes, so that no field or link points at one that is not + there; *attachment_titles* maps an attachment's anchor to its title as text.""" + + def __init__(self, plan: DocxPlan, relationships: Relationships, bookmarks: Bookmarks, placed: Collection[str], + attachment_titles: Mapping[str, str]) -> None: + self.plan = plan + self.relationships = relationships + self.bookmarks = bookmarks + self.placed = placed + self.attachment_titles = attachment_titles + + # -- plan segments -------------------------------------------------------------------- + + def segments(self, segments: Iterable[Segment], fmt: Format = PLAIN) -> list[ET.Element]: + """The runs of *segments*: literal text, and each ``REF`` field with its cached result. + A field whose bookmark the document does not write is a bug in the plan.""" + runs: list[ET.Element] = [] + for segment in segments: + if isinstance(segment, Literal_): + if segment.text: + runs.append(text_run(segment.text, props=fmt.props)) + continue + if segment.bookmark not in self.placed: + raise InternalError(f"a reference field points at the bookmark '{segment.bookmark}', which the " + "document does not contain (a bug in the DOCX plan)") + runs += complex_field(reference_instruction(segment), + [text_run(segment.cached, props=fmt.props)] if segment.cached else [], + props=fmt.props) + return runs + + # -- inlines -------------------------------------------------------------------------- + + def runs(self, inlines: Iterable[Inline], fmt: Format = PLAIN, *, in_link: bool = False) -> list[ET.Element]: + """The content of a paragraph (runs, fields, hyperlinks, bookmarks) for *inlines*. *in_link* is + True inside an external link, where no further link is made: its text stays one link.""" + content: list[ET.Element] = [] + for inline in inlines: + content += self._inline(inline, fmt, in_link) + return content + + def _inline(self, inline: Inline, fmt: Format, in_link: bool) -> list[ET.Element]: + match inline: + case Text(text=text): + return [text_run(text, props=fmt.props)] + case SoftBreak(): + return [text_run(" ", props=fmt.props)] + case HardBreak(): + return [r(el("br"), props=fmt.props)] + case Emphasis(children=children): + return self.runs(children, replace(fmt, italic=True), in_link=in_link) + case Strong(children=children): + return self.runs(children, replace(fmt, bold=True), in_link=in_link) + case Code(text=text): + return [text_run(text, props=fmt.styled(INLINE_CODE).props)] + case Link(): + return self._link(inline, fmt, in_link) + case Image(children=children): + # Alt text at conformance Level 2: no file is read (§2.3, §17.6). + alt = plain_text(children) + return [text_run(alt, props=fmt.props)] if alt else [] + case CrossRef(text=text, target_unit=unit): + reference = self.plan.refs.get(unit) if unit is not None else None + if reference is None: + return [text_run(text, props=fmt.props)] + return self.segments(reference.segments, fmt) + case TermRef(text=text, target=target): + run = text_run(text, props=fmt.styled(TERM_REFERENCE).props) + name = self._internal(target) if target is not None and not in_link else None + return [hyperlink_internal(name, run)] if name else [run] + case DefinedTerm(children=children, anchor=anchor, defining=defining): + content = self.runs(children, fmt.styled(DEFINED_TERM), in_link=in_link) + name = self.plan.anchors.get(anchor) if anchor and defining else None + return self.bookmarks.around(name, content) if name else content + case Value(): + return self._value(inline, fmt, in_link) + case Blank(text=text, prompt=prompt): + runs = [text_run(text, props=fmt.styled(BLANK).props)] + if prompt: + runs.append(text_run(f" ({prompt})", props=replace(fmt, italic=True).props)) + return runs + case Choice(options=options, separator=separator): + return [text_run("[" + separator.join(options) + "]", props=fmt.styled(CHOICE).props)] + case FailureMarker(text=text): + return [text_run(text, props=fmt.styled(FAILURE_MARKER).props)] + return [] + + def _internal(self, anchor: str) -> str | None: + """The bookmark of the output *anchor*, when the document has it.""" + name = self.plan.anchors.get(anchor) + return name if name in self.placed else None + + def _link(self, link: Link, fmt: Format, in_link: bool) -> list[ET.Element]: + """A link: external through a relationship, or to a bookmark for ``#anchor`` of this + document. One that is not a link (§8.7: a scheme links may not use) shows its text alone, as in HTML.""" + # A link with no text shows its address (§8.7: it is still a hyperlink). + shown = link.children if has_visible_content(link.children) else (Text(displayed_url(link.href)),) + if in_link: + return self.runs(shown, fmt, in_link=True) + tooltip = plain_text(link.title) or None + if link.href.startswith("#"): + name = self._internal(link.href[1:]) + if name is None: + return self.runs(link.children, fmt) + return [hyperlink_internal(name, *self.runs(shown, fmt.styled(HYPERLINK), in_link=True), tooltip=tooltip)] + identifier = self.relationships.add_external(link.href) + if identifier is None: + return self.runs(link.children, fmt) + return [hyperlink_external(identifier, *self.runs(shown, fmt.styled(HYPERLINK), in_link=True), + tooltip=tooltip)] + + def _value(self, value: Value, fmt: Format, in_link: bool) -> list[ET.Element]: + """A field value as text. An ``attach`` value leads to its attachment: a ``REF`` field when the + text is the attachment's title (so Word keeps it equal to the title), else a link with the + text it shows (``label=``).""" + text = [text_run(value.text, props=fmt.props)] + href = value.href + if not href or in_link: + return text + if href.startswith("#"): + name = self._internal(href[1:]) + if name is None: + return text + if self.attachment_titles.get(href[1:]) == value.text: + return complex_field(f"REF {name} \\h", text, props=fmt.props) + return [hyperlink_internal(name, text_run(value.text, props=fmt.styled(HYPERLINK).props))] + identifier = self.relationships.add_external(href) + if identifier is None: + return text + return [hyperlink_external(identifier, text_run(value.text, props=fmt.styled(HYPERLINK).props))] + + # -- helpers for paragraphs ----------------------------------------------------------- + + def wrap(self, names: Sequence[str], content: Sequence[Any]) -> list[ET.Element]: + """*content* inside nested bookmarks, the first of *names* outermost.""" + pairs = [self.bookmarks.pair(name) for name in names] + return [*(start for start, _ in pairs), *content, *(end for _, end in reversed(pairs))] diff --git a/src/legaldown_render/writers/docx/styles.py b/src/legaldown_render/writers/docx/styles.py index ad4a5a5..5da3d8d 100644 --- a/src/legaldown_render/writers/docx/styles.py +++ b/src/legaldown_render/writers/docx/styles.py @@ -50,6 +50,12 @@ def toc_style(level: int) -> str: return f"TOC{level}" +#: The left indent in twips that ``Quote`` and ``DraftingNote`` set, which the body adds to the paragraphs +#: that carry an indent of their own (a list item in a quote). +QUOTE_INDENT = 567 +NOTE_INDENT = 227 +CODE_INDENT = 113 + _MUTED = "666666" _RULE = "C8C8C8" _MONOSPACE = "Courier New" @@ -183,7 +189,7 @@ def half_points(factor: float) -> int: run=[el("i"), el("iCs"), el("color", {"val": _MUTED}), _size(half_points(1.1))]), _style(QUOTE, "Quote", based_on=NORMAL, next_style=NORMAL, priority=29, quick=True, props=[el("pBdr", None, _border("left", _RULE, 18, 10)), - el("spacing", {"before": em(0.5), "after": em(0.5)}), el("ind", {"left": 567, "right": 567})]), + el("spacing", {"before": em(0.5), "after": em(0.5)}), el("ind", {"left": QUOTE_INDENT, "right": 567})]), _style(CONTENTS_HEADING, "Contents Heading", based_on=NORMAL, next_style=NORMAL, priority=39, custom=True, props=[el("keepNext"), el("spacing", {"before": em(1.2), "after": em(0.4)}), el("jc", {"val": first.align if first else "left"})], @@ -192,10 +198,10 @@ def half_points(factor: float) -> int: _size(first.half_points if first else settings.half_points)]), _style(DRAFTING_NOTE, "Drafting Note", based_on=NORMAL, priority=30, custom=True, props=[el("keepLines"), el("pBdr", None, _border("left", _NOTE_RULE, 24, 8)), _shade(_NOTE_FILL), - el("spacing", {"before": em(0.4), "after": em(0.4)}), el("ind", {"left": 227, "right": 113})]), + el("spacing", {"before": em(0.4), "after": em(0.4)}), el("ind", {"left": NOTE_INDENT, "right": 113})]), _style(CODE, "Code", based_on=NORMAL, priority=31, custom=True, props=[_shade(_CODE_FILL), el("spacing", {"before": 0, "after": 0, "line": 240, "lineRule": "auto"}), - el("ind", {"left": 113, "right": 113}), el("jc", {"val": "left"})], + el("ind", {"left": CODE_INDENT, "right": 113}), el("jc", {"val": "left"})], run=[_fonts(_MONOSPACE), _size(half_points(0.9))]), _style(TABLE_GRID, "Table Grid", kind="table", based_on="TableNormal", priority=39, table=el("tblPr", None, el("tblBorders", None, *(_border(side, _RULE, 4) diff --git a/tests/test_docx_body.py b/tests/test_docx_body.py new file mode 100644 index 0000000..6ab71bf --- /dev/null +++ b/tests/test_docx_body.py @@ -0,0 +1,937 @@ +"""The body of the DOCX, written as a hand-made :class:`DocxPlan` says: numbering definitions, the +numbering of each paragraph (``w:numPr``, alternatives, indents), bookmarks and ``REF`` fields with +their cached results, hyperlinks, every inline and every block, escaping, validity against the +schemas (marker ``ooxml_schema``) and what LibreOffice shows once fields are updated (marker +``libreoffice``). The plans are written by hand, so these tests need no planner; the writer adds no +rule of its own, and what a plan does not say it does not write.""" +from __future__ import annotations + +import xml.etree.ElementTree as ET +from collections.abc import Sequence +from typing import Any + +import pytest +from docx_support import W_NS, libreoffice_export, parts, require_libreoffice, validation_errors + +from legaldown_render import render +from legaldown_render.errors import InternalError +from legaldown_render.resolve.docx_plan import ( + AbstractNum, + AltNumber, + DocxPlan, + FieldSeg, + LevelDef, + Literal_, + Num, + RefPlan, + UnitPlan, +) +from legaldown_render.style import load_style +from legaldown_render.tree import ( + Blank, + Block, + Choice, + Code, + CodeBlock, + CrossRef, + DefinedTerm, + DraftingNote, + Emphasis, + FailureMarker, + HardBreak, + Image, + Inline, + Link, + List, + ListItem, + Paragraph, + Quote, + RenderTree, + Rule, + Section, + SoftBreak, + Strong, + Table, + TermRef, + Text, + Value, +) +from legaldown_render.writers import DocxWriter + +W = f"{{{W_NS}}}" + + +# -- Building trees and plans by hand -------------------------------------------------------------------------- + + +def tree(*blocks: Block, sections: Sequence[Section] = (), **settings: Any) -> RenderTree: + return RenderTree(title=(Text("T"),), subtitle=(), language="en", preamble=tuple(blocks), sections=tuple(sections), + **settings) + + +def para(*inlines: Inline | str, **fields: Any) -> Paragraph: + return Paragraph(tuple(Text(i) if isinstance(i, str) else i for i in inlines), **fields) + + +def item(*blocks: Block | str, unit: str | None = None, **fields: Any) -> ListItem: + return ListItem(tuple(para(b) if isinstance(b, str) else b for b in blocks), unit=unit, **fields) + + +def bullets(*items: ListItem, depth: int = 0) -> List: + return List(False, tuple(items), enumerated=True, family="enumerated", depth=depth) + + +def section(title: str, *blocks: Block, unit: str | None = None, level: int = 1, **fields: Any) -> Section: + return Section(level=level, title=(Text(title),), identifier="", condition="", blocks=tuple(blocks), unit=unit, + **fields) + + +def unit(unit_id: str, kind: str = "paragraph", *, num_id: int | None = None, ilvl: int | None = None, + explicit: bool = False, ind: tuple[int, int] | None = None, alt: AltNumber | None = None, + bookmarks: tuple[str, ...] = (), toc: str | None = None) -> UnitPlan: + return UnitPlan(unit=unit_id, kind=kind, num_id=num_id, ilvl=ilvl, explicit=explicit, ind=ind, alt=alt, # type: ignore[arg-type] + label="", word_ref="", count=None, cause=(), bookmarks=bookmarks, toc_bookmark=toc) + + +def level(ilvl: int, fmt: str = "decimal", text: str | None = None, **fields: Any) -> LevelDef: + return LevelDef(ilvl, fmt, text if text is not None else f"%{ilvl + 1}.", **fields) + + +def abstract(identifier: int, kind: str = "enumerated", levels: Sequence[LevelDef] = ()) -> AbstractNum: + full = [*levels, *(level(i) for i in range(len(levels), 9))] + return AbstractNum(identifier, kind, f"{0x4C440000 + identifier:08X}", "hybridMultilevel", tuple(full)) # type: ignore[arg-type] + + +def field(bookmark: str, cached: str = "1", *, number: bool = True, suppress: bool = False, + hyperlink: bool = True) -> FieldSeg: + return FieldSeg(bookmark, number, suppress, hyperlink, cached, cached) + + +def plan(*units: UnitPlan, abstracts: Sequence[AbstractNum] = (), nums: Sequence[Num] = (), + heading: int | None = None, anchors: dict[str, str] | None = None, + refs: dict[str, tuple[Any, ...]] | None = None) -> DocxPlan: + return DocxPlan(abstract_nums=tuple(abstracts), nums=tuple(nums), heading_num_id=heading, + units={u.unit: u for u in units}, anchors=anchors or {}, + refs={key: RefPlan(tuple(segments), True) for key, segments in (refs or {}).items()}) + + +def build(document: RenderTree, the_plan: DocxPlan | None = None, **settings: Any) -> dict[str, bytes]: + data = DocxWriter(load_style("default", overrides=settings.get("overrides", {})), generator="test").write( + document, the_plan) + return parts(data) + + +def body(document: RenderTree, the_plan: DocxPlan | None = None, **settings: Any) -> list[ET.Element]: + """The children of ``w:body`` but the closing ``w:sectPr``, after the title paragraph.""" + root = ET.fromstring(build(document, the_plan, **settings)["word/document.xml"]) + children = list(root.find(f"{W}body")) # type: ignore[arg-type] + assert children[-1].tag == f"{W}sectPr" + assert children[0].find(f"{W}pPr/{W}pStyle").get(f"{W}val") == "Title" # type: ignore[union-attr] + return children[1:-1] + + +# -- Reading paragraphs ------------------------------------------------------------------------------------------ + + +def local(element: ET.Element) -> str: + return element.tag.removeprefix(W) + + +def pstyle(paragraph: ET.Element) -> str | None: + found = paragraph.find(f"{W}pPr/{W}pStyle") + return None if found is None else found.get(f"{W}val") + + +def numbering(paragraph: ET.Element) -> tuple[int | None, int] | None: + """``(ilvl, numId)`` of the paragraph's own ``w:numPr``.""" + found = paragraph.find(f"{W}pPr/{W}numPr") + if found is None: + return None + ilvl = found.find(f"{W}ilvl") + return (None if ilvl is None else int(ilvl.get(f"{W}val")), int(found.find(f"{W}numId").get(f"{W}val"))) # type: ignore[arg-type, union-attr] + + +def indent(paragraph: ET.Element) -> dict[str, int] | None: + found = paragraph.find(f"{W}pPr/{W}ind") + return None if found is None else {key.removeprefix(W): int(value) for key, value in found.attrib.items()} + + +def flow(paragraph: ET.Element) -> str: + """The paragraph's content in one line: text, ``→`` for a tab and ``↵`` for a break, a run's character style + and emphasis after it (``text@Style+b+i``), a field as ``{instruction|result}``, a bookmark as ``…`` + and a hyperlink as ``(link target: …)``.""" + names: dict[str, str] = {} + out: list[str] = [] + + def run(node: ET.Element) -> None: + pieces = [] + for child in node: + match local(child): + case "t": + pieces.append(child.text or "") + case "tab": + pieces.append("→") + case "br": + pieces.append("↵") + case "fldChar": + out.append({"begin": "{", "separate": "|", "end": "}"}[child.get(f"{W}fldCharType")]) # type: ignore[index] + case "instrText": + out.append((child.text or "").strip()) + if pieces: + suffix = "" + style = node.find(f"{W}rPr/{W}rStyle") + if style is not None: + suffix += "@" + style.get(f"{W}val") # type: ignore[operator] + for flag in ("b", "i"): + if node.find(f"{W}rPr/{W}{flag}") is not None: + suffix += f"+{flag}" + out.append("".join(pieces) + suffix) + + def walk(node: ET.Element) -> None: + for child in node: + match local(child): + case "r": + run(child) + case "bookmarkStart": + names[child.get(f"{W}id")] = child.get(f"{W}name") # type: ignore[assignment, index] + out.append(f"<{child.get(f'{W}name')}>") + case "bookmarkEnd": + out.append(f"") # type: ignore[index] + case "hyperlink": + target = child.get(f"{W}anchor") or child.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id") + out.append(f"(link {target}: ") + walk(child) + out.append(")") + + walk(paragraph) + return "".join(out) + + +# -- numbering.xml ----------------------------------------------------------------------------------------------- + + +def test_the_numbering_part_holds_each_abstract_numbering_and_each_instance_in_the_plans_order() -> None: + headings = abstract(0, "headings", [ + level(0, text="%1.", p_style="Heading1", left=567, hanging=567), + level(1, text="%1.%2", p_style="Heading2", left=680, hanging=680, suff="space"), + level(2, "lowerLetter", "(%3)", start=3)]) + bullet = abstract(1, "bullet", [LevelDef(0, "bullet", "•", left=567, hanging=360)]) + document = tree(para("x")) + the_plan = plan(abstracts=[headings, bullet], nums=[Num(1, 0), Num(2, 1, ((0, 1), (1, 1))), Num(3, 1, ((0, 4),))], + heading=1) + root = ET.fromstring(build(document, the_plan)["word/numbering.xml"]) + assert [local(child) for child in root] == ["abstractNum", "abstractNum", "num", "num", "num"] + first = root[0] + assert first.get(f"{W}abstractNumId") == "0" + assert [local(child) for child in first][:3] == ["nsid", "multiLevelType", "lvl"] + assert first.find(f"{W}nsid").get(f"{W}val") == "4C440000" # type: ignore[union-attr] + assert first.find(f"{W}multiLevelType").get(f"{W}val") == "hybridMultilevel" # type: ignore[union-attr] + assert len(first.findall(f"{W}lvl")) == 9 + lvl0, lvl1, lvl2 = first.findall(f"{W}lvl")[:3] + assert [local(child) for child in lvl0] == ["start", "numFmt", "pStyle", "lvlText", "lvlJc", "pPr"] + assert lvl0.get(f"{W}ilvl") == "0" + assert lvl0.find(f"{W}pStyle").get(f"{W}val") == "Heading1" # type: ignore[union-attr] + assert lvl0.find(f"{W}lvlText").get(f"{W}val") == "%1." # type: ignore[union-attr] + assert dict(lvl0.find(f"{W}pPr/{W}ind").attrib) == {f"{W}left": "567", f"{W}hanging": "567"} # type: ignore[union-attr] + assert [local(child) for child in lvl1] == ["start", "numFmt", "pStyle", "suff", "lvlText", "lvlJc", "pPr"] + assert lvl1.find(f"{W}suff").get(f"{W}val") == "space" # type: ignore[union-attr] + assert lvl2.find(f"{W}numFmt").get(f"{W}val") == "lowerLetter" + assert lvl2.find(f"{W}start").get(f"{W}val") == "3" # type: ignore[union-attr] + assert lvl2.find(f"{W}pPr") is None # a level with no indent writes none + assert root[1].find(f"{W}lvl/{W}numFmt").get(f"{W}val") == "bullet" # type: ignore[union-attr] + assert root[1].find(f"{W}lvl/{W}lvlText").get(f"{W}val") == "•" # type: ignore[union-attr] + nums = root.findall(f"{W}num") + assert [(n.get(f"{W}numId"), n.find(f"{W}abstractNumId").get(f"{W}val")) for n in nums] == [ # type: ignore[union-attr] + ("1", "0"), ("2", "1"), ("3", "1")] + assert [local(child) for child in nums[1]] == ["abstractNumId", "lvlOverride", "lvlOverride"] + assert [(o.get(f"{W}ilvl"), o.find(f"{W}startOverride").get(f"{W}val")) for o in nums[1].findall(f"{W}lvlOverride") # type: ignore[union-attr] + ] == [("0", "1"), ("1", "1")] + assert nums[0].find(f"{W}lvlOverride") is None + + +def test_the_headings_styles_are_linked_to_the_headings_num() -> None: + document = tree(sections=[section("A", unit="s1")]) + styles = ET.fromstring(build(document, plan(abstracts=[abstract(0, "headings")], nums=[Num(1, 0)], heading=1) + )["word/styles.xml"]) + by_id = {s.get(f"{W}styleId"): s for s in styles.findall(f"{W}style")} + for number in range(1, 6): + numpr = by_id[f"Heading{number}"].find(f"{W}pPr/{W}numPr") + assert (numpr.find(f"{W}ilvl").get(f"{W}val"), numpr.find(f"{W}numId").get(f"{W}val")) == (str(number - 1), "1") # type: ignore[union-attr] + plain = ET.fromstring(build(document, plan())["word/styles.xml"]) + assert plain.find(f".//{W}style[@{W}styleId='Heading1']/{W}pPr/{W}numPr") is None + + +def test_an_empty_plan_writes_an_empty_numbering_part() -> None: + assert list(ET.fromstring(build(tree(para("x")), plan())["word/numbering.xml"])) == [] + + +# -- The numbering of a paragraph -------------------------------------------------------------------------------- + + +def test_a_unit_writes_numpr_only_when_the_plan_says_it_is_explicit() -> None: + document = tree(bullets(item("a", unit="u1"), item("b", unit="u2")), + sections=[section("Live", unit="s1", level=2), section("Own", unit="s2")]) + the_plan = plan(unit("s1", "section", num_id=1, ilvl=1), unit("s2", "section", num_id=3, ilvl=0, explicit=True), + unit("u1", "item", num_id=2, ilvl=0, explicit=True), unit("u2", "item", num_id=2, ilvl=0), + abstracts=[abstract(0, "headings"), abstract(1)], + nums=[Num(1, 0), Num(2, 1), Num(3, 0, ((0, 5),))], heading=1) + children = body(document, the_plan) + items = [c for c in children if pstyle(c) is None] + heading_live, heading_own = (c for c in children if pstyle(c) and pstyle(c).startswith("Heading")) # type: ignore[union-attr] + assert (pstyle(heading_live), numbering(heading_live)) == ("Heading2", None) + assert (pstyle(heading_own), numbering(heading_own)) == ("Heading1", (0, 3)) + assert [numbering(i) for i in items] == [(0, 2), None] + + +def test_numpr_is_in_schema_order_among_the_other_properties() -> None: + document = tree(bullets(item("a", unit="u1"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=2, explicit=True, ind=(1134, 567)), abstracts=[abstract(0)], + nums=[Num(1, 0)]) + (paragraph,) = body(document, the_plan) + assert [local(child) for child in paragraph.find(f"{W}pPr")] == ["numPr", "ind"] # type: ignore[union-attr] + assert [local(child) for child in paragraph.find(f"{W}pPr/{W}numPr")] == ["ilvl", "numId"] # type: ignore[union-attr] + assert indent(paragraph) == {"left": 1134, "hanging": 567} + + +def test_a_unit_without_a_level_the_plan_names_writes_just_the_num() -> None: + (paragraph,) = body(tree(bullets(item("a", unit="u1"))), plan(unit("u1", "item", num_id=1, explicit=True), + abstracts=[abstract(0)], nums=[Num(1, 0)])) + assert numbering(paragraph) == (None, 1) + + +def test_an_explicit_unit_with_no_num_is_a_bug_in_the_plan() -> None: + with pytest.raises(InternalError, match="plan"): + body(tree(bullets(item("a", unit="u1"))), plan(unit("u1", "item", explicit=True))) + + +def test_an_indent_is_written_where_the_plan_sets_one_and_a_hanging_of_zero_is_left_out() -> None: + document = tree(bullets(item("a", unit="u1"), item("b", unit="u2"), item("c", unit="u3"))) + the_plan = plan(unit("u1", "item", ind=(900, 0)), unit("u2", "item", ind=(900, 300)), unit("u3", "item")) + first, second, third = body(document, the_plan) + assert indent(first) == {"left": 900} + assert indent(second) == {"left": 900, "hanging": 300} + assert indent(third) is None + + +def test_the_level_indent_is_not_repeated_on_a_paragraph_that_lives_on_its_level() -> None: + document = tree(bullets(item("a", unit="u1"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=1, explicit=True), + abstracts=[abstract(0, levels=[level(0), level(1, left=1134, hanging=567)])], nums=[Num(1, 0)]) + (paragraph,) = body(document, the_plan) + assert indent(paragraph) is None + + +def test_a_later_alternative_has_no_numbering_its_number_as_fields_and_a_tab() -> None: + alt = AltNumber((field("ld_first", "2"), Literal_(".")), "tab") + document = tree(sections=[section("Courts", unit="s1"), section("Arbitration", unit="s2")]) + the_plan = plan(unit("s1", "section", bookmarks=("ld_first",)), + unit("s2", "section", num_id=0, ilvl=0, explicit=True, ind=(567, 567), alt=alt, + bookmarks=("ld_second",))) + first, second = body(document, the_plan) + assert numbering(first) is None + assert numbering(second) == (0, 0) + assert indent(second) == {"left": 567, "hanging": 567} + assert flow(second) == "{REF ld_first \\n \\h|2}.→Arbitration" + + +def test_an_alternative_whose_plan_has_no_num_still_gets_num_zero() -> None: + alt = AltNumber((field("a", "1"),), "space") + document = tree(bullets(item("x", unit="u1")), para("a", unit="u2")) + the_plan = plan(unit("u1", bookmarks=("a",)), unit("u2", explicit=True, alt=alt)) + first, second = body(document, the_plan) + assert numbering(second) == (None, 0) + assert flow(second) == "{REF a \\n \\h|1} a" + + +# -- Bookmarks ---------------------------------------------------------------------------------------------- + + +def test_bookmarks_wrap_the_content_nested_with_the_first_outermost_and_the_toc_bookmark_innermost() -> None: + document = tree(sections=[section("Heading", unit="s1")]) + (heading,) = body(document, plan(unit("s1", "section", bookmarks=("ld_h", "_Ld1"), toc="_Toc100000001"))) + assert flow(heading) == "<_Ld1><_Toc100000001>Heading" + + +def test_bookmark_ids_are_unique_and_pair_each_start_with_its_end() -> None: + document = tree(bullets(item("a", unit="u1"), item("b", unit="u2")), sections=[section("H", unit="s1")]) + the_plan = plan(unit("s1", "section", bookmarks=("a1", "a2")), unit("u1", bookmarks=("b1",)), + unit("u2", bookmarks=("b2",), toc="_Toc1")) + root = ET.fromstring(build(document, the_plan)["word/document.xml"]) + starts = [(s.get(f"{W}id"), s.get(f"{W}name")) for s in root.iter(f"{W}bookmarkStart")] + ends = [e.get(f"{W}id") for e in root.iter(f"{W}bookmarkEnd")] + assert len({i for i, _ in starts}) == len(starts) == 5 + assert sorted(i for i, _ in starts) == sorted(ends) + assert [name for _, name in starts] == ["b1", "b2", "_Toc1", "a1", "a2"] + + +def test_condition_labels_and_an_alternatives_number_stay_outside_the_bookmarks() -> None: + alt = AltNumber((field("x", "1"),), "tab") + document = tree(para("Text", unit="u1", condition_label="if A"), bullets(item("held", unit="u2")), + sections=[section("Title", unit="s1", condition_label="if B")]) + the_plan = plan(unit("u1", bookmarks=("p",), alt=alt, explicit=True), unit("s1", "section", bookmarks=("h",)), + unit("u2", bookmarks=("x",))) + paragraph, _, heading = body(document, the_plan) + assert flow(paragraph) == "{REF x \\n \\h|1}→if A@ConditionLabel

Text

" + assert flow(heading) == "Title if B@ConditionLabel" + + +def test_an_item_that_opens_with_a_list_is_an_empty_numbered_paragraph_with_the_bookmarks_then_the_list() -> None: + nested = bullets(item("inner", unit="u2"), depth=1) + document = tree(bullets(item(nested, unit="u1", condition_label="if C"), item("plain", unit="u3"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=0, explicit=True, bookmarks=("ld_one",)), + unit("u2", "item", num_id=1, ilvl=1, explicit=True, bookmarks=("ld_inner",)), + unit("u3", "item", num_id=1, ilvl=0, explicit=True), + abstracts=[abstract(0, levels=[level(0, left=567, hanging=567), level(1, left=1134, hanging=567)])], + nums=[Num(1, 0)]) + first, inner, last = body(document, the_plan) + assert (numbering(first), flow(first)) == ((0, 1), "if C@ConditionLabel") + assert (numbering(inner), flow(inner)) == ((1, 1), "inner") + assert (numbering(last), flow(last)) == ((0, 1), "plain") + + +@pytest.mark.parametrize("blocks", [(), (Quote((para("q"),)),), (CodeBlock("c"),)]) +def test_an_item_without_a_first_paragraph_is_an_empty_numbered_paragraph(blocks: tuple[Block, ...]) -> None: + document = tree(bullets(ListItem(blocks, unit="u1"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=0, explicit=True, bookmarks=("ld_x",)), abstracts=[abstract(0)], + nums=[Num(1, 0)]) + first, *rest = body(document, the_plan) + assert (numbering(first), flow(first)) == ((0, 1), "") + assert len(rest) == len(blocks) + + +def test_the_blocks_after_an_items_first_paragraph_continue_at_its_indent_unnumbered() -> None: + document = tree(bullets(item("first", para("second"), CodeBlock("code"), bullets(item("n", unit="u2"), depth=1), + unit="u1"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=0, explicit=True), + unit("u2", "item", num_id=1, ilvl=1, explicit=True), + abstracts=[abstract(0, levels=[level(0, left=567, hanging=567), level(1, left=1134, hanging=567)])], + nums=[Num(1, 0)]) + first, second, code, nested = body(document, the_plan) + assert numbering(second) is None and indent(second) == {"left": 567} and flow(second) == "second" + assert pstyle(code) == "Code" and indent(code) == {"left": 567 + 113} + assert numbering(nested) == (1, 1) + + +def test_continuation_follows_an_indent_the_plan_sets_over_the_levels() -> None: + document = tree(bullets(item("first", "more", unit="u1"))) + first, more = body(document, plan(unit("u1", "item", ind=(2000, 400)))) + assert indent(first) == {"left": 2000, "hanging": 400} + assert indent(more) == {"left": 2000} + + +def test_a_unit_the_plan_does_not_know_is_a_plain_paragraph() -> None: + document = tree(para("a", unit="u9"), bullets(item("b", unit="u8")), sections=[section("H", unit="s9")]) + paragraph, listed, heading = body(document, plan()) + assert [pstyle(paragraph), numbering(paragraph), flow(paragraph)] == [None, None, "a"] + assert [pstyle(listed), numbering(listed), flow(listed)] == [None, None, "b"] + assert [pstyle(heading), numbering(heading)] == ["Heading1", None] + + +# -- Cross-references and fields --------------------------------------------------------------------------------- + + +def test_a_cross_reference_is_its_literal_segments_and_ref_fields_with_the_cached_results() -> None: + reference = CrossRef("Clause 1.2(b)", "x", "u9", "s2") + document = tree(para("See ", reference, "."), sections=[section("H", unit="s1")]) + refs = {"u9": (Literal_("Clause "), field("ld_s", "1.2"), Literal_("("), field("_Ld1", "b", suppress=True), + Literal_(")"))} + the_plan = plan(unit("s1", "section", bookmarks=("ld_s", "_Ld1")), refs=refs) + paragraph, _ = body(document, the_plan) + assert flow(paragraph) == "See Clause {REF ld_s \\n \\h|1.2}({REF _Ld1 \\n \\t \\h|b})." + + +def test_the_switches_follow_the_segment_in_a_fixed_order() -> None: + cases = [(field("a"), "REF a \\n \\h"), (field("a", hyperlink=False), "REF a \\n"), + (field("a", number=False), "REF a \\h"), (field("a", number=False, hyperlink=False), "REF a"), + (field("a", suppress=True, hyperlink=False), "REF a \\n \\t"), + (field("a", suppress=True), "REF a \\n \\t \\h")] + for segment, instruction in cases: + document = tree(para(CrossRef("x", "t", "u9")), bullets(item("y", unit="u1"))) + paragraph = body(document, plan(unit("u1", bookmarks=("a",)), refs={"u9": (segment,)}))[0] + assert flow(paragraph) == "{" + instruction + "|1}" + + +def test_a_field_is_five_runs_the_cached_result_between_separate_and_end() -> None: + document = tree(para(CrossRef("x", "t", "u9")), bullets(item("y", unit="u1"))) + paragraph = body(document, plan(unit("u1", bookmarks=("a",)), refs={"u9": (field("a", "7"),)}))[0] + runs = paragraph.findall(f"{W}r") + kinds = [(r.find(f"{W}fldChar").get(f"{W}fldCharType") if r.find(f"{W}fldChar") is not None + else local(r[0])) for r in runs] + assert kinds == ["begin", "instrText", "separate", "t", "end"] + + +def test_a_cached_result_that_differs_from_the_renderers_text_is_written_as_the_plan_gives_it() -> None: + reference = CrossRef("1.1(bb)", "x", "u9") + document = tree(para(reference), bullets(item("y", unit="u1"))) + refs = {"u9": (field("a", "1.1(aa)"),)} + plan_ = DocxPlan((), (), None, {"u1": unit("u1", bookmarks=("a",))}, {}, {"u9": RefPlan(tuple(refs["u9"]), False)}) + assert flow(body(document, plan_)[0]) == "{REF a \\n \\h|1.1(aa)}" + + +def test_a_cross_reference_without_fields_shows_its_text() -> None: + document = tree(para("see ", CrossRef("1.2", "x", "u9"), " and ", CrossRef("3", "", None, None), " and ", + CrossRef("4", "x", "u7"))) + (paragraph,) = body(document, plan(refs={})) + assert flow(paragraph) == "see 1.2 and 3 and 4" + + +def test_a_field_to_a_bookmark_the_document_does_not_have_is_a_bug_in_the_plan() -> None: + document = tree(para(CrossRef("x", "t", "u9"))) + with pytest.raises(InternalError, match="'nowhere'"): + body(document, plan(refs={"u9": (field("nowhere"),)})) + + +def test_a_reference_inside_a_link_stays_a_field_and_text_stays_one_link() -> None: + link = Link("https://example.com/", (Text("see "), CrossRef("1", "x", "u9"), TermRef("t", "def:t", "bold"))) + document = tree(para(link), bullets(item("y", unit="u1"))) + the_plan = plan(unit("u1", bookmarks=("a",)), anchors={"def:t": "lddef_t"}, refs={"u9": (field("a", "1"),)}) + (paragraph,) = body(document, the_plan)[:1] + assert flow(paragraph) == "(link rId5: see @Hyperlink{REF a \\n \\h|1@Hyperlink}t@TermReference)" + + +# -- Terms, definitions, attachments, markers ----------------------------------------------------------------- + + +def defined(anchor: str | None = "def:t", *, defining: bool = True) -> DefinedTerm: + return DefinedTerm((Text("Term"),), anchor, "bold", defining) + + +def test_a_defined_term_is_a_bookmark_around_text_in_the_defined_term_style() -> None: + document = tree(para("A ", defined(), " is")) + (paragraph,) = body(document, plan(anchors={"def:t": "lddef_t"})) + assert flow(paragraph) == "A Term@DefinedTerm is" + + +@pytest.mark.parametrize(("term", "anchors"), [(defined(None), {}), (defined(defining=False), {"def:t": "lddef_t"}), + (defined(), {})]) +def test_a_term_that_defines_nothing_here_or_has_no_bookmark_is_styled_text(term: DefinedTerm, + anchors: dict[str, str]) -> None: + (paragraph,) = body(tree(para(term)), plan(anchors=anchors)) + assert flow(paragraph) == "Term@DefinedTerm" + + +def test_a_term_reference_is_an_internal_hyperlink_to_the_definition_with_the_term_as_text() -> None: + document = tree(para("the ", TermRef("terms", "def:t", "bold"), " apply"), para(defined())) + first, _ = body(document, plan(anchors={"def:t": "lddef_t"})) + assert flow(first) == "the (link lddef_t: terms@TermReference) apply" + link = first.find(f"{W}hyperlink") + assert link.get(f"{W}anchor") == "lddef_t" and link.get(f"{W}history") == "1" # type: ignore[union-attr] + + +@pytest.mark.parametrize("term", [TermRef("a", None, "bold"), TermRef("a", "def:other", "bold"), + TermRef("a", "def:t", "bold")]) +def test_a_term_reference_with_no_definition_in_the_document_is_styled_text(term: TermRef) -> None: + (paragraph,) = body(tree(para(term)), plan(anchors={"def:t": "lddef_t"})) + assert flow(paragraph) == "a@TermReference" + + +def attachment_tree(value: Value, titles: Sequence[str] = ("Schedule A",)) -> RenderTree: + from legaldown_render.tree import AttachmentPart + + attachments = tuple(AttachmentPart(f"a{i}", (Text(title),), "f.lgd", f"att-{i}") for i, title in enumerate(titles)) + return tree(para("see ", value), attachments=attachments) + + +def test_an_attachment_value_is_a_field_when_it_is_the_title_else_a_link_with_its_text_and_text_when_unplaced() -> None: + def paragraph(value: Value, *, placed: bool = True) -> str: + # The attachments section is not written yet: a unit stands in for the bookmark it will place. + the_plan = plan(unit("u1", bookmarks=("ldatt_att_0",)) if placed else unit("u1"), + anchors={"att-0": "ldatt_att_0"}) + document = tree(para("see ", value), bullets(item("x", unit="u1")), + attachments=attachment_tree(value).attachments) + return flow(body(document, the_plan)[0]) + + assert paragraph(Value("attach", "Schedule A", "#att-0")) == "see {REF ldatt_att_0 \\h|Schedule A}" + assert paragraph(Value("attach", "the schedule", "#att-0")) == "see (link ldatt_att_0: the schedule@Hyperlink)" + assert paragraph(Value("attach", "Schedule A", "#att-0"), placed=False) == "see Schedule A" + assert paragraph(Value("attach", "x", "#att-9")) == "see x" + assert paragraph(Value("attach", "x", None)) == "see x" + + +def test_a_value_that_links_to_a_file_is_an_external_hyperlink_when_the_document_may_link_there() -> None: + document = tree(para(Value("attach", "price list", "https://example.com/p.pdf"), " ", + Value("attach", "other", "files/p.pdf"), " ", Value("date", "1 May", None))) + (paragraph,) = body(document, plan()) + assert flow(paragraph) == "(link rId5: price list@Hyperlink) other 1 May" + + +def test_failure_blank_and_choice_are_runs_in_their_styles() -> None: + document = tree(para(FailureMarker("[BROKEN REF: x]"), " ", Blank("b", "____", "your name"), " ", Blank("c", "[TBD]"), + " ", Choice("q", ("one", "two"), " | "))) + (paragraph,) = body(document, plan()) + assert flow(paragraph) == "[BROKEN REF: x]@FailureMarker ____@Blank (your name)+i [TBD]@Blank [one | two]@Choice" + + +# -- Inlines ---------------------------------------------------------------------------------------------------- + + +def test_emphasis_strong_and_code_are_runs_and_nest() -> None: + document = tree(para("a ", Emphasis((Text("e"), Strong((Text("both"),)))), Strong((Text("s"),)), Code("c"), + Emphasis((Code("ic"),)), SoftBreak(), "z", HardBreak(), "y")) + (paragraph,) = body(document, plan()) + assert flow(paragraph) == "a e+i" "both+b+i" "s+b" "c@InlineCode" "ic@InlineCode+i" " z↵y" + + +def test_run_properties_are_in_schema_order() -> None: + document = tree(para(Strong((Emphasis((Code("x"),)),)))) + run = body(document, plan())[0].find(f"{W}r") + assert [local(child) for child in run.find(f"{W}rPr")] == ["rStyle", "b", "bCs", "i", "iCs"] # type: ignore[union-attr] + + +def test_an_external_link_is_a_hyperlink_by_relationship_and_a_refused_one_is_its_text() -> None: + link = Link("https://example.com/a b", (Text("site"),), (Text("tip"),)) + refused = Link("javascript:alert(1)", (Text("bad"),)) + document = tree(para(link, " ", refused, " ", Link("https://example.com/a b", (Text("again"),)))) + contents = build(document, plan()) + paragraph = [c for c in ET.fromstring(contents["word/document.xml"]).find(f"{W}body") if pstyle(c) is None][0] # type: ignore[union-attr] + assert flow(paragraph) == "(link rId5: site@Hyperlink) bad (link rId5: again@Hyperlink)" + rels = ET.fromstring(contents["word/_rels/document.xml.rels"]) + external = [r for r in rels if r.get("TargetMode") == "External"] + assert [(r.get("Id"), r.get("Target")) for r in external] == [("rId5", "https://example.com/a%20b")] + assert paragraph.find(f"{W}hyperlink").get(f"{W}tooltip") == "tip" # type: ignore[union-attr] + + +def test_a_link_with_no_text_shows_its_address_and_emphasis_inside_it_keeps_the_link_style() -> None: + document = tree(para(Link("mailto:a@example.com", ()), " ", Link("https://e.com/", (Emphasis((Text("em"),)),)))) + (paragraph,) = body(document, plan()) + assert flow(paragraph) == "(link rId5: mailto:a@example.com@Hyperlink) (link rId6: em@Hyperlink+i)" + + +def test_an_internal_link_to_an_anchor_of_the_document_is_a_hyperlink_to_its_bookmark() -> None: + document = tree(para(Link("#clause", (Text("here"),)), " ", Link("#missing", (Text("there"),))), + bullets(item("x", unit="u1"))) + first, _ = body(document, plan(unit("u1", bookmarks=("ld_clause",)), anchors={"clause": "ld_clause"})) + assert flow(first) == "(link ld_clause: here@Hyperlink) there" + + +def test_an_image_is_its_alt_text() -> None: + (paragraph,) = body(tree(para("a", Image("x.png", (Text("a chart"),)), Image("y.png", ()))), plan()) + assert flow(paragraph) == "aa chart" + + +def test_text_with_tabs_and_line_breaks_becomes_tab_and_break_elements() -> None: + (paragraph,) = body(tree(para("a\tb\nc\r\nd")), plan()) + assert flow(paragraph) == "a→b↵c↵d" + + +def test_a_space_at_either_end_of_a_run_is_kept() -> None: + (paragraph,) = body(tree(para(" a ")), plan()) + assert paragraph.find(f"{W}r/{W}t").get("{http://www.w3.org/XML/1998/namespace}space") == "preserve" # type: ignore[union-attr] + + +# -- Blocks ----------------------------------------------------------------------------------------------------- + + +def test_a_quote_is_paragraphs_in_the_quote_style_and_a_note_has_its_label_run() -> None: + document = tree(Quote((para("q1"), para("q2"))), DraftingNote((para("n"),), "Drafting note"), + DraftingNote((para("m"),), None)) + q1, q2, label, note, bare = body(document, plan()) + assert [(pstyle(p), flow(p)) for p in (q1, q2, label, note, bare)] == [ + ("Quote", "q1"), ("Quote", "q2"), ("DraftingNote", "Drafting note@DraftingNoteLabel"), + ("DraftingNote", "n"), ("DraftingNote", "m")] + assert all(indent(p) is None for p in (q1, q2, label, note, bare)) + + +def test_a_list_in_a_quote_adds_the_quotes_indent_to_its_own_and_its_continuation() -> None: + inner = bullets(item("a", "more", unit="u1")) + document = tree(Quote((para("lead"), inner)), DraftingNote((inner,), "N")) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=0, explicit=True), abstracts=[abstract(0, levels=[ + level(0, left=567, hanging=567)])], nums=[Num(1, 0)]) + lead, a, more, label, a2, more2 = body(document, the_plan) + assert (pstyle(a), numbering(a), indent(a)) == ("Quote", (0, 1), {"left": 567 + 567, "hanging": 567}) + assert (pstyle(more), numbering(more), indent(more)) == ("Quote", None, {"left": 567 + 567}) + assert indent(a2) == {"left": 567 + 227, "hanging": 567} and indent(more2) == {"left": 567 + 227} + assert pstyle(a2) == pstyle(more2) == "DraftingNote" + + +def test_a_quote_inside_a_note_carries_both_indents() -> None: + document = tree(DraftingNote((Quote((para("q"),)), Rule()), None), Quote((DraftingNote((para("n"),), None),))) + q, rule, n = body(document, plan()) + assert (pstyle(q), indent(q)) == ("Quote", {"left": 227 + 567}) + assert (pstyle(rule), indent(rule)) == ("DraftingNote", None) + assert (pstyle(n), indent(n)) == ("DraftingNote", {"left": 567 + 227}) + + +def test_a_code_block_is_one_paragraph_per_line_in_the_code_style() -> None: + document = tree(CodeBlock("a\n b\n\n\tc\n"), CodeBlock("")) + *lines, empty = body(document, plan()) + assert [(pstyle(p), flow(p)) for p in lines] == [("Code", "a"), ("Code", " b"), ("Code", ""), ("Code", "→c")] + assert pstyle(empty) == "Code" + + +def test_a_rule_is_a_bottom_border() -> None: + (rule,) = body(tree(Rule()), plan()) + border = rule.find(f"{W}pPr/{W}pBdr/{W}bottom") + assert border is not None and border.get(f"{W}val") == "single" + assert flow(rule) == "" + + +def table(*rows: Sequence[str], header: Sequence[str] = ("H1", "H2"), align: tuple[str, ...] = ()) -> Table: + def cells(row: Sequence[str]) -> tuple[tuple[Inline, ...], ...]: + return tuple((Text(cell),) for cell in row) + + return Table(cells(header), tuple(cells(row) for row in rows), align) + + +def test_a_table_is_a_grid_with_a_repeated_header_row_and_aligned_columns() -> None: + document = tree(para("before"), table(("a", "b"), ("c", "d"), align=("right", "center")), para("after")) + before, tbl, after = body(document, plan()) + assert local(tbl) == "tbl" and flow(before) == "before" and flow(after) == "after" + assert [local(child) for child in tbl] == ["tblPr", "tblGrid", "tr", "tr", "tr"] + props = tbl.find(f"{W}tblPr") + assert props.find(f"{W}tblStyle").get(f"{W}val") == "TableGrid" # type: ignore[union-attr] + assert [local(child) for child in props] == ["tblStyle", "tblW", "tblCellMar", "tblLook"] # type: ignore[arg-type] + widths = [int(g.get(f"{W}w")) for g in tbl.findall(f"{W}tblGrid/{W}gridCol")] # type: ignore[arg-type] + assert len(widths) == 2 and abs(widths[0] - widths[1]) <= 1 + assert int(props.find(f"{W}tblW").get(f"{W}w")) == sum(widths) # type: ignore[union-attr, arg-type] + rows = tbl.findall(f"{W}tr") + assert rows[0].find(f"{W}trPr/{W}tblHeader") is not None + assert all(r.find(f"{W}trPr") is None for r in rows[1:]) + cells = [[(flow(c.find(f"{W}p")), c.find(f"{W}p/{W}pPr/{W}jc").get(f"{W}val")) for c in r.findall(f"{W}tc")] # type: ignore[arg-type, union-attr] + for r in rows] + assert cells == [[("H1", "right"), ("H2", "center")], [("a", "right"), ("b", "center")], + [("c", "right"), ("d", "center")]] + assert [int(c.find(f"{W}tcPr/{W}tcW").get(f"{W}w")) for c in rows[0].findall(f"{W}tc")] == widths # type: ignore[arg-type, union-attr] + + +def test_a_cell_holds_inlines_and_a_short_row_is_completed_with_empty_cells() -> None: + document = tree(Table(((Text("A"),), (Text("B"),), (Text("C"),)), + (((Strong((Text("x"),)),), (TermRef("t", None, "bold"),)),), ("", "left"))) + (tbl,) = [c for c in body(document, plan()) if local(c) == "tbl"] + row = tbl.findall(f"{W}tr")[1] + assert [flow(c.find(f"{W}p")) for c in row.findall(f"{W}tc")] == ["x+b", "t@TermReference", ""] + assert len(tbl.findall(f"{W}tblGrid/{W}gridCol")) == 3 + assert [c.find(f"{W}p/{W}pPr/{W}jc").get(f"{W}val") for c in row.findall(f"{W}tc")] == ["left"] * 3 # type: ignore[union-attr] + + +def test_two_tables_in_a_row_and_a_table_at_the_end_get_an_empty_paragraph_after_or_between() -> None: + first, between, second, last = body(tree(table(), table()), plan()) + assert [local(first), local(between), local(second), local(last)] == ["tbl", "p", "tbl", "p"] + assert flow(between) == flow(last) == "" + assert [local(c) for c in body(tree(para("x"), table()), plan())] == ["p", "tbl", "p"] + assert [local(c) for c in body(tree(table(), para("x")), plan())] == ["tbl", "p"] + + +def test_a_table_in_a_list_item_or_quote_is_indented_and_narrower() -> None: + document = tree(bullets(item("a", table(), unit="u1")), Quote((table(),))) + the_plan = plan(unit("u1", "item", ind=(1000, 0))) + _, in_item, in_quote = (c for c in body(document, the_plan) if local(c) in ("tbl", "p") and (local(c) == "tbl" or flow(c) == "a")) + for tbl, left in ((in_item, 1000), (in_quote, 567)): + props = tbl.find(f"{W}tblPr") + assert int(props.find(f"{W}tblInd").get(f"{W}w")) == left # type: ignore[union-attr, arg-type] + full = int(ET.fromstring(build(tree(table()), plan())["word/document.xml"]).find( # type: ignore[arg-type] + f".//{W}tblW").get(f"{W}w")) + assert int(props.find(f"{W}tblW").get(f"{W}w")) == full - left # type: ignore[union-attr, arg-type] + + +def test_a_table_with_no_columns_is_left_out() -> None: + assert body(tree(Table((), (), ()), para("x")), plan()) and len(body(tree(Table((), (), ())), plan())) == 0 + + +# -- Headings and condition labels ------------------------------------------------------------------------------ + + +def test_headings_use_the_heading_styles_and_are_clamped_to_five() -> None: + document = tree(sections=[section("a", level=n, unit=f"s{n}") for n in (0, 1, 3, 5, 9)]) + assert [pstyle(h) for h in body(document, plan())] == ["Heading1", "Heading1", "Heading3", "Heading5", "Heading5"] + + +def test_a_condition_label_leads_a_paragraph_and_an_item_and_follows_a_heading_title() -> None: + document = tree(para("p", condition_label="if A"), para("plain"), + bullets(item(para("first", condition_label="inner"), "second", unit="u1", + condition_label="if B")), sections=[section("H", condition_label="if C", unit="s1")]) + the_plan = plan(unit("u1", "item", ind=(567, 0))) + p, plain, item_paragraph, second, heading = body(document, the_plan) + assert flow(p) == "if A@ConditionLabel p" and flow(plain) == "plain" + assert flow(item_paragraph) == "if B@ConditionLabel inner@ConditionLabel first" + assert flow(heading) == "H if C@ConditionLabel" + + +# -- Whole documents -------------------------------------------------------------------------------------------- + + +def test_the_title_and_the_subtitle_come_first_and_the_section_properties_last() -> None: + document = RenderTree(title=(Text("My Title"),), subtitle=(Text("Sub"),), language="en", preamble=(para("x"),), + sections=()) + root = ET.fromstring(build(document, plan())["word/document.xml"]) + children = list(root.find(f"{W}body")) # type: ignore[arg-type] + assert [(pstyle(c), flow(c)) for c in children[:-1]] == [("Title", "My Title"), ("Subtitle", "Sub"), (None, "x")] + assert local(children[-1]) == "sectPr" + + +def test_without_a_plan_the_document_has_no_numbering_and_the_indent_of_each_list_depth() -> None: + result = render("---\ntitle: T\n---\n\n# A\n\n- a\n - b\n\nSee {{ref: a}}.\n", format="docx") + contents = parts(result.binary) # type: ignore[arg-type] + assert list(ET.fromstring(contents["word/numbering.xml"])) == [] + root = ET.fromstring(contents["word/document.xml"]) + assert [flow(p) for p in root.iter(f"{W}p")] == ["T", "A", "a", "b", "See 1."] + assert [indent(p) for p in root.iter(f"{W}p")] == [None, None, {"left": 567}, {"left": 1134}, None] + assert not list(root.iter(f"{W}fldChar")) + + +def test_the_same_plan_and_tree_are_the_same_bytes() -> None: + document = tree(bullets(item("a", unit="u1")), para(CrossRef("x", "t", "u9"))) + the_plan = plan(unit("u1", "item", num_id=1, ilvl=0, explicit=True, bookmarks=("a",)), abstracts=[abstract(0)], + nums=[Num(1, 0)], refs={"u9": (field("a"),)}) + assert build(document, the_plan) == build(document, the_plan) + + +# -- Escaping --------------------------------------------------------------------------------------------------- + + +EVIL = "x\x00\x01\x08\x0b\x1f\x7f\x85\ud800\ufffe<&>]]>\"'y" +CLEAN = "x<&>]]>\"'y" + + +def test_hostile_text_in_every_inline_and_block_is_dropped_by_the_sanitiser_and_the_part_stays_well_formed() -> None: + document = tree( + para(EVIL, Emphasis((Text(EVIL),)), Code(EVIL), Link("https://e.com/" + EVIL, (Text(EVIL),), (Text(EVIL),)), + Image("i", (Text(EVIL),)), CrossRef(EVIL, "t", "u9"), TermRef(EVIL, "def:t", "bold"), + DefinedTerm((Text(EVIL),), "def:t", "bold"), Value("attach", EVIL, "#att"), Blank("b", EVIL, EVIL), + Choice(EVIL, (EVIL, EVIL), EVIL), FailureMarker(EVIL), condition_label=EVIL, unit="u1"), + Quote((para(EVIL),)), DraftingNote((para(EVIL),), EVIL), CodeBlock(EVIL), table((EVIL, EVIL), header=(EVIL, EVIL)), + sections=[section(EVIL, unit="s1", condition_label=EVIL)]) + the_plan = plan(unit("u1", bookmarks=("a",)), anchors={"def:t": "lddef_t", "att": "a"}, + refs={"u9": (Literal_(EVIL), field("a", EVIL))}) + contents = build(document, the_plan) + for data in contents.values(): + ET.fromstring(data) + data.decode("utf-8") + text = contents["word/document.xml"].decode("utf-8") + for dropped in ("\x00", "\x01", "\x08", "\x0b", "\x1f", "\x7f", "\x85", "\ufffe"): + assert dropped not in text + root = ET.fromstring(contents["word/document.xml"]) + assert CLEAN in "".join(t.text or "" for t in root.iter(f"{W}t")) + assert all(CLEAN in (i.text or "") or "REF" in (i.text or "") for i in root.iter(f"{W}instrText")) + + +# -- Validity --------------------------------------------------------------------------------------------------- + +SOURCE = """--- +legaldown: '0.2' +title: Body +--- + +# Alpha {#alpha} + +Intro, see {{ref: nb}} and {{ref: beta}}. + +## Beta {#beta} + +- item one {#one} + - nested a + - nested b {#nb} + + Continued. +- item two +- - only nested +- item four + +# Gamma + +Last, see {{ref: alpha}}. +""" + + +def planned_document() -> tuple[RenderTree, DocxPlan]: + """The tree of :data:`SOURCE` and a plan written by hand for it: numbered headings, a two-level list on its own + numbering, and the references of the sentences as fields whose cached results are wrong on purpose.""" + document = render(SOURCE, format="text").tree + units = {s.unit: s for s in document.sections} + assert [s for s in units] == ["s1", "s2", "s3"] + headings = abstract(0, "headings", [ + level(0, text="%1.", p_style="Heading1", left=567, hanging=567), + level(1, text="%1.%2", p_style="Heading2", left=567, hanging=567)]) + items = abstract(1, "enumerated", [level(0, "lowerLetter", "(%1)", left=567, hanging=567), + level(1, "lowerRoman", "(%2)", left=1134, hanging=567)]) + nums = [Num(1, 0), Num(2, 1, ((0, 1), (1, 1)))] + list_units = {"u2": ("(a)", 0, ("ld_one",)), "u3": ("(i)", 1, ()), "u4": ("(ii)", 1, ("ld_nb",)), + "u5": ("(b)", 0, ()), "u6": ("(c)", 0, ()), "u7": ("(i)", 1, ()), "u8": ("(d)", 0, ())} + planned = [unit("s1", "section", bookmarks=("ld_alpha",), toc="_Toc100000001"), + unit("s2", "section", bookmarks=("ld_beta",), toc="_Toc100000002"), + unit("s3", "section", toc="_Toc100000003"), + unit("u1", "paragraph"), unit("u9", "paragraph")] + planned += [unit(u, "item", num_id=2, ilvl=ilvl, explicit=True, bookmarks=marks) + for u, (_, ilvl, marks) in list_units.items()] + return document, plan( + *planned, abstracts=[headings, items], nums=nums, heading=1, + anchors={"alpha": "ld_alpha", "beta": "ld_beta", "one": "ld_one", "nb": "ld_nb"}, + refs={"u4": (field("ld_beta", "XXX"), field("ld_one", "XXX"), field("ld_nb", "XXX")), + "s2": (field("ld_beta", "XXX"),), "s1": (Literal_("Section "), field("ld_alpha", "XXX"))}) + + +def planned_data() -> bytes: + document, the_plan = planned_document() + return DocxWriter(load_style("default"), generator="test").write(document, the_plan) + + +@pytest.mark.ooxml_schema +def test_every_hand_planned_document_validates_against_the_schemas() -> None: + from docx_support import require_schemas + + require_schemas() + documents = [planned_data(), build_hostile(), build_everything()] + for data in documents: + assert validation_errors(data) == [] + + +def build_hostile() -> bytes: + document = tree( + para(EVIL, Emphasis((Text(EVIL),)), Link("https://e.com/x", (Text(EVIL),), (Text(EVIL),)), Image("i", (Text(EVIL),)), + FailureMarker(EVIL), Blank("b", EVIL, EVIL), condition_label=EVIL), + table((EVIL, EVIL), header=(EVIL, EVIL)), sections=[section(EVIL, unit="s1")]) + return DocxWriter(load_style("default")).write(document, plan()) + + +def build_everything() -> bytes: + """Every block and inline kind, a table in a list in a quote, and every kind of unit the plan can describe.""" + alt = AltNumber((field("ld_a", "1"), Literal_(".")), "space") + nested = bullets(item("inner", table(("a", "b")), unit="u3"), depth=1) + document = tree( + para("Intro ", defined(), " ", TermRef("term", "def:t", "bold"), " ", CrossRef("1", "a", "s1"), " ", + Link("https://e.com/", (Emphasis((Strong((Text("l"),)),)),)), Image("i", (Text("alt"),)), + Blank("b", "___", "p"), Choice("q", ("o", "p"), "/"), FailureMarker("[X]"), Code("c"), SoftBreak(), + HardBreak(), Value("attach", "S", "#att-0"), condition_label="if A", unit="u1"), + Quote((para("q"), bullets(item("x", nested, unit="u2")))), + DraftingNote((para("n"), CodeBlock("a\nb"), Rule(), table(("1", "2"))), "Note"), + table(("a", "b"), align=("left", "right")), table(("c", "d")), + bullets(ListItem((nested,), unit="u4")), + sections=[section("One", para("alt", unit="u5"), unit="s1"), section("Two", unit="s2", condition_label="if B")], + attachments=attachment_tree(Value("attach", "S", "#att-0")).attachments) + the_plan = plan( + unit("s1", "section", bookmarks=("ld_a",), toc="_Toc1"), + unit("s2", "section", num_id=0, ilvl=0, explicit=True, ind=(567, 567), alt=alt), + unit("u1"), unit("u2", "item", num_id=1, ilvl=0, explicit=True, bookmarks=("ld_att",)), + unit("u3", "item", num_id=1, ilvl=1, explicit=True), unit("u4", "item", num_id=1, ilvl=0, explicit=True), + unit("u5", "paragraph", num_id=2, ilvl=0, explicit=True), + abstracts=[abstract(0, "headings", [level(0, p_style="Heading1")]), abstract(1)], nums=[Num(1, 1), Num(2, 1, ((0, 1),))], + heading=None, anchors={"def:t": "lddef_t", "att-0": "ld_att"}, refs={"s1": (field("ld_a", "1"),)}) + return DocxWriter(load_style("default")).write(document, the_plan) + + +# -- LibreOffice ------------------------------------------------------------------------------------------------ + + +@pytest.mark.libreoffice +def test_libreoffice_numbers_the_hand_planned_document_and_updates_every_reference() -> None: + """The cached results are wrong on purpose ('XXX'): what LibreOffice shows after updating fields is what the + numbering definitions and the REF fields compute, which is what a lawyer sees after inserting a clause.""" + require_libreoffice() + shown = libreoffice_export({"planned": planned_data()})["planned"] + lines = [" ".join(line.split()) for line in shown.lines if line.strip()] + assert lines == [ + "Body", + "1. Alpha", + "Intro, see 1.1(a)(ii) and 1.1.", + "1.1 Beta", + "(a) item one", + "(i) nested a", + "(ii) nested b", + "Continued.", + "(b) item two", + "(c)", + "(i) only nested", + "(d) item four", + "2. Gamma", + "Last, see Section 1.", + ] diff --git a/tests/test_docx_libreoffice.py b/tests/test_docx_libreoffice.py index 7176f40..7198d35 100644 --- a/tests/test_docx_libreoffice.py +++ b/tests/test_docx_libreoffice.py @@ -88,7 +88,7 @@ def shown() -> dict[str, Exported]: def test_the_words_come_back_in_order(shown: dict[str, Exported]) -> None: assert shown["round_trip"].lines == [ "Round Trip", "Sub", "Preamble ž€.", "First", "Body with emphasis, strong and code.", "Second line.", - "Second", "item one", "item two", "A quote", "code line 1", " code line 2", "H1\tH2", "a\tb"] + "Second", "item one", "item two", "A quote", "code line 1", " code line 2", "H1", "H2", "a", "b", ""] @pytest.mark.parametrize("name", ["features", "template"]) diff --git a/tests/test_docx_package.py b/tests/test_docx_package.py index b6c5a82..53896f2 100644 --- a/tests/test_docx_package.py +++ b/tests/test_docx_package.py @@ -396,10 +396,11 @@ def test_app_properties_name_the_program() -> None: assert app[0].text.startswith("legaldown-render ") -# -- The interim body ---------------------------------------------------------------------- +# -- The body without a plan --------------------------------------------------------------- +# (the body writer itself is in test_docx_body.py) -def test_the_interim_body_has_the_title_headings_and_text_and_no_numbers() -> None: +def test_the_body_without_a_plan_has_the_title_headings_and_text_and_no_numbers() -> None: contents = docx() body = xml(contents["word/document.xml"]).find(f"{W}body") styles_used = [(p.find(f"{W}pPr/{W}pStyle").get(f"{W}val") if p.find(f"{W}pPr/{W}pStyle") is not None else None, @@ -409,7 +410,8 @@ def test_the_interim_body_has_the_title_headings_and_text_and_no_numbers() -> No assert ("Heading2", "Nested") in styles_used assert (None, "item a") in styles_used and ("Quote", "quoted") in styles_used assert [text for style, text in styles_used if style == "Code"] == ["code 1", "code 2"] - assert (None, "A\tB") in styles_used and (None, "1\t2") in styles_used + # The cells of the table, then the paragraph of the rule that ends the document. + assert [text for _, text in styles_used][-5:] == ["A", "B", "1", "2", ""] assert all(not text.startswith(("1.", "(a)")) for _, text in styles_used) @@ -813,7 +815,7 @@ def test_code_is_left_aligned_even_when_the_body_is_justified() -> None: '_rels/.rels': 'd6f21829c0081670bdb417d3e3f644b81134a121e074fb153c7686faaa8b167b', 'docProps/core.xml': '159ed0e537106ab29739fd357a5f6d89a2c0b0154daefd4c13d878bd16dff47e', 'docProps/app.xml': 'b285204d159fd58da80b6a0242cef523448f9c3a598b357b15faf4627160dcc3', - 'word/document.xml': 'd84c7bc463bf0a89b6022fb8b39ca15601d8c722f951e8f8fb1c023da173172a', + 'word/document.xml': '973a83b30f439e08a3238504ff0d6c8df6ea9259244e7ee82e1f81a3d274bb19', 'word/_rels/document.xml.rels': 'ada136c2e7d05a317579c9503c1f0dd8dfaa8f0cce687df86fe59e36d5fa5d57', 'word/styles.xml': '83adf799de91eac3e9bf1e658b8c2f53586c700be35df179e1b2f1765f189dfc', 'word/numbering.xml': '8d1f7769c35af16db98942ba09882646b45ba404e83686d4ce7a376c6a911f0f', From 567c785b0eb622e2d815df6232d86d20171ac9a5 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:21:23 +0000 Subject: [PATCH 079/118] Document the DOCX body writer --- CONFORMANCE.md | 2 +- README.md | 14 ++++++++------ docs/architecture.md | 10 +++++++--- docs/roadmap.md | 10 ++++++---- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 2814a9d..6097d7d 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -20,7 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | -| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. The **body is interim**: the title, headings as `heading N` paragraphs and every other block as plain text, with no numbers, cross-references, table of contents, title block, attachments or signature blocks. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)); none of the numbering is built yet. Every DOCX render reports the Warning `render-docx-interim` ("DOCX output is incomplete in this version") until the body is complete; it is the renderer's own rule, not the specification's, and goes away then. HTML is the format this renderer claims §13.6 with | +| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says, but **the file is still interim**: the planner that decides the numbering is not wired in, so there are no numbers or cross-reference fields, and no table of contents, title block, attachments or signature blocks. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)); the numbering definitions, bookmarks and `REF` fields are written from a plan, and the planner that makes the plan is not built yet. Every DOCX render reports the Warning `render-docx-interim` ("DOCX output is incomplete in this version") until the body is complete; it is the renderer's own rule, not the specification's, and goes away then. HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | diff --git a/README.md b/README.md index 348fa70..0c528b0 100644 --- a/README.md +++ b/README.md @@ -141,15 +141,17 @@ computed as their cached results, so a lawyer who inserts a clause gets the numb updated by Word ([ADR 0008](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/decisions/0008-docx-word-native.md)). -**What is there today** is the package, not yet the body of the document: a valid `.docx` (checked against the -ISO/IEC 29500 schemas, and opened in LibreOffice in CI) with styles generated from the style template +**What is there today** is the package and the body writer, not yet the numbering: a valid `.docx` (checked +against the ISO/IEC 29500 schemas, and opened in LibreOffice in CI) with styles generated from the style template (`Normal`, `heading 1` to `heading 5`, `Title`, `Quote`, `toc 1` to `toc 5`, `Hyperlink`, ...), the page size and margins, a running header and footer from `page.header` and `page.footer` (the default footer is "3 / 12" as live page fields), the document's title and language, `w:updateFields` so Word refreshes fields when it -opens the file (`docx.update_fields_on_open`), and deterministic output. **The body is interim:** the title, -the headings as Word headings, and every other block as plain text, with **no numbers, references, contents, -title block, attachments or signature blocks yet**, and every render reports the Warning -`render-docx-interim` to say so. Do not send such a file as a contract. Settings Word +opens the file (`docx.update_fields_on_open`), and deterministic output. The body writer lays out headings, +paragraphs, lists (with the numbering definitions, bookmarks and `REF` fields a plan names), quotes, drafting +notes, code, tables with a repeated header row, rules, links, defined terms and the template view's labels and +choices. **The file is still interim:** until the planner that decides the numbering is wired in, it carries +**no numbers, no cross-reference fields, and no title block, contents, attachments or signature blocks**, and +every render reports the Warning `render-docx-interim` to say so. Do not send such a file as a contract. Settings Word cannot express (a `%` length, say) are an error for DOCX only. See [Styles and preferences](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/style-templates.md#docx) and the [roadmap](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/roadmap.md). diff --git a/docs/architecture.md b/docs/architecture.md index 2232608..f303356 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -251,7 +251,7 @@ references show; the Warning says so. Every other heading reads exactly as its d |---|---| | `TextWriter` | Plain text (§13.6). A soft line break inside a paragraph is a space, a hard break a new line, and lists are indented with their labels, one quote stays one quote, and tables keep escaped cells and alignment. It takes the style's `attachments.separator` (default `page-break`; `rule` and `page-break` both print `---` before the attachments heading and between attachments). It is the **test oracle** for resolution | | `HtmlWriter` | A self-contained HTML5 page, or with `standalone=False` just the `
`. Semantic markup, with labels as real text rather than CSS counters, and a stylesheet generated from the style's presentation settings, including print rules | -| `DocxWriter` | A `.docx` file, as bytes (`RenderResult.binary`; `output` is empty). OOXML written directly with `zipfile` and `xml.etree` ([ADR 0008](decisions/0008-docx-word-native.md)): the parts, the styles generated from the style template, settings, page setup, running headers and footers, properties, a deterministic ZIP, and one sanitiser for every text and attribute. **Interim body:** the title, headings (`heading N`) and plain-text paragraphs with no numbers; the numbering definitions, cross-reference fields and table of contents that make the numbers live in Word (ADR 0008) follow. The DOCX-only settings are converted and checked before rendering (`style/docx.py`) | +| `DocxWriter` | A `.docx` file, as bytes (`RenderResult.binary`; `output` is empty). OOXML written directly with `zipfile` and `xml.etree` ([ADR 0008](decisions/0008-docx-word-native.md)): the parts, the styles generated from the style template, settings, page setup, running headers and footers, properties, a deterministic ZIP, and one sanitiser for every text and attribute. **The body is laid out as a `DocxPlan` says** (`resolve/docx_plan.py`, the planner's data model): `numbering.xml` from its abstract numberings and nums; each section, list item and top-level paragraph gets `w:numPr`, `w:ind`, bookmarks and, for a later alternative that shares a number (§15.8), `numId` 0 and its number as `REF` fields, as its `UnitPlan` says; a `{{ref:}}` is the literal text and `REF` fields (cached results as planned) of its `RefPlan`; a `{{term:}}` or an `{{attach:}}` is an internal hyperlink (or a `REF` for an attachment's title), a defined term a bookmark; every block (lists with their continuation blocks, quotes, drafting notes, code, tables with a repeated header row, rules) and inline (emphasis, strong, code, external links through relationships, breaks, images as alt text, blanks, choices, failure markers) has its Word form. The writer decides none of it (ADR 0003) and a field that points at a bookmark the document does not hold is a bug in the plan (`InternalError`). **Interim, until the planner is wired in:** the writer is given a plan with no numbering (`writers/docx/interim_plan.py`), so the file has no numbers or `REF` fields, and the title block, table of contents, attachments and signature blocks are not written yet. The DOCX-only settings are converted and checked before rendering (`style/docx.py`) | | PDF | Planned ([ADR 0004](decisions/0004-html-first.md)) | ## Package layout @@ -268,6 +268,7 @@ src/legaldown_render/ ├── resolve/ │ ├── resolver.py stage 4 │ ├── numbering.py counters, the §13.1 scheme presets, label templates +│ ├── docx_plan.py the DOCX plan: the data the planner hands to the DOCX writer │ └── values.py Babel formatting of dates, money, durations ├── style/ │ ├── model.py the style dataclasses (every field has a default) @@ -278,7 +279,9 @@ src/legaldown_render/ └── writers/ ├── html.py, text.py └── docx/ the DOCX writer: ooxml.py (sanitiser, builders), package.py (parts, relationships, ZIP), - styles.py, settings.py, headers.py, properties.py, body.py (interim) + styles.py, settings.py, headers.py, properties.py, numbering.py (numbering.xml + from the plan), body.py (blocks, numbered paragraphs, tables), runs.py (inlines, fields, + hyperlinks), interim_plan.py (the plan with no numbering, temporary) ``` ## Dependencies @@ -490,7 +493,8 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_errors.py` | The exception hierarchy and pickling of errors | | `tests/test_escaping.py` | Control characters from a document or the command line never reach a diagnostic or standard error unescaped | | `tests/test_docx_style.py` | `page.header`, `page.footer` and `docx.*`: validation, and how CSS lengths, colours and fonts become Word's units, with the StyleErrors only DOCX has | -| `tests/test_docx_package.py` | The OOXML package: parts, relationships, content types, settings order, styles, page setup, headers and footers, properties, the interim body, determinism, the sanitiser and builders, the API and command line | +| `tests/test_docx_package.py` | The OOXML package: parts, relationships, content types, settings order, styles, page setup, headers and footers, properties, the body without a plan, determinism, the sanitiser and builders, the API and command line | +| `tests/test_docx_body.py` | The DOCX body from hand-made plans: numbering definitions, `w:numPr`/`w:ind`/alternatives, bookmarks and `REF` fields with cached results, hyperlinks, every inline and block, escaping, schema validity (marker `ooxml_schema`) and a LibreOffice round trip of headings, nested lists and references whose cached values are wrong on purpose (marker `libreoffice`) | | `tests/test_docx_escaping.py` | Well-formedness of every part for any text, with a seeded fuzz and the characters XML cannot carry | | `tests/test_docx_validation.py` | Every part of every generated document against the ISO/IEC 29500-4 schemas (marker `ooxml_schema`; needs lxml and `python tests/schemas/fetch.py`) | | `tests/test_docx_libreoffice.py` | The documents opened in LibreOffice through UNO (`tests/lo/uno_export.py`), fields updated: the words in order, header, footer, page count (marker `libreoffice`; skipped without `soffice` and `python3-uno`) | diff --git a/docs/roadmap.md b/docs/roadmap.md index b65ab13..2b7f907 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -112,10 +112,12 @@ values appear only as the cached results of fields. Decided in - **Verification (done).** Every part validated against the ISO/IEC 29500 schemas (fetched in CI, not stored), well-formedness fuzzing, a determinism test, and LibreOffice opening the documents through UNO as the oracle (a `docx` CI job) -- **Next.** Numbering structure in the render tree; a planner that maps the tree to Word (lists, - levels, bookmarks, cross-references composed of `REF` fields, cached values from a model of Word's - numbering); the body writer with the table of contents, title block, attachments and signature - blocks; a corpus checked against the plain-text oracle, also after edits made through UNO; a manual +- **Body writer (done, awaiting the planner).** `numbering.xml`, numbered paragraphs, bookmarks, `REF` + fields with cached results, term hyperlinks, and every block and inline, written from a `DocxPlan` + (`resolve/docx_plan.py`); a plan with no numbering stands in until the planner is wired +- **Next.** The planner that maps the tree to Word (lists, levels, bookmarks, cross-references composed + of `REF` fields, cached values from a model of Word's numbering); the table of contents, title block, + attachments and signature blocks; a corpus checked against the plain-text oracle, also after edits made through UNO; a manual Word checklist for releases ### v0.6 — PDF From e0fd796bdbbbd260bb059c72dcb66e07c525ead5 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:48:01 +0000 Subject: [PATCH 080/118] Share the numbering formats a style gives between the resolver and the DOCX checks formats_of(style) builds the NumberingFormats that the resolver writes into the tree; the DOCX style check needs the same formats before any tree exists. --- src/legaldown_render/resolve/numbering.py | 17 ++++++++++++++++- src/legaldown_render/resolve/resolver.py | 13 ++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/legaldown_render/resolve/numbering.py b/src/legaldown_render/resolve/numbering.py index de95a8e..fcfadbd 100644 --- a/src/legaldown_render/resolve/numbering.py +++ b/src/legaldown_render/resolve/numbering.py @@ -3,7 +3,8 @@ import re -from ..style.model import LevelFormat, Numbering +from ..style.model import LevelFormat, Numbering, Style +from ..tree import NumberingFormats _ROMAN = ( (1000, "m"), (900, "cm"), (500, "d"), (400, "cd"), (100, "c"), (90, "xc"), @@ -94,3 +95,17 @@ def extend(designation: str, piece: str, *, textual: bool) -> str: if textual: return f"{designation} {piece.lstrip('.')}" return designation + piece + + +def formats_of(style: Style) -> NumberingFormats: + """The numbering formats *style* gives (``RenderTree.formats``): those the resolver writes labels and + designations with, and the DOCX planner builds its numbering from.""" + enumeration = style.enumeration + return NumberingFormats( + scheme=style.numbering.scheme, + headings=() if style.numbering.scheme == "none" else heading_levels(style.numbering), + enumerated=tuple(enumeration.levels) if enumeration.enabled or enumeration.ordered == "enumerate" else (), + renumber=RENUMBERED if enumeration.ordered == "renumber" else None, + paragraphs=style.paragraphs.format if style.paragraphs.numbered else None, + reference=style.references.format, + ) diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index d237660..7a9c013 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -107,7 +107,7 @@ unlinked, ) from ..urls import file_reference_problem, is_empty_path, is_link_target, printable -from .numbering import RENUMBERED, extend, fill, format_counter, heading_levels +from .numbering import RENUMBERED, extend, fill, format_counter, formats_of, heading_levels from .values import DURATION_UNITS, Formatter #: Definition anchors are prefixed with a character no identifier can hold, @@ -410,16 +410,7 @@ def _formats(self) -> NumberingFormats: """The numbering formats in effect (``RenderTree.formats``): those the labels and designations were made with (:meth:`_number_sections`, :meth:`_structure_list`, :meth:`_number_paragraph`).""" - style = self.style - enumeration = style.enumeration - return NumberingFormats( - scheme=style.numbering.scheme, - headings=() if self.textual else heading_levels(style.numbering), - enumerated=tuple(enumeration.levels) if enumeration.enabled or enumeration.ordered == "enumerate" else (), - renumber=RENUMBERED if enumeration.ordered == "renumber" else None, - paragraphs=style.paragraphs.format if style.paragraphs.numbered else None, - reference=style.references.format, - ) + return formats_of(self.style) def _unit(self) -> str: """The unit id of the next item or paragraph, in document order.""" From 3f7a08992dbe02fa6dd1c6a1be7aa39bac685db1 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:48:01 +0000 Subject: [PATCH 081/118] Make the DOCX numbering plan place every unit as the resolver numbered it Lists that are numbered by one num get no start overrides on the levels below; a list that continues another with a fixed start, or whose level another list used in between, is reported as a warning (it drifts when items change) instead of as a restart, and each item of a clamped list keeps its number, in the format of level 9. Bullets count nothing and share one num. A label is judged on its filled-in form: it is written without the section's number, or without the number of an enclosing item that Word cannot show (a level nothing numbers reads as 0), and reported with the reading at its own depth. A run of units that Word counts off by the same amount is one finding. A '%' that meets a counter where a label and the refs it shows are put together is a style error, which docx_settings now raises before rendering; the planner takes only the tree. Diagnostics that duplicated the validator (heading-skip) or could not occur (counter) are gone, and findings are recorded in constant time. --- .../resolve/docx_numbering.py | 61 +++- src/legaldown_render/resolve/docx_plan.py | 325 +++++++++++------- src/legaldown_render/style/docx.py | 3 + 3 files changed, 249 insertions(+), 140 deletions(-) diff --git a/src/legaldown_render/resolve/docx_numbering.py b/src/legaldown_render/resolve/docx_numbering.py index 3ce794a..4bb20e6 100644 --- a/src/legaldown_render/resolve/docx_numbering.py +++ b/src/legaldown_render/resolve/docx_numbering.py @@ -175,17 +175,41 @@ def level_text(label: str, refs: Sequence[str], level: int, *, section: Sequence return tuple(parts) -def lvl_text(parts: LevelParts) -> str: - """*parts* in OOXML form: ``%k`` for level ``k-1``. A literal ``%`` before a digit, or before a counter, cannot - be told from a placeholder in ``lvlText``; style validation (:func:`.docx_plan.docx_style_errors`) refuses it before - rendering, so one reaching here is a bug.""" +def level_text_problem(parts: LevelParts) -> str | None: + """Why *parts* cannot be written as an ``lvlText``, or None. A literal ``%`` before a digit, or before a + counter, cannot be told from a placeholder; this happens where the pieces of a label are put together (a ref + ending in ``%`` before another that starts with a counter), not only inside one template.""" text = "".join(f"%{part + 1}" if isinstance(part, int) else part for part in parts) before_counter = any(isinstance(after, int) and isinstance(part, str) and part.endswith("%") for part, after in zip(parts, parts[1:], strict=False)) if before_counter or len(_PLACEHOLDER.findall(text)) != sum(isinstance(part, int) for part in parts): - raise InternalError("a label with a literal '%' before a digit reached the DOCX planner. This is a bug; " + return "a '%' meets a digit or a counter" + return None + + +def lvl_text(parts: LevelParts) -> str: + """*parts* in OOXML form: ``%k`` for level ``k-1``. Style validation (:func:`.docx_plan.docx_style_errors`) + refuses parts with a :func:`level_text_problem` before rendering, so one reaching here is a bug.""" + if level_text_problem(parts) is not None: + raise InternalError("a label that Word's level text cannot write reached the DOCX planner. This is a bug; " "style validation refuses it.") - return text + return "".join(f"%{part + 1}" if isinstance(part, int) else part for part in parts) + + +_SECTION = "section" + + +def shows_section(label: str, refs: Sequence[str], level: int, *, textual: bool = False) -> bool: + """True if the label of level *level* of a list in a section shows a counter of the section: it uses + ``{section}``, or ``{path}`` of a designation that starts from the section's. Judged on the filled-in label + (a counter of the section is in it), not on how the template is spelled.""" + section: list[Token] = [Count(_SECTION, "")] + designation: list[Token] = list(section) + for j in range(level + 1): + designation = extend_tokens(designation, fill_tokens(refs[j], n=[Count(j, "")], section=section), + textual=textual) + tokens = fill_tokens(label, n=[Count(level, "")], path=designation, section=section) + return any(isinstance(token, Count) and token.owner == _SECTION for token in tokens) def split_level_text(text: str) -> list[str | int]: @@ -227,20 +251,24 @@ class WordNumbering: :meth:`add_num` (an abstractNum's levels are read when a paragraph is stepped, so a level may be defined after the abstractNum is added), then :meth:`step` paragraphs in document order. - *skipped* is the model of a heading level that was never used: ``"lo"`` shows its start value and counts on - (what the renderer does and LibreOffice shows); ``"word"`` shows ``start - 1`` and does not advance it, which is - what Word is expected to do (the documents that skip a level have a Core Error, ``heading-skip``).""" + *skipped* is the model of a level that was never used before a deeper one: ``"lo"`` shows its start value and + counts on (what the renderer does for a skipped heading level, and LibreOffice shows); ``"word"`` shows + ``start - 1`` and does not advance it, which is what Word is expected to do. It is the default of every + abstractNum and can be set for each in :meth:`add_abstract`.""" def __init__(self, *, skipped: Literal["lo", "word"] = "lo") -> None: self.skipped = skipped + self._skipped: dict[int, Literal["lo", "word"]] = {} self._levels: dict[int, Sequence[LevelDef]] = {} self._nums: dict[int, Num] = {} self._count: dict[int, list[int]] = {} self._holder: dict[int, list[object | None]] = {} self._used: set[tuple[int, int]] = set() - def add_abstract(self, abstract_id: int, levels: Sequence[LevelDef]) -> None: + def add_abstract(self, abstract_id: int, levels: Sequence[LevelDef], *, + skipped: Literal["lo", "word"] | None = None) -> None: self._levels[abstract_id] = levels + self._skipped[abstract_id] = skipped or self.skipped self._count[abstract_id] = [0] * 9 self._holder[abstract_id] = [None] * 9 @@ -264,7 +292,10 @@ def current(self, num_id: int, ilvl: int) -> tuple[int, object | None]: return self._count[abstract_id][ilvl], self._holder[abstract_id][ilvl] def step(self, num_id: int, ilvl: int, owner: object) -> Step: - """Number the next paragraph at (*num_id*, *ilvl*), which *owner* (a unit id) is.""" + """Number the next paragraph at (*num_id*, *ilvl*), which *owner* (a unit id) is. ``numId`` 0 (no + numbering) counts nothing and restarts nothing; ``REF \\n`` of such a paragraph returns ``0``.""" + if num_id == 0: + return Step(0, "", "", "0", ()) num = self._nums[num_id] levels = self._levels[num.abstract_id] count, holder = self._count[num.abstract_id], self._holder[num.abstract_id] @@ -276,20 +307,18 @@ def step(self, num_id: int, ilvl: int, owner: object) -> Step: holder[ilvl] = owner for deeper in range(ilvl + 1, 9): count[deeper], holder[deeper] = 0, None + skipped = self._skipped[num.abstract_id] for above in range(ilvl): if count[above] == 0: holder[above] = Ghost(str(owner), above) - if self.skipped == "lo": - count[above] = levels[above].start - else: - count[above] = levels[above].start - 1 + count[above] = levels[above].start if skipped == "lo" else levels[above].start - 1 tokens: list[Token] = [] for part in split_level_text(levels[ilvl].lvl_text): if isinstance(part, int): tokens.append(Count(holder[part], word_counter(count[part], levels[part].num_fmt))) else: tokens.append(Lit(part)) - if self.skipped == "word": + if skipped == "word": # Never-used levels show start - 1 and stay unused: the next heading of that level starts at `start`. for above in range(ilvl): if isinstance(holder[above], Ghost): diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 0e51b05..a9fb97f 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -1,14 +1,15 @@ """The DOCX planner: from the render tree to the numbering Word will draw (docs/decisions/0008). -``plan_docx(tree, style)`` is the one place that knows how LegalDown numbering becomes Word numbering; the DOCX -writer serialises its :class:`DocxPlan` and decides nothing (docs/decisions/0003). It is a pure function of the -resolved tree: the numbering structure the resolver put on every section, list item and top-level paragraph -(``unit``, ``number``, ``parent_unit``, ``List.family``, ``RenderTree.formats``; §13.1, §13.2) is mapped to -abstractNums, nums and, for each unit, the (numId, ilvl) of its paragraph, and every placement is checked against the -number the resolver gave the unit, by simulating Word's counters (:mod:`.docx_numbering`). +``plan_docx(tree)`` is the one place that knows how LegalDown numbering becomes Word numbering; the DOCX writer +serialises its :class:`DocxPlan` and decides nothing (docs/decisions/0003). It is a pure function of the resolved +tree: the numbering structure the resolver put on every section, list item and top-level paragraph (``unit``, +``number``, ``parent_unit``, ``List.family``, ``RenderTree.formats``; §13.1, §13.2) is mapped to abstractNums, nums +and, for each unit, the (numId, ilvl) of its paragraph, and every placement is checked against the number the +resolver gave the unit, by simulating Word's counters (:mod:`.docx_numbering`). -What a plan holds so far: the numbering definitions and the numbering of every unit. The reference fields, -bookmarks and table of contents are further parts of the plan that are empty here. +What a plan holds so far: the numbering definitions and the numbering of every unit. ``UnitPlan.ind`` is never +set, and the reference fields, bookmarks and table of contents (``refs``, ``anchors``, ``toc``, ``bookmarks``) are +empty. :func:`docx_style_errors` is the part that runs on every DOCX render, before it (ADR 0006). **Data model.** Frozen data only; the planner assigns bookmark *names*, never ids (the writer allocates those); names are valid Word bookmark names of at most 40 characters and unique ignoring case. All sequences are tuples, all @@ -23,8 +24,7 @@ from legaldown import Diagnostic -from ..diagnostics import by_line, record -from ..errors import InternalError, StyleError +from ..diagnostics import by_line from ..style.model import LevelFormat, Style from ..tree import ( DraftingNote, @@ -36,21 +36,24 @@ Quote, RenderTree, Section, - plain_text, ) from ..urls import printable from .docx_numbering import ( NUM_FMT, SAMPLE_COUNTER, + Count, + Ghost, LevelParts, Step, WordNumbering, level_text, + level_text_problem, lvl_text, + shows_section, split_level_text, word_counter, ) -from .numbering import RENUMBERED, format_counter, heading_levels +from .numbering import format_counter, formats_of # --------------------------------------------------------------------------- # The plan (frozen data) @@ -94,8 +97,8 @@ class Num: id: int # from 1 (numId 0 means "no numbering" and is never a Num) abstract_id: int - #: ``(ilvl, value)``: the first number of the list at that level; deeper levels restart at 1. Overrides that - #: are never reached by a paragraph are harmless. + #: ``(ilvl, value)``: the first number of the list at that level; the deeper levels restart by themselves when + #: that level counts. start_overrides: tuple[tuple[int, int], ...] = () @@ -216,29 +219,21 @@ class DocxPlan: #: or before a field that becomes ``%k``. _AMBIGUOUS_PERCENT = re.compile(r"%(?=[0-9{])") - -def _formats_of(style: Style) -> NumberingFormats: - """The numbering formats *style* gives (as ``Resolver._formats`` writes them into the tree).""" - enumeration = style.enumeration - return NumberingFormats( - scheme=style.numbering.scheme, - headings=() if style.numbering.scheme == "none" else heading_levels(style.numbering), - enumerated=tuple(enumeration.levels) if enumeration.enabled or enumeration.ordered == "enumerate" else (), - renumber=RENUMBERED if enumeration.ordered == "renumber" else None, - paragraphs=style.paragraphs.format if style.paragraphs.numbered else None, - reference=style.references.format, - ) +#: The setting a kind of abstractNum is made from, for a message. +_SETTING = {"headings": "numbering.levels", "enumerated": "enumeration.levels", "renumber": "enumeration.ordered", + "paragraphs": "paragraphs.format"} def docx_style_errors(style: Style) -> list[str]: """What in *style* Word's numbering cannot express, one message per problem naming the setting; empty when the style is fine for DOCX. - Word's ``lvlText`` has no way to write a ``%`` before a digit, and a level's text is at most + Word's ``lvlText`` has no way to write a ``%`` before a digit or a counter, whether it stands in one template + or meets a counter where a label and the refs it shows are put together, and a level's text is at most :data:`MAX_LVL_TEXT` characters. These are problems of configuration, so a DOCX render calls this before - rendering and raises :class:`~legaldown_render.errors.StyleError` (ADR 0006); :func:`plan_docx` checks again.""" + rendering (``style.docx.docx_settings``) and raises :class:`~legaldown_render.errors.StyleError` (ADR 0006).""" problems: list[str] = [] - formats = _formats_of(style) + formats = formats_of(style) templates: list[tuple[str, LevelFormat]] = [] if formats.headings: templates += [(f"numbering.levels[{i}]", f) for i, f in enumerate(style.numbering.levels[:5])] @@ -249,16 +244,23 @@ def docx_style_errors(style: Style) -> list[str]: for path, level in templates: for name, template in (("label", level.label), ("ref", level.ref)): if _AMBIGUOUS_PERCENT.search(template): - problems.append( - f"{path}.{name}: a '%' before a digit or a field cannot be written in a Word numbering level " - f"(DOCX); write it without the '%'") + problems.append(f"{path}.{name}: a '%' before a digit or a field cannot be written in a Word " + f"numbering level (DOCX); write it without the '%'") if problems: return problems - for kind, levels in _definitions(formats, set()).items(): + unwritable: list[tuple[str, int]] = [] + definitions = _definitions(formats, unwritable) + first_level: dict[str, int] = {} + for kind, level in unwritable: + first_level.setdefault(kind, level) + for kind, level in first_level.items(): + problems.append(f"{_SETTING[kind]}: a '%' meets a digit or a counter where the label and the refs of level " + f"{level + 1} are put together, which a Word numbering level cannot write (DOCX)") + for kind, levels in definitions.items(): for level in levels: if len(level.lvl_text) > MAX_LVL_TEXT: - problems.append(f"the {kind} numbering level {level.ilvl + 1} is longer than {MAX_LVL_TEXT} " - f"characters in Word (DOCX): shorten its label or the refs it shows") + problems.append(f"{_SETTING.get(kind, kind)}: the numbering level {level.ilvl + 1} is longer than " + f"{MAX_LVL_TEXT} characters in Word (DOCX): shorten its label or the refs it shows") return problems @@ -273,26 +275,22 @@ def docx_style_errors(style: Style) -> list[str]: _CHARACTER = 113 -def _num_fmt(counter: str, unmapped: set[str]) -> str: - """The ``numFmt`` of *counter*; a counter Word has no format for is written as ``decimal`` and noted.""" - found = NUM_FMT.get(counter) - if found is None: - unmapped.add(counter) - return "decimal" - return found - - def _level(ilvl: int, num_fmt: str, parts: LevelParts, *, p_style: str | None = None, - widths: Mapping[int, str] | None = None) -> LevelDef: + widths: Mapping[int, str] | None = None, unwritable: list[tuple[str, int]] | None = None, + kind: str = "") -> LevelDef: """One level. Headings (*widths*: the ``numFmt`` of each level a ``%k`` names) get an indent wide enough for their widest label; items step one centimetre per level. A label with a word in it (``Article {n}``) is - followed by a space, any other by a tab.""" + followed by a space, any other by a tab. With *unwritable*, parts a level text cannot write are noted there + (and written as nothing) instead of raising.""" words = any(isinstance(part, str) and any(c.isalpha() for c in part) for part in parts) if widths is None: left, hanging = _STEP * (ilvl + 1), _STEP else: sample = sum(len(part) if isinstance(part, str) else len(SAMPLE_COUNTER[widths[part]]) for part in parts) left = hanging = max(_STEP, _CHARACTER * (sample + 2)) + if unwritable is not None and level_text_problem(parts) is not None: + unwritable.append((kind, ilvl)) + parts = () return LevelDef(ilvl, num_fmt, lvl_text(parts), p_style=p_style, suff="space" if words else "tab", left=left, hanging=hanging) @@ -302,7 +300,8 @@ def _neutral(ilvl: int) -> LevelDef: return LevelDef(ilvl, "decimal", f"%{ilvl + 1}", left=_STEP * (ilvl + 1), hanging=_STEP) -def _definitions(formats: NumberingFormats, unmapped: set[str]) -> dict[str, list[LevelDef]]: +def _definitions(formats: NumberingFormats, unwritable: list[tuple[str, int]] | None = None + ) -> dict[str, list[LevelDef]]: """The nine levels of every abstractNum *formats* needs, by kind. A label that needs a number of the section (``{section}``, or ``{path}`` of an item or paragraph in one) is written without it, as for a preamble item: Word shows only counters of the list itself in a level.""" @@ -310,37 +309,34 @@ def _definitions(formats: NumberingFormats, unmapped: set[str]) -> dict[str, lis out: dict[str, list[LevelDef]] = {} if formats.headings: refs = [f.ref for f in formats.headings] - fmts = {j: _num_fmt(f.counter, unmapped) for j, f in enumerate(formats.headings)} + fmts = {j: NUM_FMT[f.counter] for j, f in enumerate(formats.headings)} levels = [_level(k, fmts[k], level_text(f.label, refs, k, section=None), p_style=f"Heading{k + 1}", - widths=fmts) for k, f in enumerate(formats.headings)] + widths=fmts, unwritable=unwritable, kind="headings") + for k, f in enumerate(formats.headings)] out["headings"] = [*levels, *(_neutral(k) for k in range(5, 9))] if formats.enumerated: size = len(formats.enumerated) refs = [formats.enumerated[j % size].ref for j in range(9)] out["enumerated"] = [ - _level(d, _num_fmt(formats.enumerated[d % size].counter, unmapped), - level_text(formats.enumerated[d % size].label, refs, d, section=(), textual=textual)) + _level(d, NUM_FMT[formats.enumerated[d % size].counter], + level_text(formats.enumerated[d % size].label, refs, d, section=(), textual=textual), + unwritable=unwritable, kind="enumerated") for d in range(9)] if formats.renumber is not None: f = formats.renumber - out["renumber"] = [_level(d, _num_fmt(f.counter, unmapped), - level_text(f.label, [f.ref] * 9, d, section=(), textual=textual)) - for d in range(9)] + out["renumber"] = [_level(d, NUM_FMT[f.counter], + level_text(f.label, [f.ref] * 9, d, section=(), textual=textual), + unwritable=unwritable, kind="renumber") for d in range(9)] if formats.paragraphs is not None: f = formats.paragraphs out["paragraphs"] = [ - _level(0, _num_fmt(f.counter, unmapped), level_text(f.label, [f.ref], 0, section=(), textual=textual)), + _level(0, NUM_FMT[f.counter], level_text(f.label, [f.ref], 0, section=(), textual=textual), + unwritable=unwritable, kind="paragraphs"), *(_neutral(k) for k in range(1, 9))] out["bullet"] = [LevelDef(d, "bullet", _BULLETS[d % 3], left=_STEP * (d + 1), hanging=_STEP) for d in range(9)] return out -def _needs_section(level: LevelFormat) -> bool: - """True if *level*'s label shows a number of the section: ``{section}``, or ``{path}`` (which holds it) of an - item or paragraph in a section.""" - return "{section}" in level.label or "{path}" in level.label or "{section}" in level.ref - - def _reads(level: LevelDef, levels: Sequence[LevelDef]) -> str: """*level* as it reads with every counter at 1, for a message.""" return "".join(word_counter(1, levels[part].num_fmt) if isinstance(part, int) else part @@ -368,12 +364,20 @@ class _Placed(NamedTuple): abstract: _Abstract +class _Drift: + """A run of units of one level that Word counts off from the resolver by the same amount.""" + + def __init__(self, node: _Unit, kind: str, step: Step, delta: int) -> None: + self.node, self.kind, self.step, self.delta, self.more = node, kind, step, delta, 0 + + class _Planner: def __init__(self, tree: RenderTree, file: str | None) -> None: self.tree, self.file = tree, file or "" self.formats = tree.formats self.textual = tree.formats.scheme == "none" self.diagnostics: list[Diagnostic] = [] + self.found: dict[tuple[str, str, str], int] = {} self.sim = WordNumbering() self.abstracts: list[_Abstract] = [] self.nums: list[Num] = [] @@ -384,34 +388,48 @@ def __init__(self, tree: RenderTree, file: str | None) -> None: self.paragraph_nums: dict[str, Num] = {} self.flagged: set[str] = set() self.reported: set[tuple[str, object]] = set() + self.drifts: list[_Drift] = [] + self.open_drift: dict[tuple[int, int], _Drift] = {} self.heading_num: Num | None = None + self.bullet_num: Num | None = None self.depth_reported = False - self.unmapped: set[str] = set() - self.definitions = _definitions(self.formats, self.unmapped) + self.definitions = _definitions(self.formats) self.by_kind: dict[str, _Abstract] = {} # -- registry ------------------------------------------------------------------- def abstract(self, kind: str) -> _Abstract: - """The abstractNum of *kind*, made on first use (so ids follow the order of use).""" + """The abstractNum of *kind*, made on first use (so ids follow the order of use). Only headings can skip a + level (the renderer counts it as 1, as LibreOffice does); in any other list a level that nothing numbers + is read as Word reads it, 0, so that a label that needs it is reported.""" found = self.by_kind.get(kind) if found is None: found = _Abstract(len(self.abstracts), kind, self.definitions[kind]) self.abstracts.append(found) self.by_kind[kind] = found - self.sim.add_abstract(found.id, found.levels) + self.sim.add_abstract(found.id, found.levels, skipped="lo" if kind == "headings" else "word") return found def new_num(self, abstract: _Abstract, ilvl: int | None = None, first: int = 1) -> Num: - """A num on *abstract*; with *ilvl*, its list starts there at *first* and deeper levels at 1.""" - overrides = () if ilvl is None else tuple((i, first if i == ilvl else 1) for i in range(ilvl, 9)) - num = Num(len(self.nums) + 1, abstract.id, overrides) + """A num on *abstract*; with *ilvl*, its list starts there at *first* (the deeper levels restart by + themselves when that level counts).""" + num = Num(len(self.nums) + 1, abstract.id, () if ilvl is None else ((ilvl, first),)) self.nums.append(num) self.sim.add_num(num) return num def report(self, rule: str, level: str, message: str, line: int | None) -> None: - record(Diagnostic(rule=rule, level=level, message=message, line=line, file=self.file), self.diagnostics) + """Record a finding; the same one (rule, level, message) again is one, at the lowest line.""" + key = (rule, level, message) + index = self.found.get(key) + found = Diagnostic(rule=rule, level=level, message=message, line=line, file=self.file) + if index is None: + self.found[key] = len(self.diagnostics) + self.diagnostics.append(found) + else: + known = self.diagnostics[index] + if line is not None and (known.line is None or line < known.line): + self.diagnostics[index] = found # -- the pass ------------------------------------------------------------------- @@ -424,10 +442,8 @@ def run(self) -> tuple[DocxPlan, list[Diagnostic]]: self.section(section) for block in section.blocks: self.block(block, section) - for counter in sorted(self.unmapped): - self.report("render-docx-counter", "warning", - f"The counter '{printable(counter)}' has no Word number format; it is written as decimal.", - None) + for drift in self.drifts: + self.report_drift(drift) abstracts = tuple( AbstractNum(a.id, a.kind, f"{0x4C440000 + a.id:08X}", # type: ignore[arg-type] "multilevel" if a.kind == "headings" else "hybridMultilevel", tuple(a.levels)) @@ -463,25 +479,34 @@ def ancestor_flagged(self, unit: str) -> bool: parent = self.parent.get(parent) return False - def check(self, node: _Unit, kind: str, step: Step, counter: str, group: object, cause: tuple[str, ...]) -> None: - """Compare what Word gives *node* with what the resolver gave it, and report the first difference of a - chain (a unit with a flagged ancestor is not reported again). Word's count must be the resolver's; the - counter must read the same (letters past 26 and Roman numerals from 4000 do not); and the whole label must - read the same, unless a diagnostic already says why it cannot (an unsupported label, a clamped depth).""" + def check(self, node: _Unit, kind: str, step: Step, counter: str, group: object, cause: tuple[str, ...], + key: tuple[int, int]) -> None: + """Compare what Word gives *node* with what the resolver gave it. Word's count must be the resolver's (a run + of units off by the same amount is one finding, :meth:`report_drift`); the counter must read the same + (letters past 26 and Roman numerals from 4000 do not); and the whole label must read the same unless a + diagnostic already says why it cannot (a label that needs the section, a clamped depth). A unit whose + ancestor already differs is not reported again. *counter* is the one the level Word uses is written in.""" assert node.unit is not None and node.number is not None fresh = not self.ancestor_flagged(node.unit) expected = format_counter(node.number, counter) word = kind[0].upper() + kind[1:] - if step.count == node.number and "clamped" in cause: - return # a level 9 in the format of level 9: the clamp is the reason, and it is reported - if step.count == node.number and step.own == expected: - if step.label == node.label or "label-fallback" in cause: + if step.count != node.number: + self.flagged.add(node.unit) + if fresh: + delta = step.count - node.number + drift = self.open_drift.get(key) + if drift is not None and drift.delta == delta: + drift.more += 1 + else: + drift = self.open_drift[key] = _Drift(node, kind, step, delta) + self.drifts.append(drift) + return + self.open_drift.pop(key, None) + if step.own == expected: + if step.label == node.label or "label-fallback" in cause or "clamped" in cause: return rule = "render-docx-number-mismatch" message = f"Word will number the {kind} '{printable(node.label or '')}' as '{printable(step.label)}'." - elif step.count != node.number: - rule = "render-docx-number-mismatch" - message = f"Word will number the {kind} '{printable(node.label or '')}' as '{printable(step.label)}'." elif counter.endswith("roman") and node.number >= 4000: rule = "render-docx-roman-overflow" message = (f"{word} {node.number} is a Roman numeral of 4000 or more: the renderer shows digits " @@ -501,13 +526,25 @@ def check(self, node: _Unit, kind: str, step: Step, counter: str, group: object, self.reported.add((rule, group)) self.report(rule, "warning", message, node.line) - def fallback_label(self, level: LevelFormat, family: str, kind: str, line: int | None) -> None: - """Report a label that needs the section's number where Word can show only the list's own counters.""" + def report_drift(self, drift: _Drift) -> None: + """One finding for a run of units that Word counts off by the same amount: the first, and how many follow.""" + node, step = drift.node, drift.step + message = (f"Word will number the {drift.kind} '{printable(node.label or '')}' as '{printable(step.label)}', " + f"{abs(drift.delta)} {'later' if drift.delta > 0 else 'earlier'} than the renderer") + if drift.more: + message += f"; so will the following {drift.more}" + self.report("render-docx-number-mismatch", "warning", message + ".", node.line) + + def fallback_label(self, level: LevelFormat, family: str, ilvl: int, kind: str, line: int | None) -> None: + """Report a label that shows the section's number where Word can show only the list's own counters.""" levels = self.definitions[family] self.report("render-docx-section-label", "warning", f"The label '{printable(level.label)}' of {kind}s in a section shows the section's number, " - f"which Word can show only for headings: in Word it reads '{printable(_reads(levels[0], levels))}'.", - line) + f"which Word can show only for headings: in Word it reads " + f"'{printable(_reads(levels[ilvl], levels))}'.", line) + + def shared_counter(self, message: str, line: int | None) -> None: + self.report("render-docx-shared-counter", "warning", message, line) # -- units ---------------------------------------------------------------------- @@ -521,16 +558,13 @@ def section(self, section: Section) -> None: return ilvl = section.level - 1 parent = section.parent_unit - cause: tuple[str, ...] = () - if (parent is None and section.level > 1) or (parent is not None - and self.section_level[parent] + 1 < section.level): - cause = ("skipped-level",) - self.report("render-docx-heading-skip", "warning", - f"Heading level skipped before '{printable(plain_text(section.title))}': the number shows " - f"the skipped level as 1. Word may number the headings after it differently.", section.line) + skipped = (parent is None and section.level > 1) or ( + parent is not None and self.section_level[parent] + 1 < section.level) + cause = ("skipped-level",) if skipped else () step = self.sim.step(self.heading_num.id, ilvl, unit) self.add(section, "section", self.heading_num, ilvl, explicit=False, cause=cause, step=step) - self.check(section, "heading", step, self.formats.headings[ilvl].counter, ("heading", section.level), cause) + self.check(section, "heading", step, self.formats.headings[ilvl].counter, ("heading", section.level), cause, + (0, ilvl)) def paragraph(self, paragraph: Paragraph, section: Section) -> None: assert paragraph.unit is not None and section.unit is not None @@ -539,28 +573,37 @@ def paragraph(self, paragraph: Paragraph, section: Section) -> None: if paragraph.number is None or fmt is None: self.add(paragraph, "paragraph", None, None, explicit=False) return + abstract = self.abstract("paragraphs") num = self.paragraph_nums.get(section.unit) if num is None: - num = self.paragraph_nums[section.unit] = self.new_num(self.abstract("paragraphs"), 0) + num = self.paragraph_nums[section.unit] = self.new_num(abstract, 0) cause: tuple[str, ...] = () - if _needs_section(fmt): + if shows_section(fmt.label, [fmt.ref], 0, textual=self.textual): cause = ("label-fallback",) - self.fallback_label(fmt, "paragraphs", "paragraph", paragraph.line) + self.fallback_label(fmt, "paragraphs", 0, "paragraph", paragraph.line) step = self.sim.step(num.id, 0, paragraph.unit) self.add(paragraph, "paragraph", num, 0, explicit=True, cause=cause, step=step) - self.check(paragraph, "paragraph", step, fmt.counter, ("paragraphs", section.unit), cause) + self.check(paragraph, "paragraph", step, fmt.counter, ("paragraphs", section.unit), cause, (abstract.id, 0)) def list_(self, lst: List, section: Section | None) -> None: first = lst.items[0] + if lst.family == "bullet": + self.bullets(lst, section) + return + fmt = lst.format + assert fmt is not None abstract = self.abstract(lst.family) ilvl = min(lst.depth, 8) + # The counter of the level Word uses: past level 9 that is level 9's, not the resolver's cycle. + levels = self.formats.enumerated if lst.family == "enumerated" else (fmt,) + counter = levels[min(lst.depth, 8) % len(levels)].counter parent = first.parent_unit placed = self.placed.get(parent) if parent is not None else None cause: tuple[str, ...] = () # Nested in an item: that item's num, if it is of the same kind and Word counts the list from where the # resolver does (a sibling list continues, a list in a new item restarts); otherwise a num of its own. if (placed is not None and placed.abstract is abstract and lst.depth <= 8 - and (lst.family == "bullet" or self.sim.peek(placed.num.id, ilvl) == first.number)): + and self.sim.peek(placed.num.id, ilvl) == first.number): num = placed.num else: num = self.new_num(abstract, ilvl, first.number or 1) @@ -571,6 +614,12 @@ def list_(self, lst: List, section: Section | None) -> None: self.report("render-docx-list-depth", "warning", f"A list is nested {lst.depth} levels deep: Word has 9. Items from level 10 on are " f"numbered in the format of level 9 and restart by fixed values.", first.line) + elif placed is not None and first.number > 1: + cause = ("resync",) + self.shared_counter( + "A list that goes on from an earlier list in the same item (the resolver counts them as one) " + "starts at a fixed number in Word: it will not follow items added or removed before it.", + first.line) elif placed is not None: kinds = placed.abstract is not abstract cause = ("restart-family",) if kinds else ("resync",) @@ -579,35 +628,63 @@ def list_(self, lst: List, section: Section | None) -> None: "not by structure: an item added above it in Word will not restart it." if kinds else "A list that Word would continue from another list in the same item restarts by a fixed " "setting (Restart at 1), not by structure.", first.line) - fmt = lst.format - qualified = fmt is not None and section is not None and _needs_section(fmt) - if fmt is not None and qualified: - self.fallback_label(fmt, lst.family, "list item", first.line) + refs = ([f.ref for f in (self.formats.enumerated[j % len(self.formats.enumerated)] + for j in range(lst.depth + 1))] + if lst.family == "enumerated" else [fmt.ref] * (lst.depth + 1)) + qualified = section is not None and shows_section(fmt.label, refs, lst.depth, textual=self.textual) + if qualified: + self.fallback_label(fmt, lst.family, ilvl, "list item", first.line) for index, item in enumerate(lst.items): assert item.unit is not None self.parent[item.unit] = item.parent_unit - item_cause = (cause if index == 0 else ()) + (("label-fallback",) if qualified else ()) - if fmt is None: - self.units[item.unit] = UnitPlan(item.unit, "item", num.id, ilvl, True, None, None, "", "", None, - item_cause, (), None) - else: - step = self.sim.step(num.id, ilvl, item.unit) - self.add(item, "item", num, ilvl, explicit=True, cause=item_cause, step=step) - self.check(item, "list item", step, fmt.counter, ("list", first.unit), item_cause) + item_cause = cause if index == 0 else () + if lst.depth > 8 and "clamped" not in item_cause: + item_cause += ("clamped",) + if qualified: + item_cause += ("label-fallback",) + # Another list used Word's counter of this level in between: restart it where the resolver is. + if index and item.shares_number_with is None and self.sim.peek(num.id, ilvl) != item.number: + num = self.new_num(abstract, ilvl, item.number) + item_cause += ("resync",) + self.shared_counter( + "Word's counter for a level of this list was also used by another list in between; the numbers " + "are kept as the renderer shows them by restarting at a fixed number, which will not follow " + "items added or removed before it.", item.line) + step = self.sim.step(num.id, ilvl, item.unit) + if any(isinstance(token, Count) and isinstance(token.owner, Ghost) for token in step.tokens): + # The label shows a level of its list that nothing numbers here: Word shows it as 0, and the + # resolver's label is that of the enclosing item of another kind. Not left to Word's reading. + item_cause += ("label-fallback",) + self.report("render-docx-section-label", "warning", + f"The label '{printable(fmt.label)}' of list items shows the number of an enclosing item " + f"that Word cannot show here (the list is not nested in one of its own kind): in Word " + f"it reads '{printable(step.label)}'.", item.line) + self.add(item, "item", num, ilvl, explicit=True, cause=item_cause, step=step) + self.check(item, "list item", step, counter, ("list", first.unit), item_cause, (abstract.id, ilvl)) self.placed[item.unit] = _Placed(num, abstract) for child in item.blocks: self.block(child, section) + def bullets(self, lst: List, section: Section | None) -> None: + """Bullets count nothing, so every bullet list of the document is numbered by one num.""" + abstract = self.abstract("bullet") + if self.bullet_num is None: + self.bullet_num = self.new_num(abstract) + ilvl = min(lst.depth, 8) + for item in lst.items: + assert item.unit is not None + self.parent[item.unit] = item.parent_unit + cause = ("clamped",) if lst.depth > 8 else () + self.units[item.unit] = UnitPlan(item.unit, "item", self.bullet_num.id, ilvl, True, None, None, "", "", + None, cause, (), None) + self.placed[item.unit] = _Placed(self.bullet_num, abstract) + for child in item.blocks: + self.block(child, section) + -def plan_docx(tree: RenderTree, style: Style, *, file: str | None = None) -> tuple[DocxPlan, list[Diagnostic]]: +def plan_docx(tree: RenderTree, *, file: str | None = None) -> tuple[DocxPlan, list[Diagnostic]]: """The numbering plan of *tree* for Word, and the diagnostics only planning can produce (all - ``render-docx-*``, in the order of their lines, naming *file*). Pure: the same tree and style give an equal plan. - *style* is the one the tree was resolved under. Raises :class:`~legaldown_render.errors.StyleError` for a style - Word's numbering cannot express (:func:`docx_style_errors`; a render checks that first, ADR 0006).""" - problems = docx_style_errors(style) - if problems: - raise StyleError(f"'{style.name}'", problems) - if tree.formats != _formats_of(style): - raise InternalError("the tree and the style given to the DOCX planner do not belong together. This is a bug; " - "please report it with the document that triggers it.") + ``render-docx-*``, in the order of their lines, naming *file*). Pure: the same tree gives an equal plan, and + ``tree.formats`` carries the style's numbering. A style Word's numbering cannot express (:func:`docx_style_errors`) + is refused before rendering (ADR 0006), so it never reaches here.""" return _Planner(tree, file).run() diff --git a/src/legaldown_render/style/docx.py b/src/legaldown_render/style/docx.py index 93bdbcb..d4b32cf 100644 --- a/src/legaldown_render/style/docx.py +++ b/src/legaldown_render/style/docx.py @@ -20,6 +20,7 @@ has no use for), ``rgb()``, ``hsl()``, and the sixteen basic names with ``orange``; any other name is an error asking for a hexadecimal colour. * The first family of ``typography.font_family``. +* The numbering labels, which Word's level text must be able to write (``resolve.docx_plan.docx_style_errors``). A message names the setting and what is wrong with it and never quotes the value, so it is safe to show for a style from somebody else. @@ -32,6 +33,7 @@ from dataclasses import dataclass from ..errors import StyleError +from ..resolve.docx_plan import docx_style_errors from .model import Style #: A browser starts from 16 px; the size of text written in ``em`` or ``rem`` @@ -199,6 +201,7 @@ def docx_settings(style: Style, *, trusted: bool = True) -> DocxSettings: half_points=round((size if size is not None else font_points) * 2), bold=bold, italic=heading.italic, caps=heading.transform == "uppercase", small_caps=heading.transform == "small-caps", align=heading.align) + problems.items.extend(docx_style_errors(style)) if problems.items: raise StyleError(f"'{style.name}'" if trusted else "'untrusted style'", problems.items) return DocxSettings( From 673287609c13536c4f77b851d2d1b807d97c7e72 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:48:02 +0000 Subject: [PATCH 082/118] Check the DOCX numbering plan unit by unit The property test no longer lets a wrong placement explain itself: for every numbered unit Word's simulated count must be the resolver's number and its label the resolver's label, unless that unit's own counter reads differently in Word, the plan notes why its label differs, or its depth is past Word's levels; number-mismatch must not occur without an alternative that shares a number. It runs over deep lists, drafting notes, alternatives and labels with words, symbols and paths, and over mutated placements it fails. Tests for clamped depths, runs of units counted off, the numId 0 simulation, Word's counters against LibreOffice's, and the style errors through docx_settings. --- docs/architecture.md | 62 +++--- tests/test_docx_plan.py | 412 +++++++++++++++++++++++++---------- tests/test_docx_simulator.py | 48 +++- 3 files changed, 381 insertions(+), 141 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index c3047af..30a6440 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -244,44 +244,50 @@ references show; the Warning says so. Every other heading reads exactly as its d ### The DOCX planner (`resolve/docx_plan.py`) -The DOCX writer will contain no numbering rules either (ADR 0003). Between the resolved tree and that writer sits a -pure function, `plan_docx(tree, style, *, file=None) -> (DocxPlan, diagnostics)`, which maps the numbering structure -above to what Word needs: the abstractNums and nums, and for every section, list item and numbered paragraph the -`(numId, ilvl)` its paragraph takes. It is a planning step, not a stage of the pipeline: nothing calls it yet. It -reads `tree.formats` and the units' `number`, `parent_unit` and `List.family`/`depth`/`format`, and checks every -placement against the number the resolver gave the unit by simulating Word's counters -(`resolve/docx_numbering.py`), so that what Word draws is what the resolver numbered, or a diagnostic says why not. +The DOCX writer will contain no numbering rules either (ADR 0003, [ADR 0008](decisions/0008-docx-word-native.md)). +Between the resolved tree and that writer sits a pure function, `plan_docx(tree, *, file=None) -> (DocxPlan, +diagnostics)`, which maps the numbering structure above to what Word needs: the abstractNums and nums, and for every +section, list item and numbered paragraph the `(numId, ilvl)` its paragraph takes. It reads only `tree.formats` and the +units' `number`, `parent_unit` and `List.family`/`depth`/`format` (`numbering.formats_of(style)` builds `formats` for the +resolver and for the style check), and checks every placement against the number the resolver gave the unit by +simulating Word's counters (`resolve/docx_numbering.py`), so that what Word draws is what the resolver numbered, or a +diagnostic says why not. It is a planning step, not a stage of the pipeline: the writer does not call it yet. The one +part that runs on every DOCX render is the style check, `docx_style_errors(style)`, which `style/docx.py:docx_settings` +runs before rendering (ADR 0006). **Word's list semantics**, stated once (the planner and its diagnostics rely on them): - Counters belong to the abstractNum and are shared by every `w:num` of it. A num with a `startOverride` restarts the named level at the first use of that num, and the levels above keep their values: this is Word's own - "Restart at 1". -- A paragraph restarts every deeper level. A paragraph with `numId 0` counts nothing. + "Restart at 1". A level that nothing numbered before a deeper one is read by Word as 0 (the renderer counts a + skipped heading level as 1, and so does LibreOffice). +- A paragraph restarts every deeper level. A paragraph with `numId 0` counts nothing and restarts nothing. - A list nested in an item of the same kind goes on a deeper level of its parent's num, so it restarts by structure when a parent item is added. A list of another kind, a list in a quote, and a list Word would continue - where the resolver restarts have a num of their own that restarts by a fixed setting: an item added above one in - Word does not restart it. A sibling list under the same item that the resolver continues (its first item is 3) - shares the num, so that Word continues it too. + where the resolver restarts have a num of their own: it restarts at 1 by a fixed setting (an item added above it + in Word does not restart it). A list that the resolver continues (its first item is 3) but Word cannot continue + starts at that number by a fixed setting, which does not follow items added or removed before it. +- A bullet counts nothing: one num serves every bullet list. - Paragraph numbering has one num per section, and a section added in Word does not restart it by itself. - `REF \n` returns the paragraph's level text with its counters, minus trailing periods. **What is planned.** Heading numbering (one multilevel abstractNum linked to the heading styles, the label templates turned into `lvlText`, the counters into `numFmt`), list families (enumerated, renumbered, bullets) with their nesting -and restarts, depth beyond Word's nine levels (clamped), and numbered paragraphs. A label may show only counters of -its own list, so a label that needs the section's number (`{section}`, or the `{path}` of an item or paragraph in a -section: the continental style's `5.1`) is written without it, as for a preamble item, and reported -(`render-docx-section-label`). `docx_style_errors(style)` names what Word's numbering cannot express at all (a `%` -before a digit, a level text over 255 characters); a DOCX render calls it before rendering and raises `StyleError` -(ADR 0006), and `plan_docx` checks again. - -**Diagnostics** (all `render-docx-*`, with the unit's line): `heading-skip`, `section-label`, `list-depth`, +and restarts, depth beyond Word's nine levels (clamped to level 9 in that level's format), and numbered paragraphs. +A level shows only counters of its own list. A label whose filled-in form shows the section's number (`{section}`, +or the `{path}` of an item or paragraph in a section: the continental style's `5.1`), or a number of an enclosing +item that Word cannot show (a list nested in another kind of list, or in a quote), is written without it and +reported (`render-docx-section-label`). Alternatives that share a number are not planned yet (they are numbered in +sequence and reported). `UnitPlan.ind`, the references, the bookmarks and the table of contents are further parts of +the plan that this part leaves unset. + +**Diagnostics** (all `render-docx-*`, with the unit's line, once per finding): `section-label`, `list-depth`, `letter-overflow` (Word repeats letters after 26, the renderer counts on: `aa`, `ab`), `roman-overflow` (4000 and -beyond), `number-mismatch` (any other difference, which must not occur without a reason: alternatives that share a -number are not yet planned and are reported this way), `counter` (a counter Word has no format for), and, as -information, `static-restart` (a list that restarts by a fixed setting). The same finding is reported once. - -References, bookmarks and the table of contents are further parts of the plan, empty so far. +beyond), `shared-counter` (a list that goes on from another, or whose level another list used in between, starts at +a fixed number), `number-mismatch` (any other difference: the first unit of a run that Word counts off by the same +amount, with how many follow; it does not occur without an alternative that shares a number), and, as information, +`static-restart` (a list that restarts at 1 by a fixed setting). A heading level that is skipped is the Core Error +`heading-skip`; the plan only notes it on the unit. ### 5. Write (`writers/`) @@ -308,8 +314,8 @@ src/legaldown_render/ ├── build.py stage 3 ├── resolve/ │ ├── resolver.py stage 4 -│ ├── numbering.py counters, the §13.1 scheme presets, label templates -│ ├── docx_plan.py the DOCX plan: Word numbering for the resolved tree (planning step, not yet wired) +│ ├── numbering.py counters, the §13.1 scheme presets, label templates, the formats a style gives +│ ├── docx_plan.py the DOCX plan: Word numbering for the resolved tree (not yet called by the writer), and the style check DOCX runs before rendering │ ├── docx_numbering.py Word's counters, label templates as tokens, lvlText, the numbering simulator │ └── values.py Babel formatting of dates, money, durations ├── style/ @@ -516,7 +522,7 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_values.py` | Money, dates and durations in a locale: exact digits, plural of the value shown, currency display, CLDR fallback to English | | `tests/test_numbering.py` | Counter formats, label templates and the preamble's `{section}` | | `tests/test_numbering_structure.py` | The numbering structure in the tree (`formats`, unit ids, numbers, parents, alternatives, list families, cross-reference units): headings in each scheme and with skipped levels, nested lists, section-qualified items, numbered paragraphs, alternatives, the `none` scheme and its second pass; and that every label, designation and reference can be written again from the structure alone, over the tests' and the specification's documents | -| `tests/test_docx_plan.py` | The DOCX numbering plan: abstractNums per style and scheme, heading levels, top-level, nested, sibling, cross-family, quoted and clamped lists, numbered paragraphs, labels that need the section, letter and Roman overflow, diagnostics (lines, file, order, repeats), style errors, determinism; and that every numbered unit's simulated Word label is the resolver's or a diagnostic says why not, over the tests' and the specification's documents under each style and list setting, and over random documents | +| `tests/test_docx_plan.py` | The DOCX numbering plan: abstractNums per style and scheme, heading levels, top-level, nested, sibling, cross-family, quoted and clamped lists, numbered paragraphs, labels that need the section, letter and Roman overflow, diagnostics (lines, file, order, repeats, runs of units counted off), style errors (also through `docx_settings`), determinism; and, unit by unit, that Word's simulated count and label are the resolver's unless the unit's own counter reads differently in Word or the plan notes why, over the tests' and the specification's documents under each style and list setting, and over random documents and alternatives | | `tests/test_docx_simulator.py` | Word's counters, label templates as tokens (equal to filling the template as text), `lvlText`, and the simulator: restarts, start overrides, shared counters, skipped levels, `peek`/`current` | | `tests/test_build.py` | Building the tree: comment-only items and quotes, emphasis next to a directive, spacing where an inline comment was dropped | | `tests/test_terms.py` | What `{{term:}}` shows, references inside terms, terms that name themselves or each other, the cap on a term text | diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index 584e7a3..0d635ec 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -3,6 +3,7 @@ as the resolver did, or a diagnostic says why not. Everything is checked on the plan's data; no file is written.""" from __future__ import annotations +import functools import random from collections.abc import Iterator from pathlib import Path @@ -12,14 +13,11 @@ from legaldown_render import DocumentError, render from legaldown_render.errors import StyleError -from legaldown_render.resolve.docx_numbering import NUM_FMT -from legaldown_render.resolve.docx_plan import ( - DocxPlan, - _formats_of, - docx_style_errors, - plan_docx, -) +from legaldown_render.resolve.docx_numbering import NUM_FMT, word_counter +from legaldown_render.resolve.docx_plan import DocxPlan, docx_style_errors, plan_docx +from legaldown_render.resolve.numbering import format_counter, formats_of from legaldown_render.style import load_style +from legaldown_render.style.docx import docx_settings from legaldown_render.tree import DraftingNote, List, ListItem, Paragraph, Quote, RenderTree, Section NONE = {"numbering.scheme": "none"} @@ -39,12 +37,31 @@ "continental-none": {"style": "continental", "overrides": NONE}, "enumerate-ordered": {"overrides": {"enumeration.ordered": "enumerate"}}, "bullets": {"overrides": {"enumeration.enabled": False}}, + # labels with words and symbols, a label that shows the path of an item, and a paragraph number with a symbol + "article": {"overrides": {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}, + {"counter": "decimal", "label": "{path}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}]}}, + "section-sign": {"overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "\u00a7 {n}", "ref": "{n}"}, + {"counter": "lower-alpha", "label": "{n})", "ref": "({n})"}, + {"counter": "lower-roman", "label": "({n})", "ref": "({n})"}]}}, + "path-items": {"overrides": {"enumeration.levels": [ + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n}.", "ref": "({n})"}]}}, + "path-items-continental": {"style": "continental", "overrides": {"enumeration.levels": [ + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n}.", "ref": "({n})"}]}}, + "paragraph-sign": {"overrides": {"paragraphs.numbered": True, "paragraphs.format": { + "counter": "decimal", "label": "\u00b6 {n}", "ref": "{n}"}}}, } def planned(body: str, front: str = FRONT, **settings: object): # noqa: ANN201 result = render(front + body, format="text", **settings) - plan, diagnostics = plan_docx(result.tree, result.style) + plan, diagnostics = plan_docx(result.tree) return result.tree, plan, diagnostics @@ -173,15 +190,6 @@ def test_the_none_scheme_has_no_heading_abstract_num_and_sections_are_unnumbered assert plan.nums[0].abstract_id == 0 -def test_a_counter_without_a_word_format_is_written_as_decimal(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delitem(NUM_FMT, "lower-alpha") - tree, plan, diagnostics = planned("# A\n\n- x\n- y\n") - assert plan.abstract_nums[1].levels[0].num_fmt == "decimal" - assert [plan.units[i.unit].label for i in items(tree)] == ["(1)", "(2)"] - assert [(d.rule, d.level) for d in diagnostics if d.rule == "render-docx-counter"] == [ - ("render-docx-counter", "warning")] - - # -- Headings ------------------------------------------------------------------------ @@ -192,13 +200,12 @@ def test_headings_take_the_style_num_at_their_level() -> None: (1, 0, False, "2.", "2", 2)] -def test_a_skipped_heading_level_is_numbered_as_the_resolver_does_and_reported() -> None: +def test_a_skipped_heading_level_is_numbered_as_the_resolver_does() -> None: + # heading-skip is a Core Error of the document: the plan says nothing more, and notes the cause on the unit. tree, plan, diagnostics = planned("# A\n\n### B\n\n## C\n") assert [plan.units[s].label for s in ("s1", "s2", "s3")] == ["1.", "1.1.1", "1.2"] assert plan.units["s2"].cause == ("skipped-level",) and plan.units["s3"].cause == () - [found] = [d for d in diagnostics if d.rule == "render-docx-heading-skip"] - assert found.level == "warning" and found.line == tree.sections[1].line and "'B'" in found.message - assert rules(diagnostics) == {"render-docx-heading-skip"} + assert diagnostics == [] def test_a_document_that_starts_below_level_one_skips_nothing() -> None: @@ -228,7 +235,7 @@ def test_every_top_level_list_has_a_num_that_starts_it_at_one() -> None: first_list, ordered = plan.units[items[0].unit], plan.units[items[-2].unit] assert (first_list.num_id, first_list.ilvl, first_list.explicit, first_list.label) == (2, 0, True, "(a)") assert (ordered.num_id, ordered.ilvl, ordered.label) == (3, 0, "1.") - assert plan.nums[1].start_overrides == tuple((i, 1) for i in range(9)) + assert plan.nums[1].start_overrides == ((0, 1),) assert plan.nums[2].abstract_id == 2 @@ -268,11 +275,13 @@ def test_a_sibling_list_of_another_kind_that_continues_gets_a_num_starting_at_it assert [item.number for item in ordered] == [1, 2, 3] assert [plan.units[item.unit].label for item in ordered] == ["1.", "2.", "3."] # Another kind of list in between: the second ordered list cannot go on from the first by structure, so its num - # starts at its number, 3. Static, and reported as information only. + # starts at its number, 3: a fixed start that will not follow items added before it, which is a warning (the + # first ordered list restarts at 1, which is information only). last = plan.units[ordered[-1].unit] - assert last.num_id != plan.units[ordered[0].unit].num_id and last.cause == ("restart-family",) - assert plan.nums[last.num_id - 1].start_overrides[0] == (1, 3) - assert {d.level for d in diagnostics} == {"info"} + assert last.num_id != plan.units[ordered[0].unit].num_id and last.cause == ("resync",) + assert plan.nums[last.num_id - 1].start_overrides == ((1, 3),) + assert [(d.rule, d.level) for d in diagnostics] == [ + ("render-docx-static-restart", "info"), ("render-docx-shared-counter", "warning")] def test_a_nested_list_of_another_kind_has_a_num_of_its_own_and_is_reported_as_information() -> None: @@ -280,7 +289,7 @@ def test_a_nested_list_of_another_kind_has_a_num_of_its_own_and_is_reported_as_i _, x, y, _, z = items(tree) assert shown(plan, x, y, z) == [(3, 1, "1."), (3, 1, "2."), (4, 1, "1.")] assert plan.units[x.unit].cause == ("restart-family",) - assert plan.nums[2].start_overrides[0] == (1, 1) and plan.nums[2].abstract_id == 2 + assert plan.nums[2].start_overrides == ((1, 1),) and plan.nums[2].abstract_id == 2 assert [(d.rule, d.level) for d in diagnostics] == [("render-docx-static-restart", "info")] assert diagnostics[0].line == tree.sections[0].blocks[0].items[0].blocks[1].items[0].line @@ -296,17 +305,22 @@ def test_a_list_in_a_quote_inside_an_item_is_numbered_from_the_section_with_a_nu # Word's counter of that level was used by the quote's list: it starts again where the resolver is, by a # fixed setting. assert shown(plan, x, y) == [(2, 1, "(i)"), (4, 1, "(ii)")] - assert plan.nums[3].start_overrides[0] == (1, 2) and plan.units[y.unit].cause == ("resync",) - assert [(d.rule, d.level) for d in diagnostics] == [("render-docx-static-restart", "info")] + assert plan.nums[3].start_overrides == ((1, 2),) and plan.units[y.unit].cause == ("resync",) + assert [(d.rule, d.level) for d in diagnostics] == [("render-docx-shared-counter", "warning")] -def test_bullets_are_numbered_paragraphs_with_a_bullet_of_their_own() -> None: - tree, plan, diagnostics = planned("# A\n\n- a\n - b\n", overrides={"enumeration.enabled": False}) - assert [a.kind for a in plan.abstract_nums] == ["headings", "bullet"] - first, second = items(tree) - assert shown(plan, first, second) == [(2, 0, ""), (2, 1, "")] and plan.units[first.unit].count is None - assert [level.lvl_text for level in plan.abstract_nums[1].levels[:4]] == ["•", "–", "◦", "•"] - assert diagnostics == [] +def test_bullets_count_nothing_so_one_num_serves_every_bullet_list() -> None: + body = "# A\n\n- a\n - b\n\n1. o\n - c\n\n# B\n\n- d\n" + tree, plan, diagnostics = planned(body, overrides={"enumeration.enabled": False}) + first, second, _, third, fourth = items(tree) + assert [a.kind for a in plan.abstract_nums] == ["headings", "bullet", "renumber"] + bullet = plan.abstract_nums[1] + assert shown(plan, first, second, third, fourth)[:2] == [(2, 0, ""), (2, 1, "")] + assert {plan.units[n.unit].num_id for n in (first, second, third, fourth)} == {2} + assert all(num.start_overrides == () for num in plan.nums if num.abstract_id == bullet.id) + assert plan.units[first.unit].count is None + assert [level.lvl_text for level in bullet.levels[:4]] == ["\u2022", "\u2013", "\u25e6", "\u2022"] + assert diagnostics == [] # bullets under an ordered list need no restart and no report def test_ordered_lists_that_are_enumerated_take_the_enumeration_levels() -> None: @@ -339,6 +353,36 @@ def test_a_list_deeper_than_word_has_levels_is_clamped_and_reported_once() -> No assert rules(diagnostics) == {"render-docx-list-depth"} +def test_items_of_lists_past_level_nine_keep_their_numbers_and_a_sibling_at_level_nine_is_restarted() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(9)) + body += "".join(" " * 9 + f"- nine {k}\n" for k in "abc") + body += "".join(" " * 10 + f"- ten {k}\n" for k in "ab") + body += " " * 8 + "- level 8 again\n" + tree, plan, diagnostics = planned(body) + found = items(tree) + nine, ten, again = found[9:12], found[12:14], found[14] + # every item of a clamped list is Word's level 9 in the format of level 9, with the resolver's count + assert [plan.units[i.unit].cause for i in (*nine, *ten)] == [("clamped",)] * 5 + assert [(plan.units[i.unit].ilvl, plan.units[i.unit].count) for i in (*nine, *ten)] == [ + (8, 1), (8, 2), (8, 3), (8, 1), (8, 2)] + assert [plan.units[i.unit].label for i in (*nine, *ten)] == ["(A)", "(B)", "(C)", "(A)", "(B)"] + assert [i.label for i in (*nine, *ten)] == ["(a)", "(b)", "(c)", "(i)", "(ii)"] # the resolver's cycle + assert len({plan.units[i.unit].num_id for i in (nine[0], ten[0])}) == 2 + # Word's counter of level 9 was used by the clamped lists: the item after them is restarted where it is + again_plan = plan.units[again.unit] + assert (again.number, again_plan.count, again_plan.cause) == (2, 2, ("resync",)) + assert plan.nums[again_plan.num_id - 1].start_overrides == ((8, 2),) + assert rules(diagnostics) == {"render-docx-list-depth", "render-docx-shared-counter"} + assert all(d.level == "warning" for d in diagnostics) + + +def test_a_list_that_goes_on_from_a_sibling_with_a_fixed_start_is_a_warning_not_a_restart() -> None: + _, _, diagnostics = planned("# A\n\n- a\n 1. x\n\n Between.\n\n - w\n\n 1. z\n") + assert [(d.rule, d.level) for d in diagnostics] == [ + ("render-docx-static-restart", "info"), ("render-docx-shared-counter", "warning")] + assert "Restart at 1" not in diagnostics[1].message + + # -- Paragraphs ---------------------------------------------------------------------- @@ -430,22 +474,23 @@ def test_an_alternative_that_shares_a_number_is_reported_while_it_is_numbered_in tree, plan, diagnostics = planned(body, ALTERNATIVES) assert tree.sections[1].shares_number_with == "s1" assert [plan.units[s].label for s in ("s1", "s2", "s3")] == ["1.", "2.", "3."] - assert [(d.rule, d.line) for d in diagnostics] == [ - ("render-docx-number-mismatch", tree.sections[1].line), ("render-docx-number-mismatch", tree.sections[2].line)] - assert "'1.'" in diagnostics[0].message and "'2.'" in diagnostics[0].message + # the second section and the one after it are off by the same amount: one finding + [found] = diagnostics + assert (found.rule, found.line) == ("render-docx-number-mismatch", tree.sections[1].line) + assert "'1.'" in found.message and "'2.'" in found.message and "the following 1" in found.message # -- Diagnostics: lines, files, order, repeats -------------------------------------------- def test_diagnostics_name_the_file_and_the_line_and_come_in_line_order() -> None: - result = render(FRONT + "# A\n\n### B\n\n- x\n 1. y\n", format="text") - plan, diagnostics = plan_docx(result.tree, result.style, file="deal.lgd") + result = render(FRONT + "# A\n\n- x\n 1. y\n\n- p\n- q\n" + "- r\n" * 30, format="text") + plan, diagnostics = plan_docx(result.tree, file="deal.lgd") assert [(d.rule, d.file) for d in diagnostics] == [ - ("render-docx-heading-skip", "deal.lgd"), ("render-docx-static-restart", "deal.lgd")] + ("render-docx-static-restart", "deal.lgd"), ("render-docx-letter-overflow", "deal.lgd")] lines = [d.line for d in diagnostics] assert lines == sorted(lines) and None not in lines - _, anonymous = plan_docx(result.tree, result.style) + _, anonymous = plan_docx(result.tree) assert {d.file for d in anonymous} == {""} @@ -455,9 +500,10 @@ def test_the_same_finding_is_reported_once() -> None: def test_untrusted_text_in_a_message_is_escaped() -> None: - _, _, diagnostics = planned("# A\n\n### B\u0007x\n") + overrides = {"enumeration.levels": [{"counter": "lower-alpha", "label": "{section}\u0007{n}", "ref": "({n})"}]} + _, _, diagnostics = planned("# A\n\n- x\n", overrides=overrides) [found] = diagnostics - assert "\u0007" not in found.message + assert found.rule == "render-docx-section-label" and "\u0007" not in found.message # -- Determinism and the plan's data ------------------------------------------------------- @@ -465,8 +511,8 @@ def test_untrusted_text_in_a_message_is_escaped() -> None: def test_the_same_tree_gives_an_equal_plan_and_the_plan_is_frozen() -> None: result = render(FRONT + NESTED, format="text", style="continental") - first = plan_docx(result.tree, result.style) - again = plan_docx(result.tree, result.style) + first = plan_docx(result.tree) + again = plan_docx(result.tree) assert first[0] == again[0] and first[1] == again[1] with pytest.raises(AttributeError): first[0].heading_num_id = 5 # type: ignore[misc] @@ -481,20 +527,13 @@ def test_unit_plans_carry_the_whole_plan_for_a_unit() -> None: assert plan.units["s1"].kind == "section" -def test_a_tree_and_a_style_that_do_not_belong_together_are_refused() -> None: - from legaldown_render.errors import InternalError - result = render(FRONT + "# A\n", format="text") - with pytest.raises(InternalError): - plan_docx(result.tree, load_style("continental")) - - # -- Style checks (ADR 0006) ---------------------------------------------------------------- def test_formats_of_a_style_are_the_formats_in_the_tree() -> None: for settings in VARIANTS.values(): result = render(FRONT + "# A\n", format="text", **settings) - assert _formats_of(result.style) == result.tree.formats + assert formats_of(result.style) == result.tree.formats def test_the_built_in_styles_are_fine_for_docx() -> None: @@ -514,8 +553,25 @@ def test_a_percent_that_lvl_text_cannot_write_is_a_style_error(key: str, value: style = load_style(overrides=overrides) problems = docx_style_errors(style) assert len(problems) == 1 and key.split(".")[0] in problems[0] and "'%'" in problems[0] + with pytest.raises(StyleError) as caught: # the DOCX settings step refuses it before anything is rendered + docx_settings(style) + assert caught.value.problems == problems + + +@pytest.mark.parametrize(("levels", "setting"), [ + # a ref ending in '%' meets the counter that starts the next level's ref once the label is put together + ([{"counter": "decimal", "label": "{path}", "ref": "{n}%"}, {"counter": "decimal", "label": "{path}", "ref": "{n}"}], + "numbering.levels"), + ([{"counter": "decimal", "label": "{path}", "ref": "{n}%"}, {"counter": "decimal", "label": "{path}", "ref": "5{n}"}], + "numbering.levels"), +]) +def test_a_percent_that_meets_a_counter_where_the_label_is_put_together_is_a_style_error( + levels: list[dict], setting: str) -> None: + style = load_style(overrides={"numbering.levels": levels}) + [problem] = docx_style_errors(style) + assert problem.startswith(setting) and "put together" in problem with pytest.raises(StyleError): - plan_docx(render(FRONT + "# A\n", format="text", overrides=overrides).tree, style) + docx_settings(style) def test_a_percent_that_cannot_be_mistaken_is_fine() -> None: @@ -529,63 +585,164 @@ def test_settings_that_are_off_are_not_checked() -> None: overrides = {"enumeration.enabled": False, "enumeration.levels": [ {"counter": "decimal", "label": "%1{n}", "ref": "{n}"}]} assert docx_style_errors(load_style(overrides=overrides)) == [] - heads = {"numbering.scheme": "none"} - assert docx_style_errors(load_style(overrides=heads)) == [] + assert docx_style_errors(load_style(overrides={"numbering.scheme": "none"})) == [] def test_a_level_text_longer_than_word_allows_is_a_style_error() -> None: long = "x" * 260 style = load_style(overrides={"numbering.levels": [{"counter": "decimal", "label": long + "{n}", "ref": "{n}"}]}) [problem] = docx_style_errors(style) - assert "headings" in problem and "255" in problem + assert problem.startswith("numbering.levels") and "255" in problem + with pytest.raises(StyleError): + docx_settings(style) -# -- Word numbers every unit as the resolver did, or a diagnostic says why not --------------------- +# -- Labels that show what Word cannot, and runs of units that Word counts off ------------------------ + + +def test_what_a_label_shows_is_judged_on_the_filled_label_not_on_its_spelling() -> None: + # `{path}` of an item in a section starts from the section's number: it shows it. In the preamble it does not. + levels = [{"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}] + tree, plan, diagnostics = planned("# A\n\n- a\n - x\n - y\n- b\n", overrides={"enumeration.levels": levels}) + first, nested, _, _ = items(tree)[:4] + assert plan.units[first.unit].cause == () and plan.units[nested.unit].cause == ("label-fallback",) + [found] = diagnostics + # the reading is the one at the list's own depth: the level-2 text without the section ("(%1)(%2)" has only + # the list's own counters), not the level-1 text + assert found.rule == "render-docx-section-label" and "'(a)(1)'" in found.message + tree, plan, diagnostics = planned("- a\n - x\n\n# A\n", overrides={"enumeration.levels": levels}) + assert diagnostics == [] and plan.units[items(tree)[1].unit].cause == () + + +def test_a_label_that_needs_a_level_nothing_numbers_is_not_left_to_word() -> None: + # An ordered list's items are numbered by another abstractNum than the enumerated item nested in one: the level + # above the nested item is never used, which Word shows as 0, not 1. + levels = [{"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}] + tree, plan, diagnostics = planned("1. a\n - x\n\n# A\n", overrides={"enumeration.levels": levels}) + _, nested = items(tree) + assert plan.units[nested.unit].cause == ("restart-family", "label-fallback") + assert plan.units[nested.unit].label == "()(1)" # the level above reads as nothing, not as "(a)" + assert {d.rule for d in diagnostics} == {"render-docx-static-restart", "render-docx-section-label"} + + +def test_a_run_of_units_counted_off_by_the_same_amount_is_one_finding() -> None: + body = "".join(f"# S{i}\n\n- a\n- b\n\n" for i in range(4)) + body += "# D {#d when=forum:courts}\n\n- x\n\n# D {#d when=forum:arb}\n\n- y\n\n" + body += "".join(f"# T{i}\n\n- a\n\n## sub\n\n- q\n\n" for i in range(10)) + tree, plan, diagnostics = planned(body, ALTERNATIVES) + [found] = diagnostics + assert found.rule == "render-docx-number-mismatch" and found.line == tree.sections[5].line + assert "1 later than the renderer" in found.message and "the following 10" in found.message + + +def test_a_different_difference_is_a_finding_of_its_own() -> None: + body = ("# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n\n# Mid\n\n" + "# E {#e when=forum:courts}\n\n# E {#e when=forum:arb}\n\n# End\n") + tree, _, diagnostics = planned(body, ALTERNATIVES) + assert [(d.rule, d.line) for d in diagnostics] == [ + ("render-docx-number-mismatch", tree.sections[1].line), ("render-docx-number-mismatch", tree.sections[4].line)] + assert "1 later" in diagnostics[0].message and "2 later" in diagnostics[1].message -EXPLAINING = { - "label-fallback": "render-docx-section-label", - "clamped": "render-docx-list-depth", - "skipped-level": "render-docx-heading-skip", + +# -- Word's counters against LibreOffice's, observed ----------------------------------------------------- + +#: What LibreOffice 24.2 writes for these values, from a document with one list per value (it follows the OOXML +#: definition, which Word documents too): letters repeat after 26, Roman numerals go on with `m` from 4000. +LIBREOFFICE_COUNTERS = { + "lowerLetter": {1: "a", 2: "b", 25: "y", 26: "z", 27: "aa", 28: "bb", 29: "cc", 52: "zz", 53: "aaa", 54: "bbb", + 55: "ccc", 78: "zzz", 79: "aaaa", 100: "vvvv"}, + "upperLetter": {27: "AA", 28: "BB", 52: "ZZ", 53: "AAA", 100: "VVVV"}, + "lowerRoman": {1: "i", 28: "xxviii", 79: "lxxix", 100: "c", 3999: "mmmcmxcix", 4000: "mmmm"}, + "upperRoman": {29: "XXIX", 78: "LXXVIII", 3999: "MMMCMXCIX", 4000: "MMMM"}, + "decimal": {1: "1", 26: "26", 4000: "4000"}, } -MISMATCH = {"render-docx-number-mismatch", "render-docx-letter-overflow", "render-docx-roman-overflow"} + + +@pytest.mark.parametrize("num_fmt", LIBREOFFICE_COUNTERS) +def test_word_counters_are_the_ones_libreoffice_wrote(num_fmt: str) -> None: + for value, written in LIBREOFFICE_COUNTERS[num_fmt].items(): + assert word_counter(value, num_fmt) == written + + +# -- Word numbers every unit as the resolver did, or a diagnostic says why not --------------------- + +def numbered(tree: RenderTree) -> Iterator[tuple[Section | ListItem | Paragraph, str]]: + """Every numbered unit with the counter the resolver wrote its number in.""" + formats = tree.formats + + def blocks(found: tuple) -> Iterator[tuple[Section | ListItem | Paragraph, str]]: + for block in found: + match block: + case Paragraph() if block.number is not None and formats.paragraphs is not None: + yield block, formats.paragraphs.counter + case List() if block.format is not None: + for item in block.items: + yield item, block.format.counter + yield from blocks(item.blocks) + case List(): + for item in block.items: + yield from blocks(item.blocks) + case Quote(blocks=inner) | DraftingNote(blocks=inner): + yield from blocks(inner) + + yield from blocks(tree.preamble) + for section in tree.sections: + if section.number is not None: + yield section, formats.headings[section.level - 1].counter + yield from blocks(section.blocks) def assert_planned_as_resolved(result) -> None: # noqa: ANN001 - """The property: for every numbered unit, Word's label and count are the resolver's, or a diagnostic of the plan - explains the exception; and nothing is reported that did not happen.""" - plan, diagnostics = plan_docx(result.tree, result.style) + """The property, unit by unit: Word's count is the resolver's number, and Word's label is the resolver's label, + unless that unit's own counter reads differently in Word (a letter past 26, a Roman numeral from 4000: its label + and those of its descendants may differ), its label cannot show what the resolver's does (the plan says so on the + unit), or its depth is past Word's. A document without alternatives has no other exception; a diagnostic is + reported exactly where there is an exception.""" + tree = result.tree + plan, diagnostics = plan_docx(tree) found = rules(diagnostics) - differs = False - for unit in units(result.tree): - mine = plan.units[unit.unit] - if unit.number is None: - assert mine.count is None and mine.label == "" or isinstance(unit, Section) - continue - for cause, rule in EXPLAINING.items(): + shares = any(unit.shares_number_with for unit in units(tree)) + overflows: dict[str, str] = {} # unit -> the rule its counter reads differently under (itself or an ancestor) + parents = {unit.unit: unit.parent_unit for unit in units(tree)} + count_differs = False + own_rules: set[str] = set() + for node, counter in numbered(tree): + mine = plan.units[node.unit] + clamped = "clamped" in mine.cause + if not shares: + assert mine.count == node.number, (node.unit, mine.count, node.number) + count_differs = count_differs or mine.count != node.number + own = "" + if not clamped and word_counter(node.number, NUM_FMT[counter]) != format_counter(node.number, counter): + own = "render-docx-roman-overflow" if counter.endswith("roman") else "render-docx-letter-overflow" + inherited = overflows.get(parents[node.unit] or "", "") or "" + overflows[node.unit] = own or inherited + if not shares and not (own or inherited or clamped or "label-fallback" in mine.cause): + assert mine.label == node.label, (node.unit, mine.label, node.label) + for cause, rule in (("label-fallback", "render-docx-section-label"), ("clamped", "render-docx-list-depth")): if cause in mine.cause: - assert rule in found, (unit.unit, cause) - if mine.count != unit.number or (mine.label != unit.label and not {"label-fallback", "clamped"} & set(mine.cause)): - differs = True - if mine.cause == () and mine.count == unit.number: - assert mine.label == unit.label or found & MISMATCH, (unit.unit, mine.label, unit.label) - assert (not differs) or found & MISMATCH, "a difference without a diagnostic" - for rule in found & MISMATCH: - assert differs, f"{rule} without a difference" + assert rule in found, (node.unit, cause) + own_rules.update([own] if own else []) + assert found & {"render-docx-roman-overflow", "render-docx-letter-overflow"} == own_rules + if shares: + assert count_differs <= ("render-docx-number-mismatch" in found) + else: + assert "render-docx-number-mismatch" not in found, "a placement is wrong: " + "; ".join( + d.message for d in diagnostics if d.rule == "render-docx-number-mismatch") assert all(d.rule.startswith("render-docx-") for d in diagnostics) - assert {u.unit for u in units(result.tree)} == set(plan.units) - uses = {n.id for n in plan.nums} - for mine in plan.units.values(): - assert mine.num_id is None or mine.num_id in uses - # Every num with overrides is used, and the first unit on it is on a level its overrides name. + assert {u.unit for u in units(tree)} == set(plan.units) + # Every num with an override is used, by a unit whose level the override names. first: dict[int, str] = {} - for unit in units(result.tree): + for unit in units(tree): mine = plan.units[unit.unit] if mine.num_id is not None: first.setdefault(mine.num_id, unit.unit) for num in plan.nums: + assert num.id in first, f"num {num.id} is never used" if num.start_overrides: - assert num.id in first, f"num {num.id} is never used" - assert plan.units[first[num.id]].ilvl in {i for i, _ in num.start_overrides} + assert plan.units[first[num.id]].ilvl == num.start_overrides[0][0] @pytest.mark.parametrize("variant", VARIANTS) @@ -622,18 +779,19 @@ def test_the_specifications_documents_are_planned_as_resolved(variant: str) -> N # -- Random documents --------------------------------------------------------------------------- -def _random_list(rng: random.Random, depth: int, indent: str) -> list[str]: +def _random_list(rng: random.Random, depth: int, indent: str, limit: int) -> list[str]: ordered = rng.random() < 0.3 marker = "1. " if ordered else "- " lines: list[str] = [] for index in range(rng.choice([1, 2, 2, 3, 5, 30]) if depth == 0 else rng.randint(1, 3)): lines.append(f"{indent}{marker}item {depth}.{index}") inner = indent + " " * len(marker) - if depth < 4 and rng.random() < 0.45: - lines.extend(_random_list(rng, depth + 1, inner)) + chain = limit > 4 and index == 0 # a deep list is a chain: only its first item nests, or it would explode + if depth < limit and (chain or (limit <= 4 and rng.random() < 0.45)): + lines.extend(_random_list(rng, depth + 1, inner, limit)) if rng.random() < 0.3: # a sibling list after text: it goes on from the first lines += ["", f"{inner}Between.", ""] - lines.extend(_random_list(rng, depth + 1, inner)) + lines.extend(_random_list(rng, depth + 1, inner, limit)) if depth < 3 and rng.random() < 0.12: lines += ["", f"{inner}> - quoted", f"{inner}> - quoted too", ""] return lines + [""] @@ -642,6 +800,7 @@ def _random_list(rng: random.Random, depth: int, indent: str) -> list[str]: def _random_document(rng: random.Random) -> str: out: list[str] = [] level = 1 + deep = rng.random() < 0.12 # a list nested past the nine levels Word has for _ in range(rng.randint(1, 7)): move = rng.random() level = max(1, min(5, level + (1 if move < 0.4 else -1 if move < 0.55 else 2 if move < 0.6 else 0))) @@ -649,37 +808,68 @@ def _random_document(rng: random.Random) -> str: for _ in range(rng.randint(0, 3)): kind = rng.random() if kind < 0.45: - out.extend(_random_list(rng, 0, "")) - elif kind < 0.8: + out.extend(_random_list(rng, 0, "", 11 if deep else 4)) + elif kind < 0.7: out.append("A paragraph.\n") - else: + elif kind < 0.85: out.append("> quoted\n") + else: + out.append("> [!DRAFTING]\n> - a note\n> - and another\n") if rng.random() < 0.3: out.insert(0, "- preamble item\n - nested\n") return "\n".join(out) +def _random_alternatives(rng: random.Random) -> str: + """Sections, items and paragraphs that share a number, among others.""" + out: list[str] = [] + for index in range(rng.randint(2, 6)): + out.append(f"# Section {index}\n") + if rng.random() < 0.5: + out.append(f"Para. {{#p{index} when=forum:courts}}\n\nPara arb. {{#p{index} when=forum:arb}}\n") + if rng.random() < 0.5: + out.append(f"- item {{#i{index} when=forum:courts}}\n- item arb {{#i{index} when=forum:arb}}\n - nested\n" + f"- after\n") + if rng.random() < 0.4: + out.append(f"# Disputes {{#d{index} when=forum:courts}}\n\n## Sub\n\n" + f"# Disputes {{#d{index} when=forum:arb}}\n\n## Sub alt\n") + return "\n".join(out) + + +@functools.cache +def _random_renders(variant: str) -> tuple: + rng = random.Random(20261028) + return tuple(render(FRONT + _random_document(rng), format="text", **VARIANTS[variant]) for _ in range(30)) + + @pytest.mark.parametrize("variant", VARIANTS) def test_random_documents_are_planned_as_resolved(variant: str) -> None: - rng = random.Random(20261028) - for _ in range(40): - source = FRONT + _random_document(rng) + for result in _random_renders(variant): try: - assert_planned_as_resolved(render(source, format="text", **VARIANTS[variant])) + assert_planned_as_resolved(result) except AssertionError as error: - raise AssertionError(f"{error}\n--- document ---\n{source}") from error + raise AssertionError(f"{error}\n--- document ---\n{result.tree.preamble}\n{result.output}") from error + + +@pytest.mark.parametrize("variant", ["default", "continental", "paragraphs", "none"]) +def test_random_alternatives_are_planned_and_reported(variant: str) -> None: + rng = random.Random(7) + for _ in range(20): + result = render(ALTERNATIVES + _random_alternatives(rng), format="text", **VARIANTS[variant]) + assert_planned_as_resolved(result) def test_the_random_documents_reach_every_case_the_property_is_about() -> None: seen_rules: set[str] = set() seen_causes: set[str] = set() - for settings in VARIANTS.values(): - rng = random.Random(20261028) - for _ in range(40): - result = render(FRONT + _random_document(rng), format="text", **settings) - plan, diagnostics = plan_docx(result.tree, result.style) + depth = 0 + for variant in VARIANTS: + for result in _random_renders(variant): + plan, diagnostics = plan_docx(result.tree) seen_rules |= rules(diagnostics) seen_causes |= {cause for unit in plan.units.values() for cause in unit.cause} - assert {"render-docx-letter-overflow", "render-docx-section-label", "render-docx-heading-skip", - "render-docx-static-restart"} <= seen_rules - assert {"restart-family", "resync", "label-fallback", "skipped-level"} <= seen_causes + depth = max(depth, max((lst.depth for lst in lists(result.tree)), default=0)) + assert {"render-docx-letter-overflow", "render-docx-section-label", "render-docx-static-restart", + "render-docx-list-depth", "render-docx-shared-counter"} <= seen_rules + assert {"restart-family", "resync", "label-fallback", "skipped-level", "clamped"} <= seen_causes + assert depth >= 10 diff --git a/tests/test_docx_simulator.py b/tests/test_docx_simulator.py index e71cbcb..65b4206 100644 --- a/tests/test_docx_simulator.py +++ b/tests/test_docx_simulator.py @@ -5,6 +5,7 @@ import pytest +from legaldown_render.errors import InternalError from legaldown_render.resolve.docx_numbering import ( Count, Ghost, @@ -13,7 +14,9 @@ extend_tokens, fill_tokens, level_text, + level_text_problem, lvl_text, + shows_section, split_level_text, text_of, word_counter, @@ -95,13 +98,33 @@ def test_a_label_that_needs_the_section_is_written_as_in_the_preamble() -> None: def test_literal_percent_before_a_digit_cannot_be_written() -> None: assert lvl_text(level_text("100% {n}", ["{n}"], 0, section=None)) == "100% %1" - from legaldown_render.errors import InternalError with pytest.raises(InternalError): lvl_text(level_text("%1 {n}", ["{n}"], 0, section=None)) with pytest.raises(InternalError): lvl_text(level_text("%{n}", ["{n}"], 0, section=None)) +def test_a_percent_that_meets_a_counter_where_pieces_are_put_together_is_a_problem() -> None: + parts = level_text("{path}", ["{n}%", "{n}"], 1, section=None) + assert parts == (0, "%", 1) and level_text_problem(parts) is not None + assert level_text_problem(level_text("{path}", ["{n}%", " {n}"], 1, section=None)) is None + assert level_text_problem(("5%", 0)) is not None + with pytest.raises(InternalError): + lvl_text(parts) + + +def test_a_label_shows_the_section_when_the_filled_label_has_its_counter() -> None: + refs = ["({n})", "({n})"] + assert shows_section("{section}.{n}", [".{n}"], 0) + assert shows_section("{path}", refs, 1) and shows_section("{path}", refs, 0) + assert not shows_section("({n})", refs, 1) and not shows_section("{n})", refs, 0) + assert shows_section("({section}) {n}", refs, 0) + # a ref that holds the section puts it in every `{path}` below, not in a label that shows only its own counter + assert shows_section("{path}", ["{section}{n}", "({n})"], 1) + assert not shows_section("({n})", ["{section}{n}", "({n})"], 1) + assert shows_section("{section}.{n}", [".{n}"], 0, textual=True) + + def test_split_level_text() -> None: assert split_level_text("%1.%2(%10)x") == [0, ".", 1, "(", 0, "0)x"] assert split_level_text("plain") == ["plain"] @@ -164,6 +187,17 @@ def test_counters_are_shared_by_every_num_of_an_abstract_num() -> None: assert sim.step(2, 0, "s6").label == "2." +def test_a_paragraph_without_numbering_counts_nothing_and_restarts_nothing() -> None: + sim = numbering(Num(1, 0)) + sim.step(1, 0, "s1") + sim.step(1, 1, "s2") + sim.step(1, 1, "s3") + step = sim.step(0, 0, "alternative") + assert (step.count, step.label, step.word_ref, step.tokens) == (0, "", "0", ()) + assert sim.current(1, 0) == (1, "s1") and sim.current(1, 1) == (2, "s3") + assert sim.step(1, 1, "s4").label == "1.3" and sim.step(1, 0, "s5").label == "2." + + def test_peek_and_current_do_not_count() -> None: sim = numbering(Num(1, 1)) assert (sim.peek(1, 0), sim.current(1, 0)) == (1, (0, None)) @@ -188,8 +222,18 @@ def test_in_the_word_model_a_skipped_level_reads_zero_and_is_not_advanced() -> N assert sim.step(1, 1, "s3").label == "1.1" +def test_each_abstract_num_has_its_own_model_of_a_level_that_nothing_numbered() -> None: + sim = WordNumbering() + sim.add_abstract(0, HEADINGS) + sim.add_abstract(1, _levels(("lowerLetter", "(%1)"), ("decimal", "%1.%2")), skipped="word") + sim.add_num(Num(1, 0)) + sim.add_num(Num(2, 1)) + assert sim.step(1, 1, "s1").label == "1.1" # the renderer's and LibreOffice's rule + assert sim.step(2, 1, "u1").label == ".1" # Word's: a letter counter at 0 is nothing + + def test_the_first_paragraph_of_a_list_deeper_than_the_level_it_starts_at_fills_the_levels_above() -> None: - sim = numbering(Num(1, 1, ((1, 1), (2, 1)))) + sim = numbering(Num(1, 1, ((1, 1),))) assert sim.step(1, 1, "u1").label == "(i)" From fa122e66881df3335e0f264abb0a9b238a390393 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:48:10 +0000 Subject: [PATCH 083/118] Drop a stale comment from the DOCX plan module Comment only: the data model is unchanged. --- src/legaldown_render/resolve/docx_plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index f78163f..43ce963 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -14,7 +14,7 @@ from dataclasses import dataclass, field from typing import Literal -from ..tree import Inline # in the implementation: from ...tree import Inline +from ..tree import Inline @dataclass(frozen=True, slots=True) From df4175e16323da8aaac07bfc062baf90d2ad379b Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:48:10 +0000 Subject: [PATCH 084/118] Harden the DOCX body writer after review The bookmarks a reference may point at are the ones the document writes, found by a dry run, not the names in the plan: a REF to a name never written is a bug in the plan, a link or term reference to one shows its text. Indents are kept where a paragraph can be read (at most 31680 twips and 2268 short of the text width) and a table keeps 1134 twips per column, dropping its indent when it would not fit. A link that is not made, or that has no text, shows what HTML shows, and a hyperlink is never empty. An alternative always gets numId 0. Runs in a link keep its colour and underline, code and rules in a quote keep its bar, the table look matches its flags, tooltips are cut to 255 characters, bookmark names in instructions are checked, and an inline or block the writer does not know is an error. Part digest of document.xml re-pinned for the table look. --- src/legaldown_render/writers/docx/__init__.py | 4 +- src/legaldown_render/writers/docx/body.py | 84 ++++++--- src/legaldown_render/writers/docx/ooxml.py | 5 + src/legaldown_render/writers/docx/runs.py | 101 +++++++---- src/legaldown_render/writers/docx/styles.py | 7 + tests/test_docx_body.py | 169 +++++++++++++++++- tests/test_docx_package.py | 2 +- 7 files changed, 306 insertions(+), 66 deletions(-) diff --git a/src/legaldown_render/writers/docx/__init__.py b/src/legaldown_render/writers/docx/__init__.py index c4b560c..1be4a01 100644 --- a/src/legaldown_render/writers/docx/__init__.py +++ b/src/legaldown_render/writers/docx/__init__.py @@ -84,8 +84,8 @@ def write(self, tree: RenderTree, plan: DocxPlan | None = None) -> bytes: if page.footer: footer_id = rels.add(pkg.REL_FOOTER, "footer1.xml") - body = BodyWriter(tree, plan, Bookmarks(), rels, text_width).body(settings.page, header_id=header_id, - footer_id=footer_id) + body = BodyWriter(tree, plan, Bookmarks(), rels, text_width, settings.link_color).body( + settings.page, header_id=header_id, footer_id=footer_id) document = el("document", None, body) package.add("docProps/core.xml", pkg.CT_CORE_PROPERTIES, build_core(title, language)) package.add("docProps/app.xml", pkg.CT_EXTENDED_PROPERTIES, build_app(self.generator)) diff --git a/src/legaldown_render/writers/docx/body.py b/src/legaldown_render/writers/docx/body.py index 40fdaff..4444c72 100644 --- a/src/legaldown_render/writers/docx/body.py +++ b/src/legaldown_render/writers/docx/body.py @@ -32,7 +32,6 @@ from ...tree import ( Block, CodeBlock, - DefinedTerm, DraftingNote, Inline, List, @@ -43,7 +42,6 @@ Rule, Section, Table, - iter_located_body_inlines, plain_text, ) from .ooxml import Bookmarks, el, p, tbl_pr, tc_pr, text_run, tr_pr, val @@ -61,12 +59,19 @@ SUBTITLE, TABLE_GRID, TITLE, + container_bar, heading_style, ) #: The left indent the paragraph styles set, in twips: what a paragraph in that style has without an indent of its own. _STYLE_LEFT = {QUOTE: QUOTE_INDENT, DRAFTING_NOTE: NOTE_INDENT} +#: Limits on what a plan's indents can do to the page, in twips: Word does not take a left indent past 31680 +#: (22 inches), and a paragraph needs room to be read in; a column needs room for a word. +_MAX_INDENT = 31680 +_MIN_TEXT = 2268 +_MIN_COLUMN = 1134 + _RULE_COLOR = "C8C8C8" #: The space above and below the text of a table cell, in twips. _CELL_PADDING = 40 @@ -115,32 +120,42 @@ class BodyWriter: twips, which tables take.""" def __init__(self, tree: RenderTree, plan: DocxPlan, bookmarks: Bookmarks, relationships: Relationships, - text_width: int) -> None: + text_width: int, link_color: str) -> None: self.tree = tree self.plan = plan self.bookmarks = bookmarks + self.relationships = relationships self.text_width = text_width + self.link_color = link_color abstracts = {abstract.id: abstract for abstract in plan.abstract_nums} self._levels = {num.id: abstracts[num.abstract_id].levels for num in plan.nums if num.abstract_id in abstracts} attachment_titles = {a.anchor: plain_text(a.title) for a in tree.attachments if a.anchor} - self.inlines = InlineWriter(plan, relationships, bookmarks, self._placed(), attachment_titles) - - def _placed(self) -> set[str]: - """The names of the bookmarks the body writes: the plan's for every unit, and the definitions' - (a defined term's bookmark is placed where the term is). A field or link to any other name would - point at nothing.""" - names = {name for unit in self.plan.units.values() for name in (*unit.bookmarks, unit.toc_bookmark) if name} - for inline, _ in iter_located_body_inlines(self.tree): - if isinstance(inline, DefinedTerm) and inline.defining and inline.anchor: - name = self.plan.anchors.get(inline.anchor) - if name: - names.add(name) - return names + #: Not known until a first, dry run has written the document: see :meth:`body`. + self.inlines = InlineWriter(plan, relationships, bookmarks, None, attachment_titles, link_color) + + def _left(self, left: int) -> int: + """*left* twips of indent, kept where a paragraph can still be read: not past Word's limit, and leaving + :data:`_MIN_TEXT` of the text width (a deeply nested list, or a plan's wide indent, cannot push it off the page).""" + return max(0, min(left, self.text_width - _MIN_TEXT, _MAX_INDENT)) # -- the document --------------------------------------------------------------------- def body(self, page: PageGeometry, *, header_id: str | None = None, footer_id: str | None = None) -> ET.Element: - """``w:body`` for the tree, closed by the section properties of *page*.""" + """``w:body`` for the tree, closed by the section properties of *page*. + + The document is written twice. The first, dry run learns which bookmarks it writes (a name in the plan + is not enough: a defined term inside alt text or a link title is never written), and the second writes + it for real: a ``REF`` field to a name the document does not hold is then a bug in the plan + (:class:`~legaldown_render.errors.InternalError`), and a link or a term reference to one shows its text + without the link. Whatever :meth:`content` writes counts, so the attachments' bookmarks need no list of + their own.""" + dry = BodyWriter(self.tree, self.plan, Bookmarks(), Relationships(), self.text_width, self.link_color) + dry.content() + self.inlines.placed = dry.bookmarks.names + return el("body", None, self.content(), section_properties(page, header_id=header_id, footer_id=footer_id)) + + def content(self) -> list[ET.Element]: + """The block-level content of the body: the title and the subtitle, the preamble and the sections.""" tree = self.tree content: list[ET.Element] = [] title = document_title(tree) @@ -154,7 +169,7 @@ def body(self, page: PageGeometry, *, header_id: str | None = None, footer_id: s content += self.section(section) if content and content[-1].tag.endswith("}tbl"): content.append(p()) # a table cannot be the last thing before the section properties - return el("body", None, content, section_properties(page, header_id=header_id, footer_id=footer_id)) + return content def section(self, section: Section) -> list[ET.Element]: """The heading paragraph of *section* (``heading N``: the resolver has clamped the level to @@ -197,19 +212,28 @@ def block(self, block: Block, context: Context) -> Iterator[ET.Element]: yield from self.blocks(blocks, inner) case CodeBlock(text=text): # One paragraph per line: Word shades and keeps them together as one block. + # Inside a quote or a note the paragraph stays at the container's indent and keeps its bar. + bar = container_bar(context.style) if context.style else None + left = self._left(context.left if bar is not None else context.left + CODE_INDENT) for line in text.rstrip("\n").split("\n"): yield p(text_run(line), style=CODE, - props=[el("ind", {"left": context.left + CODE_INDENT}) if context.left else None]) + props=[el("ind", {"left": left}) if context.left else None, + bar is not None and el("pBdr", None, bar)]) case Table(): yield from self._table(block, context) case Rule(): + bar = container_bar(context.style) if context.style else None yield p(style=context.style, props=[ - el("pBdr", None, el("bottom", {"val": "single", "sz": 4, "space": 1, "color": _RULE_COLOR})), + el("pBdr", None, bar, el("bottom", {"val": "single", "sz": 4, "space": 1, "color": _RULE_COLOR})), self._plain_indent(context)]) + case _: + raise InternalError(f"a {type(block).__name__} reached the DOCX writer, which does not write it " + "(a bug in the DOCX writer)") def _plain_indent(self, context: Context) -> ET.Element | None: """``w:ind`` for a plain paragraph in *context*, unless its style already indents so far.""" - return el("ind", {"left": context.left}) if context.left != _STYLE_LEFT.get(context.style or "", 0) else None + left = self._left(context.left) + return el("ind", {"left": left}) if left != _STYLE_LEFT.get(context.style or "", 0) else None def _paragraph(self, block: Paragraph, context: Context) -> ET.Element: content = self.inlines.runs(block.inlines) @@ -245,7 +269,7 @@ def _indent(self, unit: UnitPlan) -> tuple[int, int] | None: def _text_left(self, unit: UnitPlan | None, context: Context) -> int: """Where the text of a unit's paragraph starts: what its continuation blocks are indented to.""" indent = self._indent(unit) if unit is not None else None - return context.left if indent is None else indent[0] + context.extra + return self._left(context.left if indent is None else indent[0] + context.extra) def _unit(self, unit_id: str | None, content: Sequence[ET.Element], *, style: str | None, context: Context, conditions: Sequence[str | None], after: bool = False) -> ET.Element: @@ -257,12 +281,13 @@ def _unit(self, unit_id: str | None, content: Sequence[ET.Element], *, style: st number: list[ET.Element] = [] names: list[str] = [] if unit is not None: - if unit.explicit: + if unit.explicit or unit.alt is not None: props.append(self._numbering(unit)) indent = self._indent(unit) if indent is not None and (unit.ind is not None or context.extra): - left, hanging = indent - props.append(el("ind", {"left": left + context.extra, "hanging": hanging or None})) + left = self._left(indent[0] + context.extra) + hanging = min(indent[1], left) + props.append(el("ind", {"left": left, "hanging": hanging or None})) if unit.alt is not None: number = self._alternative(unit.alt) names = [*unit.bookmarks, *([unit.toc_bookmark] if unit.toc_bookmark else [])] @@ -307,7 +332,10 @@ def _table(self, table: Table, context: Context) -> Iterator[ET.Element]: columns = max(len(table.header), *(len(row) for row in table.rows), 0) if not columns: return - width = max(self.text_width - context.left, columns) + left = self._left(context.left) + width = self.text_width - left + if width < columns * _MIN_COLUMN: # no room to indent: the columns come first + left, width = 0, self.text_width grid = [width // columns] * columns grid[-1] += width - sum(grid) align = (*table.align, *[""] * columns)[:columns] @@ -323,10 +351,10 @@ def row(cells: tuple[tuple[Inline, ...], ...], *, header: bool = False) -> ET.El yield el( "tbl", None, tbl_pr(val("tblStyle", TABLE_GRID), el("tblW", {"w": width, "type": "dxa"}), - context.left > 0 and el("tblInd", {"w": context.left, "type": "dxa"}), + left > 0 and el("tblInd", {"w": left, "type": "dxa"}), el("tblCellMar", None, el("top", {"w": _CELL_PADDING, "type": "dxa"}), el("bottom", {"w": _CELL_PADDING, "type": "dxa"})), - el("tblLook", {"val": "0420", "firstRow": 1, "lastRow": 0, "firstColumn": 0, "lastColumn": 0, + el("tblLook", {"val": "0620", "firstRow": 1, "lastRow": 0, "firstColumn": 0, "lastColumn": 0, "noHBand": 1, "noVBand": 1})), el("tblGrid", None, [el("gridCol", {"w": size}) for size in grid]), row(table.header, header=True) if table.header else None, diff --git a/src/legaldown_render/writers/docx/ooxml.py b/src/legaldown_render/writers/docx/ooxml.py index 4f9bc9a..c96e4c0 100644 --- a/src/legaldown_render/writers/docx/ooxml.py +++ b/src/legaldown_render/writers/docx/ooxml.py @@ -353,6 +353,11 @@ def __init__(self) -> None: self._next = 1 self._names: set[str] = set() + @property + def names(self) -> frozenset[str]: + """The names allocated so far, lower-cased (Word compares names without regard to case).""" + return frozenset(self._names) + def pair(self, name: str) -> tuple[ET.Element, ET.Element]: """The ``w:bookmarkStart`` and ``w:bookmarkEnd`` of a new bookmark *name*.""" if not _BOOKMARK_RE.fullmatch(name): diff --git a/src/legaldown_render/writers/docx/runs.py b/src/legaldown_render/writers/docx/runs.py index 6f14895..5de155b 100644 --- a/src/legaldown_render/writers/docx/runs.py +++ b/src/legaldown_render/writers/docx/runs.py @@ -35,8 +35,8 @@ has_visible_content, plain_text, ) -from ...urls import displayed_url -from .ooxml import Bookmarks, complex_field, el, hyperlink_external, hyperlink_internal, r, text_run, val +from ...urls import displayed_url, is_link_target +from .ooxml import _BOOKMARK_RE, Bookmarks, complex_field, el, hyperlink_external, hyperlink_internal, r, text_run, val from .package import Relationships from .styles import ( BLANK, @@ -52,22 +52,29 @@ @dataclass(frozen=True, slots=True) class Format: """The character formatting of the runs being written: a character style and the emphasis - that enclosing ``*``/``**`` add.""" + that enclosing ``*``/``**`` add. *link_color* is set inside a link, where a run has one character + style and it is not ``Hyperlink``: the run then carries the link's colour and underline directly.""" style: str | None = None bold: bool = False italic: bool = False + link_color: str | None = None @property def props(self) -> list[Any]: """The run properties (``w:rPr`` children, in any order, :func:`~.ooxml.rpr` sorts them).""" + linked = self.link_color is not None and self.style != HYPERLINK return [val("rStyle", self.style) if self.style else None, self.bold and [el("b"), el("bCs")], - self.italic and [el("i"), el("iCs")]] + self.italic and [el("i"), el("iCs")], linked and el("color", {"val": self.link_color}), + linked and el("u", {"val": "single"})] def styled(self, style: str) -> Format: return replace(self, style=style) +#: The longest screen tip of a hyperlink, in characters (Word's limit). +_TOOLTIP = 255 + #: No character formatting. PLAIN = Format() @@ -84,32 +91,46 @@ class InlineWriter: *plan* names the bookmarks and the reference fields; *relationships* receives the external hyperlinks (:meth:`~.package.Relationships.add_external`: a link the document may not - make stays text); *bookmarks* allocates the ids of the bookmarks written here; *placed* holds the - names of the bookmarks the document writes, so that no field or link points at one that is not - there; *attachment_titles* maps an attachment's anchor to its title as text.""" + make stays text); *bookmarks* allocates the ids of the bookmarks written here; *placed* is the + lower-cased names of the bookmarks the document writes, so that no field or link points at one that is + not there (None: not known yet, as in the dry run that finds them out, :meth:`.body.BodyWriter.body`); + *attachment_titles* maps an attachment's anchor to its title as text; *link_color* is the colour of a + hyperlink (``w:color``), which runs inside a link carry when their own style is not ``Hyperlink``.""" - def __init__(self, plan: DocxPlan, relationships: Relationships, bookmarks: Bookmarks, placed: Collection[str], - attachment_titles: Mapping[str, str]) -> None: + def __init__(self, plan: DocxPlan, relationships: Relationships, bookmarks: Bookmarks, + placed: Collection[str] | None, attachment_titles: Mapping[str, str], link_color: str) -> None: self.plan = plan self.relationships = relationships self.bookmarks = bookmarks self.placed = placed self.attachment_titles = attachment_titles + self.link_color = link_color + + def _is_placed(self, name: str) -> bool: + return self.placed is None or name.lower() in self.placed + + def _reference_target(self, name: str) -> str: + """*name*, a bookmark a field points at: it must be a valid name and written by the document, or the plan + is wrong (Word would show "Error! Reference source not found").""" + if not _BOOKMARK_RE.fullmatch(name): + raise InternalError("a reference field names a bookmark that is not a valid name (a bug in the DOCX plan)") + if not self._is_placed(name): + raise InternalError(f"a reference field points at the bookmark '{name}', which the document does not " + "contain (a bug in the DOCX plan)") + return name # -- plan segments -------------------------------------------------------------------- def segments(self, segments: Iterable[Segment], fmt: Format = PLAIN) -> list[ET.Element]: """The runs of *segments*: literal text, and each ``REF`` field with its cached result. - A field whose bookmark the document does not write is a bug in the plan.""" + A field whose bookmark the document does not write, or that is not a bookmark name, is a bug in the plan.""" runs: list[ET.Element] = [] for segment in segments: if isinstance(segment, Literal_): if segment.text: runs.append(text_run(segment.text, props=fmt.props)) continue - if segment.bookmark not in self.placed: - raise InternalError(f"a reference field points at the bookmark '{segment.bookmark}', which the " - "document does not contain (a bug in the DOCX plan)") + self._reference_target(segment.bookmark) runs += complex_field(reference_instruction(segment), [text_run(segment.cached, props=fmt.props)] if segment.cached else [], props=fmt.props) @@ -169,31 +190,44 @@ def _inline(self, inline: Inline, fmt: Format, in_link: bool) -> list[ET.Element return [text_run("[" + separator.join(options) + "]", props=fmt.styled(CHOICE).props)] case FailureMarker(text=text): return [text_run(text, props=fmt.styled(FAILURE_MARKER).props)] - return [] + case _: + raise InternalError(f"a {type(inline).__name__} reached the DOCX writer, which does not write it " + "(a bug: the tree is not resolved)") def _internal(self, anchor: str) -> str | None: """The bookmark of the output *anchor*, when the document has it.""" name = self.plan.anchors.get(anchor) - return name if name in self.placed else None + return name if name is not None and self._is_placed(name) else None + + def _linked(self, fmt: Format) -> Format: + """*fmt* inside a link: the ``Hyperlink`` style, and the link's colour for runs that take another style.""" + return replace(fmt, style=HYPERLINK, link_color=self.link_color) + + def _link_runs(self, link: Link, fmt: Format) -> list[ET.Element]: + """What a link shows (§8.7): its text, or its address when it has none, as HTML does. An image with no + alt text writes no run, so it shows the address too: a hyperlink is never empty.""" + shown = link.children if has_visible_content(link.children) else () + runs = self.runs(shown, fmt, in_link=True) + return runs or self.runs((Text(displayed_url(link.href)),), fmt, in_link=True) def _link(self, link: Link, fmt: Format, in_link: bool) -> list[ET.Element]: - """A link: external through a relationship, or to a bookmark for ``#anchor`` of this - document. One that is not a link (§8.7: a scheme links may not use) shows its text alone, as in HTML.""" - # A link with no text shows its address (§8.7: it is still a hyperlink). - shown = link.children if has_visible_content(link.children) else (Text(displayed_url(link.href)),) + """A link: external through a relationship, or to a bookmark for ``#anchor`` of this document. A link that + cannot be made one here shows what HTML shows: its text (address when it has none) when the URL is a link + at all (§8.7), else its children alone.""" if in_link: - return self.runs(shown, fmt, in_link=True) - tooltip = plain_text(link.title) or None + return self._link_runs(link, fmt) + tooltip = plain_text(link.title)[:_TOOLTIP] or None if link.href.startswith("#"): name = self._internal(link.href[1:]) - if name is None: - return self.runs(link.children, fmt) - return [hyperlink_internal(name, *self.runs(shown, fmt.styled(HYPERLINK), in_link=True), tooltip=tooltip)] - identifier = self.relationships.add_external(link.href) - if identifier is None: - return self.runs(link.children, fmt) - return [hyperlink_external(identifier, *self.runs(shown, fmt.styled(HYPERLINK), in_link=True), - tooltip=tooltip)] + if name is not None: + return [hyperlink_internal(name, *self._link_runs(link, self._linked(fmt)), tooltip=tooltip)] + else: + identifier = self.relationships.add_external(link.href) + if identifier is not None: + return [hyperlink_external(identifier, *self._link_runs(link, self._linked(fmt)), tooltip=tooltip)] + if is_link_target(link.href): + return self._link_runs(link, fmt) + return self.runs(link.children, fmt) def _value(self, value: Value, fmt: Format, in_link: bool) -> list[ET.Element]: """A field value as text. An ``attach`` value leads to its attachment: a ``REF`` field when the @@ -203,17 +237,16 @@ def _value(self, value: Value, fmt: Format, in_link: bool) -> list[ET.Element]: href = value.href if not href or in_link: return text + linked = text_run(value.text, props=self._linked(fmt).props) if href.startswith("#"): name = self._internal(href[1:]) if name is None: return text if self.attachment_titles.get(href[1:]) == value.text: - return complex_field(f"REF {name} \\h", text, props=fmt.props) - return [hyperlink_internal(name, text_run(value.text, props=fmt.styled(HYPERLINK).props))] + return complex_field(f"REF {self._reference_target(name)} \\h", text, props=fmt.props) + return [hyperlink_internal(name, linked)] identifier = self.relationships.add_external(href) - if identifier is None: - return text - return [hyperlink_external(identifier, text_run(value.text, props=fmt.styled(HYPERLINK).props))] + return text if identifier is None else [hyperlink_external(identifier, linked)] # -- helpers for paragraphs ----------------------------------------------------------- diff --git a/src/legaldown_render/writers/docx/styles.py b/src/legaldown_render/writers/docx/styles.py index 5da3d8d..e219404 100644 --- a/src/legaldown_render/writers/docx/styles.py +++ b/src/legaldown_render/writers/docx/styles.py @@ -92,6 +92,13 @@ def _border(side: str, color: str, size: int = 4, space: int = 0) -> ET.Element: return el(side, {"val": "single", "sz": size, "space": space, "color": color}) +def container_bar(style_id: str) -> ET.Element | None: + """The left border that ``Quote`` and ``DraftingNote`` draw, for the paragraphs inside them that are not + in that style (code, rules), so that the bar runs unbroken; None for any other style.""" + return {QUOTE: lambda: _border("left", _RULE, 18, 10), + DRAFTING_NOTE: lambda: _border("left", _NOTE_RULE, 24, 8)}.get(style_id, lambda: None)() + + def _style(style_id: str, name: str, *, kind: str = "paragraph", based_on: str | None = None, next_style: str | None = None, priority: int | None = None, quick: bool = False, custom: bool = False, hidden_until_used: bool = False, default: bool = False, props: Iterable[Any] = (), diff --git a/tests/test_docx_body.py b/tests/test_docx_body.py index 6ab71bf..55e0951 100644 --- a/tests/test_docx_body.py +++ b/tests/test_docx_body.py @@ -747,6 +747,139 @@ def test_a_condition_label_leads_a_paragraph_and_an_item_and_follows_a_heading_t assert flow(heading) == "H if C@ConditionLabel" +# -- Bookmarks the document does not write, indents and widths, links with no text ---------------------------- + + +def test_a_bookmark_that_is_only_in_text_the_document_never_writes_is_not_a_target() -> None: + """A defined term inside alt text or a link's title is plan, not document: a reference to it has no target.""" + term = DefinedTerm((Text("Term"),), "def:t", "bold") + document = tree(para(Image("i.png", (Text("alt "), term)), TermRef("Term", "def:t", "bold"), + Link("https://e.com/", (Text("l"),), (Text("tip "), term)), TermRef("Term", "def:t", "bold"))) + (paragraph,) = body(document, plan(anchors={"def:t": "lddef_t"})) + assert flow(paragraph) == "alt TermTerm@TermReference(link rId5: l@Hyperlink)Term@TermReference" + assert not list(paragraph.iter(f"{W}bookmarkStart")) + with pytest.raises(InternalError, match="lddef_t"): + body(tree(para(Image("i.png", (term,)), CrossRef("x", "t", "u9"))), + plan(anchors={"def:t": "lddef_t"}, refs={"u9": (field("lddef_t"),)})) + + +def test_a_term_reference_before_its_definition_still_links_and_a_reference_after_it_too() -> None: + document = tree(para(TermRef("T", "def:t", "bold"), DefinedTerm((Text("Term"),), "def:t", "bold"), + CrossRef("x", "t", "u9"))) + (paragraph,) = body(document, plan(anchors={"def:t": "lddef_t"}, refs={"u9": (field("lddef_t", number=False),)})) + assert flow(paragraph) == "(link lddef_t: T@TermReference)Term@DefinedTerm{REF lddef_t \\h|1}" + + +def test_a_field_may_not_name_what_is_not_a_bookmark() -> None: + for name in ("bad name", "1x", "a" * 41, "a\\n"): + with pytest.raises(InternalError, match="valid"): + body(tree(para(CrossRef("x", "t", "u9"))), plan(refs={"u9": (field(name),)})) + + +def test_every_hyperlink_has_content_and_a_link_that_is_not_made_still_shows_what_html_shows() -> None: + nothing = Link("https://e.com/", (Image("p.png", ()),)) + document = tree(para(Link("docs/a.pdf", ()), " ", Link("#nope", ()), " ", nothing, " ", + Link("javascript:x", (Text("bad"),)), Link("javascript:x", ()))) + (paragraph,) = body(document, plan()) + assert flow(paragraph) == "docs/a.pdf #nope (link rId5: https://e.com/@Hyperlink) bad" + assert all(len(link) for link in paragraph.iter(f"{W}hyperlink")) + + +def test_a_tooltip_is_cut_to_255_characters() -> None: + (paragraph,) = body(tree(para(Link("https://e.com/", (Text("l"),), (Text("t" * 300),)))), plan()) + assert len(paragraph.find(f"{W}hyperlink").get(f"{W}tooltip")) == 255 # type: ignore[union-attr, arg-type] + + +def test_runs_in_a_link_that_take_another_style_keep_the_links_colour_and_underline() -> None: + link = Link("https://e.com/", (Code("c"), TermRef("t", None, "bold"), Blank("b", "__"), Text("x"))) + (paragraph,) = body(tree(para(link)), plan()) + runs = paragraph.find(f"{W}hyperlink").findall(f"{W}r") # type: ignore[union-attr] + styles = [r.find(f"{W}rPr/{W}rStyle").get(f"{W}val") for r in runs] # type: ignore[union-attr] + assert styles == ["InlineCode", "TermReference", "Blank", "Hyperlink"] + for run in runs[:3]: + assert run.find(f"{W}rPr/{W}u").get(f"{W}val") == "single" # type: ignore[union-attr] + assert run.find(f"{W}rPr/{W}color") is not None + assert runs[3].find(f"{W}rPr/{W}u") is None # the Hyperlink style has them + + +def test_an_alternative_is_never_numbered_by_a_heading_style() -> None: + """The plan sets numId 0 for an alternative; a heading that names an alternative but is not marked explicit still + gets it, or the heading style's own numbering would count it.""" + alt = AltNumber((field("a", "1"),), "tab") + document = tree(bullets(item("x", unit="u1")), sections=[section("H", unit="s1")]) + the_plan = plan(unit("u1", bookmarks=("a",)), unit("s1", "section", alt=alt, ind=(567, 567))) + _, heading = body(document, the_plan) + assert numbering(heading) == (None, 0) + + +def test_an_inline_or_block_the_writer_does_not_know_is_a_bug() -> None: + from legaldown_render.tree import DirectiveSource + + with pytest.raises(InternalError, match="DirectiveSource"): + body(tree(para(DirectiveSource(None))), plan()) # type: ignore[arg-type] + with pytest.raises(InternalError, match="int"): + body(tree(42), plan()) # type: ignore[arg-type] + + +def test_code_and_rules_in_a_quote_or_a_note_keep_the_bar_of_the_container() -> None: + document = tree(Quote((para("q"), CodeBlock("c"), Rule())), DraftingNote((CodeBlock("n"), Rule()), None), Rule()) + q, code, rule, note_code, note_rule, plain_rule = body(document, plan()) + assert code.find(f"{W}pPr/{W}pBdr/{W}left").get(f"{W}color") == "C8C8C8" # type: ignore[union-attr] + assert indent(code) == {"left": 567} + assert [local(c) for c in rule.find(f"{W}pPr/{W}pBdr")] == ["left", "bottom"] # type: ignore[arg-type, union-attr] + assert note_code.find(f"{W}pPr/{W}pBdr/{W}left").get(f"{W}color") == "D9A400" # type: ignore[union-attr] + assert [local(c) for c in note_rule.find(f"{W}pPr/{W}pBdr")] == ["left", "bottom"] # type: ignore[arg-type, union-attr] + assert [local(c) for c in plain_rule.find(f"{W}pPr/{W}pBdr")] == ["bottom"] # type: ignore[arg-type, union-attr] + + +def test_the_table_look_says_what_its_flags_say() -> None: + (tbl, _) = body(tree(table()), plan()) + look = tbl.find(f"{W}tblPr/{W}tblLook") + flags = {"firstRow": 0x20, "lastRow": 0x40, "firstColumn": 0x80, "noHBand": 0x200, "noVBand": 0x400} + assert int(look.get(f"{W}val"), 16) == sum(bit for name, bit in flags.items() if look.get(f"{W}{name}") == "1") # type: ignore[union-attr, arg-type] + + +TEXT_WIDTH = 9072 # the default style's page: A4 with 2.5 cm margins + + +def lefts(root: ET.Element) -> list[int]: + return [int(e.get(f"{W}left")) for e in root.iter(f"{W}ind") if e.get(f"{W}left")] # type: ignore[arg-type] + + +def test_a_wide_indent_in_the_plan_cannot_push_the_text_off_the_page_or_squeeze_a_table() -> None: + document = tree(bullets(item("a", table(header=("x", "y", "z")), unit="u1"), item("b", "c", unit="u2"))) + the_plan = plan(unit("u1", "item", ind=(9000, 567)), unit("u2", "item", ind=(99999, 99999))) + result = body(document, the_plan) + root = ET.Element("x") + root.extend(result) + assert all(left <= TEXT_WIDTH - 2268 for left in lefts(root)) + assert max(lefts(root)) == TEXT_WIDTH - 2268 + assert all(int(h.get(f"{W}hanging")) <= int(h.get(f"{W}left")) for h in root.iter(f"{W}ind") if h.get(f"{W}hanging")) # type: ignore[arg-type] + tbl = next(c for c in result if local(c) == "tbl") + assert tbl.find(f"{W}tblPr/{W}tblInd") is None # three columns do not fit beside it: the indent goes + widths = [int(g.get(f"{W}w")) for g in tbl.findall(f"{W}tblGrid/{W}gridCol")] # type: ignore[arg-type] + assert min(widths) >= 1134 or sum(widths) == TEXT_WIDTH + + +def test_a_table_beside_an_indent_keeps_at_least_a_column_width_per_column() -> None: + wide = Table(tuple((Text(c),) for c in "abc"), (), ()) + for left in (0, 1000, 5000, 8000): + document = tree(bullets(item("a", wide, unit="u1"))) + tbl = next(c for c in body(document, plan(unit("u1", "item", ind=(left, 0)))) if local(c) == "tbl") + widths = [int(g.get(f"{W}w")) for g in tbl.findall(f"{W}tblGrid/{W}gridCol")] # type: ignore[arg-type] + assert min(widths) >= 1134 and sum(widths) <= TEXT_WIDTH + + +def test_a_deeply_nested_list_without_a_plan_stays_on_the_page() -> None: + inner: Block = bullets(item("deepest", unit="u70"), depth=70) + for depth in range(69, -1, -1): + inner = bullets(item(f"d{depth}", inner, unit=f"u{depth}"), depth=depth) + document = tree(inner) + data = DocxWriter(load_style("default")).write(document) + root = ET.fromstring(parts(data)["word/document.xml"]) + assert max(lefts(root)) <= min(31680, TEXT_WIDTH - 2268) + + # -- Whole documents -------------------------------------------------------------------------------------------- @@ -802,7 +935,8 @@ def test_hostile_text_in_every_inline_and_block_is_dropped_by_the_sanitiser_and_ assert dropped not in text root = ET.fromstring(contents["word/document.xml"]) assert CLEAN in "".join(t.text or "" for t in root.iter(f"{W}t")) - assert all(CLEAN in (i.text or "") or "REF" in (i.text or "") for i in root.iter(f"{W}instrText")) + # A field's instruction is the bookmark name and the switches, whatever the text around it says. + assert {(i.text or "").strip() for i in root.iter(f"{W}instrText")} == {"REF a \\n \\h"} # -- Validity --------------------------------------------------------------------------------------------------- @@ -935,3 +1069,36 @@ def test_libreoffice_numbers_the_hand_planned_document_and_updates_every_referen "2. Gamma", "Last, see Section 1.", ] + + +def alternative_data() -> bytes: + """A heading, a later alternative that shares its number (numId 0 and the number as a REF field, cached wrong), a + paragraph with a hyperlink, a table and a closing paragraph.""" + link = Link("https://example.com/", (Text("site"),)) + document = tree(sections=[ + section("One", unit="s1"), + section("Two", para("See ", link, "."), table(("a", "b"), ("c", "d")), para("End"), unit="s2")]) + the_plan = plan( + unit("s1", "section", bookmarks=("ld_one",)), + unit("s2", "section", num_id=0, ilvl=0, explicit=True, ind=(567, 567), + alt=AltNumber((field("ld_one", "XXX"), Literal_(".")), "tab")), + abstracts=[abstract(0, "headings", [level(0, text="%1.", p_style="Heading1", left=567, hanging=567)])], + nums=[Num(1, 0)], heading=1) + return DocxWriter(load_style("default"), generator="test").write(document, the_plan) + + +@pytest.mark.ooxml_schema +def test_the_alternative_document_validates() -> None: + from docx_support import require_schemas + + require_schemas() + assert validation_errors(alternative_data()) == [] + + +@pytest.mark.libreoffice +def test_libreoffice_shows_an_alternatives_number_as_its_holders_a_table_and_a_hyperlinks_text() -> None: + require_libreoffice() + shown = libreoffice_export({"alt": alternative_data()})["alt"] + lines = [" ".join(line.split()) for line in shown.lines if line.strip()] + # "Two" is not counted: it shows the number of "One" as a field, not a counter of its own. + assert lines == ["T", "1. One", "1. Two", "See site.", "H1", "H2", "a", "b", "c", "d", "End"] diff --git a/tests/test_docx_package.py b/tests/test_docx_package.py index 53896f2..cb16b06 100644 --- a/tests/test_docx_package.py +++ b/tests/test_docx_package.py @@ -815,7 +815,7 @@ def test_code_is_left_aligned_even_when_the_body_is_justified() -> None: '_rels/.rels': 'd6f21829c0081670bdb417d3e3f644b81134a121e074fb153c7686faaa8b167b', 'docProps/core.xml': '159ed0e537106ab29739fd357a5f6d89a2c0b0154daefd4c13d878bd16dff47e', 'docProps/app.xml': 'b285204d159fd58da80b6a0242cef523448f9c3a598b357b15faf4627160dcc3', - 'word/document.xml': '973a83b30f439e08a3238504ff0d6c8df6ea9259244e7ee82e1f81a3d274bb19', + 'word/document.xml': '58f0081f76a6a2092397770d50ae151e38f7aacdadf3525379b8112f964e0b8a', 'word/_rels/document.xml.rels': 'ada136c2e7d05a317579c9503c1f0dd8dfaa8f0cce687df86fe59e36d5fa5d57', 'word/styles.xml': '83adf799de91eac3e9bf1e658b8c2f53586c700be35df179e1b2f1765f189dfc', 'word/numbering.xml': '8d1f7769c35af16db98942ba09882646b45ba404e83686d4ce7a376c6a911f0f', From da8647d20183441e6464db0df11bf55e1cfb6498 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:42:07 +0000 Subject: [PATCH 085/118] Name the section each contents entry lists ContentsEntry gets a unit field: the unit of the section it lists, None for the attachments heading and for an attachment. A writer that builds the table of contents from Word's numbers keys each entry on it instead of counting positions. The HTML and text output are unchanged. --- src/legaldown_render/resolve/resolver.py | 2 +- src/legaldown_render/tree.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/legaldown_render/resolve/resolver.py b/src/legaldown_render/resolve/resolver.py index 7a9c013..7b849b7 100644 --- a/src/legaldown_render/resolve/resolver.py +++ b/src/legaldown_render/resolve/resolver.py @@ -575,7 +575,7 @@ def _contents( return () entries = [ ContentsEntry(section.level, section.label, unlinked(section.title), section.anchor, - section.condition_label) + section.condition_label, section.unit) for section in sections if section.level <= settings.depth ] diff --git a/src/legaldown_render/tree.py b/src/legaldown_render/tree.py index 0a255c9..2fddfd8 100644 --- a/src/legaldown_render/tree.py +++ b/src/legaldown_render/tree.py @@ -450,6 +450,9 @@ class ContentsEntry: title: tuple[Inline, ...] anchor: str | None condition_label: str | None = None + #: The section's :attr:`Section.unit` for an entry that lists a section, so that a writer keys the entry on the + #: unit and does not count positions; None for the attachments heading and for an attachment. + unit: str | None = None @dataclass(frozen=True, slots=True) From 82d51829eb825b4016522564eae8b024da05e440 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:42:07 +0000 Subject: [PATCH 086/118] Plan the cross-references, bookmarks and contents of a DOCX The planner now covers the renderer's designation of every {{ref:}} target with REF fields and the delimiters between them (fewest fields, then fewest \t fields, then fewest literal characters; a literal never stands for a counter), inside the text of references.format, with Word's value as each field's cached result and LibreOffice's as cached_lo. A designation no combination of fields reads like falls back to one field for the target's own number and is reported (render-docx-reference), as is a reference that reads differently in Word for a reason no unit already gives; a cover that relies on \t is an information. Every anchor of the tree gets a bookmark (ld_, lddef_, ldatt_ prefixes, at most 40 characters, unique ignoring case, a hidden _Ld name otherwise and for a unit a field needs that has no anchor), the table of contents is planned from the numbers Word draws, and each contents entry has a _Toc bookmark. --- .../resolve/docx_bookmarks.py | 68 ++ src/legaldown_render/resolve/docx_cover.py | 159 +++++ src/legaldown_render/resolve/docx_plan.py | 292 ++++++++- tests/test_docx_plan.py | 616 +++++++++++++++++- 4 files changed, 1120 insertions(+), 15 deletions(-) create mode 100644 src/legaldown_render/resolve/docx_bookmarks.py create mode 100644 src/legaldown_render/resolve/docx_cover.py diff --git a/src/legaldown_render/resolve/docx_bookmarks.py b/src/legaldown_render/resolve/docx_bookmarks.py new file mode 100644 index 0000000..5e23e4b --- /dev/null +++ b/src/legaldown_render/resolve/docx_bookmarks.py @@ -0,0 +1,68 @@ +"""Bookmark names for the DOCX plan (docs/decisions/0008, §6.3). + +Word reaches a paragraph by a bookmark: a ``REF`` field, an internal link and a table of contents entry all name one. +The planner gives every anchor of the tree a bookmark name; the writer allocates the ids (``writers/docx/ooxml.py`` +checks that a name is valid, at most 40 characters and unique ignoring case, which is how Word compares them). + +* A name is a prefix for what it marks and the anchor: ``ld_`` for a section, list item or paragraph, ``lddef_`` for a + defined term, ``ldatt_`` for an attachment, with ``-`` and ``:`` written as ``_`` (an identifier holds only + letters, digits and ``-``, so nothing else is lost). It is used only if it is a valid name of at most 40 + characters, nothing but the dashes was changed, and no name taken before is the same ignoring case. +* Otherwise, and for a unit a field needs that has no anchor, the name is hidden: ``_Ld``, ``k`` counting from 1 in + the order the names are given out. Word does not list names that start with ``_`` in its Bookmark dialog. +* The table of contents uses ``_Toc`` and nine digits, from 100000001, as Word does. +""" +from __future__ import annotations + +import re + +#: The longest bookmark name Word accepts, in characters. +MAX_LENGTH = 40 + +_LOSSLESS = re.compile(r"[A-Za-z0-9_:\-]+") +_VALID = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") + +#: The name of the attachments heading (the anchor ``ld:attachments``). +ATTACHMENTS_HEADING = "_LdAttachments" + + +class BookmarkNames: + """Hands out bookmark names, each unique ignoring case. Give names in a fixed order and the result is the same + on every run.""" + + def __init__(self) -> None: + self._taken: set[str] = set() + self._hidden = 0 + self._toc = 100000000 + + def _take(self, name: str) -> str | None: + if len(name) > MAX_LENGTH or not _VALID.fullmatch(name) or name.lower() in self._taken: + return None + self._taken.add(name.lower()) + return name + + def anchor(self, prefix: str, anchor: str) -> str: + """The name for *anchor* with *prefix* (``ld_``, ``lddef_``, ``ldatt_``), or a hidden one when the anchor + cannot be written as one.""" + name = self._take(prefix + re.sub(r"[-:]", "_", anchor)) if _LOSSLESS.fullmatch(anchor) else None + return name if name is not None else self.hidden() + + def fixed(self, name: str) -> str: + """*name*, a name of the renderer's own (taken so that no other can be the same).""" + taken = self._take(name) + assert taken is not None, name + return taken + + def hidden(self) -> str: + """The next hidden name, ``_Ld1``, ``_Ld2``, ...""" + self._hidden += 1 + name = self._take(f"_Ld{self._hidden}") + assert name is not None + return name + + def toc(self) -> str: + """The next name for a table of contents entry, ``_Toc100000001``, ...""" + self._toc += 1 + name = self._take(f"_Toc{self._toc}") + assert name is not None + return name diff --git a/src/legaldown_render/resolve/docx_cover.py b/src/legaldown_render/resolve/docx_cover.py new file mode 100644 index 0000000..f2d17e9 --- /dev/null +++ b/src/legaldown_render/resolve/docx_cover.py @@ -0,0 +1,159 @@ +"""The designation cover: how a cross-reference is built from Word's ``REF`` fields (docs/decisions/0008, §6.3, §13.3). + +A reference shows the designation of its target: the number of the section, the item or the paragraph written +with the ``ref`` templates of every unit on the way (``1.1.1``, ``I.A.1.a(i)``, ``1(b)(ii)``). In Word every part of +it that Word can compute must be a field, so that it stays right when a clause is inserted: ``REF \\n`` +returns the number Word draws for the bookmarked paragraph (its whole ``lvlText``, trailing periods dropped), and +nothing else Word could know may be written as literal text. This module finds, for one designation, the +combination of such fields and literal delimiters that reads exactly like it. + +Everything here works on token strings (:mod:`.docx_numbering`): the designation the renderer wrote, and what each +available field returns, are lists of :class:`~.docx_numbering.Lit` (text that is not a counter) and +:class:`~.docx_numbering.Count` (the counter of one owner, as formatted); a :class:`Title` is the text of a heading +under the ``none`` scheme (§13.3), the one thing besides counters a designation can hold that Word keeps up to date +(``REF \\h`` returns the bookmarked text). The designation is cut into **atoms** (one per character of +literal text, one per counter or title), a field is a sequence of atoms too, and a field **fits** at a position when +every atom of it equals the designation's there: characters equal, counters and titles of the same owner (their +text may differ: the field is then structurally right and its cached result is what Word will compute, which the +planner reports). A literal may stand only for characters of literal text, never for a counter or a title. Among the +covers the one with the fewest fields wins, then the fewest fields using ``\\t``, then the fewest literal +characters; of equal ones the first in the order the fields were given. + +``\\t`` (Word only; LibreOffice ignores it) suppresses the text around the counters of a number. It is modelled only +where its result is certain: when every literal of the field is text to drop (``Article I`` gives ``I``, +``§ 3`` gives ``3``), and not when one of them is a delimiter (``(a)``, ``a)``, ``1.2``), where what Word does +is not known (:func:`suppressed`). +""" +from __future__ import annotations + +from collections.abc import Sequence +from typing import NamedTuple + +from .docx_numbering import Count, Lit, Token + +#: The characters Word keeps when ``\t`` drops the text of a number: a field whose literal text holds one of them +#: has no ``\t`` form (what Word would make of it is not known). +DELIMITERS = frozenset(".,;:()[]/-–") + + +class Title(NamedTuple): + """The text of the heading of *owner* (a unit id), as the ``none`` scheme designates a section with it.""" + + owner: object + text: str + + +Shown = Token | Title + + +class Field(NamedTuple): + """A ``REF`` field that can stand in a designation: the bookmark of unit *owner*, the switches that decide what it + returns (*number*: ``\\n``, else the bookmarked text; *suppress_text*: ``\\t``), and what it returns as tokens + (Word's value, which is its cached result).""" + + owner: str + number: bool + suppress_text: bool + tokens: tuple[Shown, ...] + + +def text_of(tokens: Sequence[Shown]) -> str: + return "".join(token.text for token in tokens) + + +def atoms(tokens: Sequence[Shown]) -> list[tuple[str, object]]: + """*tokens* as atoms: ``("d", character)`` for each character of literal text, ``("c", owner)`` for a counter and + ``("t", owner)`` for a title.""" + out: list[tuple[str, object]] = [] + for token in tokens: + if isinstance(token, Lit): + out.extend(("d", character) for character in token.text) + elif isinstance(token, Count): + out.append(("c", token.owner)) + else: + out.append(("t", token.owner)) + return out + + +def suppressed(tokens: Sequence[Shown]) -> tuple[Shown, ...] | None: + """What ``REF \\n \\t`` returns for a field that returns *tokens* with ``\\n``: the counters alone, when every + literal text between them is text Word drops; None when one holds a delimiter or there is no counter, since what + Word keeps is then not known.""" + kept: list[Shown] = [] + for token in tokens: + if isinstance(token, Lit): + if any(character in DELIMITERS for character in token.text): + return None + else: + kept.append(token) + return tuple(kept) if kept else None + + +def _fits(wanted: Sequence[tuple[str, object]], at: int, field: Sequence[tuple[str, object]]) -> bool: + """True if the atoms of a field equal those of the designation from *at* on. Atoms are equal when they are the + same kind and owner or character: the *text* of a counter is not compared (that is the cached result's business).""" + return at + len(field) <= len(wanted) and all(wanted[at + k] == atom for k, atom in enumerate(field)) + + +class _Cost(NamedTuple): + fields: int + suppressing: int + literal: int + + +def cover(designation: Sequence[Shown], fields: Sequence[Field]) -> list[str | Field] | None: + """The cheapest cover of *designation* by *fields* and literal text, as literal strings and fields in order, + or None when there is none (a counter or a title no field stands for, or text around a counter that no field + drops).""" + wanted = atoms(designation) + shapes = [atoms(field.tokens) for field in fields] + size = len(wanted) + best: list[tuple[_Cost, list[str | Field]] | None] = [None] * (size + 1) + best[size] = (_Cost(0, 0, 0), []) + for at in range(size - 1, -1, -1): + options: list[tuple[_Cost, list[str | Field]]] = [] + later = best[at + 1] + if wanted[at][0] == "d" and later is not None: + cost, rest = later + options.append((_Cost(cost.fields, cost.suppressing, cost.literal + 1), [str(wanted[at][1]), *rest])) + for field, shape in zip(fields, shapes, strict=True): + if shape and shape[0] == wanted[at] and _fits(wanted, at, shape): + onward = best[at + len(shape)] + if onward is not None: + cost, rest = onward + options.append((_Cost(cost.fields + 1, cost.suppressing + field.suppress_text, cost.literal), + [field, *rest])) + if options: + best[at] = min(options, key=lambda option: option[0]) + if best[0] is None: + return None + merged: list[str | Field] = [] + for piece in best[0][1]: + if merged and isinstance(piece, str) and isinstance(merged[-1], str): + merged[-1] += piece + else: + merged.append(piece) + return merged + + +def uncovered(designation: Sequence[Shown], fields: Sequence[Field]) -> int | None: + """Why :func:`cover` finds none: the index (among the atoms of *designation*) of the last atom that a cover of what + comes before it reaches and that neither a literal nor a field can stand for; None when there is a cover.""" + wanted = atoms(designation) + shapes = [atoms(field.tokens) for field in fields] + size = len(wanted) + onward = [False] * (size + 1) + onward[size] = True + steps: list[list[int]] = [[] for _ in range(size)] + for at in range(size - 1, -1, -1): + if wanted[at][0] == "d": + steps[at].append(at + 1) + steps[at] += [at + len(shape) for shape in shapes if shape and shape[0] == wanted[at] and _fits(wanted, at, shape)] + onward[at] = any(onward[after] for after in steps[at]) + if onward[0]: + return None + reached = {0} + for at in range(size): + if at in reached: + reached.update(steps[at]) + return max(at for at in reached if at < size and not onward[at]) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index a9fb97f..2eea4fd 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -7,9 +7,13 @@ and, for each unit, the (numId, ilvl) of its paragraph, and every placement is checked against the number the resolver gave the unit, by simulating Word's counters (:mod:`.docx_numbering`). -What a plan holds so far: the numbering definitions and the numbering of every unit. ``UnitPlan.ind`` is never -set, and the reference fields, bookmarks and table of contents (``refs``, ``anchors``, ``toc``, ``bookmarks``) are -empty. :func:`docx_style_errors` is the part that runs on every DOCX render, before it (ADR 0006). +A plan holds the numbering definitions and the numbering of every unit, the bookmark of every anchor of the tree +(``anchors``, ``UnitPlan.bookmarks``; :mod:`.docx_bookmarks`), the fields of every cross-reference (``refs``: the +renderer's designation of the target covered by ``REF`` fields and the delimiters between them, with Word's value as +each field's cached result; :mod:`.docx_cover`) and the entries of the table of contents (``toc``, +``UnitPlan.toc_bookmark``), with the diagnostics for what Word will not show as the renderer does. +``UnitPlan.ind`` is never set. :func:`docx_style_errors` is the part that runs on every DOCX render, before it +(ADR 0006). **Data model.** Frozen data only; the planner assigns bookmark *names*, never ids (the writer allocates those); names are valid Word bookmark names of at most 40 characters and unique ignoring case. All sequences are tuples, all @@ -19,7 +23,7 @@ import re from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Literal, NamedTuple from legaldown import Diagnostic @@ -27,6 +31,8 @@ from ..diagnostics import by_line from ..style.model import LevelFormat, Style from ..tree import ( + CrossRef, + DefinedTerm, DraftingNote, Inline, List, @@ -36,16 +42,23 @@ Quote, RenderTree, Section, + iter_located_body_inlines, + iter_tree_inlines, ) from ..urls import printable +from .docx_bookmarks import ATTACHMENTS_HEADING, BookmarkNames +from .docx_cover import Field, Shown, Title, atoms, cover, suppressed, text_of, uncovered from .docx_numbering import ( NUM_FMT, SAMPLE_COUNTER, Count, Ghost, LevelParts, + Lit, Step, WordNumbering, + extend_tokens, + fill_tokens, level_text, level_text_problem, lvl_text, @@ -162,8 +175,8 @@ class UnitPlan: #: Why the unit is not on the plain path: ``resync``, ``alternative``, ``reserved-level``, ``restart-family``, #: ``restart-section``, ``restart-alternative``, ``label-fallback``, ``clamped``, ``skipped-level``. cause: tuple[str, ...] - #: Names to wrap around the paragraph's title/first paragraph content: the anchor bookmark first (when the - #: unit has an anchor), then the helper bookmark a cover needs. + #: Names to wrap around the paragraph's title/first paragraph content: the bookmark of the unit's anchor when it + #: has one, else a hidden one when a reference field points at it, else none. bookmarks: tuple[str, ...] #: ``_Toc`` name of a heading listed in the contents. toc_bookmark: str | None @@ -364,6 +377,26 @@ class _Placed(NamedTuple): abstract: _Abstract +class _Reference(NamedTuple): + """The ``{{ref:}}`` to one unit that the plan builds fields for: the unit, its section, what the renderer shows and + the first line it stands at (None outside the body).""" + + target: str + section: str | None + text: str + line: int | None + + +class _Built(NamedTuple): + """What the plan makes of one reference: its pieces (literal text and fields), whether they read exactly as the + renderer's text, and what LibreOffice shows for them when that is not what Word shows (a field with ``\\t``), + else None.""" + + pieces: list[str | Field] + exact: bool + lo_text: str | None + + class _Drift: """A run of units of one level that Word counts off from the resolver by the same amount.""" @@ -395,6 +428,14 @@ def __init__(self, tree: RenderTree, file: str | None) -> None: self.depth_reported = False self.definitions = _definitions(self.formats) self.by_kind: dict[str, _Abstract] = {} + #: Every unit's node, in document order; what Word numbers each as (:class:`Step`); and its designation as + #: the resolver wrote it, as tokens (§13.1-§13.3), each counter with the unit that holds it. + self.nodes: dict[str, _Unit] = {} + self.steps: dict[str, Step] = {} + self.desig: dict[str, list[Shown]] = {} + #: The parts of the number of the open section, one per level: the unit that holds it (a skipped level has a + #: :class:`Ghost`) and its value. + self.owners: list[tuple[object, int]] = [] # -- registry ------------------------------------------------------------------- @@ -431,6 +472,201 @@ def report(self, rule: str, level: str, message: str, line: int | None) -> None: if line is not None and (known.line is None or line < known.line): self.diagnostics[index] = found + # -- references, bookmarks and the table of contents ----------------------------- + + def words(self, unit: str) -> tuple[Shown, ...]: + """What ``REF \\n`` returns, as tokens: the number Word draws for the unit without its trailing periods + (:attr:`Step.word_ref` with the unit that holds each counter).""" + tokens = list(self.steps[unit].tokens) + while tokens and isinstance(tokens[-1], Lit): + stripped = tokens[-1].text.rstrip(".") + if stripped == tokens[-1].text: + break + if stripped: + tokens[-1] = Lit(stripped) + break + tokens.pop() + return tuple(tokens) + + def fields(self, designation: Sequence[Shown]) -> list[Field]: + """The fields that can stand for parts of *designation*: for each unit that holds a counter of it, ``REF \\n`` + of its bookmark, and ``REF \\n \\t`` where that leaves the counters bare (:func:`.docx_cover.suppressed`); for a + heading under the ``none`` scheme the bookmarked text. A later alternative takes its number from the unit that + holds it and is never a field's target (§15.8).""" + fields: list[Field] = [] + seen: set[str] = set() + for token in designation: + owner = None if isinstance(token, Lit) else token.owner + if not isinstance(owner, str) or owner in seen or owner not in self.units: + continue + seen.add(owner) + if self.units[owner].num_id == 0 or self.units[owner].alt is not None: + continue + if isinstance(token, Title): + fields.append(Field(owner, False, False, (token,))) + elif owner in self.steps: + words = self.words(owner) + fields.append(Field(owner, True, False, words)) + bare = suppressed(words) + if bare is not None and atoms(bare) != atoms(words): + fields.append(Field(owner, True, True, bare)) + return fields + + def why_uncovered(self, designation: Sequence[Shown], fields: Sequence[Field]) -> str: + """Why no combination of *fields* and delimiters reads like *designation* (§13.1).""" + wanted = atoms(designation) + kind, owner = wanted[uncovered(designation, fields) or 0] + if kind == "c" and isinstance(owner, Ghost): + return "a heading level is skipped before it, and no field stands for the level that is missing" + for candidate in fields: + if candidate.number and any(isinstance(t, Count) and t.owner == owner for t in candidate.tokens): + return (f"Word draws its number as '{printable(text_of(candidate.tokens))}', with text or punctuation " + f"around the counter that the designation does not have and that Word's \\t switch cannot " + f"leave out") + return "no combination of Word's fields reads like it" + + def own_field(self, unit: str | None) -> Field | None: + """The one field for the number Word draws for *unit* (its heading text under the ``none`` scheme), which a + reference falls back to when its designation cannot be covered; None for a unit with neither.""" + if unit is None or unit not in self.units: + return None + if unit in self.steps: + return Field(unit, True, False, self.words(unit)) + title = self.desig.get(unit, []) + return Field(unit, False, False, (title[0],)) if title and isinstance(title[0], Title) else None + + def reference(self, found: _Reference) -> _Built | None: + """The fields of the reference *found*: its designation covered by fields and delimiters, inside the text of + ``references.format``; None for a target the plan knows nothing of. A designation that cannot be covered is one + field for the number Word draws for the target, and is reported, as is a reference that reads differently + in Word for a reason no diagnostic of a unit already gives.""" + node = self.nodes.get(found.target) + if node is None or found.target not in self.units: + return None + numbered = isinstance(node, Section) or node.number is not None + designation = self.desig[found.target] if numbered else self.desig.get(found.section or "", []) + fields = self.fields(designation) + covered = cover(designation, fields) + reason = None + if covered is None: + reason = self.why_uncovered(designation, fields) + own = self.own_field(found.target if numbered else found.section) + if own is None: + return None + covered = [own] + pieces: list[str | Field] = [] + parts = self.formats.reference.split("{designation}") + for index, literal in enumerate(parts): + for piece in ([literal] if literal else []) + (covered if index < len(parts) - 1 else []): + if pieces and isinstance(piece, str) and isinstance(pieces[-1], str): + pieces[-1] += piece + else: + pieces.append(piece) + shown = "".join(piece if isinstance(piece, str) else text_of(piece.tokens) for piece in pieces) + text = printable(found.text) + if reason is not None: + self.report("render-docx-reference", "warning", + f"The cross-reference '{text}' cannot be built from Word's numbering fields: {reason}. It is " + f"one field for the target's own number, which reads '{printable(shown)}' in Word.", + found.line) + elif shown != found.text: + chain = {t.owner for t in designation if not isinstance(t, Lit) and isinstance(t.owner, str)} + if not chain & self.flagged and found.target not in self.flagged: + self.report("render-docx-reference", "warning", + f"The cross-reference '{text}' will read '{printable(shown)}' in Word: a part of it is " + f"numbered or written differently there.", found.line) + lo_text = None + if any(isinstance(piece, Field) and piece.suppress_text for piece in pieces): + lo_text = "".join(piece if isinstance(piece, str) else text_of(self.words(piece.owner) if piece.number + else piece.tokens) for piece in pieces) + return _Built(pieces, shown == found.text, lo_text) + + def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan], tuple[TocEntryPlan, ...]]: + """The bookmarks, the cross-reference fields and the table of contents, which need the numbering of every + unit: the bookmark of every anchor (:class:`.docx_bookmarks.BookmarkNames`, given out in the order of the + document so that the names are the same on every run), a hidden one for a unit a field needs that has no + anchor, the fields of every ``{{ref:}}`` target, and the table of contents entries.""" + found: dict[str, _Reference] = {} + for inline in iter_tree_inlines(self.tree): + if isinstance(inline, CrossRef) and inline.target_unit is not None: + found.setdefault(inline.target_unit, _Reference(inline.target_unit, inline.section_unit, + inline.text, None)) + for inline, line in iter_located_body_inlines(self.tree): + if isinstance(inline, CrossRef) and inline.target_unit in found and line is not None: + known = found[inline.target_unit] + if known.line is None or line < known.line: + found[inline.target_unit] = known._replace(line=line) + built: dict[str, _Built] = {} + for target, reference in found.items(): + made = self.reference(reference) + if made is not None: + built[target] = made + needed = {piece.owner for made in built.values() for piece in made.pieces if isinstance(piece, Field)} + word_only = [(found[target].text, made.lo_text) for target, made in built.items() if made.lo_text is not None] + if word_only: + first, lo = word_only[0] + more = f"; so do {len(word_only) - 1} more" if len(word_only) > 1 else "" + self.report("render-docx-reference-word-only", "info", + f"The cross-reference '{printable(first)}' leaves out the words of a number with Word's \\t " + f"switch, which LibreOffice ignores: there it reads '{printable(lo)}'{more}.", None) + + names = BookmarkNames() + anchors: dict[str, str] = {} + bookmark: dict[str, str] = {} + units = dict(self.units) + for unit, node in self.nodes.items(): + anchor = node.anchor + if anchor: + name = names.anchor("ld_", anchor) + anchors.setdefault(anchor, name) + elif unit in needed: + name = names.hidden() + else: + continue + bookmark[unit] = name + units[unit] = replace(units[unit], bookmarks=(name,)) + for inline in iter_tree_inlines(self.tree): + if isinstance(inline, DefinedTerm) and inline.defining and inline.anchor and inline.anchor not in anchors: + anchors[inline.anchor] = names.anchor("lddef_", inline.anchor.removeprefix("def:")) + for attachment in self.tree.attachments: + if attachment.anchor and attachment.anchor not in anchors: + anchors[attachment.anchor] = names.anchor("ldatt_", attachment.anchor) + if self.tree.attachments_anchor and self.tree.attachments_anchor not in anchors: + anchors[self.tree.attachments_anchor] = names.fixed(ATTACHMENTS_HEADING) + + refs: dict[str, RefPlan] = {} + for target, made in built.items(): + segments: list[Segment] = [] + for piece in made.pieces: + if isinstance(piece, str): + segments.append(Literal_(piece)) + continue + cached = text_of(piece.tokens) + plain = text_of(self.words(piece.owner)) if piece.number else cached + segments.append(FieldSeg(bookmark[piece.owner], piece.number, piece.suppress_text, True, cached, + plain if piece.suppress_text else cached)) + refs[target] = RefPlan(tuple(segments), made.exact) + + toc: list[TocEntryPlan] = [] + for entry in self.tree.contents: + name = names.toc() + number = "" + sep: Literal["\t", " ", ""] = "" + mine = units.get(entry.unit) if entry.unit is not None else None + if mine is not None: + mine = units[mine.unit] = replace(mine, toc_bookmark=name) + suff: str | None = None + if mine.alt is not None: + number = "".join(piece.text if isinstance(piece, Literal_) else piece.cached + for piece in mine.alt.segments) + suff = mine.alt.suff + elif mine.ilvl is not None: + number, suff = mine.label, self.definitions["headings"][mine.ilvl].suff + if number: + sep = "\t" if suff == "tab" else " " + toc.append(TocEntryPlan(entry.unit, entry.level, number, sep, entry.title, entry.condition_label, name)) + return units, anchors, refs, tuple(toc) + # -- the pass ------------------------------------------------------------------- def run(self) -> tuple[DocxPlan, list[Diagnostic]]: @@ -444,12 +680,14 @@ def run(self) -> tuple[DocxPlan, list[Diagnostic]]: self.block(block, section) for drift in self.drifts: self.report_drift(drift) + units, anchors, refs, toc = self.finish() abstracts = tuple( AbstractNum(a.id, a.kind, f"{0x4C440000 + a.id:08X}", # type: ignore[arg-type] "multilevel" if a.kind == "headings" else "hybridMultilevel", tuple(a.levels)) for a in self.abstracts) heading_id = self.heading_num.id if self.heading_num else None - return DocxPlan(abstracts, tuple(self.nums), heading_id, units=self.units), by_line(self.diagnostics) + return (DocxPlan(abstracts, tuple(self.nums), heading_id, units=units, anchors=anchors, refs=refs, toc=toc), + by_line(self.diagnostics)) def block(self, block: object, section: Section | None) -> None: match block: @@ -466,6 +704,9 @@ def block(self, block: object, section: Section | None) -> None: def add(self, node: _Unit, kind: Literal["section", "item", "paragraph"], num: Num | None, ilvl: int | None, *, explicit: bool, cause: tuple[str, ...] = (), step: Step | None = None) -> None: assert node.unit is not None + self.nodes[node.unit] = node + if step is not None: + self.steps[node.unit] = step self.units[node.unit] = UnitPlan( node.unit, kind, num.id if num else None, ilvl, explicit, None, None, step.label if step else "", step.word_ref if step else "", step.count if step else None, @@ -548,11 +789,33 @@ def shared_counter(self, message: str, line: int | None) -> None: # -- units ---------------------------------------------------------------------- + def section_designation(self, section: Section) -> list[Shown]: + """The designation of *section* as tokens, written as the resolver writes it (§13.1): the ``ref`` of every part + of its number after the one before; a part the document skipped is a :class:`Ghost` that no unit holds. Under + the ``none`` scheme it is the heading's text (§13.3), and empty when that is.""" + assert section.unit is not None + if self.textual: + text = section.designation or "" + return [Title(section.unit, text)] if text else [] + assert section.number is not None + del self.owners[section.level - 1:] + while len(self.owners) < section.level - 1: + self.owners.append((Ghost(section.unit, len(self.owners)), 1)) + self.owners.append((section.unit, section.number)) + designation: list[Shown] = [] + for part, (owner, value) in enumerate(self.owners): + level = self.formats.headings[min(part, len(self.formats.headings) - 1)] + designation = extend_tokens( + designation, fill_tokens(level.ref, n=[Count(owner, format_counter(value, level.counter))]), + textual=False) + return designation + def section(self, section: Section) -> None: unit = section.unit assert unit is not None self.parent[unit] = section.parent_unit self.section_level[unit] = section.level + self.desig[unit] = self.section_designation(section) if self.heading_num is None or section.number is None: self.add(section, "section", None, None, explicit=False) return @@ -570,9 +833,14 @@ def paragraph(self, paragraph: Paragraph, section: Section) -> None: assert paragraph.unit is not None and section.unit is not None self.parent[paragraph.unit] = paragraph.parent_unit fmt = self.formats.paragraphs + base = self.desig[section.unit] if paragraph.number is None or fmt is None: + self.desig[paragraph.unit] = base self.add(paragraph, "paragraph", None, None, explicit=False) return + piece = fill_tokens(fmt.ref, n=[Count(paragraph.unit, format_counter(paragraph.number, fmt.counter))], + section=base) + self.desig[paragraph.unit] = extend_tokens(base, piece, textual=self.textual) abstract = self.abstract("paragraphs") num = self.paragraph_nums.get(section.unit) if num is None: @@ -634,9 +902,14 @@ def list_(self, lst: List, section: Section | None) -> None: qualified = section is not None and shows_section(fmt.label, refs, lst.depth, textual=self.textual) if qualified: self.fallback_label(fmt, lst.family, ilvl, "list item", first.line) + parent_des = self.desig.get(parent, []) if parent is not None else [] + section_des = self.desig[section.unit] if section is not None and section.unit is not None else [] for index, item in enumerate(lst.items): - assert item.unit is not None + assert item.unit is not None and item.number is not None self.parent[item.unit] = item.parent_unit + piece = fill_tokens(fmt.ref, n=[Count(item.unit, format_counter(item.number, fmt.counter))], + section=section_des) + self.desig[item.unit] = extend_tokens(parent_des, piece, textual=self.textual) item_cause = cause if index == 0 else () if lst.depth > 8 and "clamped" not in item_cause: item_cause += ("clamped",) @@ -671,9 +944,12 @@ def bullets(self, lst: List, section: Section | None) -> None: if self.bullet_num is None: self.bullet_num = self.new_num(abstract) ilvl = min(lst.depth, 8) + parent = lst.items[0].parent_unit for item in lst.items: assert item.unit is not None self.parent[item.unit] = item.parent_unit + self.nodes[item.unit] = item + self.desig[item.unit] = self.desig.get(parent, []) if parent is not None else [] cause = ("clamped",) if lst.depth > 8 else () self.units[item.unit] = UnitPlan(item.unit, "item", self.bullet_num.id, ilvl, True, None, None, "", "", None, cause, (), None) diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index 0d635ec..b84e340 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -1,11 +1,14 @@ -"""The DOCX numbering plan (docs/architecture.md, "The DOCX planner"): which abstractNums and nums a document needs, +"""The DOCX plan (docs/architecture.md, "The DOCX planner"): which abstractNums and nums a document needs, which (numId, ilvl) each section, list item and numbered paragraph takes, and that Word, simulated, numbers every unit -as the resolver did, or a diagnostic says why not. Everything is checked on the plan's data; no file is written.""" +as the resolver did, or a diagnostic says why not; and, from the numbering, the fields of every cross-reference, the +bookmarks and the table of contents. Everything is checked on the plan's data; no file is written (the LibreOffice +round trip is in test_docx_roundtrip.py).""" from __future__ import annotations import functools import random -from collections.abc import Iterator +import re +from collections.abc import Iterator, Sequence from pathlib import Path import pytest @@ -13,12 +16,31 @@ from legaldown_render import DocumentError, render from legaldown_render.errors import StyleError -from legaldown_render.resolve.docx_numbering import NUM_FMT, word_counter -from legaldown_render.resolve.docx_plan import DocxPlan, docx_style_errors, plan_docx +from legaldown_render.resolve.docx_bookmarks import BookmarkNames +from legaldown_render.resolve.docx_cover import Field, Title, cover, suppressed, uncovered +from legaldown_render.resolve.docx_numbering import NUM_FMT, Count, Ghost, Lit, word_counter +from legaldown_render.resolve.docx_plan import ( + DocxPlan, + FieldSeg, + Literal_, + Segment, + docx_style_errors, + plan_docx, +) from legaldown_render.resolve.numbering import format_counter, formats_of from legaldown_render.style import load_style from legaldown_render.style.docx import docx_settings -from legaldown_render.tree import DraftingNote, List, ListItem, Paragraph, Quote, RenderTree, Section +from legaldown_render.tree import ( + CrossRef, + DraftingNote, + List, + ListItem, + Paragraph, + Quote, + RenderTree, + Section, + iter_tree_inlines, +) NONE = {"numbering.scheme": "none"} PARAGRAPHS = {"paragraphs.numbered": True} @@ -517,7 +539,7 @@ def test_the_same_tree_gives_an_equal_plan_and_the_plan_is_frozen() -> None: with pytest.raises(AttributeError): first[0].heading_num_id = 5 # type: ignore[misc] assert list(first[0].units) == [u.unit for u in units(result.tree)] - assert first[0].refs == {} and first[0].anchors == {} and first[0].toc == () + assert first[0].refs == {} and first[0].toc == () and first[0].anchors == {"a": "ld_a"} def test_unit_plans_carry_the_whole_plan_for_a_unit() -> None: @@ -873,3 +895,583 @@ def test_the_random_documents_reach_every_case_the_property_is_about() -> None: "render-docx-list-depth", "render-docx-shared-counter"} <= seen_rules assert {"restart-family", "resync", "label-fallback", "skipped-level", "clamped"} <= seen_causes assert depth >= 10 + + +# ===================================================================================================== +# Cross-references, bookmarks and the table of contents (docs/architecture.md, "The DOCX planner") +# ===================================================================================================== + + +def show(plan: DocxPlan, segments: Sequence[Segment]) -> str: + """*segments* as one line: literal text in quotes, a field as ``REF`` and the anchor of the bookmark it points at, + ``\\t`` after it when it suppresses the text around the counter, ``\\h`` for the text of a heading.""" + anchors = {name: anchor for anchor, name in plan.anchors.items()} + pieces = [] + for segment in segments: + if isinstance(segment, Literal_): + pieces.append(f'"{segment.text}"') + else: + pieces.append(f"REF {anchors[segment.bookmark]}" + (" \\t" if segment.suppress_text else "") + + ("" if segment.number else " \\h")) + return " + ".join(pieces) + + +def cached(segments: Sequence[Segment]) -> str: + return "".join(s.text if isinstance(s, Literal_) else s.cached for s in segments) + + +#: A document that has every kind of target; the references at the end are the ones the tests read. +TARGETS = """# Alpha {#alpha} + +Intro paragraph one. {#p1} + +- first item {#i1} +- second item {#i2} + - nested one {#n1} + - nested two {#n2} + - deep one {#d1} +- third item {#i3} + +## Beta {#beta} + +### Gamma {#gamma} + +1. ordered one {#o1} +2. ordered two {#o2} + - bullet in ordered {#b1} + +#### Delta {#delta} + +##### Epsilon {#eps} + +- eps item {#ei} + +# Second {#second} + +- a {#sa} +""" +REFERENCES = "\n" + " ".join(f"{{{{ref: {anchor}}}}}" for anchor in ( + "alpha", "p1", "i1", "n2", "d1", "beta", "gamma", "o2", "b1", "delta", "eps", "ei", "second", "sa", "i3")) + +ARTICLE = {"numbering.levels": [{"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}]} +SECTION_SIGN = {"numbering.levels": [{"counter": "decimal", "label": "§ {n}", "ref": "{n}"}], + "references.format": "§ {designation}"} + + +def covers(variant: str, **settings: object) -> dict[str, tuple[str, str]]: + """Anchor of each target to (what the renderer shows, the plan's cover), for the document of :data:`TARGETS`.""" + result = render(FRONT + TARGETS + REFERENCES, format="text", **settings) + plan, _ = plan_docx(result.tree) + units_of = {n.unit: n for n in units(result.tree)} + found = {} + for inline in iter_tree_inlines(result.tree): + if isinstance(inline, CrossRef): + anchor = units_of[inline.target_unit].anchor + found[anchor] = (inline.text, show(plan, plan.refs[inline.target_unit].segments)) + return found + + +# What each style's references are made of: fields wherever Word draws the number, literal text only for the +# delimiters between them. A field for the target alone covers a ``{path}`` label (``1.1.1``), one field per +# level covers a ``{n}`` label (``I.A.1``). +COVERS = [ + # default: decimal headings, (a) (i) (A) items, 1.1.1(2) for an ordered item in a section + ("default", {}, "alpha", "1", "REF alpha"), + ("default", {}, "i1", "1(a)", "REF alpha + REF i1"), + ("default", {}, "d1", "1(b)(ii)(A)", "REF alpha + REF i2 + REF n2 + REF d1"), + ("default", {}, "gamma", "1.1.1", "REF gamma"), + ("default", {}, "o2", "1.1.1(2)", 'REF gamma + "(" + REF o2 + ")"'), + ("default", {}, "eps", "1.1.1.1.1", "REF eps"), + ("default", {}, "ei", "1.1.1.1.1(a)", "REF eps + REF ei"), + # outline: every part of the number is a counter of its own level + ("outline", {"style": "outline"}, "alpha", "I", "REF alpha"), + ("outline", {"style": "outline"}, "beta", "I.A", 'REF alpha + "." + REF beta'), + ("outline", {"style": "outline"}, "eps", "I.A.1.a(i)", + 'REF alpha + "." + REF beta + "." + REF gamma + "." + REF delta + REF eps'), + ("outline", {"style": "outline"}, "i3", "I(c)", "REF alpha + REF i3"), + # mixed: 1(a)(i) + ("mixed", {"overrides": {"numbering.scheme": "mixed"}}, "gamma", "1(a)(i)", "REF alpha + REF beta + REF gamma"), + ("mixed", {"overrides": {"numbering.scheme": "mixed"}}, "o2", "1(a)(i)(2)", + 'REF alpha + REF beta + REF gamma + "(" + REF o2 + ")"'), + # continental: 1.1 for an item (the number of the section is not part of the item's own level in Word yet), + # a(n) item nested in an item written "b)" and referred to as "(b)" + ("continental", {"style": "continental"}, "i1", "1.1", 'REF alpha + "." + REF i1'), + ("continental", {"style": "continental"}, "n2", "1.2(b)", 'REF alpha + "." + REF i2 + "(" + REF n2'), + ("continental", {"style": "continental"}, "p1", "1(1)", "REF alpha + REF p1"), + ("continental", {"style": "continental"}, "gamma", "1.1.1", "REF gamma"), + # the none scheme: the text of the heading, then the pieces after a space + ("none", {"overrides": {"numbering.scheme": "none"}}, "alpha", "Alpha", "REF alpha \\h"), + ("none", {"overrides": {"numbering.scheme": "none"}}, "d1", "Alpha (b) (ii) (A)", + 'REF alpha \\h + " " + REF i2 + " " + REF n2 + " " + REF d1'), + ("none", {"overrides": {"numbering.scheme": "none"}}, "o2", "Gamma (2)", 'REF gamma \\h + " (" + REF o2 + ")"'), + # words and symbols in a label that the reference leaves out: \t drops them, the format puts them back + ("article", {"overrides": {**ARTICLE, "references.format": "Article {designation}"}}, "i1", "Article I(a)", + '"Article " + REF alpha \\t + REF i1'), + ("article", {"overrides": {**ARTICLE, "references.format": "Article {designation}"}}, "beta", "Article I.1", + '"Article " + REF beta'), + ("section-sign", {"overrides": SECTION_SIGN}, "d1", "§ 1(b)(ii)(A)", + '"§ " + REF alpha \\t + REF i2 + REF n2 + REF d1'), +] + + +@pytest.mark.parametrize(("variant", "settings", "anchor", "text", "expected"), COVERS) +def test_a_reference_is_the_renderers_designation_covered_by_fields_and_delimiters( + variant: str, settings: dict, anchor: str, text: str, expected: str) -> None: + assert covers(variant, **settings)[anchor] == (text, expected) + + +def test_a_cover_reads_as_the_renderer_does_and_a_field_is_cached_as_word_draws_it() -> None: + result = render(FRONT + TARGETS + REFERENCES, format="text", style="outline") + plan, diagnostics = plan_docx(result.tree) + for inline in iter_tree_inlines(result.tree): + if isinstance(inline, CrossRef): + reference = plan.refs[inline.target_unit] + assert reference.exact and cached(reference.segments) == inline.text + for segment in reference.segments: + if isinstance(segment, FieldSeg): + assert segment.number and segment.hyperlink and not segment.suppress_text + assert segment.cached_lo == segment.cached + # a section's own field returns its number without the period of its label + assert plan.units["s1"].label == "I." and plan.units["s1"].word_ref == "I" + assert next(s for s in plan.refs["s1"].segments if isinstance(s, FieldSeg)).cached == "I" + assert diagnostics and all(d.rule == "render-docx-static-restart" for d in diagnostics) + + +def test_a_reference_to_an_item_in_another_section_is_built_from_the_chain_of_its_own_section() -> None: + body = "# One {#one}\n\n## Inner {#inner}\n\n- deep {#deep}\n - deeper {#deeper}\n\n# Two {#two}\n\n" \ + "See {{ref: deeper}} and {{ref: inner}}, {{ref: one}}.\n" + tree, plan, diagnostics = planned(body, style="outline") + refs = {i.text: plan.refs[i.target_unit] for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)} + assert [(text, show(plan, r.segments)) for text, r in refs.items()] == [ + ("I.A(a)(i)", 'REF one + "." + REF inner + REF deep + REF deeper'), + ("I.A", 'REF one + "." + REF inner'), ("I", "REF one")] + assert all(r.exact for r in refs.values()) and not [d for d in diagnostics if "reference" in d.rule] + + +def test_a_reference_to_a_unit_the_style_does_not_number_shows_the_designation_of_its_section() -> None: + body = "# Alpha {#alpha}\n\n## Beta {#beta}\n\n- bullet {#bullet}\n\nSee {{ref: bullet}}.\n" + tree, plan, diagnostics = planned(body, overrides={"enumeration.enabled": False}) + [reference] = [i for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + assert reference.text == "1.1" and reference.target_unit != reference.section_unit + assert show(plan, plan.refs[reference.target_unit].segments) == "REF beta" + assert plan.refs[reference.target_unit].exact and cached(plan.refs[reference.target_unit].segments) == "1.1" + # the bullet itself is a target nobody's field points at: it keeps its own bookmark for links + assert plan.units[reference.target_unit].bookmarks == ("ld_bullet",) + + +def test_references_format_wraps_every_occurrence_of_the_designation() -> None: + body = "# Alpha {#alpha}\n\n- x {#x}\n\nSee {{ref: alpha}} {{ref: x}}.\n" + for fmt, expected in (("Section {designation} hereof", ["Section 1 hereof", "Section 1(a) hereof"]), + ("{designation} and {designation}", ["1 and 1", "1(a) and 1(a)"]), + ("[{designation}]", ["[1]", "[1(a)]"])): + tree, plan, _ = planned(body, overrides={"references.format": fmt}) + found = [i for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + assert [i.text for i in found] == expected + assert [cached(plan.refs[i.target_unit].segments) for i in found] == expected + assert all(plan.refs[i.target_unit].exact for i in found) + tree, plan, _ = planned(body, overrides={"references.format": "{designation} and {designation}"}) + [first, second] = [plan.refs[i.target_unit] for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + assert show(plan, first.segments) == 'REF alpha + " and " + REF alpha' + assert show(plan, second.segments) == 'REF alpha + REF x + " and " + REF alpha + REF x' + + +def test_a_reference_to_the_same_target_is_planned_once() -> None: + tree, plan, _ = planned("# A {#a}\n\n{{ref: a}} {{ref: a}} and again {{ref: a}}.\n") + assert list(plan.refs) == ["s1"] + + +def test_the_fields_of_a_unit_that_is_not_a_target_have_a_bookmark_of_their_own() -> None: + """Every unit a field points at has a bookmark: its anchor's, or a hidden one when it has no anchor.""" + tree, plan, _ = planned("# A\n\n- x\n - y {#y}\n\nSee {{ref: y}}.\n") + assert plan.units["u2"].bookmarks == ("ld_y",) + assert plan.units["u1"].bookmarks == ("_Ld1",) and plan.units["s1"].bookmarks == ("ld_a",) + assert plan.units["s1"].bookmarks[0] in plan.anchors.values() and "_Ld1" not in plan.anchors.values() + [reference] = plan.refs.values() + assert [s.bookmark for s in reference.segments] == ["ld_a", "_Ld1", "ld_y"] + + +# -- Why a reference cannot be built from fields ------------------------------------------------------------- + + +def references_of(tree: RenderTree, plan: DocxPlan) -> list[tuple[str, str, bool]]: + return [(i.text, show(plan, plan.refs[i.target_unit].segments), plan.refs[i.target_unit].exact) + for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + + +def test_a_skipped_heading_level_leaves_a_part_of_the_designation_no_field_stands_for() -> None: + body = "# A {#a}\n\n### B {#b}\n\nSee {{ref: b}}.\n" + tree, plan, diagnostics = planned(body, style="outline") + assert references_of(tree, plan) == [("I.A.1", "REF b", False)] + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert found.level == "warning" and found.line == tree.sections[1].blocks[0].line + assert found.message.startswith("The cross-reference 'I.A.1' cannot be built from Word's numbering fields: a " + "heading level is skipped before it") + assert "reads '1' in Word" in found.message + # the decimal scheme writes the whole number in the heading's own field, so nothing is missing there + tree, plan, diagnostics = planned(body) + assert references_of(tree, plan) == [("1.1.1", "REF b", True)] and not diagnostics + + +def test_text_around_a_counter_that_word_cannot_drop_leaves_the_reference_as_one_field() -> None: + body = "# A\n\n- x {#x}\n\nSee {{ref: x}}.\n" + for label, shown in (("({n})", "(1)"), ("{n}:", "1:"), ("No. {n}", "No. 1")): + overrides = {"enumeration.levels": [{"counter": "decimal", "label": label, "ref": "{n}"}]} + tree, plan, diagnostics = planned(body, overrides=overrides) + [(text, cover_, exact)] = references_of(tree, plan) + assert (text, cover_, exact) == ("11", "REF x", False) + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert f"Word draws its number as '{shown}'" in found.message and "'11'" in found.message + + +def test_a_cover_that_reads_differently_in_word_is_reported_unless_a_unit_already_says_why() -> None: + many = "\n".join(f"- i{i} {{#m{i}}}" for i in range(1, 30)) + tree, plan, diagnostics = planned(f"# A {{#a}}\n\n{many}\n\nSee {{{{ref: m28}}}} {{{{ref: m1}}}}.\n") + # the letter counter past 26 is a finding of its own: the references to it are not reported again + assert references_of(tree, plan) == [("1(ab)", "REF a + REF m28", False), ("1(a)", "REF a + REF m1", True)] + assert [d.rule for d in diagnostics] == ["render-docx-letter-overflow"] + assert cached(plan.refs[tree.sections[0].blocks[0].items[27].unit].segments) == "1(bb)" + + +def test_a_reference_whose_text_the_renderer_cut_is_reported() -> None: + title = "word " * 300 + body = f"# {title.strip()} {{#long}}\n\nSee {{{{ref: long}}}}.\n" + tree, plan, diagnostics = planned(body, overrides=NONE) + [reference] = [i for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + assert len(reference.text) < len(title) - 100 + assert not plan.refs["s1"].exact + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert "will read" in found.message and "numbered or written differently" in found.message + + +def test_a_cover_with_the_switch_word_ignores_in_libreoffice_is_an_information_for_the_document() -> None: + body = "# A {#a}\n\n# B {#b}\n\nSee {{ref: a}} {{ref: b}}.\n" + settings = {"overrides": {**ARTICLE, "references.format": "Article {designation}"}} + tree, plan, diagnostics = planned(body, **settings) + first, second = (plan.refs[i.target_unit].segments for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)) + assert [(s.suppress_text, s.cached, s.cached_lo) for s in first if isinstance(s, FieldSeg)] == [(True, "I", "Article I")] + assert [(s.suppress_text, s.cached, s.cached_lo) for s in second if isinstance(s, FieldSeg)] == [(True, "II", "Article II")] + [found] = [d for d in diagnostics if d.rule == "render-docx-reference-word-only"] + assert found.level == "info" and found.line is None and "'Article I'" in found.message + assert "reads 'Article Article I'" in found.message and "so do 1 more" in found.message + assert all(plan.refs[i].exact for i in plan.refs) + + +def test_reference_findings_name_the_file_and_the_first_line_of_a_repeated_reference() -> None: + body = "# A {#a}\n\n### B {#b}\n\nFirst {{ref: b}}.\n\nSecond {{ref: b}}.\n" + result = render(FRONT + body, format="text", style="outline") + _, diagnostics = plan_docx(result.tree, file="deal.lgd") + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert found.file == "deal.lgd" and found.line == result.tree.sections[1].blocks[0].line + + +def test_untrusted_text_in_a_reference_message_is_escaped() -> None: + body = "# A\n\n### B {#b}\n\nSee {{ref: b}}.\n" + levels = [{"counter": "decimal", "label": "{n}\u0007", "ref": "{n}\u0007"}, {"counter": "decimal", "label": "{n}", + "ref": "{n}"}] + _, _, diagnostics = planned(body, overrides={"numbering.levels": levels}) + assert all("\u0007" not in d.message for d in diagnostics) + + +# -- The cover on its own ---------------------------------------------------------------------------------- + + +def field(owner: str, *tokens: object, number: bool = True, suppress: bool = False) -> Field: + return Field(owner, number, suppress, tuple(tokens)) # type: ignore[arg-type] + + +def test_fewest_fields_win_then_fewest_switches_then_fewest_literals() -> None: + designation = [Count("a", "1"), Lit("."), Count("b", "2")] + whole = field("b", Count("a", "1"), Lit("."), Count("b", "2")) + parts = [field("a", Count("a", "1")), field("b", Count("b", "2"))] + assert cover(designation, [*parts, whole]) == [whole] + assert cover(designation, parts) == [parts[0], ".", parts[1]] + plain = field("a", Lit("Article "), Count("a", "I")) + bare = field("a", Count("a", "I"), suppress=True) + assert cover([Count("a", "I")], [plain, bare]) == [bare] + assert cover([Lit("Article "), Count("a", "I")], [plain, bare]) == [plain] # the words are in the field + assert cover([Lit("Article "), Count("a", "I")], [bare]) == ["Article ", bare] # or come from the literal + assert cover([], []) == [] + + +def test_a_literal_stands_for_delimiters_only_never_for_a_counter_or_a_title() -> None: + assert cover([Count("a", "1")], []) is None + assert cover([Title("a", "Alpha")], []) is None + assert cover([Lit("(")], []) == ["("] + assert cover([Title("a", "Alpha")], [field("a", Title("a", "Alpha"), number=False)]) is not None + + +def test_a_field_fits_where_the_counters_belong_to_the_same_units_whatever_their_text() -> None: + designation = [Count("a", "ab")] + assert cover(designation, [field("a", Count("a", "bb"))]) == [field("a", Count("a", "bb"))] + assert cover(designation, [field("a", Count("other", "ab"))]) is None + assert cover([Count(Ghost("s2", 1), "1")], [field("s2", Count("s2", "1"))]) is None + + +def test_the_atom_nothing_can_stand_for_is_found() -> None: + designation = [Count("a", "I"), Lit("."), Count(Ghost("s2", 1), "1"), Lit("."), Count("b", "1")] + fields = [field("a", Count("a", "I")), field("b", Count("b", "1"))] + assert cover(designation, fields) is None and uncovered(designation, fields) == 2 + assert uncovered([Count("a", "I")], fields) is None + + +def test_the_switch_that_drops_text_is_modelled_only_where_it_leaves_bare_counters() -> None: + assert suppressed([Lit("Article "), Count("a", "I")]) == (Count("a", "I"),) + assert suppressed([Lit("§ "), Count("a", "3")]) == (Count("a", "3"),) + assert suppressed([Lit("("), Count("a", "a"), Lit(")")]) is None + assert suppressed([Count("a", "1"), Lit("."), Count("b", "2")]) is None + assert suppressed([Lit("No. "), Count("a", "1")]) is None + assert suppressed([Lit("Article")]) is None + + +# -- Bookmarks ---------------------------------------------------------------------------------------------- + + +def test_bookmark_names_have_a_prefix_for_what_they_mark_and_write_dashes_as_underscores() -> None: + names = BookmarkNames() + assert names.anchor("ld_", "delay-cause") == "ld_delay_cause" + assert names.anchor("lddef_", "business-day") == "lddef_business_day" + assert names.anchor("ldatt_", "schedule-a") == "ldatt_schedule_a" + assert names.fixed("_LdAttachments") == "_LdAttachments" + assert names.toc() == "_Toc100000001" and names.toc() == "_Toc100000002" + + +def test_a_name_that_cannot_be_used_is_a_hidden_one_counted_in_the_order_they_are_given() -> None: + names = BookmarkNames() + assert names.anchor("ld_", "a-b") == "ld_a_b" + assert names.anchor("ld_", "a_b") == "_Ld1" # the same name once the dash is an underscore: not lossless + assert names.anchor("ld_", "A-B") == "_Ld2" # the same ignoring case + assert names.anchor("ld_", "clanek-č") == "_Ld3" # a letter a name cannot hold + assert names.anchor("ld_", "x" * 37) == "ld_" + "x" * 37 # 40 characters + assert names.anchor("ld_", "y" * 38) == "_Ld4" # 41 + assert names.hidden() == "_Ld5" + + +def test_hidden_names_are_numbered_in_document_order_across_anchors_and_units() -> None: + tree, plan, _ = planned("# A\n\n- x\n - y {#y}\n\nSee {{ref: y}} and " + "{{ref: " + "z" * 50 + "}}.\n") + assert plan.units["u1"].bookmarks == ("_Ld1",) + + +def test_an_anchor_too_long_for_a_name_is_hidden_and_still_found_by_the_plan() -> None: + anchor = "a-very-long-identifier-that-goes-on-and-on-past-forty" + tree, plan, _ = planned(f"# A {{#{anchor}}}\n\n{{{{ref: {anchor}}}}}\n") + assert plan.anchors == {anchor: "_Ld1"} and plan.units["s1"].bookmarks == ("_Ld1",) + assert [s.bookmark for s in plan.refs["s1"].segments if isinstance(s, FieldSeg)] == ["_Ld1"] + + +def test_the_names_do_not_depend_on_what_was_rendered_before() -> None: + body = "# A {#a-b}\n\n- x {#x}\n\nSee {{ref: x}}.\n" + first = planned(body)[1] + render(FRONT + "# Other {#a}\n", format="text") + assert planned(body)[1] == first + + +def test_definitions_and_attachments_have_bookmarks_too() -> None: + result = render((DOCUMENTS / "features.lgd").read_text(encoding="utf-8"), format="text") + plan, _ = plan_docx(result.tree) + assert {a: n for a, n in plan.anchors.items() if not n.startswith("ld_")} == { + "def:agreement": "lddef_agreement", "def:services": "lddef_services", "def:business-day": "lddef_business_day", + "schedule-a": "ldatt_schedule_a", "exhibit-1": "ldatt_exhibit_1", "ld:attachments": "_LdAttachments"} + assert plan.anchors["services"] == "ld_services" and plan.anchors["delay-cause"] == "ld_delay_cause" + assert all(plan.units[unit.unit].bookmarks == (plan.anchors[unit.anchor],) + for unit in units(result.tree) if unit.anchor) + + +def test_every_name_is_valid_and_unique_ignoring_case() -> None: + for name in ("features.lgd", "template.lgd"): + for settings in VARIANTS.values(): + result = render((DOCUMENTS / name).read_text(encoding="utf-8"), format="text", **settings) + plan, _ = plan_docx(result.tree) + assert_names_are_sound(plan) + + +def assert_names_are_sound(plan: DocxPlan) -> None: + names = [*(n for u in plan.units.values() for n in u.bookmarks), *plan.anchors.values(), + *(t.bookmark for t in plan.toc)] + names = list(dict.fromkeys(names)) # an anchor's name is the first of its unit's + assert all(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,39}", n) for n in names), names + assert len({n.lower() for n in names}) == len(names), names + + +# -- The table of contents --------------------------------------------------------------------------------------- + + +ATTACHED = """--- +title: T +attachments: + - id: schedule-a + title: Schedule A + file: a.lgd + - id: exhibit-1 + title: Exhibit 1 + file: b.pdf +--- +""" + + +def test_a_contents_entry_names_the_section_it_lists() -> None: + result = render(ATTACHED + "# A\n\n## B\n\n### C\n\n# D\n", format="text", + overrides={"contents.enabled": True, "contents.depth": 2}) + assert [(e.level, e.unit) for e in result.tree.contents] == [ + (1, "s1"), (2, "s2"), (1, "s4"), (1, None), (2, None), (2, None)] + result = render(ATTACHED + "# A\n", format="text") + assert result.tree.contents == () + + +def test_the_contents_come_from_the_numbers_word_draws() -> None: + body = "# Alpha\n\n## Beta\n\n### Gamma\n\n# Delta\n" + tree, plan, _ = planned(body, ATTACHED, overrides={"contents.enabled": True, "contents.depth": 2}) + assert [(e.unit, e.level, e.number, e.sep, e.condition_label, e.cached_page) for e in plan.toc] == [ + ("s1", 1, "1.", "\t", None, ""), ("s2", 2, "1.1", "\t", None, ""), ("s4", 1, "2.", "\t", None, ""), + (None, 1, "", "", None, ""), (None, 2, "", "", None, ""), (None, 2, "", "", None, "")] + assert [e.title for e in plan.toc[:3]] == [tree.sections[0].title, tree.sections[1].title, tree.sections[3].title] + assert [e.bookmark for e in plan.toc] == [f"_Toc10000000{k}" for k in range(1, 7)] + assert [plan.units[s].toc_bookmark for s in ("s1", "s2", "s3", "s4")] == [ + "_Toc100000001", "_Toc100000002", None, "_Toc100000003"] + assert [e.title for e in plan.toc[3:]] == [tree.contents[3].title, tree.contents[4].title, tree.contents[5].title] + assert_names_are_sound(plan) + + +def test_a_contents_entry_has_the_separator_the_level_has_and_the_number_of_the_style() -> None: + body = "# Alpha\n\n## Beta\n" + _, plan, _ = planned(body, overrides={"contents.enabled": True, **ARTICLE}) + assert [(e.number, e.sep) for e in plan.toc] == [("Article I", " "), ("I.1", "\t")] + _, plan, _ = planned(body, overrides={"contents.enabled": True, "numbering.scheme": "legal-outline"}) + assert [(e.number, e.sep) for e in plan.toc] == [("I.", "\t"), ("A.", "\t")] + _, plan, _ = planned(body, overrides={"contents.enabled": True, **NONE}) + assert [(e.number, e.sep) for e in plan.toc] == [("", ""), ("", "")] + _, plan, _ = planned(body, overrides={"contents.enabled": True, "contents.attachments": False}, front=ATTACHED) + assert [e.unit for e in plan.toc] == ["s1"] * 0 + ["s1", "s2"] + + +def test_the_contents_carry_the_condition_label_of_a_conditional_heading() -> None: + _, plan, _ = planned("# A {when=forum:courts}\n\n# B\n", ALTERNATIVES, overrides={"contents.enabled": True}) + assert [e.condition_label for e in plan.toc] == ["Only if: forum:courts", None] + + +def test_there_are_no_entries_unless_the_style_asks_for_contents() -> None: + _, plan, _ = planned("# A\n", ATTACHED) + assert plan.toc == () and plan.units["s1"].toc_bookmark is None + + +# -- The property: the plan's references read as the renderer wrote them, or a diagnostic says why not --------- + + +def with_anchors(source: str) -> str: + """*source* with an anchor on every heading, item, quoted item and plain paragraph the random documents hold.""" + out, count = [], 0 + for line in source.splitlines(): + if re.match(r"^\s*(#+ Heading|[-]|\d+\.) ", line) or line == "A paragraph." or re.match(r"^\s*> - ", line): + count += 1 + line += f" {{#a{count}}}" + out.append(line) + return "\n".join(out) + + +def with_references(source: str) -> str: + """*source* with a paragraph at its end that refers to every anchor in it.""" + anchors = dict.fromkeys(re.findall(r"\{#([a-z][a-z0-9-]*)", source)) + return source + "\n\nRefs: " + " ".join(f"{{{{ref: {anchor}}}}}" for anchor in anchors) + "\n" + + +def assert_references_as_resolved(result) -> None: # noqa: ANN001 + """For every reference: its cover reads as the renderer's text (``exact``, and the cached results say so), or the + plan says why not: a unit on the way is numbered differently in Word (a diagnostic of the numbering says so, as the + property of the numbering above has it), or a ``render-docx-reference`` finding names the reference. Every field + points at a bookmark the plan makes, the names are sound, and the same tree gives the same plan.""" + tree = result.tree + plan, diagnostics = plan_docx(tree) + assert plan_docx(tree) == (plan, diagnostics) + assert_names_are_sound(plan) + nodes = {u.unit: u for u in units(tree)} + counters = {node.unit: counter for node, counter in numbered(tree)} + marked = {n for u in plan.units.values() for n in u.bookmarks} + reported = [d for d in diagnostics if d.rule == "render-docx-reference"] + texts = {i.target_unit: i.text for i in iter_tree_inlines(tree) if isinstance(i, CrossRef) and i.target_unit} + assert set(plan.refs) == set(texts) + findings = 0 + for target, text in texts.items(): + reference = plan.refs[target] + assert all(s.bookmark in marked for s in reference.segments if isinstance(s, FieldSeg)) + assert reference.exact == (cached(reference.segments) == text) + for segment in reference.segments: + if isinstance(segment, FieldSeg) and segment.number and not segment.suppress_text: + assert segment.cached == plan.units[next(u for u, p in plan.units.items() + if segment.bookmark in p.bookmarks)].word_ref + mentioned = any(d.message.startswith(f"The cross-reference '{text}' ") for d in reported) + findings += mentioned + if reference.exact: + assert mentioned <= any(not plan.refs[t].exact for t in texts if texts[t] == text), (target, text) + continue + chain, unit = [], target + while unit is not None: + chain.append(unit) + unit = nodes[unit].parent_unit + differs = False + for unit in chain: + mine, node = plan.units[unit], nodes[unit] + if node.number is not None and mine.count is not None: + differs |= mine.count != node.number or ( + unit in counters and word_counter(node.number, NUM_FMT[counters[unit]]) + != format_counter(node.number, counters[unit])) + assert differs or mentioned, (target, text, cached(reference.segments), show(plan, reference.segments)) + assert len(reported) <= findings + + +@pytest.mark.parametrize("variant", VARIANTS) +@pytest.mark.parametrize("name", ["features.lgd", "template.lgd"]) +def test_the_test_documents_references_are_planned_as_resolved(name: str, variant: str) -> None: + source = (DOCUMENTS / name).read_text(encoding="utf-8") + assert_references_as_resolved(render(source, format="text", **VARIANTS[variant])) + + +@pytest.mark.conformance +@pytest.mark.skipif(SPEC is None, reason="LegalDown specification checkout not found") +@pytest.mark.parametrize("variant", VARIANTS) +def test_the_specifications_documents_references_are_planned_as_resolved(variant: str) -> None: + for path in _spec_documents(): + try: + result = render(path.read_text(encoding="utf-8"), format="text", **VARIANTS[variant]) + except DocumentError: + continue + try: + assert_references_as_resolved(result) + except AssertionError as error: + raise AssertionError(f"{path.relative_to(SPEC)}: {error}") from error + + +@functools.cache +def _referenced_renders(variant: str) -> tuple: + rng = random.Random(20261029) + sources = [with_references(with_anchors(_random_document(rng))) for _ in range(30)] + return tuple((source, render(FRONT + source, format="text", **VARIANTS[variant])) for source in sources) + + +@pytest.mark.parametrize("variant", VARIANTS) +def test_random_documents_references_are_planned_as_resolved(variant: str) -> None: + for source, result in _referenced_renders(variant): + try: + assert_references_as_resolved(result) + except AssertionError as error: + raise AssertionError(f"{error}\n--- document ---\n{source}") from error + + +@pytest.mark.parametrize("variant", ["default", "continental", "paragraphs", "none"]) +def test_random_alternatives_references_are_planned_as_resolved(variant: str) -> None: + rng = random.Random(11) + for _ in range(20): + source = ALTERNATIVES + with_references(_random_alternatives(rng)) + assert_references_as_resolved(render(source, format="text", **VARIANTS[variant])) + + +def test_the_random_documents_reach_every_kind_of_cover() -> None: + shapes: set[str] = set() + inexact = switches = 0 + for variant in VARIANTS: + for _, result in _referenced_renders(variant): + plan, _ = plan_docx(result.tree) + for reference in plan.refs.values(): + shapes.add("".join("L" if isinstance(s, Literal_) else "F" for s in reference.segments)) + inexact += not reference.exact + switches += any(isinstance(s, FieldSeg) and s.suppress_text for s in reference.segments) + assert {"F", "FF", "FFF", "FFFF", "FLF", "FLFL", "FLFLF"} <= shapes + assert inexact > 0 and switches > 0 From 2d87b3b76c7a6b648e0507f03607d9654a073ed6 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:42:15 +0000 Subject: [PATCH 087/118] Write DOCX from the planner's plan render(format="docx") plans the resolved tree, passes the plan to the writer and adds the planner's diagnostics to the result: named with the document's file, each finding once, in the order of their lines, after the renderer's own. The writer plans a tree itself when it is used directly without a plan, so the plan with no numbering is gone. The render-docx-interim warning now names what is still missing: the title block, the table of contents, the attachments and the signature blocks. The tests that expected a file without numbers expect the numbering, and the digests of the fixed document are updated. --- src/legaldown_render/api.py | 25 ++++++---- src/legaldown_render/writers/docx/__init__.py | 12 ++--- .../writers/docx/interim_plan.py | 47 ------------------- tests/test_docx_body.py | 18 +++---- tests/test_docx_libreoffice.py | 28 ++++++++--- tests/test_docx_package.py | 44 ++++++++++++++--- 6 files changed, 90 insertions(+), 84 deletions(-) delete mode 100644 src/legaldown_render/writers/docx/interim_plan.py diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index e552af2..465c53d 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -25,6 +25,7 @@ from .diagnostics import by_line, key_line, repeats from .errors import DocumentError, OptionsError, RenderRefused, StyleError from .resolve import Formatter, parse_locale, resolve_tree +from .resolve.docx_plan import plan_docx from .style import Style, load_style from .style.docx import DocxSettings, docx_settings from .tree import RenderTree, assert_resolved @@ -121,14 +122,13 @@ def ok(self) -> bool: def _interim_warning(filename: str) -> Diagnostic: - """The Warning that DOCX output is not complete yet: the writer's body is interim (no numbers, - references, contents, title block, attachments or signature blocks). It goes when the body is - complete.""" + """The Warning that DOCX output is not complete yet: the writer does not yet write the title block, the + table of contents, the attachments or the signature blocks. It goes when the body is complete.""" return Diagnostic( rule="render-docx-interim", level="warning", file=filename, - message="DOCX output is incomplete in this version: headings and list items carry no numbers, and " - "cross-references, the table of contents, the title block, attachments and signature blocks are " - "not written yet. Do not use the file as the finished document.") + message="DOCX output is incomplete in this version: the title block, the table of contents, the " + "attachments and the signature blocks are not written yet (the numbering and the " + "cross-references are). Do not use the file as the finished document.") def render(source: str, options: RenderOptions | None = None, /, **settings: Any) -> RenderResult: @@ -186,10 +186,17 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any if options.strict and any(d.level == "error" for d in diagnostics): raise RenderRefused(diagnostics) - if options.format == "docx": - diagnostics.append(_interim_warning(document.filename)) assert_resolved(tree) - written = _writer(options, style, docx).write(tree) + writer = _writer(options, style, docx) + if isinstance(writer, DocxWriter): + # What Word will draw differently from the other formats (docs/decisions/0008): the planner's findings follow + # the renderer's, in the order of their lines, none repeating one the document has already. + plan, planned = plan_docx(tree, file=document.filename) + diagnostics += [d for d in planned if not repeats(d, diagnostics)] + diagnostics.append(_interim_warning(document.filename)) + written: str | bytes = writer.write(tree, plan) + else: + written = writer.write(tree) if isinstance(written, bytes): return RenderResult("", diagnostics, options.format, tree, style, binary=written) return RenderResult(written, diagnostics, options.format, tree, style) diff --git a/src/legaldown_render/writers/docx/__init__.py b/src/legaldown_render/writers/docx/__init__.py index 1be4a01..c45df66 100644 --- a/src/legaldown_render/writers/docx/__init__.py +++ b/src/legaldown_render/writers/docx/__init__.py @@ -13,21 +13,19 @@ * :mod:`.styles`, :mod:`.settings`, :mod:`.headers`, :mod:`.properties` -- the parts that come from the style template; * :mod:`.numbering` -- ``numbering.xml``, from the plan's numbering definitions; * :mod:`.body` and :mod:`.runs` -- the body of the document and its inlines, laid out as the plan says (interim - for the title block, contents, attachments and signatures, see there); -* :mod:`.interim_plan` -- the plan with no numbering that stands in until the planner is wired. + for the title block, contents, attachments and signatures, see there). """ from __future__ import annotations from collections.abc import Mapping -from ...resolve.docx_plan import DocxPlan +from ...resolve.docx_plan import DocxPlan, plan_docx from ...style import Style from ...style.docx import DocxSettings, docx_settings from ...tree import RenderTree from . import package as pkg from .body import BodyWriter, document_title from .headers import build_footer, build_header -from .interim_plan import _unnumbered_plan from .numbering import numbering_part from .ooxml import Bookmarks, el, serialize from .properties import build_app, build_core @@ -58,10 +56,10 @@ def __init__(self, style: Style, *, generator: str = "", settings: DocxSettings def write(self, tree: RenderTree, plan: DocxPlan | None = None) -> bytes: """The ``.docx`` file for *tree*, numbered, bookmarked and cross-referenced as *plan* says. Without - one the document has no numbering (:func:`~.interim_plan._unnumbered_plan`): temporary, until the - planner is wired in.""" + one the tree is planned here (:func:`~legaldown_render.resolve.docx_plan.plan_docx`), whose diagnostics + are then lost: :func:`~legaldown_render.render` plans first and reports them.""" if plan is None: - plan = _unnumbered_plan(tree) + plan = plan_docx(tree)[0] settings = self.settings language = language_tag(tree.language, self.style.locale) title = document_title(tree) diff --git a/src/legaldown_render/writers/docx/interim_plan.py b/src/legaldown_render/writers/docx/interim_plan.py deleted file mode 100644 index f8b127d..0000000 --- a/src/legaldown_render/writers/docx/interim_plan.py +++ /dev/null @@ -1,47 +0,0 @@ -"""A plan with no numbering, for as long as the DOCX planner is not wired to the writer. - -**Temporary.** :func:`_unnumbered_plan` builds a valid :class:`~legaldown_render.resolve.docx_plan.DocxPlan` -that has no numbering definitions, no bookmarks and no reference fields: every unit is unnumbered and -every cross-reference shows its text, so the document reads in the right order but carries no numbers -(the ``render-docx-interim`` warning says so). A list item is indented by its depth, so that the -structure of nested lists shows. The planner (``resolve/docx_plan.py``) -replaces it: it is the only place with the Word numbering rules (docs/decisions/0003). -""" -from __future__ import annotations - -from collections.abc import Iterable, Iterator -from typing import Literal - -from ...resolve.docx_plan import DocxPlan, UnitPlan -from ...tree import Block, DraftingNote, List, Paragraph, Quote, RenderTree - -#: The left indent of one level of nesting, in twips (1 cm). -_STEP = 567 - - -def _unit(unit: str, kind: Literal["section", "item", "paragraph"], ind: tuple[int, int] | None = None) -> UnitPlan: - return UnitPlan(unit=unit, kind=kind, num_id=None, ilvl=None, explicit=False, ind=ind, alt=None, label="", word_ref="", count=None, cause=(), bookmarks=(), toc_bookmark=None) - - -def _units(blocks: Iterable[Block]) -> Iterator[UnitPlan]: - for block in blocks: - match block: - case Paragraph(unit=unit) if unit: - yield _unit(unit, "paragraph") - case List(items=items, depth=depth): - for item in items: - if item.unit: - yield _unit(item.unit, "item", ((depth + 1) * _STEP, 0)) - yield from _units(item.blocks) - case Quote(blocks=inner) | DraftingNote(blocks=inner): - yield from _units(inner) - - -def _unnumbered_plan(tree: RenderTree) -> DocxPlan: - """A :class:`DocxPlan` for *tree* with no numbering: see the module docstring.""" - units = [*_units(tree.preamble)] - for section in tree.sections: - if section.unit: - units.append(_unit(section.unit, "section")) - units += _units(section.blocks) - return DocxPlan(abstract_nums=(), nums=(), heading_num_id=None, units={unit.unit: unit for unit in units}) diff --git a/tests/test_docx_body.py b/tests/test_docx_body.py index 55e0951..c9b8188 100644 --- a/tests/test_docx_body.py +++ b/tests/test_docx_body.py @@ -870,12 +870,13 @@ def test_a_table_beside_an_indent_keeps_at_least_a_column_width_per_column() -> assert min(widths) >= 1134 and sum(widths) <= TEXT_WIDTH -def test_a_deeply_nested_list_without_a_plan_stays_on_the_page() -> None: +def test_a_deeply_indented_list_stays_on_the_page() -> None: inner: Block = bullets(item("deepest", unit="u70"), depth=70) for depth in range(69, -1, -1): inner = bullets(item(f"d{depth}", inner, unit=f"u{depth}"), depth=depth) document = tree(inner) - data = DocxWriter(load_style("default")).write(document) + indents = [unit(f"u{depth}", "item", ind=((depth + 1) * 567, 0)) for depth in range(71)] + data = DocxWriter(load_style("default")).write(document, plan(*indents)) root = ET.fromstring(parts(data)["word/document.xml"]) assert max(lefts(root)) <= min(31680, TEXT_WIDTH - 2268) @@ -892,14 +893,15 @@ def test_the_title_and_the_subtitle_come_first_and_the_section_properties_last() assert local(children[-1]) == "sectPr" -def test_without_a_plan_the_document_has_no_numbering_and_the_indent_of_each_list_depth() -> None: - result = render("---\ntitle: T\n---\n\n# A\n\n- a\n - b\n\nSee {{ref: a}}.\n", format="docx") +def test_a_rendered_document_is_numbered_cross_referenced_and_bookmarked_as_the_planner_says() -> None: + result = render("---\ntitle: T\n---\n\n# A\n\n- a {#x}\n - b\n\nSee {{ref: x}}.\n", format="docx") contents = parts(result.binary) # type: ignore[arg-type] - assert list(ET.fromstring(contents["word/numbering.xml"])) == [] + assert [child.tag.removeprefix(W) for child in ET.fromstring(contents["word/numbering.xml"])] == [ + "abstractNum", "abstractNum", "num", "num"] root = ET.fromstring(contents["word/document.xml"]) - assert [flow(p) for p in root.iter(f"{W}p")] == ["T", "A", "a", "b", "See 1."] - assert [indent(p) for p in root.iter(f"{W}p")] == [None, None, {"left": 567}, {"left": 1134}, None] - assert not list(root.iter(f"{W}fldChar")) + assert [flow(p) for p in root.iter(f"{W}p")] == [ + "T", "A", "a", "b", "See {REF ld_a \\n \\h|1}{REF ld_x \\n \\h|(a)}."] + assert [indent(p) for p in root.iter(f"{W}p")] == [None] * 5 def test_the_same_plan_and_tree_are_the_same_bytes() -> None: diff --git a/tests/test_docx_libreoffice.py b/tests/test_docx_libreoffice.py index 7198d35..bf77c40 100644 --- a/tests/test_docx_libreoffice.py +++ b/tests/test_docx_libreoffice.py @@ -14,6 +14,7 @@ from docx_support import Exported, libreoffice_export, parts, require_libreoffice, texts from legaldown_render import render +from legaldown_render.resolve.docx_plan import plan_docx pytestmark = pytest.mark.libreoffice @@ -61,6 +62,16 @@ def docx(source: str, **settings: object) -> bytes: return result.binary +def without_number(line: str, labels: list[str]) -> str: + """*line* without the number LibreOffice drew before the text of a numbered paragraph (one of *labels*, + the numbers Word draws, with any indent of a nested item).""" + text = line.lstrip() + for label in labels: + if text.startswith(f"{label} "): + return text.removeprefix(f"{label} ") + return line + + def expected_lines(data: bytes) -> list[str]: """What the paragraphs of the document part say, one line per line, without blank ones.""" lines = [] @@ -86,16 +97,21 @@ def shown() -> dict[str, Exported]: def test_the_words_come_back_in_order(shown: dict[str, Exported]) -> None: + # the numbers are drawn by LibreOffice from the numbering definitions (test_docx_roundtrip.py compares them) assert shown["round_trip"].lines == [ - "Round Trip", "Sub", "Preamble ž€.", "First", "Body with emphasis, strong and code.", "Second line.", - "Second", "item one", "item two", "A quote", "code line 1", " code line 2", "H1", "H2", "a", "b", ""] + "Round Trip", "Sub", "Preamble ž€.", "1. First", "Body with emphasis, strong and code.", "Second line.", + "1.1 Second", " (a) item one", " (b) item two", "A quote", "code line 1", " code line 2", "H1", "H2", + "a", "b", ""] @pytest.mark.parametrize("name", ["features", "template"]) def test_the_test_documents_read_as_their_paragraphs(shown: dict[str, Exported], name: str) -> None: - data = docx((DOCUMENTS / f"{name}.lgd").read_text(encoding="utf-8"), **({"style": "continental"} - if name == "template" else {})) - assert [line for line in shown[name].lines if line] == expected_lines(data) + settings = {"style": "continental"} if name == "template" else {} + source = (DOCUMENTS / f"{name}.lgd").read_text(encoding="utf-8") + labels = sorted({unit.label for unit in plan_docx(render(source, format="text", **settings).tree)[0].units.values() + if unit.label}, key=len, reverse=True) + words = [without_number(line, labels) for line in shown[name].lines if line] + assert words == expected_lines(docx(source, **settings)) def test_the_header_shows_the_templates_and_the_page_field(shown: dict[str, Exported]) -> None: @@ -130,5 +146,5 @@ def test_a_landscape_page_with_a_right_aligned_page_number_opens(shown: dict[str def test_hostile_text_opens_with_the_unwritable_characters_dropped(shown: dict[str, Exported]) -> None: lines = shown["hostile"].lines assert lines[0] == "ab" - assert lines[1] == "Hx" + assert lines[1] == "1. Hx" assert lines[2] == "body <&> ]]>" diff --git a/tests/test_docx_package.py b/tests/test_docx_package.py index cb16b06..0d3b7a7 100644 --- a/tests/test_docx_package.py +++ b/tests/test_docx_package.py @@ -128,8 +128,9 @@ def test_update_fields_on_open_can_be_turned_off() -> None: assert settings.find(f"{W}compat") is not None -def test_the_numbering_part_is_an_empty_stub_that_later_work_fills() -> None: - assert list(xml(docx()["word/numbering.xml"])) == [] +def test_the_numbering_part_holds_the_plans_definitions() -> None: + assert [child.tag.removeprefix(W) for child in xml(docx()["word/numbering.xml"])] == [ + "abstractNum", "abstractNum", "num", "num"] from legaldown_render.writers.docx.settings import build_numbering filled = xml(build_numbering([ooxml.el("abstractNum", {"abstractNumId": 0})], [ooxml.el("num", {"numId": 1})])) @@ -163,13 +164,16 @@ def test_the_styles_the_body_and_word_need_exist_under_the_names_word_knows() -> assert by_id["Heading1"].get(f"{W}customStyle") is None -def test_headings_are_outline_levels_zero_to_four_and_not_numbered_yet() -> None: +def test_headings_are_outline_levels_zero_to_four_numbered_by_the_heading_list() -> None: by_id = style_map(docx()) for level in range(1, 6): heading = by_id[f"Heading{level}"] assert heading.find(f"{W}pPr/{W}outlineLvl").get(f"{W}val") == str(level - 1) assert heading.find(f"{W}basedOn").get(f"{W}val") == "Normal" - assert heading.find(f"{W}pPr/{W}numPr") is None + numbering = heading.find(f"{W}pPr/{W}numPr") + assert (numbering.find(f"{W}ilvl").get(f"{W}val"), numbering.find(f"{W}numId").get(f"{W}val")) == ( + str(level - 1), "1") + assert all(style.find(f"{W}pPr/{W}numPr") is None for key, style in by_id.items() if not key.startswith("Heading")) assert by_id["ContentsHeading"].find(f"{W}pPr/{W}outlineLvl") is None # in no table of contents @@ -628,6 +632,32 @@ def test_the_interim_warning_is_one_warning_that_does_not_refuse_under_strict() assert not [d for d in render(FRONT + "# A\n", format="html").diagnostics if d.rule == "render-docx-interim"] +def test_the_planners_findings_follow_the_renderers_in_the_order_of_their_lines_with_the_documents_file() -> None: + source = FRONT + "# A {#a}\n\n- x {#x}\n\nSee {{ref: nowhere}}.\n\n" + "- r\n" * 30 + result = render(source, format="docx", filename="deal.lgd") + text = render(source, format="text", filename="deal.lgd").diagnostics + assert [d.rule for d in result.diagnostics] == [ + *(d.rule for d in text), "render-docx-letter-overflow", "render-docx-interim"] + assert result.diagnostics[:len(text)] == text and {d.file for d in result.diagnostics} == {"deal.lgd"} + assert "ref-broken" in {d.rule for d in text} + + +def test_the_interim_warning_names_what_is_still_missing() -> None: + warning = render(FRONT + "# A\n", format="docx").diagnostics[-1] + assert warning.rule == "render-docx-interim" + for missing in ("title block", "table of contents", "attachments", "signature blocks"): + assert missing in warning.message + assert "carry no numbers" not in warning.message + + +def test_the_writer_gets_the_plan_the_render_reported_on() -> None: + result = render(FRONT + "# A {#a}\n\n- x {#x}\n\nSee {{ref: x}}.\n", format="docx") + assert result.binary is not None + document = parts(result.binary)["word/document.xml"].decode("utf-8") + assert document.count("REF ld_a \\n \\h") == 1 and document.count("REF ld_x \\n \\h") == 1 + assert 'w:name="ld_x"' in document + + def test_the_docx_writer_can_be_used_directly() -> None: from legaldown_render.style import load_style @@ -815,10 +845,10 @@ def test_code_is_left_aligned_even_when_the_body_is_justified() -> None: '_rels/.rels': 'd6f21829c0081670bdb417d3e3f644b81134a121e074fb153c7686faaa8b167b', 'docProps/core.xml': '159ed0e537106ab29739fd357a5f6d89a2c0b0154daefd4c13d878bd16dff47e', 'docProps/app.xml': 'b285204d159fd58da80b6a0242cef523448f9c3a598b357b15faf4627160dcc3', - 'word/document.xml': '58f0081f76a6a2092397770d50ae151e38f7aacdadf3525379b8112f964e0b8a', + 'word/document.xml': '79fb3712f0b4bef4a7139fa2b1049339381e98bb7f2114441638145daece15a8', 'word/_rels/document.xml.rels': 'ada136c2e7d05a317579c9503c1f0dd8dfaa8f0cce687df86fe59e36d5fa5d57', - 'word/styles.xml': '83adf799de91eac3e9bf1e658b8c2f53586c700be35df179e1b2f1765f189dfc', - 'word/numbering.xml': '8d1f7769c35af16db98942ba09882646b45ba404e83686d4ce7a376c6a911f0f', + 'word/styles.xml': 'cdb4ca1d3be44bd27761763c040ee637516ed7aa7cf995e2cc0b60cc80d7f8ac', + 'word/numbering.xml': '368f179b5e26a34c1f23db597966dfd96b43dc96801a9a7ded728c379db07f59', 'word/settings.xml': 'b3c72bc5720862c1a9c1449d89903673777f30df6ff38ceb23da73e21cad22e1', 'word/footer1.xml': 'e693781f92e5a7ccf747fd2ed3facb627e448f389b021882cac589d746aa709b', } From 5b6b70e08ff3c6b32da21452cf568b17dacc8f11 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:42:15 +0000 Subject: [PATCH 088/118] Check the DOCX numbers and references against LibreOffice Every document of the corpus (the test documents, the specification's examples and valid fixtures, and documents written to stress the numbering) is rendered under the default, outline, continental, mixed and none schemes, its cached field results are made wrong, its numbered paragraphs are marked and a line per reference is added. The number LibreOffice draws and each reference it computes must be the renderer's, except where the plan reports that Word draws something else, and then the plan's. tests/lo/uno_export.py can export without updating the fields, which shows that LibreOffice computes the results when it opens the file. --- tests/docx_support.py | 8 +- tests/lo/uno_export.py | 17 +- tests/test_docx_roundtrip.py | 435 +++++++++++++++++++++++++++++++++++ 3 files changed, 452 insertions(+), 8 deletions(-) create mode 100644 tests/test_docx_roundtrip.py diff --git a/tests/docx_support.py b/tests/docx_support.py index cc09f67..cd0e28c 100644 --- a/tests/docx_support.py +++ b/tests/docx_support.py @@ -220,16 +220,18 @@ def repackage(data: bytes, name: str, change: Callable[[str], str]) -> bytes: return buffer.getvalue() -def libreoffice_export(documents: dict[str, bytes]) -> dict[str, Exported]: +def libreoffice_export(documents: dict[str, bytes], *, update: bool = True) -> dict[str, Exported]: """Open each document (name to DOCX bytes) in one LibreOffice instance, update its fields - and indexes, and return its text, header, footer and page count.""" + and indexes (unless *update* is False: the fields as the file caches them), and return its + text, header, footer and page count.""" python = require_libreoffice() with tempfile.TemporaryDirectory(prefix="ld-lo-") as folder: out = Path(folder, "out") out.mkdir() for name, data in documents.items(): Path(folder, f"{name}.docx").write_bytes(data) - done = subprocess.run([python, str(UNO_EXPORT), str(out), *(str(Path(folder, f"{n}.docx")) for n in documents)], + done = subprocess.run([python, str(UNO_EXPORT), *([] if update else ["--no-update"]), str(out), + *(str(Path(folder, f"{n}.docx")) for n in documents)], capture_output=True, text=True, timeout=300, check=False) if done.returncode != 0: pytest.fail(f"LibreOffice export failed:\n{done.stdout}\n{done.stderr}") diff --git a/tests/lo/uno_export.py b/tests/lo/uno_export.py index cae746b..4167298 100644 --- a/tests/lo/uno_export.py +++ b/tests/lo/uno_export.py @@ -7,10 +7,15 @@ Run it with the system Python that has the ``uno`` module (``/usr/bin/python3`` once ``python3-uno`` is installed), **not** with the project's virtual environment: - /usr/bin/python3 tests/lo/uno_export.py OUTDIR FILE.docx [FILE.docx ...] + /usr/bin/python3 tests/lo/uno_export.py [--no-update] OUTDIR FILE.docx [FILE.docx ...] The same LibreOffice instance opens every file, so a test module pays the start-up once. Exit status 0 when every file was exported. + +With ``--no-update`` the fields are left as the file caches them: what the export says +is then what the writer wrote (a document whose cached results were made wrong on +purpose shows the wrong results there, and the right ones only after the update, which +is how a round trip proves that LibreOffice computed the numbers and did not read them). """ from __future__ import annotations @@ -66,7 +71,7 @@ def _page_texts(document: object) -> tuple[str, str]: return header, footer -def export(out_dir: str, files: list[str]) -> None: +def export(out_dir: str, files: list[str], *, update: bool = True) -> None: profile = tempfile.mkdtemp(prefix="lo-profile-") pipe = f"legaldown_{uuid.uuid4().hex}" process = subprocess.Popen( @@ -85,7 +90,8 @@ def export(out_dir: str, files: list[str]) -> None: if document is None: raise SystemExit(f"LibreOffice could not open {path}") try: - _update(document) + if update: + _update(document) header, footer = _page_texts(document) document.storeToURL(uno.systemPathToFileUrl(os.path.join(out_dir, f"{stem}.txt")), (_value("FilterName", "Text (encoded)"), _value("FilterOptions", "UTF8"))) @@ -105,6 +111,7 @@ def export(out_dir: str, files: list[str]) -> None: if __name__ == "__main__": - if len(sys.argv) < 3: + arguments = [argument for argument in sys.argv[1:] if argument != "--no-update"] + if len(arguments) < 2: raise SystemExit(__doc__) - export(sys.argv[1], sys.argv[2:]) + export(arguments[0], arguments[1:], update="--no-update" not in sys.argv[1:]) diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py new file mode 100644 index 0000000..05ac9a1 --- /dev/null +++ b/tests/test_docx_roundtrip.py @@ -0,0 +1,435 @@ +"""LibreOffice as the oracle of the numbering and the cross-references (docs/architecture.md, "The DOCX planner"). + +Every document of the test corpus (the test documents, the specification's examples and valid fixtures, and +documents written to stress the numbering) is rendered to DOCX under the default, outline, continental and mixed +schemes and under ``none`` (where a reference is a field for the text of a heading), then changed so that the file cannot be right by accident: + +* every cached result of a field is replaced by ``XXX``, so that what LibreOffice shows after it has updated the + fields is what it computed from the numbering definitions and the bookmarks, not what the writer cached; +* a marker ``⟦u3⟧`` leads the paragraph of every numbered unit, so that the number LibreOffice drew before it can be + read from the exported text, and a paragraph per ``{{ref:}}`` target is added, made of exactly the fields the plan + gives that reference, preceded by ``⟦ref u3⟧``. + +The exported text is then compared with what the renderer wrote (the labels and designations that the text writer +prints), **modulo the gaps the plan documents**, each of which the plan reports with a diagnostic: + +* a unit whose number Word draws differently from the renderer's (a letter counter past 26, a label that needs the + section's number, a list deeper than Word's nine levels, alternatives, which the next stage of the planner + places): LibreOffice must agree with the plan (the simulator), and the plan must say so; +* ``\\t`` is ignored by LibreOffice (it is Word's switch), so a reference built with it reads there as + ``FieldSeg.cached_lo`` says; +* LibreOffice fills a skipped heading level with its start value, which is what the simulator models for headings. + +Skipped when LibreOffice or its Python bindings are missing, a failure when ``LEGALDOWN_REQUIRE_DOCX_TOOLS`` is set. +""" +from __future__ import annotations + +import functools +import re +import xml.etree.ElementTree as ET +from dataclasses import dataclass + +import pytest +from conftest import DOCUMENTS, FRONT, spec_root +from docx_support import Exported, W, libreoffice_export, parts, repackage, require_libreoffice +from test_docx_plan import ALTERNATIVES, units + +from legaldown_render import DocumentError, render +from legaldown_render.resolve.docx_plan import DocxPlan, FieldSeg, Literal_, plan_docx +from legaldown_render.tree import CrossRef, ListItem, Paragraph, RenderTree, Section, iter_tree_inlines +from legaldown_render.writers.docx.ooxml import Bookmarks, p, text_run +from legaldown_render.writers.docx.package import Relationships +from legaldown_render.writers.docx.runs import InlineWriter + +pytestmark = pytest.mark.libreoffice + +VARIANTS: dict[str, dict] = { + "default": {}, + "outline": {"style": "outline"}, + "continental": {"style": "continental"}, + "mixed": {"overrides": {"numbering.scheme": "mixed"}}, + "none": {"overrides": {"numbering.scheme": "none"}}, +} + +#: Documents that stress the numbering and the references, written for this test. +STRESS: dict[str, str] = { + "nested": FRONT + """ +# Alpha {#alpha} + +Intro paragraph one. {#p1} + +Intro paragraph two. {#p2} + +- first item {#i1} +- second item {#i2} + - nested one {#n1} + - nested two {#n2} + - deep one {#d1} +- third item {#i3} + +## Beta {#beta} + +### Gamma {#gamma} + +1. ordered one {#o1} +2. ordered two {#o2} + - bullet in ordered {#b1} + +#### Delta {#delta} + +##### Epsilon {#eps} + +- eps item {#ei} + +# Second {#second} + +- a {#sa} + +Refs: {{ref: alpha}} {{ref: p1}} {{ref: i1}} {{ref: n2}} {{ref: d1}} {{ref: beta}} {{ref: gamma}} {{ref: o2}} +{{ref: b1}} {{ref: delta}} {{ref: eps}} {{ref: ei}} {{ref: second}} {{ref: sa}} {{ref: i3}} {{ref: p2}} +""", + "lists": FRONT + """ +# A {#a} + +- x {#x} + - n1 {#n1} + - n2 {#n2} + + Text between. + + - n3 {#n3} + 1. o1 {#o1} + 1. o2 {#o2} + > - quoted {#q1} + > - quoted two +- y {#y} + 1. oy {#oy} + - ny {#ny} + +Para. + +- z {#z} + +Ref {{ref: n3}} {{ref: o2}} {{ref: oy}} {{ref: ny}} {{ref: z}} {{ref: q1}} +""", + "preamble": FRONT + """- pre one {#p1} +- pre two {#p2} + - pre nested {#pn} + +# A {#a} + +- in {#in1} + +Ref {{ref: p1}} {{ref: pn}} {{ref: in1}} +""", + "letters": FRONT + "# A {#a}\n\n" + "\n".join(f"- item {i} {{#m{i}}}" for i in range(1, 32)) + ( + "\n\nRef {{ref: m1}} {{ref: m26}} {{ref: m27}} {{ref: m28}} {{ref: m31}}\n"), + "level five": FRONT + """ +# A + +## B + +### C + +#### D + +##### E {#e} + +- five item {#fi} +- five item two {#fj} + +Para {#pp} + +Ref {{ref: fi}} {{ref: fj}} {{ref: e}} {{ref: pp}} +""", + "skipped": FRONT + """ +# One {#one} + +### Deep {#deep} + +## Two {#two} + +#### Deeper {#deeper} + +Ref: {{ref: deep}} {{ref: two}} {{ref: deeper}} +""", + "helpers": FRONT + """ +# A {#a} + +- unanchored + - also unanchored + - anchored {#deep} + +- unanchored ordered + 1. unanchored nested + 1. anchored too {#too} + +Ref {{ref: deep}} {{ref: too}} +""", + "alternatives": ALTERNATIVES + """ +# Intro {#intro} + +# Disputes {#d when=forum:courts} + +## Sub one {#s1} + +- courts item {#ci when=forum:courts} +- arbitration item {#ai when=forum:arb} + - nested under alt {#an} +- common item {#cm} + +# Disputes {#d when=forum:arb} + +## Sub alt {#sa} + +# Last {#last} + +Ref {{ref: d}} {{ref: s1}} {{ref: sa}} {{ref: ci}} {{ref: an}} {{ref: cm}} {{ref: last}} +""", +} + +#: The settings that give the numbering words and symbols, which Word's ``\t`` switch drops from a reference. +WORDS: dict[str, dict] = { + "article": {"overrides": {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}], "references.format": "Article {designation}"}}, + "section-sign": {"overrides": {"numbering.levels": [ + {"counter": "decimal", "label": "§ {n}", "ref": "{n}"}], "references.format": "§ {designation}"}}, + "paragraphs": {"overrides": {"paragraphs.numbered": True}}, +} + +#: What a diagnostic of the plan has to say when Word numbers a unit differently from the renderer. +EXPLAINING = {"render-docx-number-mismatch", "render-docx-section-label", "render-docx-letter-overflow", + "render-docx-roman-overflow", "render-docx-list-depth", "render-docx-shared-counter"} + +WRONG = "XXX" +OPEN, CLOSE = "⟦", "⟧" +MARKED = re.compile(rf"^(?P.*?){OPEN}(?P[^{CLOSE}]+){CLOSE}(?P.*)$") + + +# -- Making the files --------------------------------------------------------------------------------------------- + + +def _is_field_part(run: ET.Element) -> str | None: + """``begin``, ``separate`` or ``end`` for a run that holds that field character, else None.""" + mark = run.find(f"{W}fldChar") + return None if mark is None else mark.get(f"{W}fldCharType") + + +def seed_wrong_results(root: ET.Element, wrong: str = WRONG) -> int: + """Replace the cached result of every complex field under *root* by *wrong*; returns how many there are.""" + count = 0 + for paragraph in root.iter(f"{W}p"): + state = None + first: ET.Element | None = None + for run in list(paragraph.iter(f"{W}r")): + part = _is_field_part(run) + if part == "begin": + state, first = "instruction", None + count += 1 + elif part == "separate": + state = "result" + elif part == "end": + if state == "result" and first is None: + spare = ET.Element(f"{W}r") + ET.SubElement(spare, f"{W}t").text = wrong + run_list = list(paragraph) + paragraph.insert(run_list.index(run), spare) + state = None + elif state == "result": + text = run.find(f"{W}t") + if text is None: + continue + if first is None: + text.text, first = wrong, run + else: + paragraph.remove(run) + return count + + +def marked_paragraphs(root: ET.Element, tree: RenderTree, plan: DocxPlan) -> list[Section | ListItem | Paragraph]: + """The units whose paragraph carries a number: every heading and every unit the plan says is explicit, in the + order of the document, which is the order of the paragraphs of the body that carry a heading style or ``w:numPr``.""" + expected = [unit for unit in units(tree) if isinstance(unit, Section) or plan.units[unit.unit].explicit] + body = root.find(f"{W}body") + assert body is not None + found = [] + for paragraph in body.findall(f"{W}p"): + style = paragraph.find(f"{W}pPr/{W}pStyle") + heading = style is not None and re.fullmatch(r"Heading[1-5]", style.get(f"{W}val", "")) is not None + if heading or paragraph.find(f"{W}pPr/{W}numPr") is not None: + found.append(paragraph) + assert len(found) == len(expected), (len(found), len(expected)) + for paragraph, unit in zip(found, expected, strict=True): + marker = ET.Element(f"{W}r") + ET.SubElement(marker, f"{W}t").text = f"{OPEN}{unit.unit}{CLOSE}" + position = 1 if paragraph.find(f"{W}pPr") is not None else 0 + paragraph.insert(position, marker) + return expected + + +def reference_lines(root: ET.Element, plan: DocxPlan) -> None: + """Add a paragraph for every reference target of the plan, made of the fields of that reference.""" + body = root.find(f"{W}body") + assert body is not None + writer = InlineWriter(plan, Relationships(), Bookmarks(), None, {}, "0563C1") + end = len(body) - 1 # before the section properties + for offset, (target, reference) in enumerate(plan.refs.items()): + line = p(text_run(f"{OPEN}ref {target}{CLOSE}"), writer.segments(reference.segments)) + body.insert(end + offset, line) + + +def prepared(data: bytes, tree: RenderTree, plan: DocxPlan) -> bytes: + """*data* with the results of its fields made wrong, its unit paragraphs marked and a line per reference.""" + def change(document: str) -> str: + ET.register_namespace("w", W.strip("{}")) + ET.register_namespace("r", "http://schemas.openxmlformats.org/officeDocument/2006/relationships") + root = ET.fromstring(document.encode("utf-8")) + seed_wrong_results(root) + marked_paragraphs(root, tree, plan) + reference_lines(root, plan) + return ET.tostring(root, encoding="unicode", xml_declaration=False) + + return repackage(data, "word/document.xml", change) + + +@dataclass +class Case: + name: str + tree: RenderTree + plan: DocxPlan + rules: set[str] + data: bytes + shown: Exported | None = None + + +def corpus() -> dict[str, str]: + """The documents: tests/documents, the specification's examples and valid fixtures, and the stress documents.""" + found = {f"documents/{path.name}": path.read_text(encoding="utf-8") for path in sorted(DOCUMENTS.glob("*.lgd"))} + spec = spec_root() + if spec is not None: + for path in sorted((spec / "examples").rglob("*.lgd")) + sorted((spec / "fixtures" / "valid").rglob("*.lgd")): + found[str(path.relative_to(spec))] = path.read_text(encoding="utf-8") + found.update({f"stress/{name}": source for name, source in STRESS.items()}) + return found + + +@functools.cache +def round_trip(variant: str, update: bool = True) -> tuple[Case, ...]: + """Every document under *variant*, rendered, made wrong and marked, and shown by LibreOffice.""" + require_libreoffice() + settings = {**VARIANTS, **WORDS}[variant] + cases: dict[str, Case] = {} + for name, source in corpus().items(): + if variant in WORDS and not name.startswith("stress/"): + continue + try: + result = render(source, format="docx", **settings) + except DocumentError: + continue + assert result.binary is not None + plan, diagnostics = plan_docx(result.tree) + cases[f"{variant}/{name}"] = Case(name, result.tree, plan, {d.rule for d in diagnostics}, + prepared(result.binary, result.tree, plan)) + key = {f"d{index}": case for index, case in enumerate(cases.values())} + shown = libreoffice_export({name: case.data for name, case in key.items()}, update=update) + for name, case in key.items(): + case.shown = shown[name] + return tuple(cases.values()) + + +# -- Reading what LibreOffice says ---------------------------------------------------------------------------------- + + +def marked_lines(shown: Exported) -> dict[str, tuple[str, str]]: + """Tag to (the number LibreOffice drew before the marker, the text after it), for every marked line.""" + found: dict[str, tuple[str, str]] = {} + for line in shown.text.lstrip("").splitlines(): + match = MARKED.match(line) + if match: + found[match["tag"]] = (match["number"].strip(), match["rest"].strip()) + return found + + +def lo_reference(segments: tuple) -> str: + """What LibreOffice shows for *segments*: the cached results, but for a field written with ``\\t`` (which it + ignores) the result of the same field without it.""" + return "".join(s.text if isinstance(s, Literal_) else s.cached_lo for s in segments) + + +def renderer_references(tree: RenderTree) -> dict[str, str]: + return {i.target_unit: i.text for i in iter_tree_inlines(tree) if isinstance(i, CrossRef) and i.target_unit} + + +def check(case: Case) -> None: + """The property: every number LibreOffice draws is the renderer's, or the plan's where the plan says Word differs + (with a diagnostic), and every reference reads as the renderer wrote it, or as the plan says.""" + assert case.shown is not None + lines = marked_lines(case.shown) + excused = False + for node in units(case.tree): + mine = case.plan.units[node.unit] + if not (isinstance(node, Section) or mine.explicit): + continue + number, _ = lines[node.unit] + label = node.label or "" + if mine.num_id is None: + assert number == "", (node.unit, number) + elif mine.count is None: # a bullet: LibreOffice draws its own symbol + assert number != "", node.unit + else: + assert number == mine.label, (node.unit, "LibreOffice", number, "plan", mine.label) + if mine.label != label: + excused = True + else: + assert number == label, (node.unit, number, label) + if excused: + assert case.rules & EXPLAINING, "the plan says Word numbers a unit differently but reports nothing" + shown_by_renderer = renderer_references(case.tree) + for target, reference in case.plan.refs.items(): + _, text = lines[f"ref {target}"] + assert text == lo_reference(reference.segments), (target, "LibreOffice", text, "plan", reference.segments) + suppressing = any(isinstance(s, FieldSeg) and s.suppress_text for s in reference.segments) + if reference.exact and not suppressing: + assert text == shown_by_renderer[target], (target, text, shown_by_renderer[target]) + + +@pytest.mark.parametrize("variant", list(VARIANTS) + list(WORDS)) +def test_libreoffice_draws_the_numbers_and_reads_the_references_as_the_renderer_wrote_them(variant: str) -> None: + cases = round_trip(variant) + assert cases + for case in cases: + try: + check(case) + except AssertionError as error: + raise AssertionError(f"{variant}/{case.name}: {error}") from error + + +def test_the_round_trip_reaches_numbers_references_and_every_gap() -> None: + """The corpus is not vacuous: it numbers headings, items and paragraphs, it has references of every kind, and it + reaches each documented gap (``\\t``, an overflowing letter counter, a label that needs the section).""" + seen_rules: set[str] = set() + references = suppressing = numbered = hidden = 0 + for variant in list(VARIANTS) + list(WORDS): + for case in round_trip(variant): + seen_rules |= case.rules + references += len(case.plan.refs) + numbered += sum(1 for unit in case.plan.units.values() if unit.count is not None) + suppressing += sum(any(isinstance(s, FieldSeg) and s.suppress_text for s in r.segments) + for r in case.plan.refs.values()) + hidden += sum(isinstance(s, FieldSeg) and s.bookmark.startswith("_Ld") + for r in case.plan.refs.values() for s in r.segments) + assert numbered > 200 and references > 100 and suppressing > 0 and hidden > 0 + assert {"render-docx-letter-overflow", "render-docx-section-label"} <= seen_rules + + +def test_the_cached_results_are_wrong_in_the_file_and_libreoffice_does_not_read_them() -> None: + """The seed works: the file holds ``XXX`` as the result of every reference field, and LibreOffice shows the right + results all the same, even before the fields are updated, because it computes them from the numbering and the + bookmarks when it opens the file (so that what the comparison above checks is its computation, not the cache).""" + for update in (True, False): + document = next(case for case in round_trip("default", update=update) if case.name == "stress/nested") + written = parts(document.data)["word/document.xml"].decode("utf-8") + assert written.count(f">{WRONG}<") >= len(document.plan.refs) > 10 + lines = marked_lines(document.shown) # type: ignore[arg-type] + assert not any(WRONG in text for _, text in lines.values()) + check(document) From 3345033a268bb3fb967a6e4375ef756347d211e0 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:42:16 +0000 Subject: [PATCH 089/118] Document the DOCX cross-references, bookmarks and diagnostics Architecture: the designation cover, bookmark names, the contents plan and the planner's diagnostics. Conformance and README: numbering and cross-references are live in the DOCX, with the render-docx-* rules a user can now see; the file is still interim for the title block, contents, attachments and signature blocks. --- CONFORMANCE.md | 4 +- README.md | 29 ++++++++------ docs/architecture.md | 90 +++++++++++++++++++++++++++++++++----------- docs/roadmap.md | 20 ++++++---- 4 files changed, 99 insertions(+), 44 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 6097d7d..98e78d3 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -20,7 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | -| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says, but **the file is still interim**: the planner that decides the numbering is not wired in, so there are no numbers or cross-reference fields, and no table of contents, title block, attachments or signature blocks. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)); the numbering definitions, bookmarks and `REF` fields are written from a plan, and the planner that makes the plan is not built yet. Every DOCX render reports the Warning `render-docx-interim` ("DOCX output is incomplete in this version") until the body is complete; it is the renderer's own rule, not the specification's, and goes away then. HTML is the format this renderer claims §13.6 with | +| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says, but **the file is still interim**: there is no table of contents, title block, attachments or signature blocks, alternatives that share a number (§15.8) are numbered in sequence (reported), and a label that shows the section's number in a list item (the continental `5.1`) is drawn without it (reported). The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a list continued by a fixed number, which will not follow edits), `render-docx-number-mismatch` (any other difference, including alternatives), `render-docx-reference` (a cross-reference that cannot be built from fields, or reads differently in Word); Infos `render-docx-static-restart` (a list that restarts by a fixed setting) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). Every DOCX render also reports the Warning `render-docx-interim` ("DOCX output is incomplete in this version") until the body is complete; it goes away then. HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | @@ -56,7 +56,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes - **Line numbers (§16.9).** Every diagnostic names its line, the validator's and the renderer's own (`render-ref-ambiguous`, `ref-not-enumerated`, `render-not-processed`, `render-link-refused`, `path-not-relative`, `path-outside-root`, `render-term-limit`, `render-term-too-long`, - `render-designation-cycle`, `render-designation-limit`, `render-locale-fallback`), and the CLI prints it as `file:line:`. The renderer's line is the + `render-designation-cycle`, `render-designation-limit`, `render-locale-fallback`, and the `render-docx-*` rules of the DOCX planner), and the CLI prints it as `file:line:`. The renderer's line is the **first line of the block it is in**, as `Document.line_of` gives it, whatever line the directive itself is on: a paragraph's, a table's header row, a quote's first `>`, a code block's opening fence, a heading's (a section title); for a list item, the line of **its marker** diff --git a/README.md b/README.md index 0c528b0..62f1d13 100644 --- a/README.md +++ b/README.md @@ -141,18 +141,23 @@ computed as their cached results, so a lawyer who inserts a clause gets the numb updated by Word ([ADR 0008](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/decisions/0008-docx-word-native.md)). -**What is there today** is the package and the body writer, not yet the numbering: a valid `.docx` (checked -against the ISO/IEC 29500 schemas, and opened in LibreOffice in CI) with styles generated from the style template -(`Normal`, `heading 1` to `heading 5`, `Title`, `Quote`, `toc 1` to `toc 5`, `Hyperlink`, ...), the page size -and margins, a running header and footer from `page.header` and `page.footer` (the default footer is "3 / 12" -as live page fields), the document's title and language, `w:updateFields` so Word refreshes fields when it -opens the file (`docx.update_fields_on_open`), and deterministic output. The body writer lays out headings, -paragraphs, lists (with the numbering definitions, bookmarks and `REF` fields a plan names), quotes, drafting -notes, code, tables with a repeated header row, rules, links, defined terms and the template view's labels and -choices. **The file is still interim:** until the planner that decides the numbering is wired in, it carries -**no numbers, no cross-reference fields, and no title block, contents, attachments or signature blocks**, and -every render reports the Warning `render-docx-interim` to say so. Do not send such a file as a contract. Settings Word -cannot express (a `%` length, say) are an error for DOCX only. See +**What is there today:** a valid `.docx` (checked against the ISO/IEC 29500 schemas, and opened in LibreOffice in CI) +with styles generated from the style template (`Normal`, `heading 1` to `heading 5`, `Title`, `Quote`, `toc 1` to +`toc 5`, `Hyperlink`, ...), the page size and margins, a running header and footer from `page.header` and +`page.footer` (the default footer is "3 / 12" as live page fields), the document's title and language, +`w:updateFields` so Word refreshes fields when it opens the file (`docx.update_fields_on_open`), and deterministic +output. **Numbering and cross-references are live:** headings, list items and numbered paragraphs are numbered by Word +numbering definitions (headings by the heading styles, nested lists restarting by structure), and a +`{{ref: clause}}` is `REF` fields on bookmarks whose cached results are the text the plain-text writer prints, so +"1(b)(ii)" or "I.A.1" stays right when a lawyer inserts a clause and updates fields. A defined term or an attachment +reference is an internal link. Where Word cannot draw what the renderer does (letters past 26, a label that shows the +section's number in a list item, alternatives that share a number, a list restarted by a fixed setting), the render +reports it as a `render-docx-*` Warning or Info with the line, see +[the conformance notes](https://github.com/ForLegalAI/legaldown-render/blob/main/CONFORMANCE.md). The body writer also +lays out paragraphs, quotes, drafting notes, code, tables with a repeated header row, rules, links, blanks and +choices. **The file is still interim:** it has **no title block, table of contents, attachments or signature +blocks**, and every render reports the Warning `render-docx-interim` to say so. Do not send such a file as a +contract. Settings Word cannot express (a `%` length, say) are an error for DOCX only. See [Styles and preferences](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/style-templates.md#docx) and the [roadmap](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/roadmap.md). diff --git a/docs/architecture.md b/docs/architecture.md index be45689..aba8ca7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -244,16 +244,18 @@ references show; the Warning says so. Every other heading reads exactly as its d ### The DOCX planner (`resolve/docx_plan.py`) -The DOCX writer will contain no numbering rules either (ADR 0003, [ADR 0008](decisions/0008-docx-word-native.md)). +The DOCX writer contains no numbering rules either (ADR 0003, [ADR 0008](decisions/0008-docx-word-native.md)). Between the resolved tree and that writer sits a pure function, `plan_docx(tree, *, file=None) -> (DocxPlan, -diagnostics)`, which maps the numbering structure above to what Word needs: the abstractNums and nums, and for every -section, list item and numbered paragraph the `(numId, ilvl)` its paragraph takes. It reads only `tree.formats` and the +diagnostics)`, which maps the numbering structure above to what Word needs: the abstractNums and nums, for every +section, list item and numbered paragraph the `(numId, ilvl)` its paragraph takes, the bookmark of every anchor, the +fields of every cross-reference, and the entries of the table of contents. It reads only `tree.formats` and the units' `number`, `parent_unit` and `List.family`/`depth`/`format` (`numbering.formats_of(style)` builds `formats` for the resolver and for the style check), and checks every placement against the number the resolver gave the unit by simulating Word's counters (`resolve/docx_numbering.py`), so that what Word draws is what the resolver numbered, or a -diagnostic says why not. It is a planning step, not a stage of the pipeline: the writer does not call it yet. The one -part that runs on every DOCX render is the style check, `docx_style_errors(style)`, which `style/docx.py:docx_settings` -runs before rendering (ADR 0006). +diagnostic says why not. It is a planning step, not a stage of the pipeline: `render(format="docx")` calls it after +resolution, passes the plan to the writer, and adds its diagnostics (named with the document's file, each finding once, +in the order of their lines) to the result's. The style check, `docx_style_errors(style)`, runs earlier still, in +`style/docx.py:docx_settings`, before rendering (ADR 0006). **Word's list semantics**, stated once (the planner and its diagnostics rely on them): @@ -278,16 +280,57 @@ A level shows only counters of its own list. A label whose filled-in form shows or the `{path}` of an item or paragraph in a section: the continental style's `5.1`), or a number of an enclosing item that Word cannot show (a list nested in another kind of list, or in a quote), is written without it and reported (`render-docx-section-label`). Alternatives that share a number are not planned yet (they are numbered in -sequence and reported). `UnitPlan.ind`, the references, the bookmarks and the table of contents are further parts of -the plan that this part leaves unset. - -**Diagnostics** (all `render-docx-*`, with the unit's line, once per finding): `section-label`, `list-depth`, -`letter-overflow` (Word repeats letters after 26, the renderer counts on: `aa`, `ab`), `roman-overflow` (4000 and -beyond), `shared-counter` (a list that goes on from another, or whose level another list used in between, starts at -a fixed number), `number-mismatch` (any other difference: the first unit of a run that Word counts off by the same -amount, with how many follow; it does not occur without an alternative that shares a number), and, as information, -`static-restart` (a list that restarts at 1 by a fixed setting). A heading level that is skipped is the Core Error -`heading-skip`; the plan only notes it on the unit. +sequence and reported). `UnitPlan.ind` is the one part of the plan this leaves unset. + +**Cross-references: the designation cover** (`resolve/docx_cover.py`, §6.3, §13.3). The renderer's text of a +`{{ref:}}` is the designation of the target (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)`) inside `references.format`; the planner +writes the designation of every unit as a *token string* (a counter with the unit that holds it, literal text, or a +heading's text under the `none` scheme) in the same pass that numbers it, ported one to one from the resolver's `fill` +and `extend`. In Word every part of it that Word can compute must be a field, and nothing Word can compute may be +literal text, so the plan covers the designation by `REF \n \h` fields and the delimiters between them. +A field returns the full level text of the bookmarked paragraph with its counters, minus trailing periods (what the +simulator gives as `Step.word_ref`), and fits where its counters belong to the same units as the designation's: a +`{path}` label (`1.1.1`) is covered by one field on the target, a `{n}` label (`I.A.1`) by one field per level with +literal delimiters between (`I` `.` `A` `.` `1`), a nested item (`(b)(ii)`) by one field per level, an ordered +list in a section by `REF section` + `(` + `REF item` + `)`. Literal text may stand only for delimiters, never for a +counter. The cover with the fewest fields wins, then the fewest using `\t`, then the fewest literal characters; +the text of `references.format` is literal around it, and with two `{designation}` the same cover is written twice. +`\t` (Word's switch that suppresses the words around a counter; LibreOffice ignores it) is used only where it leaves +the bare counter (`Article I` gives `I`, `§ 3` gives `3`), never where the field's text holds a delimiter +(`(a)`, `a)`, `1.2`, `No. 1`). Each field's cached result is **Word's** value (the simulator's), and `cached_lo` what +LibreOffice shows. A unit that a field points at has a bookmark (its anchor's, or a hidden one); a reference to an +item or paragraph the style does not number is the designation of its section (`ref-not-enumerated`), covered as one. +Under the `none` scheme a heading is a field for the bookmarked text (`REF bookmark \h`) and the item's pieces +follow after a space. A `RefPlan` is made once per target, keyed by `CrossRef.target_unit`; `exact` says that +the cached results read as the renderer's text. + +**Bookmarks** (`resolve/docx_bookmarks.py`). The planner names bookmarks and the writer gives them ids. Every anchor of +the tree has one, referenced or not: `ld_` for a section, list item or paragraph, `lddef_` for a +defined term, `ldatt_` for an attachment, `_LdAttachments` for the attachments heading, with `-` and `:` written +as `_`. A name is used only if it is valid, at most 40 characters, nothing but dashes was changed, and it is unique +ignoring case (Word compares names so); otherwise, and for a unit a field needs that has no anchor, it is hidden: +`_Ld`, counted in document order. The table of contents uses `_Toc` from 100000001. The writer wraps +the paragraph's title or first paragraph in `UnitPlan.bookmarks`, then `toc_bookmark`; a term reference and an +internal link point at `plan.anchors`. + +**The table of contents.** `ContentsEntry.unit` (the section's unit; None for the attachments heading and an +attachment) keys each entry of `tree.contents` to its section. `plan.toc` lists the entries in order with +the number Word draws for the heading (`UnitPlan.label`, so a style's `Article I` is the number Word shows, not +a rendering of the renderer's), the separator the level has (`suff`: tab or space; none for an unnumbered entry), the +title, the condition label, the `_Toc` bookmark (also `UnitPlan.toc_bookmark`) and an empty cached page number. The +complex `TOC` field that uses them is written with the title block. + +**Diagnostics** (all `render-docx-*`, with the line of the unit or of the reference, once per finding): `section-label`, +`list-depth`, `letter-overflow` (Word repeats letters after 26, the renderer counts on: `aa`, `ab`), `roman-overflow` +(4000 and beyond), `shared-counter` (a list that goes on from another, or whose level another list used in between, +starts at a fixed number), `number-mismatch` (any other difference: the first unit of a run that Word counts off by the +same amount, with how many follow; it does not occur without an alternative that shares a number), `reference` (a +cross-reference that cannot be built from fields, because a heading level is skipped before the target with a +`{n}`-style scheme, or Word draws text around a counter that `\t` cannot drop (`({n})`, `{n}:`, `No. {n}`): it is one +field for the target's own number, which Word reads differently; or one that reads differently in Word for a reason +that no diagnostic of a unit already gives), and, as information, `static-restart` (a list that restarts at 1 by a +fixed setting) and `reference-word-only` (a cover relies on `\t`, which LibreOffice ignores, once per document). A +heading level that is skipped is the Core Error `heading-skip`; the plan only notes it on the unit. ### 5. Write (`writers/`) @@ -298,7 +341,7 @@ amount, with how many follow; it does not occur without an alternative that shar |---|---| | `TextWriter` | Plain text (§13.6). A soft line break inside a paragraph is a space, a hard break a new line, and lists are indented with their labels, one quote stays one quote, and tables keep escaped cells and alignment. It takes the style's `attachments.separator` (default `page-break`; `rule` and `page-break` both print `---` before the attachments heading and between attachments). It is the **test oracle** for resolution | | `HtmlWriter` | A self-contained HTML5 page, or with `standalone=False` just the `
`. Semantic markup, with labels as real text rather than CSS counters, and a stylesheet generated from the style's presentation settings, including print rules | -| `DocxWriter` | A `.docx` file, as bytes (`RenderResult.binary`; `output` is empty). OOXML written directly with `zipfile` and `xml.etree` ([ADR 0008](decisions/0008-docx-word-native.md)): the parts, the styles generated from the style template, settings, page setup, running headers and footers, properties, a deterministic ZIP, and one sanitiser for every text and attribute. **The body is laid out as a `DocxPlan` says** (`resolve/docx_plan.py`, the planner's data model): `numbering.xml` from its abstract numberings and nums; each section, list item and top-level paragraph gets `w:numPr`, `w:ind`, bookmarks and, for a later alternative that shares a number (§15.8), `numId` 0 and its number as `REF` fields, as its `UnitPlan` says; a `{{ref:}}` is the literal text and `REF` fields (cached results as planned) of its `RefPlan`; a `{{term:}}` or an `{{attach:}}` is an internal hyperlink (or a `REF` for an attachment's title), a defined term a bookmark; every block (lists with their continuation blocks, quotes, drafting notes, code, tables with a repeated header row, rules) and inline (emphasis, strong, code, external links through relationships, breaks, images as alt text, blanks, choices, failure markers) has its Word form. The writer decides none of it (ADR 0003) and a field that points at a bookmark the document does not hold is a bug in the plan (`InternalError`). **Interim, until the planner is wired in:** the writer is given a plan with no numbering (`writers/docx/interim_plan.py`), so the file has no numbers or `REF` fields, and the title block, table of contents, attachments and signature blocks are not written yet. The DOCX-only settings are converted and checked before rendering (`style/docx.py`) | +| `DocxWriter` | A `.docx` file, as bytes (`RenderResult.binary`; `output` is empty). OOXML written directly with `zipfile` and `xml.etree` ([ADR 0008](decisions/0008-docx-word-native.md)): the parts, the styles generated from the style template, settings, page setup, running headers and footers, properties, a deterministic ZIP, and one sanitiser for every text and attribute. **The body is laid out as a `DocxPlan` says** (`resolve/docx_plan.py`, the planner's data model): `numbering.xml` from its abstract numberings and nums; each section, list item and top-level paragraph gets `w:numPr`, `w:ind`, bookmarks and, for a later alternative that shares a number (§15.8), `numId` 0 and its number as `REF` fields, as its `UnitPlan` says; a `{{ref:}}` is the literal text and `REF` fields (cached results as planned) of its `RefPlan`; a `{{term:}}` or an `{{attach:}}` is an internal hyperlink (or a `REF` for an attachment's title), a defined term a bookmark; every block (lists with their continuation blocks, quotes, drafting notes, code, tables with a repeated header row, rules) and inline (emphasis, strong, code, external links through relationships, breaks, images as alt text, blanks, choices, failure markers) has its Word form. The writer decides none of it (ADR 0003) and a field that points at a bookmark the document does not hold is a bug in the plan (`InternalError`). **Interim:** the title block, the table of contents, the attachments and the signature blocks are not written yet; the numbering, bookmarks and cross-reference fields are, from the plan `render` computes (the writer plans a tree itself when it is used directly and given none). The DOCX-only settings are converted and checked before rendering (`style/docx.py`) | | PDF | Planned ([ADR 0004](decisions/0004-html-first.md)) | ## Package layout @@ -315,8 +358,10 @@ src/legaldown_render/ ├── resolve/ │ ├── resolver.py stage 4 │ ├── numbering.py counters, the §13.1 scheme presets, label templates, the formats a style gives -│ ├── docx_plan.py the DOCX plan: Word numbering for the resolved tree (not yet called by the writer), and the style check DOCX runs before rendering +│ ├── docx_plan.py the DOCX plan: Word numbering, bookmarks, cross-reference fields and contents for the resolved tree, and the style check DOCX runs before rendering │ ├── docx_numbering.py Word's counters, label templates as tokens, lvlText, the numbering simulator +│ ├── docx_cover.py the cover of a designation by REF fields and delimiters +│ ├── docx_bookmarks.py bookmark names │ └── values.py Babel formatting of dates, money, durations ├── style/ │ ├── model.py the style dataclasses (every field has a default) @@ -329,7 +374,7 @@ src/legaldown_render/ └── docx/ the DOCX writer: ooxml.py (sanitiser, builders), package.py (parts, relationships, ZIP), styles.py, settings.py, headers.py, properties.py, numbering.py (numbering.xml from the plan), body.py (blocks, numbered paragraphs, tables), runs.py (inlines, fields, - hyperlinks), interim_plan.py (the plan with no numbering, temporary) + hyperlinks) ``` ## Dependencies @@ -366,7 +411,7 @@ Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can ev `ref-not-enumerated`, `path-not-relative` and `path-outside-root` (Errors for a file reference that is not relative, or that leaves the document root, §2.3; reported here because validator 0.4 checks neither), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and `render-docx-interim` (a Warning on every DOCX render, while the DOCX body is incomplete). The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and `render-docx-interim` (a Warning on every DOCX render, while the DOCX body is incomplete), and the rules the DOCX planner reports on what Word draws differently from the other formats (`render-docx-section-label`, `-list-depth`, `-letter-overflow`, `-roman-overflow`, `-shared-counter`, `-number-mismatch`, `-reference`, as Warnings, `-static-restart` and `-reference-word-only` as Infos; see "The DOCX planner"). The validator's diagnostics carry their line (§16.9), and so do the renderer's own, and the CLI prints it as `file:line:`. @@ -524,7 +569,7 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_values.py` | Money, dates and durations in a locale: exact digits, plural of the value shown, currency display, CLDR fallback to English | | `tests/test_numbering.py` | Counter formats, label templates and the preamble's `{section}` | | `tests/test_numbering_structure.py` | The numbering structure in the tree (`formats`, unit ids, numbers, parents, alternatives, list families, cross-reference units): headings in each scheme and with skipped levels, nested lists, section-qualified items, numbered paragraphs, alternatives, the `none` scheme and its second pass; and that every label, designation and reference can be written again from the structure alone, over the tests' and the specification's documents | -| `tests/test_docx_plan.py` | The DOCX numbering plan: abstractNums per style and scheme, heading levels, top-level, nested, sibling, cross-family, quoted and clamped lists, numbered paragraphs, labels that need the section, letter and Roman overflow, diagnostics (lines, file, order, repeats, runs of units counted off), style errors (also through `docx_settings`), determinism; and, unit by unit, that Word's simulated count and label are the resolver's unless the unit's own counter reads differently in Word or the plan notes why, over the tests' and the specification's documents under each style and list setting, and over random documents and alternatives | +| `tests/test_docx_plan.py` | The DOCX plan: abstractNums per style and scheme, heading levels, top-level, nested, sibling, cross-family, quoted and clamped lists, numbered paragraphs, labels that need the section, letter and Roman overflow, diagnostics (lines, file, order, repeats, runs of units counted off), style errors (also through `docx_settings`), determinism; and, unit by unit, that Word's simulated count and label are the resolver's unless the unit's own counter reads differently in Word or the plan notes why, over the tests' and the specification's documents under each style and list setting, and over random documents and alternatives. Also the cross-references: the worked table of covers for each style (default, outline, mixed, continental, `none`, `Article {n}`, `§ {n}`), the cover algorithm on its own, references to other sections and to unnumbered units, `references.format`, every reason a reference cannot be covered, bookmark names (prefixes, length, collisions, hidden names, determinism), the table of contents, the render's diagnostics, and the property that every cached reference reads as the renderer's text or a diagnostic says why not, over the same documents and random documents with an anchor on every unit | | `tests/test_docx_simulator.py` | Word's counters, label templates as tokens (equal to filling the template as text), `lvlText`, and the simulator: restarts, start overrides, shared counters, skipped levels, `peek`/`current` | | `tests/test_build.py` | Building the tree: comment-only items and quotes, emphasis next to a directive, spacing where an inline comment was dropped | | `tests/test_terms.py` | What `{{term:}}` shows, references inside terms, terms that name themselves or each other, the cap on a term text | @@ -543,9 +588,10 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_errors.py` | The exception hierarchy and pickling of errors | | `tests/test_escaping.py` | Control characters from a document or the command line never reach a diagnostic or standard error unescaped | | `tests/test_docx_style.py` | `page.header`, `page.footer` and `docx.*`: validation, and how CSS lengths, colours and fonts become Word's units, with the StyleErrors only DOCX has | -| `tests/test_docx_package.py` | The OOXML package: parts, relationships, content types, settings order, styles, page setup, headers and footers, properties, the body without a plan, determinism, the sanitiser and builders, the API and command line | +| `tests/test_docx_package.py` | The OOXML package: parts, relationships, content types, settings order, styles, page setup, headers and footers, properties, the body of a planned document, determinism, the sanitiser and builders, the API and command line | | `tests/test_docx_body.py` | The DOCX body from hand-made plans: numbering definitions, `w:numPr`/`w:ind`/alternatives, bookmarks and `REF` fields with cached results, hyperlinks, every inline and block, escaping, schema validity (marker `ooxml_schema`) and a LibreOffice round trip of headings, nested lists and references whose cached values are wrong on purpose (marker `libreoffice`) | | `tests/test_docx_escaping.py` | Well-formedness of every part for any text, with a seeded fuzz and the characters XML cannot carry | | `tests/test_docx_validation.py` | Every part of every generated document against the ISO/IEC 29500-4 schemas (marker `ooxml_schema`; needs lxml and `python tests/schemas/fetch.py`) | +| `tests/test_docx_roundtrip.py` | LibreOffice as the oracle of the numbering and the cross-references: every document of the corpus (the tests', the specification's, stress documents) under the default, outline, continental, mixed and `none` schemes, with every cached field result made wrong (`XXX`), every numbered paragraph marked and a line per reference added; the number LibreOffice draws and every reference it computes equal the renderer's, modulo the gaps the plan reports (marker `libreoffice`) | | `tests/test_docx_libreoffice.py` | The documents opened in LibreOffice through UNO (`tests/lo/uno_export.py`), fields updated: the words in order, header, footer, page count (marker `libreoffice`; skipped without `soffice` and `python3-uno`) | | `tests/conformance/` | Every example and fixture in the specification repository renders, every internal link has a target, and each invalid fixture renders its rule's failure marker. Needs `$LEGALDOWN_SPEC_DIR` or a sibling `../LegalDown` checkout | diff --git a/docs/roadmap.md b/docs/roadmap.md index 2b7f907..69515fd 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -101,7 +101,7 @@ values appear only as the cached results of fields. Decided in `[docx]` extra): the OOXML parts, styles generated from the style template, settings (compatibility mode 15, `w:updateFields`), page setup, running headers and footers with `PAGE`/`NUMPAGES` fields, document properties, deterministic output. `RenderResult.binary`, `-o x.docx`, `-o -`. The body is - an interim one: the title, `heading N` paragraphs and plain-text paragraphs, with no numbers yet + an interim one at that point: the title, `heading N` paragraphs and plain-text paragraphs, with no numbers yet - **Behaviour changes this brings.** `-o -` now writes to standard output for every format (it used to create a file named `-`); `.docx` is a writable extension, and a DOCX with no `-o` goes to standard output when that is not a terminal; `--fragment` with DOCX is a usage error; `RenderTree` gains `version` and @@ -112,13 +112,17 @@ values appear only as the cached results of fields. Decided in - **Verification (done).** Every part validated against the ISO/IEC 29500 schemas (fetched in CI, not stored), well-formedness fuzzing, a determinism test, and LibreOffice opening the documents through UNO as the oracle (a `docx` CI job) -- **Body writer (done, awaiting the planner).** `numbering.xml`, numbered paragraphs, bookmarks, `REF` - fields with cached results, term hyperlinks, and every block and inline, written from a `DocxPlan` - (`resolve/docx_plan.py`); a plan with no numbering stands in until the planner is wired -- **Next.** The planner that maps the tree to Word (lists, levels, bookmarks, cross-references composed - of `REF` fields, cached values from a model of Word's numbering); the table of contents, title block, - attachments and signature blocks; a corpus checked against the plain-text oracle, also after edits made through UNO; a manual - Word checklist for releases +- **Body writer (done).** `numbering.xml`, numbered paragraphs, bookmarks, `REF` fields with cached results, + term hyperlinks, and every block and inline, written from a `DocxPlan` (`resolve/docx_plan.py`) +- **Planner: numbering, cross-references and contents plan (done).** The tree is mapped to Word numbering + (heading list, list families, paragraphs) and checked against the resolver with a model of Word's counters; + every `{{ref:}}` is covered by `REF` fields and delimiters (the designation cover), every anchor has a bookmark, + and the table of contents is planned from Word's numbers; `render` runs the planner and reports its + `render-docx-*` diagnostics. A LibreOffice round trip with wrong cached results checks the numbers and references + against the plain-text oracle +- **Next.** Qualified labels (the continental `5.1`) on the heading list, alternatives that share a number, + restarts Word would otherwise get wrong; the table of contents field, title block, attachments and signature + blocks; the corpus checked also after edits made through UNO; a manual Word checklist for releases ### v0.6 — PDF From 2fff7459fe1b9d3ad056d38237878cb84d57c921 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:56:38 +0000 Subject: [PATCH 090/118] Keep a reference that has no cover as text, and keep fields out of their own bookmarks A designation that Word's fields cannot reproduce is now the renderer's text (RefPlan.fixed): right when the file is opened, where one field for the target's own number was wrong. render-docx-reference says it will not follow edits, and names the first of the references that fail alike and how many more; the planner's findings are ordered by line with the renderer's own. A bookmark is a point at the start of its paragraph unless a field returns the text it holds (UnitPlan.collapsed), so no REF field stands inside the bookmark it points at. Under the none scheme a heading whose title refers back to the heading, directly or through other titles, has those references written as their text (UnitPlan.literal_refs) and render-docx-reference-cycle reports it: Word shows a self-reference there and LibreOffice expanded it without end. The text a heading field caches is what the runs of the title show, not the renderer's collapsed designation. The property tests now require that no counter is literal text in a cover. They share one set of random documents and one planning of each document, and rotate the schemes over the random documents, which brings the suite back to its earlier length. --- src/legaldown_render/api.py | 16 +- src/legaldown_render/resolve/docx_plan.py | 197 +++++++++++++++---- src/legaldown_render/writers/docx/body.py | 13 +- src/legaldown_render/writers/docx/runs.py | 18 +- tests/test_docx_body.py | 27 ++- tests/test_docx_package.py | 10 +- tests/test_docx_plan.py | 228 +++++++++++++++++----- 7 files changed, 398 insertions(+), 111 deletions(-) diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index 465c53d..6d2053f 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -179,6 +179,14 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any raise RenderRefused(diagnostics) tree, extra = _resolve(document, result, style, base, options.untrusted) + assert_resolved(tree) + writer = _writer(options, style, docx) + plan = None + if isinstance(writer, DocxWriter): + # What Word will draw differently from the other formats (docs/decisions/0008) is the planner's to report; its + # findings are the renderer's, and are ordered with them. + plan, planned = plan_docx(tree, file=document.filename) + extra = [*extra, *planned] # The renderer's findings that the validator's or the assembly's already # name (the same rule, level and message), wherever they are, are not repeated; # the rest follow in the order of their lines. @@ -186,13 +194,7 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any if options.strict and any(d.level == "error" for d in diagnostics): raise RenderRefused(diagnostics) - assert_resolved(tree) - writer = _writer(options, style, docx) - if isinstance(writer, DocxWriter): - # What Word will draw differently from the other formats (docs/decisions/0008): the planner's findings follow - # the renderer's, in the order of their lines, none repeating one the document has already. - plan, planned = plan_docx(tree, file=document.filename) - diagnostics += [d for d in planned if not repeats(d, diagnostics)] + if plan is not None: diagnostics.append(_interim_warning(document.filename)) written: str | bytes = writer.write(tree, plan) else: diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 2eea4fd..a7a4f1d 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -31,10 +31,18 @@ from ..diagnostics import by_line from ..style.model import LevelFormat, Style from ..tree import ( + Blank, + Choice, + Code, CrossRef, DefinedTerm, DraftingNote, + Emphasis, + FailureMarker, + HardBreak, + Image, Inline, + Link, List, ListItem, NumberingFormats, @@ -42,10 +50,18 @@ Quote, RenderTree, Section, + SoftBreak, + Strong, + TermRef, + Text, + Value, + has_visible_content, + iter_inlines, iter_located_body_inlines, iter_tree_inlines, + plain_text, ) -from ..urls import printable +from ..urls import displayed_url, printable from .docx_bookmarks import ATTACHMENTS_HEADING, BookmarkNames from .docx_cover import Field, Shown, Title, atoms, cover, suppressed, text_of, uncovered from .docx_numbering import ( @@ -180,6 +196,12 @@ class UnitPlan: bookmarks: tuple[str, ...] #: ``_Toc`` name of a heading listed in the contents. toc_bookmark: str | None + #: True: *bookmarks* are points at the start of the paragraph, not around its text (no field returns that + #: text, and a field must never stand inside the bookmark it points at). + collapsed: bool = False + #: Targets of the ``{{ref:}}`` in this heading's title that are written as their text, not as fields, because + #: their fields would lead back into the title (see :meth:`_Planner.cyclic_titles`). + literal_refs: tuple[str, ...] = () @dataclass(frozen=True, slots=True) @@ -190,6 +212,9 @@ class RefPlan: segments: tuple[Segment, ...] #: ``concat(cached) == CrossRef.text``. exact: bool + #: No cover exists, so the reference is the renderer's text with no field: right when the file is opened, but + #: fixed (an edit in Word does not change it). The plan reports each (``render-docx-reference``). + fixed: bool = False @dataclass(frozen=True, slots=True) @@ -360,6 +385,33 @@ def _reads(level: LevelDef, levels: Sequence[LevelDef]) -> str: # The planner # --------------------------------------------------------------------------- + +def _shown(inlines: tuple[Inline, ...]) -> str: + """What the runs of *inlines* show in the document (``writers/docx/runs.py``): the text of each, a hard break as a + line break, a blank with its prompt. A heading's bookmark returns this, which is not always the heading's + designation (white space is collapsed in that).""" + out: list[str] = [] + for inline in inlines: + match inline: + case Text(text=text) | Code(text=text) | CrossRef(text=text) | TermRef(text=text) | Value(text=text) \ + | FailureMarker(text=text): + out.append(text) + case SoftBreak(): + out.append(" ") + case HardBreak(): + out.append("\n") + case Blank(text=text, prompt=prompt): + out.append(text + (f" ({prompt})" if prompt else "")) + case Choice(options=options, separator=separator): + out.append("[" + separator.join(options) + "]") + case Image(children=children): + out.append(plain_text(children)) + case Link(href=href, children=children): + out.append(_shown(children) if has_visible_content(children) else displayed_url(href)) + case Emphasis(children=children) | Strong(children=children) | DefinedTerm(children=children): + out.append(_shown(children)) + return "".join(out) + _Unit = Section | ListItem | Paragraph @@ -389,12 +441,22 @@ class _Reference(NamedTuple): class _Built(NamedTuple): """What the plan makes of one reference: its pieces (literal text and fields), whether they read exactly as the - renderer's text, and what LibreOffice shows for them when that is not what Word shows (a field with ``\\t``), - else None.""" + renderer's text, what LibreOffice shows for them when that is not what Word shows (a field with ``\\t``), else + None, and whether it is fixed text for want of a cover.""" pieces: list[str | Field] exact: bool lo_text: str | None + fixed: bool + + +class _Finding(NamedTuple): + """A reference that is fixed text (*kind* ``fixed``, *reason* why) or reads differently in Word (``differs``).""" + + kind: str + reason: str + found: _Reference + shown: str class _Drift: @@ -436,6 +498,9 @@ def __init__(self, tree: RenderTree, file: str | None) -> None: #: The parts of the number of the open section, one per level: the unit that holds it (a skipped level has a #: :class:`Ghost`) and its value. self.owners: list[tuple[object, int]] = [] + #: What a heading's bookmark returns (``REF bookmark \\h``): its title as the runs show it. + self.heading_text: dict[str, str] = {} + self.unreferable: list[_Finding] = [] # -- registry ------------------------------------------------------------------- @@ -503,7 +568,7 @@ def fields(self, designation: Sequence[Shown]) -> list[Field]: if self.units[owner].num_id == 0 or self.units[owner].alt is not None: continue if isinstance(token, Title): - fields.append(Field(owner, False, False, (token,))) + fields.append(Field(owner, False, False, (Title(owner, self.heading_text.get(owner, token.text)),))) elif owner in self.steps: words = self.words(owner) fields.append(Field(owner, True, False, words)) @@ -525,21 +590,11 @@ def why_uncovered(self, designation: Sequence[Shown], fields: Sequence[Field]) - f"leave out") return "no combination of Word's fields reads like it" - def own_field(self, unit: str | None) -> Field | None: - """The one field for the number Word draws for *unit* (its heading text under the ``none`` scheme), which a - reference falls back to when its designation cannot be covered; None for a unit with neither.""" - if unit is None or unit not in self.units: - return None - if unit in self.steps: - return Field(unit, True, False, self.words(unit)) - title = self.desig.get(unit, []) - return Field(unit, False, False, (title[0],)) if title and isinstance(title[0], Title) else None - def reference(self, found: _Reference) -> _Built | None: """The fields of the reference *found*: its designation covered by fields and delimiters, inside the text of - ``references.format``; None for a target the plan knows nothing of. A designation that cannot be covered is one - field for the number Word draws for the target, and is reported, as is a reference that reads differently - in Word for a reason no diagnostic of a unit already gives.""" + ``references.format``; None for a target the plan knows nothing of. A designation that cannot be covered is + written as the renderer's text, which is right at open and not a field (``fixed``), and noted for the + report, as is a cover that reads differently in Word for a reason no diagnostic of a unit already gives.""" node = self.nodes.get(found.target) if node is None or found.target not in self.units: return None @@ -547,13 +602,9 @@ def reference(self, found: _Reference) -> _Built | None: designation = self.desig[found.target] if numbered else self.desig.get(found.section or "", []) fields = self.fields(designation) covered = cover(designation, fields) - reason = None if covered is None: - reason = self.why_uncovered(designation, fields) - own = self.own_field(found.target if numbered else found.section) - if own is None: - return None - covered = [own] + self.unreferable.append(_Finding("fixed", self.why_uncovered(designation, fields), found, found.text)) + return _Built([found.text] if found.text else [], True, None, True) pieces: list[str | Field] = [] parts = self.formats.reference.split("{designation}") for index, literal in enumerate(parts): @@ -563,23 +614,70 @@ def reference(self, found: _Reference) -> _Built | None: else: pieces.append(piece) shown = "".join(piece if isinstance(piece, str) else text_of(piece.tokens) for piece in pieces) - text = printable(found.text) - if reason is not None: - self.report("render-docx-reference", "warning", - f"The cross-reference '{text}' cannot be built from Word's numbering fields: {reason}. It is " - f"one field for the target's own number, which reads '{printable(shown)}' in Word.", - found.line) - elif shown != found.text: + if shown != found.text: chain = {t.owner for t in designation if not isinstance(t, Lit) and isinstance(t.owner, str)} if not chain & self.flagged and found.target not in self.flagged: - self.report("render-docx-reference", "warning", - f"The cross-reference '{text}' will read '{printable(shown)}' in Word: a part of it is " - f"numbered or written differently there.", found.line) + self.unreferable.append(_Finding("differs", "", found, shown)) lo_text = None if any(isinstance(piece, Field) and piece.suppress_text for piece in pieces): lo_text = "".join(piece if isinstance(piece, str) else text_of(self.words(piece.owner) if piece.number else piece.tokens) for piece in pieces) - return _Built(pieces, shown == found.text, lo_text) + return _Built(pieces, shown == found.text, lo_text, False) + + def report_references(self) -> None: + """One finding for the references that fail in the same way (the first, by line, and how many more), as + for a run of units counted off by the same amount.""" + groups: dict[tuple[str, str], list[_Finding]] = {} + for finding in self.unreferable: + groups.setdefault((finding.kind, finding.reason), []).append(finding) + for (kind, reason), members in groups.items(): + members.sort(key=lambda m: (m.found.line is None, m.found.line or 0)) + first = members[0] + text, shown = printable(first.found.text), printable(first.shown) + more = f" {len(members) - 1} more cross-references fail in the same way." if len(members) > 1 else "" + if kind == "fixed": + message = (f"The cross-reference '{text}' cannot be built from Word's numbering fields: {reason}. It is " + f"written as the text '{text}', which is right now but is not a field: it will not follow " + f"edits made in Word.{more}") + else: + message = (f"The cross-reference '{text}' will read '{shown}' in Word: a part of it is numbered or " + f"written differently there.{more}") + self.report("render-docx-reference", "warning", message, first.found.line) + + def cyclic_titles(self, built: dict[str, _Built]) -> dict[str, tuple[str, ...]]: + """The cross-references in a heading's title that must be text, by heading unit. Under the ``none`` scheme a + heading is designated by its text, and a field for it (``REF bookmark \\h``) returns what its bookmark holds; + a reference in the title that leads back to the heading (directly, or through the titles of other headings) + would be a field inside the bookmark it points at, which Word cannot show (a self-reference) and + LibreOffice expands without end. Such a reference is written as its text and reported.""" + if not any(not p.number for made in built.values() for p in made.pieces if isinstance(p, Field)): + return {} # no field returns the text of a heading: nothing can lead back into one + inside: dict[str, list[str]] = {} + for section in self.tree.sections: + if section.unit is not None: + targets = [i.target_unit for i in iter_inlines(section.title) + if isinstance(i, CrossRef) and i.target_unit in built] + inside[section.unit] = list(dict.fromkeys(t for t in targets if t is not None)) + owners = {target: {p.owner for p in made.pieces if isinstance(p, Field) and not p.number} + for target, made in built.items()} + + def leads_to(start: str, goal: str) -> bool: + seen, pending = set(), [start] + while pending: + heading = pending.pop() + if heading == goal: + return True + if heading not in seen: + seen.add(heading) + pending += [t for target in inside.get(heading, ()) for t in owners[target]] + return False + + found: dict[str, tuple[str, ...]] = {} + for heading, targets in inside.items(): + cyclic = tuple(t for t in targets if any(leads_to(owner, heading) for owner in owners[t])) + if cyclic: + found[heading] = cyclic + return found def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan], tuple[TocEntryPlan, ...]]: """The bookmarks, the cross-reference fields and the table of contents, which need the numbering of every @@ -591,17 +689,24 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan if isinstance(inline, CrossRef) and inline.target_unit is not None: found.setdefault(inline.target_unit, _Reference(inline.target_unit, inline.section_unit, inline.text, None)) - for inline, line in iter_located_body_inlines(self.tree): - if isinstance(inline, CrossRef) and inline.target_unit in found and line is not None: - known = found[inline.target_unit] - if known.line is None or line < known.line: - found[inline.target_unit] = known._replace(line=line) + if found: + for inline, line in iter_located_body_inlines(self.tree): + if isinstance(inline, CrossRef) and inline.target_unit in found and line is not None: + known = found[inline.target_unit] + if known.line is None or line < known.line: + found[inline.target_unit] = known._replace(line=line) built: dict[str, _Built] = {} for target, reference in found.items(): made = self.reference(reference) if made is not None: built[target] = made + self.report_references() needed = {piece.owner for made in built.values() for piece in made.pieces if isinstance(piece, Field)} + #: A bookmark holds its paragraph's text only if a field returns that text; any other is a point at the start + #: of the paragraph, so that no field ever stands inside the bookmark it points at. + enclosing = {piece.owner for made in built.values() for piece in made.pieces + if isinstance(piece, Field) and not piece.number} + cyclic = self.cyclic_titles(built) word_only = [(found[target].text, made.lo_text) for target, made in built.items() if made.lo_text is not None] if word_only: first, lo = word_only[0] @@ -609,6 +714,13 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan self.report("render-docx-reference-word-only", "info", f"The cross-reference '{printable(first)}' leaves out the words of a number with Word's \\t " f"switch, which LibreOffice ignores: there it reads '{printable(lo)}'{more}.", None) + for section in self.tree.sections: + if section.unit in cyclic: + self.report("render-docx-reference-cycle", "warning", + f"The heading '{printable(plain_text(section.title)[:40])}' holds a cross-reference that " + f"leads back to the heading itself, which a field cannot show (the heading's text is what " + f"the reference returns): it is written as text, so it will not follow edits made in Word.", + section.line) names = BookmarkNames() anchors: dict[str, str] = {} @@ -624,7 +736,9 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan else: continue bookmark[unit] = name - units[unit] = replace(units[unit], bookmarks=(name,)) + units[unit] = replace(units[unit], bookmarks=(name,), collapsed=unit not in enclosing) + for unit, targets in cyclic.items(): + units[unit] = replace(units[unit], literal_refs=targets) for inline in iter_tree_inlines(self.tree): if isinstance(inline, DefinedTerm) and inline.defining and inline.anchor and inline.anchor not in anchors: anchors[inline.anchor] = names.anchor("lddef_", inline.anchor.removeprefix("def:")) @@ -645,7 +759,7 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan plain = text_of(self.words(piece.owner)) if piece.number else cached segments.append(FieldSeg(bookmark[piece.owner], piece.number, piece.suppress_text, True, cached, plain if piece.suppress_text else cached)) - refs[target] = RefPlan(tuple(segments), made.exact) + refs[target] = RefPlan(tuple(segments), made.exact, made.fixed) toc: list[TocEntryPlan] = [] for entry in self.tree.contents: @@ -816,6 +930,7 @@ def section(self, section: Section) -> None: self.parent[unit] = section.parent_unit self.section_level[unit] = section.level self.desig[unit] = self.section_designation(section) + self.heading_text[unit] = _shown(section.title) if self.heading_num is None or section.number is None: self.add(section, "section", None, None, explicit=False) return diff --git a/src/legaldown_render/writers/docx/body.py b/src/legaldown_render/writers/docx/body.py index 4444c72..3f3af68 100644 --- a/src/legaldown_render/writers/docx/body.py +++ b/src/legaldown_render/writers/docx/body.py @@ -176,7 +176,9 @@ def section(self, section: Section) -> list[ET.Element]: 1-5) and its blocks. The condition label (§15.8) is a run inside the heading, after the title, so that Word's table of contents keeps it.""" level = min(max(section.level, 1), 5) - heading = self._unit(section.unit, self.inlines.runs(section.title), style=heading_style(level), + unit = self.plan.units.get(section.unit or "") + title = self.inlines.title_runs(section.title, unit.literal_refs if unit is not None else ()) + heading = self._unit(section.unit, title, style=heading_style(level), context=Context(), conditions=(section.condition_label,), after=True) return [heading, *self.blocks(section.blocks, Context())] @@ -275,11 +277,13 @@ def _unit(self, unit_id: str | None, content: Sequence[ET.Element], *, style: st conditions: Sequence[str | None], after: bool = False) -> ET.Element: """The paragraph of the unit *unit_id* (None, or a unit the plan does not know, has no numbering) with *content*, the title or first paragraph's runs. The condition labels lead the paragraph, or follow the - content with *after* (a heading); the unit's bookmarks wrap the content alone.""" + content with *after* (a heading); the unit's bookmarks wrap the content alone, or, where the plan says they are + collapsed, stand as points at the start of the paragraph.""" unit = self.plan.units.get(unit_id) if unit_id else None props: list[ET.Element | None] = [] number: list[ET.Element] = [] names: list[str] = [] + points: list[str] = [] if unit is not None: if unit.explicit or unit.alt is not None: props.append(self._numbering(unit)) @@ -290,10 +294,11 @@ def _unit(self, unit_id: str | None, content: Sequence[ET.Element], *, style: st props.append(el("ind", {"left": left, "hanging": hanging or None})) if unit.alt is not None: number = self._alternative(unit.alt) - names = [*unit.bookmarks, *([unit.toc_bookmark] if unit.toc_bookmark else [])] + (points if unit.collapsed else names).extend(unit.bookmarks) + names.extend([unit.toc_bookmark] if unit.toc_bookmark else []) else: props.append(self._plain_indent(context)) - wrapped = self.inlines.wrap(names, content) if names else list(content) + wrapped = [*self.inlines.points(points), *(self.inlines.wrap(names, content) if names else content)] labels = self._conditions(conditions, content, after=after) return p(number, wrapped + labels if after else [*labels, *wrapped], style=style, props=props) diff --git a/src/legaldown_render/writers/docx/runs.py b/src/legaldown_render/writers/docx/runs.py index 5de155b..22e5660 100644 --- a/src/legaldown_render/writers/docx/runs.py +++ b/src/legaldown_render/writers/docx/runs.py @@ -105,6 +105,9 @@ def __init__(self, plan: DocxPlan, relationships: Relationships, bookmarks: Book self.placed = placed self.attachment_titles = attachment_titles self.link_color = link_color + #: Targets of the cross-references that are written as their text (the plan's ``literal_refs``), while + #: a heading's title is written. + self._literal: frozenset[str] = frozenset() def _is_placed(self, name: str) -> bool: return self.placed is None or name.lower() in self.placed @@ -146,6 +149,15 @@ def runs(self, inlines: Iterable[Inline], fmt: Format = PLAIN, *, in_link: bool content += self._inline(inline, fmt, in_link) return content + def title_runs(self, inlines: Iterable[Inline], literal: Collection[str] = ()) -> list[ET.Element]: + """The runs of a heading's title, in which a cross-reference to any of the units *literal* is its text + and no field (the plan says its field would lead back into the title).""" + self._literal = frozenset(literal) + try: + return self.runs(inlines) + finally: + self._literal = frozenset() + def _inline(self, inline: Inline, fmt: Format, in_link: bool) -> list[ET.Element]: match inline: case Text(text=text): @@ -167,7 +179,7 @@ def _inline(self, inline: Inline, fmt: Format, in_link: bool) -> list[ET.Element alt = plain_text(children) return [text_run(alt, props=fmt.props)] if alt else [] case CrossRef(text=text, target_unit=unit): - reference = self.plan.refs.get(unit) if unit is not None else None + reference = self.plan.refs.get(unit) if unit is not None and unit not in self._literal else None if reference is None: return [text_run(text, props=fmt.props)] return self.segments(reference.segments, fmt) @@ -250,6 +262,10 @@ def _value(self, value: Value, fmt: Format, in_link: bool) -> list[ET.Element]: # -- helpers for paragraphs ----------------------------------------------------------- + def points(self, names: Sequence[str]) -> list[ET.Element]: + """Bookmarks that mark a point: each start is followed by its end.""" + return [element for name in names for element in self.bookmarks.pair(name)] + def wrap(self, names: Sequence[str], content: Sequence[Any]) -> list[ET.Element]: """*content* inside nested bookmarks, the first of *names* outermost.""" pairs = [self.bookmarks.pair(name) for name in names] diff --git a/tests/test_docx_body.py b/tests/test_docx_body.py index c9b8188..d842ce6 100644 --- a/tests/test_docx_body.py +++ b/tests/test_docx_body.py @@ -88,9 +88,11 @@ def section(title: str, *blocks: Block, unit: str | None = None, level: int = 1, def unit(unit_id: str, kind: str = "paragraph", *, num_id: int | None = None, ilvl: int | None = None, explicit: bool = False, ind: tuple[int, int] | None = None, alt: AltNumber | None = None, - bookmarks: tuple[str, ...] = (), toc: str | None = None) -> UnitPlan: + bookmarks: tuple[str, ...] = (), toc: str | None = None, collapsed: bool = False, + literal_refs: tuple[str, ...] = ()) -> UnitPlan: return UnitPlan(unit=unit_id, kind=kind, num_id=num_id, ilvl=ilvl, explicit=explicit, ind=ind, alt=alt, # type: ignore[arg-type] - label="", word_ref="", count=None, cause=(), bookmarks=bookmarks, toc_bookmark=toc) + label="", word_ref="", count=None, cause=(), bookmarks=bookmarks, toc_bookmark=toc, + collapsed=collapsed, literal_refs=literal_refs) def level(ilvl: int, fmt: str = "decimal", text: str | None = None, **fields: Any) -> LevelDef: @@ -352,6 +354,25 @@ def test_bookmarks_wrap_the_content_nested_with_the_first_outermost_and_the_toc_ assert flow(heading) == "<_Ld1><_Toc100000001>Heading" +def test_a_collapsed_bookmark_is_a_point_at_the_start_of_the_paragraph_and_the_toc_bookmark_still_wraps() -> None: + document = tree(sections=[section("Heading", unit="s1")]) + (heading,) = body(document, plan(unit("s1", "section", bookmarks=("ld_h",), toc="_Toc1", collapsed=True))) + assert flow(heading) == "<_Toc1>Heading" + + +def test_a_cross_reference_in_a_title_that_the_plan_makes_literal_is_its_text_and_elsewhere_a_field() -> None: + reference = CrossRef("See B", "t", "s2") + other = CrossRef("C", "t", "s3") + document = tree(sections=[Section(1, (Text("A "), reference, Text(" and "), other), "", "", (), unit="s1"), + section("B", unit="s2"), section("C", unit="s3")]) + refs = {"s2": (field("b", "B", number=False),), "s3": (field("c", "C"),)} + units_ = (unit("s2", "section", bookmarks=("b",)), unit("s3", "section", bookmarks=("c",))) + literal = plan(unit("s1", "section", bookmarks=("a",), literal_refs=("s2",)), *units_, refs=refs) + assert flow(body(document, literal)[0]) == "A See B and {REF c \\n \\h|C}" + fields = plan(unit("s1", "section", bookmarks=("a",)), *units_, refs=refs) + assert flow(body(document, fields)[0]) == "A {REF b \\h|B} and {REF c \\n \\h|C}" + + def test_bookmark_ids_are_unique_and_pair_each_start_with_its_end() -> None: document = tree(bullets(item("a", unit="u1"), item("b", unit="u2")), sections=[section("H", unit="s1")]) the_plan = plan(unit("s1", "section", bookmarks=("a1", "a2")), unit("u1", bookmarks=("b1",)), @@ -900,7 +921,7 @@ def test_a_rendered_document_is_numbered_cross_referenced_and_bookmarked_as_the_ "abstractNum", "abstractNum", "num", "num"] root = ET.fromstring(contents["word/document.xml"]) assert [flow(p) for p in root.iter(f"{W}p")] == [ - "T", "A", "a", "b", "See {REF ld_a \\n \\h|1}{REF ld_x \\n \\h|(a)}."] + "T", "A", "a", "b", "See {REF ld_a \\n \\h|1}{REF ld_x \\n \\h|(a)}."] assert [indent(p) for p in root.iter(f"{W}p")] == [None] * 5 diff --git a/tests/test_docx_package.py b/tests/test_docx_package.py index 0d3b7a7..3130626 100644 --- a/tests/test_docx_package.py +++ b/tests/test_docx_package.py @@ -642,6 +642,14 @@ def test_the_planners_findings_follow_the_renderers_in_the_order_of_their_lines_ assert "ref-broken" in {d.rule for d in text} +def test_the_planners_findings_are_ordered_with_the_renderers_own_by_line() -> None: + source = FRONT + "# A\n\n" + "- r\n" * 30 + "\nA [link](javascript:alert) here.\n" + result = render(source, format="docx") + found = [(d.rule, d.line) for d in result.diagnostics if d.rule.startswith("render-")] + assert [rule for rule, _ in found] == ["render-docx-letter-overflow", "render-link-refused", "render-docx-interim"] + assert found[0][1] < found[1][1] and found[2][1] is None + + def test_the_interim_warning_names_what_is_still_missing() -> None: warning = render(FRONT + "# A\n", format="docx").diagnostics[-1] assert warning.rule == "render-docx-interim" @@ -845,7 +853,7 @@ def test_code_is_left_aligned_even_when_the_body_is_justified() -> None: '_rels/.rels': 'd6f21829c0081670bdb417d3e3f644b81134a121e074fb153c7686faaa8b167b', 'docProps/core.xml': '159ed0e537106ab29739fd357a5f6d89a2c0b0154daefd4c13d878bd16dff47e', 'docProps/app.xml': 'b285204d159fd58da80b6a0242cef523448f9c3a598b357b15faf4627160dcc3', - 'word/document.xml': '79fb3712f0b4bef4a7139fa2b1049339381e98bb7f2114441638145daece15a8', + 'word/document.xml': 'fe9fef7508bd6260d6c6fb707080e5e6ef8987d26e4a0088933c8cd19ce664a4', 'word/_rels/document.xml.rels': 'ada136c2e7d05a317579c9503c1f0dd8dfaa8f0cce687df86fe59e36d5fa5d57', 'word/styles.xml': 'cdb4ca1d3be44bd27761763c040ee637516ed7aa7cf995e2cc0b60cc80d7f8ac', 'word/numbering.xml': '368f179b5e26a34c1f23db597966dfd96b43dc96801a9a7ded728c379db07f59', diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index b84e340..a7b5b55 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -716,14 +716,22 @@ def blocks(found: tuple) -> Iterator[tuple[Section | ListItem | Paragraph, str]] yield from blocks(section.blocks) -def assert_planned_as_resolved(result) -> None: # noqa: ANN001 +@functools.cache +def planned_run(source: str, variant: str) -> tuple: + """*source* rendered under *variant* and planned: (result, plan, diagnostics). The property tests of the numbering + and of the references look at the same documents, so each is rendered and planned once.""" + result = render(source, format="text", **VARIANTS[variant]) + plan, diagnostics = plan_docx(result.tree) + return result, plan, diagnostics + + +def assert_planned_as_resolved(result, plan, diagnostics) -> None: # noqa: ANN001 """The property, unit by unit: Word's count is the resolver's number, and Word's label is the resolver's label, unless that unit's own counter reads differently in Word (a letter past 26, a Roman numeral from 4000: its label and those of its descendants may differ), its label cannot show what the resolver's does (the plan says so on the unit), or its depth is past Word's. A document without alternatives has no other exception; a diagnostic is reported exactly where there is an exception.""" tree = result.tree - plan, diagnostics = plan_docx(tree) found = rules(diagnostics) shares = any(unit.shares_number_with for unit in units(tree)) overflows: dict[str, str] = {} # unit -> the rule its counter reads differently under (itself or an ancestor) @@ -771,7 +779,7 @@ def assert_planned_as_resolved(result) -> None: # noqa: ANN001 @pytest.mark.parametrize("name", ["features.lgd", "template.lgd"]) def test_the_test_documents_are_planned_as_resolved(name: str, variant: str) -> None: source = (DOCUMENTS / name).read_text(encoding="utf-8") - assert_planned_as_resolved(render(source, format="text", **VARIANTS[variant])) + assert_planned_as_resolved(*planned_run(source, variant)) SPEC = spec_root() @@ -789,11 +797,11 @@ def _spec_documents() -> list[Path]: def test_the_specifications_documents_are_planned_as_resolved(variant: str) -> None: for path in _spec_documents(): try: - result = render(path.read_text(encoding="utf-8"), format="text", **VARIANTS[variant]) + run = planned_run(path.read_text(encoding="utf-8"), variant) except DocumentError: continue try: - assert_planned_as_resolved(result) + assert_planned_as_resolved(*run) except AssertionError as error: raise AssertionError(f"{path.relative_to(SPEC)}: {error}") from error @@ -858,27 +866,53 @@ def _random_alternatives(rng: random.Random) -> str: return "\n".join(out) +#: The random documents: one set for the numbering and the references. Each has an anchor on every unit and a +#: paragraph that refers to up to 40 of them, and is planned under every fourth variant (rotating, so that each variant +#: sees a different quarter), once. +RANDOM_DOCUMENTS = 36 +ROTATION = 4 + + @functools.cache -def _random_renders(variant: str) -> tuple: +def _random_sources() -> tuple[str, ...]: rng = random.Random(20261028) - return tuple(render(FRONT + _random_document(rng), format="text", **VARIANTS[variant]) for _ in range(30)) + return tuple(with_references(with_anchors(_random_document(rng))) for _ in range(RANDOM_DOCUMENTS)) + + +@functools.cache +def random_runs(variant: str) -> tuple: + """(source, result, plan, diagnostics) of the random documents planned under *variant*.""" + offset = list(VARIANTS).index(variant) + return tuple((source, *planned_run(FRONT + source, variant)) + for index, source in enumerate(_random_sources()) if (index + offset) % ROTATION == 0) + + +@functools.cache +def alternative_runs(variant: str) -> tuple: + rng = random.Random(7) + sources = [ALTERNATIVES + with_references(_random_alternatives(rng)) for _ in range(15)] + return tuple((source, *planned_run(source, variant)) for source in sources) @pytest.mark.parametrize("variant", VARIANTS) def test_random_documents_are_planned_as_resolved(variant: str) -> None: - for result in _random_renders(variant): + for source, result, plan, diagnostics in random_runs(variant): try: - assert_planned_as_resolved(result) + assert_planned_as_resolved(result, plan, diagnostics) except AssertionError as error: - raise AssertionError(f"{error}\n--- document ---\n{result.tree.preamble}\n{result.output}") from error + raise AssertionError(f"{error}\n--- document ---\n{source}") from error @pytest.mark.parametrize("variant", ["default", "continental", "paragraphs", "none"]) def test_random_alternatives_are_planned_and_reported(variant: str) -> None: - rng = random.Random(7) - for _ in range(20): - result = render(ALTERNATIVES + _random_alternatives(rng), format="text", **VARIANTS[variant]) - assert_planned_as_resolved(result) + for _, result, plan, diagnostics in alternative_runs(variant): + assert_planned_as_resolved(result, plan, diagnostics) + + +def test_planning_twice_gives_the_same_plan() -> None: + for variant in ("default", "continental", "none"): + for _, result, plan, diagnostics in random_runs(variant)[:4]: + assert plan_docx(result.tree) == (plan, diagnostics) def test_the_random_documents_reach_every_case_the_property_is_about() -> None: @@ -886,8 +920,7 @@ def test_the_random_documents_reach_every_case_the_property_is_about() -> None: seen_causes: set[str] = set() depth = 0 for variant in VARIANTS: - for result in _random_renders(variant): - plan, diagnostics = plan_docx(result.tree) + for _, result, plan, diagnostics in random_runs(variant): seen_rules |= rules(diagnostics) seen_causes |= {cause for unit in plan.units.values() for cause in unit.cause} depth = max(depth, max((lst.depth for lst in lists(result.tree)), default=0)) @@ -1101,15 +1134,16 @@ def references_of(tree: RenderTree, plan: DocxPlan) -> list[tuple[str, str, bool def test_a_skipped_heading_level_leaves_a_part_of_the_designation_no_field_stands_for() -> None: body = "# A {#a}\n\n### B {#b}\n\nSee {{ref: b}}.\n" tree, plan, diagnostics = planned(body, style="outline") - assert references_of(tree, plan) == [("I.A.1", "REF b", False)] + # no field can stand for the missing level: the reference is the renderer's text, right now and not a field + assert references_of(tree, plan) == [("I.A.1", '"I.A.1"', True)] and plan.refs["s2"].fixed [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] assert found.level == "warning" and found.line == tree.sections[1].blocks[0].line assert found.message.startswith("The cross-reference 'I.A.1' cannot be built from Word's numbering fields: a " "heading level is skipped before it") - assert "reads '1' in Word" in found.message + assert "written as the text 'I.A.1'" in found.message and "not a field" in found.message # the decimal scheme writes the whole number in the heading's own field, so nothing is missing there tree, plan, diagnostics = planned(body) - assert references_of(tree, plan) == [("1.1.1", "REF b", True)] and not diagnostics + assert references_of(tree, plan) == [("1.1.1", "REF b", True)] and not diagnostics and not plan.refs["s2"].fixed def test_text_around_a_counter_that_word_cannot_drop_leaves_the_reference_as_one_field() -> None: @@ -1118,9 +1152,9 @@ def test_text_around_a_counter_that_word_cannot_drop_leaves_the_reference_as_one overrides = {"enumeration.levels": [{"counter": "decimal", "label": label, "ref": "{n}"}]} tree, plan, diagnostics = planned(body, overrides=overrides) [(text, cover_, exact)] = references_of(tree, plan) - assert (text, cover_, exact) == ("11", "REF x", False) + assert (text, cover_, exact) == ("11", '"11"', True) and plan.refs[next(iter(plan.refs))].fixed [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] - assert f"Word draws its number as '{shown}'" in found.message and "'11'" in found.message + assert f"Word draws its number as '{shown}'" in found.message and "written as the text '11'" in found.message def test_a_cover_that_reads_differently_in_word_is_reported_unless_a_unit_already_says_why() -> None: @@ -1172,6 +1206,74 @@ def test_untrusted_text_in_a_reference_message_is_escaped() -> None: assert all("\u0007" not in d.message for d in diagnostics) +# -- Bookmarks that never hold a field, and references that would ----------------------------------------------- + + +def test_a_bookmark_is_a_point_unless_a_field_returns_the_text_it_holds() -> None: + body = "# A {#a}\n\n- x {#x}\n\nSee {{ref: x}} and {{ref: a}}.\n" + _, plan, _ = planned(body) + assert plan.units["s1"].collapsed and plan.units["u1"].collapsed # numbers only: \n fields + _, plan, _ = planned(body, overrides=NONE) + # the heading's text is what the field for it returns: its bookmark holds it; the item's is a point + assert not plan.units["s1"].collapsed and plan.units["u1"].collapsed + + +def test_a_heading_that_refers_to_itself_or_in_a_circle_has_those_references_written_as_text() -> None: + body = ("# A {{ref: b}} {#a}\n\n# B {{ref: a}} {#b}\n\n# C {{ref: c}} {#c}\n\n# D {{ref: e}} {#d}\n\n" + "# E {#e}\n\nSee {{ref: a}}.\n") + tree, plan, diagnostics = planned(body, overrides=NONE) + assert [(u, plan.units[u].literal_refs) for u in ("s1", "s2", "s3", "s4", "s5")] == [ + ("s1", ("s2",)), ("s2", ("s1",)), ("s3", ("s3",)), ("s4", ()), ("s5", ())] # D's reference to E is no cycle + found = [d for d in diagnostics if d.rule == "render-docx-reference-cycle"] + assert [(d.level, d.line) for d in found] == [("warning", tree.sections[i].line) for i in range(3)] + assert "written as text" in found[0].message and "will not follow edits" in found[0].message + # outside a heading, and where the number is what is referred to, nothing is a cycle + _, plan, diagnostics = planned(body) + assert not any(u.literal_refs for u in plan.units.values()) + assert "render-docx-reference-cycle" not in rules(diagnostics) + + +def test_a_reference_in_a_heading_to_a_unit_inside_another_heading_is_still_a_field() -> None: + body = "# A {{ref: x}} {#a}\n\n# B {#b}\n\n- x {#x}\n\n# C {{ref: b}} {#c}\n" + _, plan, diagnostics = planned(body, overrides=NONE) + assert not any(u.literal_refs for u in plan.units.values()) + assert "render-docx-reference-cycle" not in rules(diagnostics) + + +# -- What the bookmark returns is what the runs show --------------------------------------------------------------- + + +def test_a_heading_field_returns_the_text_of_the_runs_not_the_renderers_collapsed_designation() -> None: + tree, plan, diagnostics = planned("# The {{def: foo}} clause {#c}\n\nSee {{ref: c}}.\n", overrides=NONE) + [reference] = [i for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + assert reference.text == "The clause" # the renderer collapses the white space of a designation + [segment] = plan.refs[reference.target_unit].segments + assert segment.cached == "The clause" and not plan.refs[reference.target_unit].exact # what the runs show + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert "'The clause'" in found.message and "'The clause'" in found.message + + +def test_the_text_of_the_runs_has_a_blanks_prompt_and_a_hard_break() -> None: + from legaldown_render.resolve.docx_plan import _shown + from legaldown_render.tree import Blank, Emphasis, HardBreak, SoftBreak, Text + + assert _shown((Text("a"), SoftBreak(), Emphasis((Text("b"),)), HardBreak(), Blank("x", "[_]", "How much"), + Blank("y", "[_]"))) == "a b\n[_] (How much)[_]" + + +# -- One finding for references that fail alike ---------------------------------------------------------------------- + + +def test_references_that_fail_in_the_same_way_are_one_finding_naming_the_first_and_how_many_more() -> None: + body = "# A {#a}\n\n### B {#b}\n\n### C {#c}\n\n# D {#d}\n\n### E {#e}\n\n{{ref: b}} {{ref: c}} {{ref: e}}\n" + tree, plan, diagnostics = planned(body, style="outline") + found = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert len(found) == 1 and found[0].message.startswith("The cross-reference 'I.A.1' cannot be built") + assert found[0].message.endswith(" 2 more cross-references fail in the same way.") + assert all(plan.refs[t].fixed for t in ("s2", "s3", "s5")) + assert found[0].line == tree.sections[-1].blocks[-1].line # the lowest line of the three: all in one paragraph + + # -- The cover on its own ---------------------------------------------------------------------------------- @@ -1368,19 +1470,29 @@ def with_anchors(source: str) -> str: def with_references(source: str) -> str: - """*source* with a paragraph at its end that refers to every anchor in it.""" - anchors = dict.fromkeys(re.findall(r"\{#([a-z][a-z0-9-]*)", source)) + """*source* with a paragraph at its end that refers to the anchors in it (at most 40, evenly spread: resolving + a reference costs more than numbering a unit).""" + anchors = list(dict.fromkeys(re.findall(r"\{#([a-z][a-z0-9-]*)", source))) + anchors = anchors[::-(-len(anchors) // 40) or 1] return source + "\n\nRefs: " + " ".join(f"{{{{ref: {anchor}}}}}" for anchor in anchors) + "\n" -def assert_references_as_resolved(result) -> None: # noqa: ANN001 - """For every reference: its cover reads as the renderer's text (``exact``, and the cached results say so), or the - plan says why not: a unit on the way is numbered differently in Word (a diagnostic of the numbering says so, as the - property of the numbering above has it), or a ``render-docx-reference`` finding names the reference. Every field - points at a bookmark the plan makes, the names are sound, and the same tree gives the same plan.""" +def literal_text_of_a_cover(tree: RenderTree, segments: Sequence[Segment]) -> str: + """The literal text of *segments* without the text of ``references.format`` around the designation.""" + text = "".join(s.text for s in segments if isinstance(s, Literal_)) + for part in tree.formats.reference.split("{designation}"): + assert part in text or not part, (part, text) + text = text.replace(part, "", 1) + return text + + +def assert_references_as_resolved(result, plan, diagnostics) -> None: # noqa: ANN001 + """For every reference: it is a cover of fields and delimiters, with no counter written as text; and it reads as + the renderer's text (``exact``, and the cached results say so), or the plan says why not: a unit on the way is + numbered differently in Word (a diagnostic of the numbering says so, as the property of the numbering above has + it), or a ``render-docx-reference`` finding names it. A reference that has no cover is the renderer's text, marked + ``fixed`` and reported as such. Every field points at a bookmark the plan makes, and the names are sound.""" tree = result.tree - plan, diagnostics = plan_docx(tree) - assert plan_docx(tree) == (plan, diagnostics) assert_names_are_sound(plan) nodes = {u.unit: u for u in units(tree)} counters = {node.unit: counter for node, counter in numbered(tree)} @@ -1388,7 +1500,13 @@ def assert_references_as_resolved(result) -> None: # noqa: ANN001 reported = [d for d in diagnostics if d.rule == "render-docx-reference"] texts = {i.target_unit: i.text for i in iter_tree_inlines(tree) if isinstance(i, CrossRef) and i.target_unit} assert set(plan.refs) == set(texts) - findings = 0 + # The ``ref`` templates write delimiters only: any letter or digit that is literal text in a cover is a counter + # that should have been a field. + templates = [f.ref for f in (*tree.formats.headings, *tree.formats.enumerated, tree.formats.renumber, + tree.formats.paragraphs) if f is not None] + delimiters_only = not any(c.isalnum() for t in templates for c in re.sub(r"\{(n|path|section)\}", "", t)) + needing_a_finding = 0 + fixed = 0 for target, text in texts.items(): reference = plan.refs[target] assert all(s.bookmark in marked for s in reference.segments if isinstance(s, FieldSeg)) @@ -1397,10 +1515,15 @@ def assert_references_as_resolved(result) -> None: # noqa: ANN001 if isinstance(segment, FieldSeg) and segment.number and not segment.suppress_text: assert segment.cached == plan.units[next(u for u, p in plan.units.items() if segment.bookmark in p.bookmarks)].word_ref - mentioned = any(d.message.startswith(f"The cross-reference '{text}' ") for d in reported) - findings += mentioned + if reference.fixed: + fixed += 1 + assert reference.segments == ((Literal_(text),) if text else ()) and reference.exact + continue + if delimiters_only: + assert not any(c.isalnum() for c in literal_text_of_a_cover(tree, reference.segments)), ( + target, text, show(plan, reference.segments)) + assert any(isinstance(s, FieldSeg) for s in reference.segments) or not text, (target, text) if reference.exact: - assert mentioned <= any(not plan.refs[t].exact for t in texts if texts[t] == text), (target, text) continue chain, unit = [], target while unit is not None: @@ -1413,15 +1536,22 @@ def assert_references_as_resolved(result) -> None: # noqa: ANN001 differs |= mine.count != node.number or ( unit in counters and word_counter(node.number, NUM_FMT[counters[unit]]) != format_counter(node.number, counters[unit])) - assert differs or mentioned, (target, text, cached(reference.segments), show(plan, reference.segments)) - assert len(reported) <= findings + needing_a_finding += not differs + # Every fixed text and every other difference is in a finding, named or counted among "N more". + counted = 0 + for finding in reported: + more = re.search(r" (\d+) more cross-references fail in the same way\.$", finding.message) + counted += 1 + (int(more.group(1)) if more else 0) + assert needing_a_finding + fixed <= counted <= len(texts) + if fixed: + assert any("not a field" in d.message for d in reported) @pytest.mark.parametrize("variant", VARIANTS) @pytest.mark.parametrize("name", ["features.lgd", "template.lgd"]) def test_the_test_documents_references_are_planned_as_resolved(name: str, variant: str) -> None: source = (DOCUMENTS / name).read_text(encoding="utf-8") - assert_references_as_resolved(render(source, format="text", **VARIANTS[variant])) + assert_references_as_resolved(*planned_run(source, variant)) @pytest.mark.conformance @@ -1430,45 +1560,35 @@ def test_the_test_documents_references_are_planned_as_resolved(name: str, varian def test_the_specifications_documents_references_are_planned_as_resolved(variant: str) -> None: for path in _spec_documents(): try: - result = render(path.read_text(encoding="utf-8"), format="text", **VARIANTS[variant]) + run = planned_run(path.read_text(encoding="utf-8"), variant) except DocumentError: continue try: - assert_references_as_resolved(result) + assert_references_as_resolved(*run) except AssertionError as error: raise AssertionError(f"{path.relative_to(SPEC)}: {error}") from error -@functools.cache -def _referenced_renders(variant: str) -> tuple: - rng = random.Random(20261029) - sources = [with_references(with_anchors(_random_document(rng))) for _ in range(30)] - return tuple((source, render(FRONT + source, format="text", **VARIANTS[variant])) for source in sources) - - @pytest.mark.parametrize("variant", VARIANTS) def test_random_documents_references_are_planned_as_resolved(variant: str) -> None: - for source, result in _referenced_renders(variant): + for source, result, plan, diagnostics in random_runs(variant): try: - assert_references_as_resolved(result) + assert_references_as_resolved(result, plan, diagnostics) except AssertionError as error: raise AssertionError(f"{error}\n--- document ---\n{source}") from error @pytest.mark.parametrize("variant", ["default", "continental", "paragraphs", "none"]) def test_random_alternatives_references_are_planned_as_resolved(variant: str) -> None: - rng = random.Random(11) - for _ in range(20): - source = ALTERNATIVES + with_references(_random_alternatives(rng)) - assert_references_as_resolved(render(source, format="text", **VARIANTS[variant])) + for _, result, plan, diagnostics in alternative_runs(variant): + assert_references_as_resolved(result, plan, diagnostics) def test_the_random_documents_reach_every_kind_of_cover() -> None: shapes: set[str] = set() inexact = switches = 0 for variant in VARIANTS: - for _, result in _referenced_renders(variant): - plan, _ = plan_docx(result.tree) + for _, _, plan, _ in random_runs(variant): for reference in plan.refs.values(): shapes.add("".join("L" if isinstance(s, Literal_) else "F" for s in reference.segments)) inexact += not reference.exact From edd16d35ac5b55aed3e1d563b6d20077b9827ce4 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:56:47 +0000 Subject: [PATCH 091/118] Round-trip body paragraphs and edits made in LibreOffice The round trip also compares every body paragraph with what the file cached for it, and a second one inserts an item, a heading and an ordered item before the targets through UNO as a lawyer does (tests/lo/uno_edit.py), updates the fields and compares every reference and number with the text writer's output for the edited source: a reference written as text would not move. The corpus runs in one LibreOffice session, with the specification's documents taking turns under the schemes, and has a document with headings that refer to each other. --- tests/docx_support.py | 23 +++- tests/lo/uno_edit.py | 104 ++++++++++++++++ tests/test_docx_roundtrip.py | 233 ++++++++++++++++++++++++++++++----- 3 files changed, 328 insertions(+), 32 deletions(-) create mode 100644 tests/lo/uno_edit.py diff --git a/tests/docx_support.py b/tests/docx_support.py index cd0e28c..4675ade 100644 --- a/tests/docx_support.py +++ b/tests/docx_support.py @@ -24,6 +24,7 @@ ROOT = Path(__file__).parent UNO_EXPORT = ROOT / "lo" / "uno_export.py" +UNO_EDIT = ROOT / "lo" / "uno_edit.py" W_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" W = f"{{{W_NS}}}" @@ -232,7 +233,7 @@ def libreoffice_export(documents: dict[str, bytes], *, update: bool = True) -> d Path(folder, f"{name}.docx").write_bytes(data) done = subprocess.run([python, str(UNO_EXPORT), *([] if update else ["--no-update"]), str(out), *(str(Path(folder, f"{n}.docx")) for n in documents)], - capture_output=True, text=True, timeout=300, check=False) + capture_output=True, text=True, timeout=300 + 3 * len(documents), check=False) if done.returncode != 0: pytest.fail(f"LibreOffice export failed:\n{done.stdout}\n{done.stderr}") result = {} @@ -241,3 +242,23 @@ def libreoffice_export(documents: dict[str, bytes], *, update: bool = True) -> d info = json.loads(Path(out, f"{name}.json").read_text(encoding="utf-8")) result[name] = Exported(text, info["header"], info["footer"], info["pages"]) return result + + +def libreoffice_edit(jobs: dict[str, tuple[bytes, list[tuple[str, str]]]]) -> dict[str, str]: + """Open each document (name to DOCX bytes and edits) in one LibreOffice instance, insert a paragraph with the text + ``new`` before the first paragraph that holds ``match`` for each ``(match, new)``, update the fields and indexes, + and return the text of each (tests/lo/uno_edit.py).""" + python = require_libreoffice() + with tempfile.TemporaryDirectory(prefix="ld-lo-") as folder: + out = Path(folder, "out") + out.mkdir() + specification = [] + for name, (data, edits) in jobs.items(): + Path(folder, f"{name}.docx").write_bytes(data) + specification.append({"name": name, "file": str(Path(folder, f"{name}.docx")), "edits": edits}) + Path(folder, "jobs.json").write_text(json.dumps(specification), encoding="utf-8") + done = subprocess.run([python, str(UNO_EDIT), str(out), str(Path(folder, "jobs.json"))], + capture_output=True, text=True, timeout=300, check=False) + if done.returncode != 0: + pytest.fail(f"LibreOffice edit failed:\n{done.stdout}\n{done.stderr}") + return {name: Path(out, f"{name}.txt").read_text(encoding="utf-8-sig") for name in jobs} diff --git a/tests/lo/uno_edit.py b/tests/lo/uno_edit.py new file mode 100644 index 0000000..6fe3bee --- /dev/null +++ b/tests/lo/uno_edit.py @@ -0,0 +1,104 @@ +#!/usr/bin/python3 +"""Edit DOCX files in headless LibreOffice through UNO the way a lawyer does, update the fields, and write what +the document says: for each job, a paragraph is inserted before the first paragraph that holds some text (so that +it takes that paragraph's numbering and style, as pressing Enter at its start does), then every field and index is +updated and the body is exported as plain text (``JOB.txt``, UTF-8). + +Run it with the system Python that has the ``uno`` module, **not** with the project's virtual environment: + + /usr/bin/python3 tests/lo/uno_edit.py OUTDIR JOBS.json + +``JOBS.json`` is a list of ``{"name": "job", "file": "/path/doc.docx", "edits": [["text of a paragraph", "new"], ...]}``. +One LibreOffice instance serves every job. Exit status 0 when every job was exported. +""" +from __future__ import annotations + +import contextlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import uuid + +import uno # type: ignore[import-not-found] # noqa: I001 - provided by python3-uno, not by pip +from com.sun.star.beans import PropertyValue # type: ignore[import-not-found] +from com.sun.star.text.ControlCharacter import PARAGRAPH_BREAK # type: ignore[import-not-found] + + +def _value(name: str, value: object) -> PropertyValue: + prop = PropertyValue() + prop.Name, prop.Value = name, value + return prop + + +def _connect(pipe: str, process: subprocess.Popen[bytes]) -> object: + local = uno.getComponentContext() + resolver = local.ServiceManager.createInstanceWithContext("com.sun.star.bridge.UnoUrlResolver", local) + for _ in range(120): + if process.poll() is not None: + raise SystemExit("soffice exited before it accepted a connection") + try: + return resolver.resolve(f"uno:pipe,name={pipe};urp;StarOffice.ComponentContext") + except Exception: # not listening yet + time.sleep(0.5) + raise SystemExit("soffice did not accept a connection in time") + + +def _insert_before(document: object, match: str, new: str) -> None: + text = document.getText() + paragraphs = text.createEnumeration() + while paragraphs.hasMoreElements(): + paragraph = paragraphs.nextElement() + if paragraph.supportsService("com.sun.star.text.Paragraph") and match in paragraph.getString(): + cursor = text.createTextCursorByRange(paragraph.getStart()) + text.insertControlCharacter(cursor, PARAGRAPH_BREAK, False) + cursor.goLeft(1, False) + cursor.setString(new) + return + raise SystemExit(f"no paragraph holds {match!r}") + + +def edit(out_dir: str, jobs: list[dict]) -> None: + profile = tempfile.mkdtemp(prefix="lo-profile-") + pipe = f"legaldown_{uuid.uuid4().hex}" + process = subprocess.Popen( + ["soffice", "--headless", "--invisible", "--norestore", "--nologo", "--nodefault", + f"--accept=pipe,name={pipe};urp;", f"-env:UserInstallation=file://{profile}"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + desktop = None + try: + context = _connect(pipe, process) + desktop = context.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", context) + for job in jobs: + document = desktop.loadComponentFromURL(uno.systemPathToFileUrl(os.path.abspath(job["file"])), "_blank", 0, + (_value("Hidden", True),)) + if document is None: + raise SystemExit(f"LibreOffice could not open {job['file']}") + try: + for match, new in job["edits"]: + _insert_before(document, match, new) + for _ in range(2): # page numbers and indexes settle after the first layout + document.getTextFields().refresh() + document.refresh() + document.storeToURL(uno.systemPathToFileUrl(os.path.join(out_dir, f"{job['name']}.txt")), + (_value("FilterName", "Text (encoded)"), _value("FilterOptions", "UTF8"))) + finally: + document.close(True) + finally: + if desktop is not None: + with contextlib.suppress(Exception): # already gone + desktop.terminate() + time.sleep(0.5) + process.kill() + process.wait() + shutil.rmtree(profile, ignore_errors=True) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + raise SystemExit(__doc__) + with open(sys.argv[2], encoding="utf-8") as handle: + edit(sys.argv[1], json.load(handle)) diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index 05ac9a1..d7b49c0 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -2,7 +2,8 @@ Every document of the test corpus (the test documents, the specification's examples and valid fixtures, and documents written to stress the numbering) is rendered to DOCX under the default, outline, continental and mixed -schemes and under ``none`` (where a reference is a field for the text of a heading), then changed so that the file cannot be right by accident: +schemes and under ``none`` (where a reference is a field for the text of a heading; the specification's documents +take turns, each under half of them), then changed so that the file cannot be right by accident: * every cached result of a field is replaced by ``XXX``, so that what LibreOffice shows after it has updated the fields is what it computed from the numbering definitions and the bookmarks, not what the writer cached; @@ -10,6 +11,11 @@ read from the exported text, and a paragraph per ``{{ref:}}`` target is added, made of exactly the fields the plan gives that reference, preceded by ``⟦ref u3⟧``. +Every body paragraph's text must also be what the file cached for it, and a second round trip makes the edits a +lawyer makes (an item, a heading and an ordered item inserted before the targets, in LibreOffice) and updates the +fields: every reference and number must read as the renderer writes them for the edited source. The table of +contents is not written yet, so its entries and the ``_Toc`` bookmarks are not exercised. + The exported text is then compared with what the renderer wrote (the labels and designations that the text writer prints), **modulo the gaps the plan documents**, each of which the plan reports with a diagnostic: @@ -24,14 +30,15 @@ """ from __future__ import annotations +import difflib import functools import re import xml.etree.ElementTree as ET -from dataclasses import dataclass +from dataclasses import dataclass, replace import pytest from conftest import DOCUMENTS, FRONT, spec_root -from docx_support import Exported, W, libreoffice_export, parts, repackage, require_libreoffice +from docx_support import Exported, W, libreoffice_edit, libreoffice_export, parts, repackage, require_libreoffice from test_docx_plan import ALTERNATIVES, units from legaldown_render import DocumentError, render @@ -165,6 +172,21 @@ 1. anchored too {#too} Ref {{ref: deep}} {{ref: too}} +""", + "cycles": FRONT + """ +# A {{ref: b}} {#a} + +# B {{ref: a}} {#b} + +# C {{ref: c}} {#c} + +# D {{ref: e}} {#d} + +# E plain {#e} + +- item in {{ref: it}} {#it} + +See {{ref: a}} {{ref: b}} {{ref: c}} {{ref: d}} {{ref: it}}. """, "alternatives": ALTERNATIVES + """ # Intro {#intro} @@ -294,10 +316,12 @@ def change(document: str) -> str: @dataclass class Case: + variant: str name: str tree: RenderTree plan: DocxPlan rules: set[str] + original: bytes data: bytes shown: Exported | None = None @@ -313,28 +337,41 @@ def corpus() -> dict[str, str]: return found +def make_case(variant: str, name: str, source: str) -> Case | None: + try: + result = render(source, format="docx", **{**VARIANTS, **WORDS}[variant]) + except DocumentError: + return None + assert result.binary is not None + plan, diagnostics = plan_docx(result.tree) + return Case(variant, name, result.tree, plan, {d.rule for d in diagnostics}, result.binary, + prepared(result.binary, result.tree, plan)) + + @functools.cache -def round_trip(variant: str, update: bool = True) -> tuple[Case, ...]: - """Every document under *variant*, rendered, made wrong and marked, and shown by LibreOffice.""" +def round_trips() -> dict[str, tuple[Case, ...]]: + """Every document under every variant, rendered, made wrong and marked, and shown by LibreOffice: one LibreOffice + instance for all of them.""" require_libreoffice() - settings = {**VARIANTS, **WORDS}[variant] - cases: dict[str, Case] = {} - for name, source in corpus().items(): - if variant in WORDS and not name.startswith("stress/"): - continue - try: - result = render(source, format="docx", **settings) - except DocumentError: - continue - assert result.binary is not None - plan, diagnostics = plan_docx(result.tree) - cases[f"{variant}/{name}"] = Case(name, result.tree, plan, {d.rule for d in diagnostics}, - prepared(result.binary, result.tree, plan)) - key = {f"d{index}": case for index, case in enumerate(cases.values())} - shown = libreoffice_export({name: case.data for name, case in key.items()}, update=update) - for name, case in key.items(): - case.shown = shown[name] - return tuple(cases.values()) + cases: dict[str, list[Case]] = {variant: [] for variant in (*VARIANTS, *WORDS)} + for offset, variant in enumerate(cases): + for index, (name, source) in enumerate(corpus().items()): + if variant in WORDS and not name.startswith("stress/"): + continue + if name.startswith(("examples/", "fixtures/")) and (index + offset) % 2: + continue # the specification's documents take turns under the schemes: each is shown under half + case = make_case(variant, name, source) + if case is not None: + cases[variant].append(case) + everything = {f"d{index}": case for index, case in enumerate(case for group in cases.values() for case in group)} + shown = libreoffice_export({key: case.data for key, case in everything.items()}) + for key, case in everything.items(): + case.shown = shown[key] + return {variant: tuple(group) for variant, group in cases.items()} + + +def round_trip(variant: str) -> tuple[Case, ...]: + return round_trips()[variant] # -- Reading what LibreOffice says ---------------------------------------------------------------------------------- @@ -343,7 +380,7 @@ def round_trip(variant: str, update: bool = True) -> tuple[Case, ...]: def marked_lines(shown: Exported) -> dict[str, tuple[str, str]]: """Tag to (the number LibreOffice drew before the marker, the text after it), for every marked line.""" found: dict[str, tuple[str, str]] = {} - for line in shown.text.lstrip("").splitlines(): + for line in shown.text.lstrip("\ufeff").splitlines(): match = MARKED.match(line) if match: found[match["tag"]] = (match["number"].strip(), match["rest"].strip()) @@ -360,9 +397,62 @@ def renderer_references(tree: RenderTree) -> dict[str, str]: return {i.target_unit: i.text for i in iter_tree_inlines(tree) if isinstance(i, CrossRef) and i.target_unit} +def squeeze(line: str) -> str: + return re.sub(r"\s+", " ", line).strip() + + +def cached_lines(data: bytes, plan: DocxPlan) -> list[str]: + """What the paragraphs of the file say, as the writer cached it, one entry per line without the blank ones. A field + written with ``\\t`` is what LibreOffice shows for it (it ignores the switch), by ``FieldSeg.cached_lo``.""" + lo = {(s.bookmark, s.suppress_text): s.cached_lo for r in plan.refs.values() for s in r.segments + if isinstance(s, FieldSeg)} + root = ET.fromstring(parts(data)["word/document.xml"]) + lines: list[str] = [] + for paragraph in root.iter(f"{W}p"): + text, state, instruction, replacement = "", None, "", None + for run in paragraph.iter(f"{W}r"): + mark = _is_field_part(run) + if mark == "begin": + state, instruction, replacement = "instruction", "", None + elif mark == "separate": + state = "result" + switches = instruction.split() + if "\\t" in switches: + replacement = lo.get((switches[1], True)) + elif mark == "end": + text += replacement or "" + state = None + elif state == "instruction": + instruction += "".join(node.text or "" for node in run.findall(f"{W}instrText")) + elif state == "result" and replacement is not None: + continue + else: + for node in run: + text += (node.text or "") if node.tag == f"{W}t" else "\t" if node.tag == f"{W}tab" else ( + "\n" if node.tag == f"{W}br" else "") + lines += [squeeze(line) for line in text.split("\n") if squeeze(line)] + return lines + + +def shown_lines(shown: Exported) -> list[str]: + """What LibreOffice says, line by line, without the blank ones, the marker lines of the references and the + markers and numbers of the numbered paragraphs.""" + lines = [] + for line in shown.text.lstrip("\ufeff").splitlines(): + match = MARKED.match(line) + if match and match["tag"].startswith("ref "): + continue + text = squeeze(match["rest"] if match else line) + if text: + lines.append(text) + return lines + + def check(case: Case) -> None: """The property: every number LibreOffice draws is the renderer's, or the plan's where the plan says Word differs - (with a diagnostic), and every reference reads as the renderer wrote it, or as the plan says.""" + (with a diagnostic), every reference reads as the renderer wrote it, or as the plan says, and every body paragraph + says what the file cached for it. The table of contents is not written yet, so its entries and the ``_Toc`` + bookmarks are not exercised.""" assert case.shown is not None lines = marked_lines(case.shown) excused = False @@ -391,6 +481,8 @@ def check(case: Case) -> None: suppressing = any(isinstance(s, FieldSeg) and s.suppress_text for s in reference.segments) if reference.exact and not suppressing: assert text == shown_by_renderer[target], (target, text, shown_by_renderer[target]) + cached, said = cached_lines(case.original, case.plan), shown_lines(case.shown) + assert said == cached, "\n".join(difflib.unified_diff(cached, said, "cached", "LibreOffice", lineterm="", n=1)) @pytest.mark.parametrize("variant", list(VARIANTS) + list(WORDS)) @@ -426,10 +518,89 @@ def test_the_cached_results_are_wrong_in_the_file_and_libreoffice_does_not_read_ """The seed works: the file holds ``XXX`` as the result of every reference field, and LibreOffice shows the right results all the same, even before the fields are updated, because it computes them from the numbering and the bookmarks when it opens the file (so that what the comparison above checks is its computation, not the cache).""" - for update in (True, False): - document = next(case for case in round_trip("default", update=update) if case.name == "stress/nested") - written = parts(document.data)["word/document.xml"].decode("utf-8") - assert written.count(f">{WRONG}<") >= len(document.plan.refs) > 10 - lines = marked_lines(document.shown) # type: ignore[arg-type] + document = next(case for case in round_trip("default") if case.name == "stress/nested") + written = parts(document.data)["word/document.xml"].decode("utf-8") + assert written.count(f">{WRONG}<") >= len(document.plan.refs) > 10 + unupdated = libreoffice_export({"nested": document.data}, update=False)["nested"] + for shown in (document.shown, unupdated): + lines = marked_lines(shown) # type: ignore[arg-type] assert not any(WRONG in text for _, text in lines.values()) - check(document) + check(replace(document, shown=shown)) + + +# -- Edits in LibreOffice, then the fields updated ------------------------------------------------------------------- + + +EDITED = FRONT + """ +# Alpha {#alpha} + +Para {#p1} + +- first {#i1} +- second item {#i2} + - nested one {#n1} + - nested two {#n2} +- third {#i3} + +## Beta {#beta} + +1. ord one {#o1} +2. ord two {#o2} + +# Gamma {#gamma} + +- g item {#gi} + +Refs: {{ref: alpha}} {{ref: p1}} {{ref: i1}} {{ref: i2}} {{ref: n1}} {{ref: n2}} {{ref: i3}} {{ref: beta}} {{ref: o2}} +{{ref: gamma}} {{ref: gi}} +""" + +#: What a lawyer does in Word: press Enter at the start of a paragraph and type (an item, a heading, an ordered item). +#: Each is (the text of the paragraph it goes before, what is typed, the source line it goes before, the new line). +INSERTIONS = [ + ("second item", "NEW item", "- second item {#i2}", "- NEW item"), + ("Gamma", "NEWH", "# Gamma {#gamma}", "# NEWH\n"), + ("ord two", "NEW ord", "2. ord two {#o2}", "1. NEW ord"), +] + +EDIT_VARIANTS = {name: VARIANTS[name] for name in ("default", "outline", "continental", "none")} + + +def after_the_edits() -> str: + source = EDITED + for _, _, before, new in INSERTIONS: + source = source.replace(before, f"{new}\n{before}") + return source + + +@functools.cache +def edited() -> dict[str, tuple[str, str]]: + """Variant to (what LibreOffice says after the insertions and the update, what the text writer prints for the + edited source).""" + require_libreoffice() + jobs = {} + for variant, settings in EDIT_VARIANTS.items(): + jobs[variant] = (render(EDITED, format="docx", **settings).binary, [(m, n) for m, n, _, _ in INSERTIONS]) + said = libreoffice_edit(jobs) # type: ignore[arg-type] + return {variant: (said[variant], render(after_the_edits(), format="text", **EDIT_VARIANTS[variant]).output) + for variant in EDIT_VARIANTS} + + +@pytest.mark.parametrize("variant", EDIT_VARIANTS) +def test_the_references_follow_the_insertions_a_lawyer_makes(variant: str) -> None: + """The point of the whole design: after items and headings are inserted before the targets and the fields are + updated, every reference reads as the renderer writes it for the edited document (a reference that was text + would still say what it said before).""" + said, written = edited()[variant] + refs = [squeeze(line) for line in said.splitlines() if "Refs:" in line] + expected = [squeeze(line) for line in written.splitlines() if "Refs:" in line] + assert refs == expected and refs + before = render(EDITED, format="text", **EDIT_VARIANTS[variant]).output + assert [squeeze(line) for line in before.splitlines() if "Refs:" in line] != expected # the edits moved them + + +@pytest.mark.parametrize("variant", ["default", "outline", "none"]) +def test_the_numbers_follow_the_insertions_a_lawyer_makes(variant: str) -> None: + said, written = edited()[variant] + assert [squeeze(line) for line in said.lstrip("\ufeff").splitlines() if squeeze(line)] == [ + squeeze(line) for line in written.splitlines() if squeeze(line)] From f4a0e3490c485380459bff1b277355aece916bcd Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:56:47 +0000 Subject: [PATCH 092/118] Document fixed references, point bookmarks and the reference cycle A reference with no cover is fixed text, a bookmark holds text only where a field returns it, and a heading that refers back to itself under the none scheme has the reference written as text; a period counts as a delimiter for \t, so No. {n} has no \t form. --- CONFORMANCE.md | 2 +- README.md | 2 +- docs/architecture.md | 40 +++++++++++++++++++++++++++------------- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 98e78d3..4d24853 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -20,7 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | -| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says, but **the file is still interim**: there is no table of contents, title block, attachments or signature blocks, alternatives that share a number (§15.8) are numbered in sequence (reported), and a label that shows the section's number in a list item (the continental `5.1`) is drawn without it (reported). The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a list continued by a fixed number, which will not follow edits), `render-docx-number-mismatch` (any other difference, including alternatives), `render-docx-reference` (a cross-reference that cannot be built from fields, or reads differently in Word); Infos `render-docx-static-restart` (a list that restarts by a fixed setting) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). Every DOCX render also reports the Warning `render-docx-interim` ("DOCX output is incomplete in this version") until the body is complete; it goes away then. HTML is the format this renderer claims §13.6 with | +| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference for which Word's fields cannot reproduce the designation (a heading level skipped before the target under a `{n}`-style scheme, text around the counter that Word's `\t` switch cannot drop such as `(a)`, `{n}:` or `No. {n}`) is fixed text, reported. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says, but **the file is still interim**: there is no table of contents, title block, attachments or signature blocks, alternatives that share a number (§15.8) are numbered in sequence (reported), and a label that shows the section's number in a list item (the continental `5.1`) is drawn without it (reported). The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a list continued by a fixed number, which will not follow edits), `render-docx-number-mismatch` (any other difference, including alternatives), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-static-restart` (a list that restarts by a fixed setting) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). Every DOCX render also reports the Warning `render-docx-interim` ("DOCX output is incomplete in this version") until the body is complete; it goes away then. HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | diff --git a/README.md b/README.md index 62f1d13..cf84196 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ output. **Numbering and cross-references are live:** headings, list items and nu numbering definitions (headings by the heading styles, nested lists restarting by structure), and a `{{ref: clause}}` is `REF` fields on bookmarks whose cached results are the text the plain-text writer prints, so "1(b)(ii)" or "I.A.1" stays right when a lawyer inserts a clause and updates fields. A defined term or an attachment -reference is an internal link. Where Word cannot draw what the renderer does (letters past 26, a label that shows the +reference is an internal link. A reference that Word's fields cannot reproduce is written as its text (it will not follow edits) and reported. Where Word cannot draw what the renderer does (letters past 26, a label that shows the section's number in a list item, alternatives that share a number, a list restarted by a fixed setting), the render reports it as a `render-docx-*` Warning or Info with the line, see [the conformance notes](https://github.com/ForLegalAI/legaldown-render/blob/main/CONFORMANCE.md). The body writer also diff --git a/docs/architecture.md b/docs/architecture.md index aba8ca7..6f72381 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -254,7 +254,7 @@ resolver and for the style check), and checks every placement against the number simulating Word's counters (`resolve/docx_numbering.py`), so that what Word draws is what the resolver numbered, or a diagnostic says why not. It is a planning step, not a stage of the pipeline: `render(format="docx")` calls it after resolution, passes the plan to the writer, and adds its diagnostics (named with the document's file, each finding once, -in the order of their lines) to the result's. The style check, `docx_style_errors(style)`, runs earlier still, in +ordered by line with the renderer's own) to the result's. The style check, `docx_style_errors(style)`, runs earlier still, in `style/docx.py:docx_settings`, before rendering (ADR 0006). **Word's list semantics**, stated once (the planner and its diagnostics rely on them): @@ -297,12 +297,19 @@ counter. The cover with the fewest fields wins, then the fewest using `\t`, then the text of `references.format` is literal around it, and with two `{designation}` the same cover is written twice. `\t` (Word's switch that suppresses the words around a counter; LibreOffice ignores it) is used only where it leaves the bare counter (`Article I` gives `I`, `§ 3` gives `3`), never where the field's text holds a delimiter -(`(a)`, `a)`, `1.2`, `No. 1`). Each field's cached result is **Word's** value (the simulator's), and `cached_lo` what -LibreOffice shows. A unit that a field points at has a bookmark (its anchor's, or a hidden one); a reference to an -item or paragraph the style does not number is the designation of its section (`ref-not-enumerated`), covered as one. -Under the `none` scheme a heading is a field for the bookmarked text (`REF bookmark \h`) and the item's pieces -follow after a space. A `RefPlan` is made once per target, keyed by `CrossRef.target_unit`; `exact` says that -the cached results read as the renderer's text. +(`(a)`, `a)`, `1.2`, and also a period: `No. 1`, so `No. {n}` has no `\t` form; this is conservative, because +what Word does with a delimiter is not known, and is on the manual Word checklist). Each field's cached result is +**Word's** value (the simulator's), and `cached_lo` what LibreOffice shows. A unit that a field points at has a +bookmark (its anchor's, or a hidden one); a reference to an item or paragraph the style does not number is the +designation of its section (`ref-not-enumerated`), covered as one. Under the `none` scheme a heading is a field for the +bookmarked text (`REF bookmark \h`) and the item's pieces follow after a space; what the field returns is what the +heading's runs show (`_shown`), which can differ from the renderer's designation by white space, and then the +reference is reported. A `RefPlan` is made once per target, keyed by `CrossRef.target_unit`; `exact` says that the +cached results read as the renderer's text. **When no cover exists** (a heading level skipped before the target under +a `{n}`-style scheme, text around a counter that `\t` cannot drop) the reference is the renderer's text with no field +(`RefPlan.fixed`): it reads right when the file is opened and does not follow edits, and `render-docx-reference` says +so (one finding for the references that fail alike, naming the first and how many more). Nothing that Word could +compute is written as literal text otherwise. **Bookmarks** (`resolve/docx_bookmarks.py`). The planner names bookmarks and the writer gives them ids. Every anchor of the tree has one, referenced or not: `ld_` for a section, list item or paragraph, `lddef_` for a @@ -311,7 +318,13 @@ as `_`. A name is used only if it is valid, at most 40 characters, nothing but d ignoring case (Word compares names so); otherwise, and for a unit a field needs that has no anchor, it is hidden: `_Ld`, counted in document order. The table of contents uses `_Toc` from 100000001. The writer wraps the paragraph's title or first paragraph in `UnitPlan.bookmarks`, then `toc_bookmark`; a term reference and an -internal link point at `plan.anchors`. +internal link point at `plan.anchors`. **A bookmark holds text only when a field returns that text** (`REF \h`, a +heading under the `none` scheme): every other bookmark is a point at the start of the paragraph +(`UnitPlan.collapsed`), because `REF \n` needs only the paragraph, and a field inside the bookmark it points at is +a self-reference that Word cannot show and LibreOffice expands without end. Under the `none` scheme a heading whose +title holds a reference that leads back to the heading, directly or through the titles of other headings, would have +exactly that; the plan lists those references (`UnitPlan.literal_refs`, by target) and the writer writes them as +their text, and `render-docx-reference-cycle` says so (the renderer already warns with `render-designation-cycle`). **The table of contents.** `ContentsEntry.unit` (the section's unit; None for the attachments heading and an attachment) keys each entry of `tree.contents` to its section. `plan.toc` lists the entries in order with @@ -326,9 +339,10 @@ complex `TOC` field that uses them is written with the title block. starts at a fixed number), `number-mismatch` (any other difference: the first unit of a run that Word counts off by the same amount, with how many follow; it does not occur without an alternative that shares a number), `reference` (a cross-reference that cannot be built from fields, because a heading level is skipped before the target with a -`{n}`-style scheme, or Word draws text around a counter that `\t` cannot drop (`({n})`, `{n}:`, `No. {n}`): it is one -field for the target's own number, which Word reads differently; or one that reads differently in Word for a reason -that no diagnostic of a unit already gives), and, as information, `static-restart` (a list that restarts at 1 by a +`{n}`-style scheme, or Word draws text around a counter that `\t` cannot drop (`({n})`, `{n}:`, `No. {n}`): it is +written as fixed text; or one that reads differently in Word for a reason that no diagnostic of a unit already +gives; references that fail alike are one finding), `reference-cycle` (a heading's title holds a reference that +leads back to it: written as text), and, as information, `static-restart` (a list that restarts at 1 by a fixed setting) and `reference-word-only` (a cover relies on `\t`, which LibreOffice ignores, once per document). A heading level that is skipped is the Core Error `heading-skip`; the plan only notes it on the unit. @@ -411,7 +425,7 @@ Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can ev `ref-not-enumerated`, `path-not-relative` and `path-outside-root` (Errors for a file reference that is not relative, or that leaves the document root, §2.3; reported here because validator 0.4 checks neither), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and `render-docx-interim` (a Warning on every DOCX render, while the DOCX body is incomplete), and the rules the DOCX planner reports on what Word draws differently from the other formats (`render-docx-section-label`, `-list-depth`, `-letter-overflow`, `-roman-overflow`, `-shared-counter`, `-number-mismatch`, `-reference`, as Warnings, `-static-restart` and `-reference-word-only` as Infos; see "The DOCX planner"). The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and `render-docx-interim` (a Warning on every DOCX render, while the DOCX body is incomplete), and the rules the DOCX planner reports on what Word draws differently from the other formats (`render-docx-section-label`, `-list-depth`, `-letter-overflow`, `-roman-overflow`, `-shared-counter`, `-number-mismatch`, `-reference`, `-reference-cycle`, as Warnings, `-static-restart` and `-reference-word-only` as Infos; see "The DOCX planner"). The validator's diagnostics carry their line (§16.9), and so do the renderer's own, and the CLI prints it as `file:line:`. @@ -592,6 +606,6 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_docx_body.py` | The DOCX body from hand-made plans: numbering definitions, `w:numPr`/`w:ind`/alternatives, bookmarks and `REF` fields with cached results, hyperlinks, every inline and block, escaping, schema validity (marker `ooxml_schema`) and a LibreOffice round trip of headings, nested lists and references whose cached values are wrong on purpose (marker `libreoffice`) | | `tests/test_docx_escaping.py` | Well-formedness of every part for any text, with a seeded fuzz and the characters XML cannot carry | | `tests/test_docx_validation.py` | Every part of every generated document against the ISO/IEC 29500-4 schemas (marker `ooxml_schema`; needs lxml and `python tests/schemas/fetch.py`) | -| `tests/test_docx_roundtrip.py` | LibreOffice as the oracle of the numbering and the cross-references: every document of the corpus (the tests', the specification's, stress documents) under the default, outline, continental, mixed and `none` schemes, with every cached field result made wrong (`XXX`), every numbered paragraph marked and a line per reference added; the number LibreOffice draws and every reference it computes equal the renderer's, modulo the gaps the plan reports (marker `libreoffice`) | +| `tests/test_docx_roundtrip.py` | LibreOffice as the oracle of the numbering and the cross-references, in one LibreOffice session: every document of the corpus (the tests', the specification's, stress documents with cycles and unanchored ancestors) under the default, outline, continental, mixed and `none` schemes, with every cached field result made wrong (`XXX`), every numbered paragraph marked and a line per reference added; the number LibreOffice draws, every reference it computes and every body paragraph's text equal what the file cached and the renderer wrote, modulo the gaps the plan reports (`\t` ignored, letters past 26, labels that need the section, alternatives; the table of contents is not written yet, so its entries and the `_Toc` bookmarks are not exercised); and edit-then-update (`tests/lo/uno_edit.py`): items and headings inserted before the targets as a lawyer does, fields updated, every reference and the numbers equal the renderer's for the edited source (marker `libreoffice`) | | `tests/test_docx_libreoffice.py` | The documents opened in LibreOffice through UNO (`tests/lo/uno_export.py`), fields updated: the words in order, header, footer, page count (marker `libreoffice`; skipped without `soffice` and `python3-uno`) | | `tests/conformance/` | Every example and fixture in the specification repository renders, every internal link has a target, and each invalid fixture renders its rule's failure marker. Needs `$LEGALDOWN_SPEC_DIR` or a sibling `../LegalDown` checkout | From bcc09bc6a37eb9e8e802beb8a0ea6ba66e068f4f Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:35:57 +0000 Subject: [PATCH 093/118] Write the front and back matter of a DOCX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The title block (header rows and parties), a live TOC field with the planned entries as its cached result, the attachments (bookmarked so that references to them resolve, separated by a page break or a rule) and the signature blocks (§13.6, §13.8, §3.6) are written in the order the HTML shows them. The attachments are listed in the contents by outline level, so the field holds no locale-dependent style list. With the matter complete the render-docx-interim Warning is gone. LibreOffice checks that the updated contents lists the HTML's entries, also after a heading is inserted. --- src/legaldown_render/api.py | 11 - src/legaldown_render/writers/docx/__init__.py | 9 +- src/legaldown_render/writers/docx/body.py | 37 +- src/legaldown_render/writers/docx/matter.py | 357 ++++++++ src/legaldown_render/writers/docx/ooxml.py | 22 +- src/legaldown_render/writers/docx/styles.py | 54 +- tests/lo/uno_edit.py | 9 +- tests/test_docx_body.py | 12 +- tests/test_docx_libreoffice.py | 2 +- tests/test_docx_matter.py | 865 ++++++++++++++++++ tests/test_docx_package.py | 43 +- 11 files changed, 1359 insertions(+), 62 deletions(-) create mode 100644 src/legaldown_render/writers/docx/matter.py create mode 100644 tests/test_docx_matter.py diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index 6d2053f..9eff8d6 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -121,16 +121,6 @@ def ok(self) -> bool: return not any(diagnostic.level == "error" for diagnostic in self.diagnostics) -def _interim_warning(filename: str) -> Diagnostic: - """The Warning that DOCX output is not complete yet: the writer does not yet write the title block, the - table of contents, the attachments or the signature blocks. It goes when the body is complete.""" - return Diagnostic( - rule="render-docx-interim", level="warning", file=filename, - message="DOCX output is incomplete in this version: the title block, the table of contents, the " - "attachments and the signature blocks are not written yet (the numbering and the " - "cross-references are). Do not use the file as the finished document.") - - def render(source: str, options: RenderOptions | None = None, /, **settings: Any) -> RenderResult: """Render LegalDown *source*. @@ -195,7 +185,6 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any raise RenderRefused(diagnostics) if plan is not None: - diagnostics.append(_interim_warning(document.filename)) written: str | bytes = writer.write(tree, plan) else: written = writer.write(tree) diff --git a/src/legaldown_render/writers/docx/__init__.py b/src/legaldown_render/writers/docx/__init__.py index c45df66..f31838b 100644 --- a/src/legaldown_render/writers/docx/__init__.py +++ b/src/legaldown_render/writers/docx/__init__.py @@ -12,8 +12,9 @@ * :mod:`.package` -- parts, relationships, content types and the deterministic ZIP; * :mod:`.styles`, :mod:`.settings`, :mod:`.headers`, :mod:`.properties` -- the parts that come from the style template; * :mod:`.numbering` -- ``numbering.xml``, from the plan's numbering definitions; -* :mod:`.body` and :mod:`.runs` -- the body of the document and its inlines, laid out as the plan says (interim - for the title block, contents, attachments and signatures, see there). +* :mod:`.body` and :mod:`.runs` -- the body of the document and its inlines, laid out as the plan says; +* :mod:`.matter` -- the front and back matter: header rows and parties, the table of contents, the attachments and + the signature blocks. """ from __future__ import annotations @@ -26,6 +27,7 @@ from . import package as pkg from .body import BodyWriter, document_title from .headers import build_footer, build_header +from .matter import MatterSettings from .numbering import numbering_part from .ooxml import Bookmarks, el, serialize from .properties import build_app, build_core @@ -82,7 +84,8 @@ def write(self, tree: RenderTree, plan: DocxPlan | None = None) -> bytes: if page.footer: footer_id = rels.add(pkg.REL_FOOTER, "footer1.xml") - body = BodyWriter(tree, plan, Bookmarks(), rels, text_width, settings.link_color).body( + matter = MatterSettings(text_width, settings.points, self.style.contents.depth, self.style.attachments.separator) + body = BodyWriter(tree, plan, Bookmarks(), rels, text_width, settings.link_color, matter).body( settings.page, header_id=header_id, footer_id=footer_id) document = el("document", None, body) package.add("docProps/core.xml", pkg.CT_CORE_PROPERTIES, build_core(title, language)) diff --git a/src/legaldown_render/writers/docx/body.py b/src/legaldown_render/writers/docx/body.py index 3f3af68..5ba8672 100644 --- a/src/legaldown_render/writers/docx/body.py +++ b/src/legaldown_render/writers/docx/body.py @@ -16,9 +16,9 @@ * the other blocks of an item, and a quote's or a drafting note's, are plain paragraphs at the indent of what they belong to. -**Interim.** The title block (the header rows and parties), the table of contents, the attachments -and the signature blocks are not written yet: only the title and the subtitle are. The -:func:`section_properties` and the builders in :mod:`.ooxml` are what the rest will keep. +The title block, the table of contents, the attachments and the signature blocks are laid out by +:mod:`.matter`, in the order the HTML writer shows them: title, subtitle, header rows and parties, contents, +preamble, sections, attachments, signatures. """ from __future__ import annotations @@ -44,6 +44,7 @@ Table, plain_text, ) +from .matter import MatterSettings, MatterWriter from .ooxml import Bookmarks, el, p, tbl_pr, tc_pr, text_run, tr_pr, val from .package import Relationships from .runs import InlineWriter @@ -117,21 +118,24 @@ class BodyWriter: *bookmarks* and *relationships* are the document's (the bookmark ids are unique across the package; the relationships receive the external links); *text_width* is the width between the margins in - twips, which tables take.""" + twips, which tables take; *matter* says what the front and back matter need of the style (it defaults + to what the built-in styles give).""" def __init__(self, tree: RenderTree, plan: DocxPlan, bookmarks: Bookmarks, relationships: Relationships, - text_width: int, link_color: str) -> None: + text_width: int, link_color: str, matter: MatterSettings | None = None) -> None: self.tree = tree self.plan = plan self.bookmarks = bookmarks self.relationships = relationships self.text_width = text_width self.link_color = link_color + self.matter_settings = matter if matter is not None else MatterSettings(text_width) abstracts = {abstract.id: abstract for abstract in plan.abstract_nums} self._levels = {num.id: abstracts[num.abstract_id].levels for num in plan.nums if num.abstract_id in abstracts} attachment_titles = {a.anchor: plain_text(a.title) for a in tree.attachments if a.anchor} #: Not known until a first, dry run has written the document: see :meth:`body`. self.inlines = InlineWriter(plan, relationships, bookmarks, None, attachment_titles, link_color) + self.matter = MatterWriter(tree, plan, self.inlines, bookmarks, self.matter_settings) def _left(self, left: int) -> int: """*left* twips of indent, kept where a paragraph can still be read: not past Word's limit, and leaving @@ -149,13 +153,16 @@ def body(self, page: PageGeometry, *, header_id: str | None = None, footer_id: s (:class:`~legaldown_render.errors.InternalError`), and a link or a term reference to one shows its text without the link. Whatever :meth:`content` writes counts, so the attachments' bookmarks need no list of their own.""" - dry = BodyWriter(self.tree, self.plan, Bookmarks(), Relationships(), self.text_width, self.link_color) + dry = BodyWriter(self.tree, self.plan, Bookmarks(), Relationships(), self.text_width, self.link_color, + self.matter_settings) dry.content() self.inlines.placed = dry.bookmarks.names return el("body", None, self.content(), section_properties(page, header_id=header_id, footer_id=footer_id)) def content(self) -> list[ET.Element]: - """The block-level content of the body: the title and the subtitle, the preamble and the sections.""" + """The block-level content of the body, in the order the HTML writer shows it: the title and the subtitle, + the header rows and the parties, the table of contents, the preamble, the sections, the attachments and + the signature blocks.""" tree = self.tree content: list[ET.Element] = [] title = document_title(tree) @@ -164,13 +171,29 @@ def content(self) -> list[ET.Element]: subtitle = plain_text(tree.subtitle).strip() if subtitle: content.append(p(text_run(subtitle), style=SUBTITLE)) + content += self.matter.title_block() + content += self.matter.contents() content += self.blocks(tree.preamble, Context()) for section in tree.sections: content += self.section(section) + content += self.matter.attachments() + content += self.matter.signatures() + content = self._apart(content) if content and content[-1].tag.endswith("}tbl"): content.append(p()) # a table cannot be the last thing before the section properties return content + @staticmethod + def _apart(content: list[ET.Element]) -> list[ET.Element]: + """*content* with an empty paragraph between two tables that follow each other (Word merges them + otherwise), as :meth:`blocks` does inside a block.""" + out: list[ET.Element] = [] + for element in content: + if element.tag.endswith("}tbl") and out and out[-1].tag.endswith("}tbl"): + out.append(p()) + out.append(element) + return out + def section(self, section: Section) -> list[ET.Element]: """The heading paragraph of *section* (``heading N``: the resolver has clamped the level to 1-5) and its blocks. The condition label (§15.8) is a run inside the heading, after the title, diff --git a/src/legaldown_render/writers/docx/matter.py b/src/legaldown_render/writers/docx/matter.py new file mode 100644 index 0000000..eb15d82 --- /dev/null +++ b/src/legaldown_render/writers/docx/matter.py @@ -0,0 +1,357 @@ +"""The front and back matter of the document (§13.8, §3.6, §13.6): the header rows and the parties, +the table of contents, the attachments and the signature blocks, laid out as the HTML writer shows them +and in the order it shows them. + +Like the rest of the writer (docs/decisions/0008) it holds no rule: the entries of the table of contents, +their numbers and their ``_Toc`` bookmarks are the plan's (:attr:`~legaldown_render.resolve.docx_plan.DocxPlan.toc`), +and what the tree says is what is written. + +* **Table of contents (§2.7 of the plan).** One complex field, ``TOC \\o "1-N" \\h \\z \\u``, whose cached + result is the planned entries: one ``TOC1`` to ``TOC5`` paragraph each, a hyperlink to the entry's bookmark + holding the number, its separator, the title, the condition label and a ``PAGEREF`` field whose cached page + is empty (Word fills it in when it updates the field). The field opens in the first entry's paragraph and + closes in a paragraph of its own after the last, as Word writes it. The attachments are listed by outline + level, not by a style list: the attachments heading and each attachment's title are paragraphs of styles + that carry one (``\\u``), so that no list separator (a comma or a semicolon, by locale) is in the field. The + contents label is a paragraph of the ``ContentsHeading`` style: no outline level, no numbering, so not listed. +* **Attachments.** The heading and each title are bookmarked: by the plan's name for the anchor, so that + ``{{attach:}}`` is a ``REF`` field or a link that resolves, and by the ``_Toc`` name of the contents entry. + ``attachments.separator`` is a page break before the part (a paragraph property: no empty paragraph is left + on the page before, and no blank page when the part falls at a page top) or a rule above each attachment. +* **Signature blocks.** Borderless tables (each party a cell, two to a row) kept on one page: a row does not split + and every paragraph but the last keeps with the next. +""" +from __future__ import annotations + +import xml.etree.ElementTree as ET +from collections.abc import Iterable, Sequence +from dataclasses import dataclass + +from ...errors import InternalError +from ...resolve.docx_plan import DocxPlan, TocEntryPlan +from ...tree import ( + AttachmentPart, + ContentsEntry, + Inline, + PartyInfo, + RenderTree, + SideInfo, + SignatureParty, + plain_text, +) +from .ooxml import ( + Bookmarks, + el, + field_end, + field_start, + hyperlink_internal, + p, + r, + tbl_pr, + tc_pr, + text_run, + tr_pr, + val, +) +from .runs import InlineWriter +from .styles import ( + ATTACHMENT_FILE, + ATTACHMENT_TITLE, + ATTACHMENTS_HEADING, + CONDITION_LABEL, + CONTENTS_HEADING, + PARTY_NAME, + PARTY_SIDE, + SIGNATURE_LINE, + SIGNATURE_SIDE, + SIGNATURE_TEXT, + TITLE_BLOCK_LABEL, + TITLE_BLOCK_TEXT, + toc_style, +) + +#: The share of the text width the label column of the header rows and party details takes, in percent. +_LABEL_SHARE = 30 +#: The gap between two cells, in twips, and the share of a signature cell the signing line leaves free, in percent. +_GUTTER = 283 +_LINE_FREE = 20 +#: Signature blocks in a row. +_SIGNATURE_COLUMNS = 2 +#: What a character of a number takes in an entry of the contents, and the gap after it, in em; where the text +#: starts after a number's tab stop is the longest number of its level (a guess for a font the writer does +#: not measure: Word recomputes the stops when it updates the field). +_NUMBER_EM = 0.7 +_NUMBER_GAP_EM = 0.6 + + +@dataclass(frozen=True, slots=True) +class MatterSettings: + """What the matter needs besides the tree and the plan: the page and the style's contents and attachments.""" + + #: The width between the margins, in twips: where the page numbers' right tab goes, and the width of tables. + text_width: int + #: The body font size in points (the width of a number in the contents is estimated from it). + points: float = 11.0 + #: ``contents.depth``: the deepest heading level the ``TOC`` field lists. + depth: int = 2 + #: ``attachments.separator``: ``page-break``, ``rule`` or ``none``. + separator: str = "page-break" + + +class MatterWriter: + """Writes the matter around the sections. *inlines* and *bookmarks* are the body writer's, so that + every bookmark written here counts for the links and references that point at it (the document is + written twice, :meth:`.body.BodyWriter.body`).""" + + def __init__(self, tree: RenderTree, plan: DocxPlan, inlines: InlineWriter, bookmarks: Bookmarks, + settings: MatterSettings) -> None: + self.tree = tree + self.plan = plan + self.inlines = inlines + self.bookmarks = bookmarks + self.settings = settings + if tree.contents and len(tree.contents) != len(plan.toc): + raise InternalError("the plan's table of contents does not hold the entries of the tree's " + "(a bug in the DOCX plan)") + #: The ``_Toc`` name of the contents entries that list the attachments heading and an attachment, by anchor + #: (a section's is the plan's ``UnitPlan.toc_bookmark``, written with its heading). + self._toc_of: dict[str, str] = { + entry.anchor: planned.bookmark for entry, planned in zip(tree.contents, plan.toc, strict=False) + if entry.unit is None and entry.anchor} + + # -- header rows and parties ---------------------------------------------------------- + + def title_block(self) -> list[ET.Element]: + """The header rows (label and value, as ``dt`` and ``dd``) and the parties: each side's label, then + each party's name and its details (§13.5).""" + tree = self.tree + content: list[ET.Element] = [] + if tree.header: + content.append(self._rows([(label, value) for label, value in tree.header])) + for side in tree.sides: + content += self._side(side) + return content + + def _side(self, side: SideInfo) -> list[ET.Element]: + content = [p(self.inlines.runs(side.label), style=PARTY_SIDE)] + for party in side.parties: + content += self._party(party) + return content + + def _party(self, party: PartyInfo) -> list[ET.Element]: + content = [p(self.inlines.runs(party.name), style=PARTY_NAME)] + if party.details: + content.append(self._rows(list(party.details))) + return content + + def _rows(self, rows: Sequence[tuple[str, tuple[Inline, ...]]]) -> ET.Element: + """A borderless table of label and value rows: the label in italics (a style blanked by the label + leaves its cell empty, as the ``dt`` is).""" + label = self.settings.text_width * _LABEL_SHARE // 100 + widths = (label, self.settings.text_width - label) + return _table(widths, [ + [[p(text_run(name, style=TITLE_BLOCK_LABEL), style=TITLE_BLOCK_TEXT)] if name + else [p(style=TITLE_BLOCK_TEXT)], + [p(self.inlines.runs(value), style=TITLE_BLOCK_TEXT)]] + for name, value in rows]) + + # -- table of contents ---------------------------------------------------------------- + + def contents(self) -> list[ET.Element]: + """The contents label and the ``TOC`` field with its cached entries; nothing when the tree lists + nothing (the HTML writer writes no contents then, label included).""" + tree = self.tree + if not tree.contents: + return [] + content: list[ET.Element] = [] + if tree.contents_label: + content.append(p(text_run(tree.contents_label), style=CONTENTS_HEADING)) + depth = max(1, min(self.settings.depth, 5)) + instruction = f'TOC \\o "1-{depth}" \\h \\z \\u' + stops = self._number_stops(tree.contents, depth) + for index, (entry, planned) in enumerate(zip(tree.contents, self.plan.toc, strict=True)): + level = self._level(entry, depth) + content.append(self._entry(planned, level, stops.get(level), + lead=field_start(instruction) if index == 0 else ())) + content.append(p(field_end())) + return content + + @staticmethod + def _level(entry: ContentsEntry, depth: int) -> int: + """The level Word lists *entry* at (``TOCn``): a section's own, and an attachment's outline level + clamped to the field's depth (below, the field does not list it at all), so that the cached result is the + one an update gives.""" + return entry.level if entry.unit is not None else max(1, min(entry.level, depth)) + + def _outline(self, level: int) -> ET.Element | None: + """The outline level of an attachments paragraph the contents lists at *level* (``\\u`` lists by it), never + deeper than the field's depth, which would drop it from the list. Where the contents exists and does not + list the attachments (``contents.attachments``), the paragraphs are body text, as their style's outline + level would list them; with no contents the style's level stands (the navigation pane).""" + if not self.tree.contents: + return None + if not any(entry.unit is None for entry in self.tree.contents): + return val("outlineLvl", 9) + return val("outlineLvl", max(0, min(level - 1, self.settings.depth - 1))) + + def _number_stops(self, entries: Iterable[ContentsEntry], depth: int) -> dict[int, int]: + """The position of the tab after the number, by level: left of the text of every entry of that level + whose number is followed by a tab, so that their titles line up.""" + stops: dict[int, int] = {} + em = self.settings.points * 20 + for entry, planned in zip(entries, self.plan.toc, strict=True): + if planned.sep != "\t" or not planned.number: + continue + level = self._level(entry, depth) + width = len(planned.number) * _NUMBER_EM * em + _NUMBER_GAP_EM * em + 240 * (level - 1) + stops[level] = min(max(stops.get(level, 0), round(width)), self.settings.text_width // 2) + return stops + + def _entry(self, planned: TocEntryPlan, level: int, stop: int | None, + lead: Iterable[ET.Element] = ()) -> ET.Element: + """One entry's paragraph: a hyperlink to its bookmark with the number, the title, the condition label, + and the page number as a ``PAGEREF`` field with an empty cached result. *lead* are runs before the + hyperlink (the opening of the ``TOC`` field, in the first entry).""" + name = self._placed(planned.bookmark) + runs: list[ET.Element] = [] + if planned.number: + runs.append(text_run(planned.number, props=[el("noProof")])) + runs.append(r(el("tab"), props=[el("noProof")]) if planned.sep == "\t" + else text_run(" ", props=[el("noProof")])) + runs += self.inlines.runs(planned.title) + if planned.condition_label: + runs += [text_run(" "), text_run(planned.condition_label, style=CONDITION_LABEL)] + runs.append(r(el("tab"), props=[el("noProof")])) + runs += [*field_start(f"PAGEREF {name} \\h"), field_end()] + tabs = [stop is not None and bool(planned.number) and planned.sep == "\t" and el("tab", {"val": "left", "pos": stop}), + el("tab", {"val": "right", "leader": "dot", "pos": self.settings.text_width})] + return p(*lead, hyperlink_internal(name, *runs), style=toc_style(level), props=[el("tabs", None, tabs)]) + + def _placed(self, name: str) -> str: + """*name*, a bookmark an entry points at: the document must write it (not known in the dry run).""" + placed = self.inlines.placed + if placed is not None and name.lower() not in placed: + raise InternalError(f"a table of contents entry points at the bookmark '{name}', which the document " + "does not contain (a bug in the DOCX plan)") + return name + + # -- attachments ---------------------------------------------------------------------- + + def attachments(self) -> list[ET.Element]: + """The attachments heading and each attachment's title, condition label and file (§13.8).""" + tree = self.tree + if not tree.attachments: + return [] + content: list[ET.Element] = [] + separator = self.settings.separator + # The page break goes before the first thing of the part, whichever that is, and before every title after the + # first: the heading stays with the first attachment. + breaking = separator == "page-break" + level = 1 + if tree.attachments_label: + level = 2 + names = self._names(tree.attachments_anchor) + content.append(p(self.inlines.wrap(names, [text_run(tree.attachments_label)]) if names + else text_run(tree.attachments_label), + style=ATTACHMENTS_HEADING, + props=[breaking and el("pageBreakBefore"), + self._outline(1)])) + breaking = False + for attachment in tree.attachments: + content += self._attachment(attachment, level, page_break=breaking, rule=separator == "rule") + breaking = separator == "page-break" + return content + + def _names(self, anchor: str | None) -> list[str]: + """The bookmarks of an attachments paragraph: the plan's name for its anchor, and the contents + entry's ``_Toc``.""" + names = [self.plan.anchors[anchor]] if anchor and anchor in self.plan.anchors else [] + if anchor and anchor in self._toc_of: + names.append(self._toc_of[anchor]) + return names + + def _attachment(self, attachment: AttachmentPart, level: int, *, page_break: bool, rule: bool) -> list[ET.Element]: + names = self._names(attachment.anchor) + title = self.inlines.runs(attachment.title) + shown = self.inlines.wrap(names, title) if names else title + if attachment.condition_label: + shown += [text_run(" "), text_run(attachment.condition_label, style=CONDITION_LABEL)] + content = [p(shown, style=ATTACHMENT_TITLE, props=[ + page_break and el("pageBreakBefore"), + rule and el("pBdr", None, el("top", {"val": "single", "sz": 4, "space": 8, "color": "C8C8C8"})), + self._outline(level)])] + if attachment.file: + content.append(p(text_run(attachment.file), style=ATTACHMENT_FILE)) + return content + + # -- signature blocks ----------------------------------------------------------------- + + def signatures(self) -> list[ET.Element]: + """The signature blocks (§3.6) as a borderless table, a party to a cell, two to a row, kept together.""" + tree = self.tree + if not tree.signatures: + return [] + columns = min(_SIGNATURE_COLUMNS, len(tree.signatures)) + cell = self.settings.text_width // _SIGNATURE_COLUMNS + cells = [self._signature(signature, cell) for signature in tree.signatures] + rows = [cells[start:start + columns] for start in range(0, len(cells), columns)] + for row in rows: + row += [[p(style=SIGNATURE_TEXT)] for _ in range(columns - len(row))] + # Keep the table on one page: every paragraph of it but the last keeps with the next. + for paragraph in [paragraph for row in rows for blocks in row for paragraph in blocks][:-1]: + _keep_next(paragraph) + return [_table((cell,) * columns, rows)] + + def _signature(self, signature: SignatureParty, cell: int) -> list[ET.Element]: + """One party's block: its side, its name, a signing line with the signatory's name and title (one blank + line when none is declared) for each representative, then the date and place to be written in.""" + labels, colon = self.tree.signature_labels, self.tree.colon + blocks = [p(self.inlines.runs(signature.side), style=SIGNATURE_SIDE), + p(self.inlines.runs(signature.name), style=SIGNATURE_TEXT)] + + def row(key: str, value: tuple[Inline, ...] = (), *, write_in: bool = False) -> list[ET.Element]: + # As the text writer: a blanked label leaves the value alone and a row with neither is left out. Date + # and place (*write_in*) are to be written in, so they keep a no-break space of the colon; a name and + # a title without a value lose all the space after the label. + shown = plain_text(value) + label = f"{labels.get(key, '')}{colon}" if labels.get(key) else "" + if not (label + shown).strip(): + return [] + lead = label if shown else label.rstrip(" " if write_in else None) + return [p(text_run(lead) if lead else None, self.inlines.runs(value), style=SIGNATURE_TEXT)] + + for name, title in signature.signatories or (((), ()),): + blocks.append(p(style=SIGNATURE_LINE, props=[ + el("ind", {"right": cell * _LINE_FREE // 100 + _GUTTER})])) + blocks += [*row("name", name), *row("title", title)] + blocks += [*row("date", write_in=True), *row("place", write_in=True)] + return blocks + + +_PPR = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}pPr" + + +def _keep_next(paragraph: ET.Element) -> None: + """Add ``w:keepNext`` to *paragraph*'s properties, in the place the schema gives it (right after the style).""" + properties = paragraph.find(_PPR) + if properties is None: + properties = el("pPr") + paragraph.insert(0, properties) + keep = el("keepNext") + properties.insert(1 if properties.find("{*}pStyle") is not None else 0, keep) + + +def _table(widths: Sequence[int], rows: Sequence[Sequence[Sequence[ET.Element]]]) -> ET.Element: + """A borderless table: one ``w:tc`` per entry of a row, holding its paragraphs, *widths* twips wide; + rows do not split across pages.""" + return el( + "tbl", None, + tbl_pr(el("tblW", {"w": sum(widths), "type": "dxa"}), el("tblLayout", {"type": "fixed"}), + el("tblCellMar", None, el("left", {"w": 0, "type": "dxa"}), el("right", {"w": _GUTTER, "type": "dxa"})), + el("tblLook", {"val": "0000", "firstRow": 0, "lastRow": 0, "firstColumn": 0, "lastColumn": 0, + "noHBand": 1, "noVBand": 1})), + el("tblGrid", None, [el("gridCol", {"w": width}) for width in widths]), + [el("tr", None, tr_pr(el("cantSplit")), + [el("tc", None, tc_pr(el("tcW", {"w": width, "type": "dxa"})), paragraphs) + for paragraphs, width in zip(row, widths, strict=True)]) + for row in rows]) diff --git a/src/legaldown_render/writers/docx/ooxml.py b/src/legaldown_render/writers/docx/ooxml.py index c96e4c0..fdb09f1 100644 --- a/src/legaldown_render/writers/docx/ooxml.py +++ b/src/legaldown_render/writers/docx/ooxml.py @@ -310,6 +310,22 @@ def text_run(text: str, *, style: str | None = None, props: Iterable[Any] = ()) return r(content, style=style, props=props) +def field_start(instruction: str, *, props: Iterable[Any] = (), dirty: bool = False) -> list[ET.Element]: + """The runs that open a complex field: ``begin``, the instruction and ``separate``. The result runs + follow, then :func:`field_end`; a field that spans paragraphs (the table of contents) is written in + these parts.""" + def char(kind: str, **extra: Any) -> ET.Element: + return r(el("fldChar", {"fldCharType": kind, **extra}), props=props) + + instr = r(el("instrText", {"xml:space": "preserve"}, text=f" {_t(instruction).strip()} "), props=props) + return [char("begin", dirty=True if dirty else None), instr, char("separate")] + + +def field_end(*, props: Iterable[Any] = ()) -> ET.Element: + """The run that closes a complex field.""" + return r(el("fldChar", {"fldCharType": "end"}), props=props) + + def complex_field(instruction: str, result: Iterable[ET.Element] = (), *, props: Iterable[Any] = (), dirty: bool = False) -> list[ET.Element]: """The runs of a complex field: ``begin``, the instruction, ``separate``, @@ -318,11 +334,7 @@ def complex_field(instruction: str, result: Iterable[ET.Element] = (), *, props: computed for it, shown until Word updates the field. *props* format the field's own runs; *dirty* asks Word to update this one field when the file opens.""" - def char(kind: str, **extra: Any) -> ET.Element: - return r(el("fldChar", {"fldCharType": kind, **extra}), props=props) - - instr = r(el("instrText", {"xml:space": "preserve"}, text=f" {_t(instruction).strip()} "), props=props) - return [char("begin", dirty=True if dirty else None), instr, char("separate"), *result, char("end")] + return [*field_start(instruction, props=props, dirty=dirty), *result, field_end(props=props)] def hyperlink_external(relationship_id: str, *runs: ET.Element, tooltip: str | None = None) -> ET.Element: diff --git a/src/legaldown_render/writers/docx/styles.py b/src/legaldown_render/writers/docx/styles.py index e219404..fe1ab48 100644 --- a/src/legaldown_render/writers/docx/styles.py +++ b/src/legaldown_render/writers/docx/styles.py @@ -15,7 +15,7 @@ from typing import Any from ...style import Style -from ...style.docx import DocxSettings +from ...style.docx import DocxSettings, HeadingFormat from .ooxml import el, ppr, rpr, serialize, val # Style ids. A paragraph or a run names a style by id; Word shows the name. @@ -38,6 +38,16 @@ CHOICE = "Choice" FAILURE_MARKER = "FailureMarker" CONDITION_LABEL = "ConditionLabel" +TITLE_BLOCK_TEXT = "TitleBlockText" +TITLE_BLOCK_LABEL = "TitleBlockLabel" +PARTY_SIDE = "PartySide" +PARTY_NAME = "PartyName" +ATTACHMENTS_HEADING = "AttachmentsHeading" +ATTACHMENT_TITLE = "AttachmentTitle" +ATTACHMENT_FILE = "AttachmentFile" +SIGNATURE_SIDE = "SignatureSide" +SIGNATURE_TEXT = "SignatureText" +SIGNATURE_LINE = "SignatureLine" def heading_style(level: int) -> str: @@ -55,6 +65,12 @@ def toc_style(level: int) -> str: QUOTE_INDENT = 567 NOTE_INDENT = 227 CODE_INDENT = 113 +#: The outline level (0-based) of the attachments heading and of an attachment's title: the levels the +#: contents lists them at, as the HTML contents does (the heading at 1, the titles under it at 2). +ATTACHMENTS_OUTLINE = 0 +ATTACHMENT_OUTLINE = 1 +#: The space above a signing line, in twips: room to sign in. +SIGNATURE_SPACE = 720 _MUTED = "666666" _RULE = "C8C8C8" @@ -132,6 +148,15 @@ def _text_style(kind: str) -> list[ET.Element]: }.get(kind, []) +def _heading_run(heading: HeadingFormat | None, settings: DocxSettings) -> list[Any]: + """The run properties of a heading level (None: a style that sets none: bold at the body size), for the + paragraph styles that look like a heading and are not one (so that no heading numbering reaches them).""" + if heading is None: + return [el("b"), el("bCs"), _size(settings.half_points)] + return [heading.bold and [el("b"), el("bCs")], heading.italic and [el("i"), el("iCs")], + heading.caps and el("caps"), heading.small_caps and el("smallCaps"), _size(heading.half_points)] + + def build_styles(style: Style, settings: DocxSettings, language: str, *, heading_num_id: int | None = None) -> bytes: """``word/styles.xml`` for *style*, converted to Word units as *settings*, in the document *language* (``w:lang``). *heading_num_id* links ``heading 1`` to ``heading 5`` @@ -187,6 +212,7 @@ def half_points(factor: float) -> int: heading.caps and el("caps"), heading.small_caps and el("smallCaps"), _size(heading.half_points)])) first = settings.headings.get(1) + second = settings.headings.get(2) or first styles += [ _style(TITLE, "Title", based_on=NORMAL, next_style=NORMAL, priority=10, quick=True, props=[el("keepNext"), el("spacing", {"before": 0, "after": em(0.5)}), el("jc", {"val": "center"})], @@ -203,6 +229,32 @@ def half_points(factor: float) -> int: run=[(first is None or first.bold) and [el("b"), el("bCs")], first and first.caps and el("caps"), first and first.small_caps and el("smallCaps"), _size(first.half_points if first else settings.half_points)]), + _style(TITLE_BLOCK_TEXT, "Title Block Text", based_on=NORMAL, priority=41, custom=True, + props=[el("spacing", {"before": 0, "after": em(0.2)}), el("jc", {"val": "left"})]), + _style(PARTY_SIDE, "Party Side", based_on=NORMAL, next_style=NORMAL, priority=42, custom=True, + props=[el("keepNext"), el("spacing", {"before": em(1), "after": em(0.2)}), el("jc", {"val": "left"})], + run=[el("b"), el("bCs")]), + _style(PARTY_NAME, "Party Name", based_on=NORMAL, priority=43, custom=True, + props=[el("keepNext"), el("spacing", {"before": em(0.5), "after": em(0.2)}), el("jc", {"val": "left"})]), + _style(ATTACHMENTS_HEADING, "Attachments Heading", based_on=NORMAL, next_style=NORMAL, priority=44, + custom=True, props=[el("keepNext"), el("keepLines"), el("spacing", {"before": em(1.2), "after": em(0.4)}), + el("jc", {"val": first.align if first else "left"}), + val("outlineLvl", ATTACHMENTS_OUTLINE)], + run=_heading_run(first, settings)), + _style(ATTACHMENT_TITLE, "Attachment Title", based_on=NORMAL, next_style=NORMAL, priority=45, custom=True, + props=[el("keepNext"), el("keepLines"), el("spacing", {"before": em(0.9), "after": em(0.4)}), + el("jc", {"val": second.align if second else "left"}), val("outlineLvl", ATTACHMENT_OUTLINE)], + run=_heading_run(second, settings)), + _style(ATTACHMENT_FILE, "Attachment File", based_on=NORMAL, priority=46, custom=True, + run=[el("color", {"val": _MUTED})]), + _style(SIGNATURE_SIDE, "Signature Side", based_on=NORMAL, priority=47, custom=True, + props=[el("keepLines"), el("spacing", {"before": em(1.5), "after": em(0.2)}), el("jc", {"val": "left"})], + run=[el("b"), el("bCs")]), + _style(SIGNATURE_TEXT, "Signature Text", based_on=NORMAL, priority=48, custom=True, + props=[el("keepLines"), el("spacing", {"before": 0, "after": em(0.2)}), el("jc", {"val": "left"})]), + _style(SIGNATURE_LINE, "Signature Line", based_on=NORMAL, priority=49, custom=True, + props=[el("keepLines"), el("pBdr", None, _border("bottom", "auto", 4, 1)), + el("spacing", {"before": SIGNATURE_SPACE, "after": em(0.3)})]), _style(DRAFTING_NOTE, "Drafting Note", based_on=NORMAL, priority=30, custom=True, props=[el("keepLines"), el("pBdr", None, _border("left", _NOTE_RULE, 24, 8)), _shade(_NOTE_FILL), el("spacing", {"before": em(0.4), "after": em(0.4)}), el("ind", {"left": NOTE_INDENT, "right": 113})]), diff --git a/tests/lo/uno_edit.py b/tests/lo/uno_edit.py index 6fe3bee..8e2fa60 100644 --- a/tests/lo/uno_edit.py +++ b/tests/lo/uno_edit.py @@ -2,7 +2,7 @@ """Edit DOCX files in headless LibreOffice through UNO the way a lawyer does, update the fields, and write what the document says: for each job, a paragraph is inserted before the first paragraph that holds some text (so that it takes that paragraph's numbering and style, as pressing Enter at its start does), then every field and index is -updated and the body is exported as plain text (``JOB.txt``, UTF-8). +updated (the indexes too, so a table of contents lists what was inserted) and the body is exported as plain text (``JOB.txt``, UTF-8). Run it with the system Python that has the ``uno`` module, **not** with the project's virtual environment: @@ -52,7 +52,9 @@ def _insert_before(document: object, match: str, new: str) -> None: paragraphs = text.createEnumeration() while paragraphs.hasMoreElements(): paragraph = paragraphs.nextElement() - if paragraph.supportsService("com.sun.star.text.Paragraph") and match in paragraph.getString(): + # A line of a table of contents holds the heading's words too: only the document's own paragraphs count. + if (paragraph.supportsService("com.sun.star.text.Paragraph") and match in paragraph.getString() + and not paragraph.ParaStyleName.startswith("Contents")): cursor = text.createTextCursorByRange(paragraph.getStart()) text.insertControlCharacter(cursor, PARAGRAPH_BREAK, False) cursor.goLeft(1, False) @@ -81,6 +83,9 @@ def edit(out_dir: str, jobs: list[dict]) -> None: for match, new in job["edits"]: _insert_before(document, match, new) for _ in range(2): # page numbers and indexes settle after the first layout + indexes = document.getDocumentIndexes() + for position in range(indexes.getCount()): + indexes.getByIndex(position).update() document.getTextFields().refresh() document.refresh() document.storeToURL(uno.systemPathToFileUrl(os.path.join(out_dir, f"{job['name']}.txt")), diff --git a/tests/test_docx_body.py b/tests/test_docx_body.py index d842ce6..a32fff4 100644 --- a/tests/test_docx_body.py +++ b/tests/test_docx_body.py @@ -555,11 +555,9 @@ def attachment_tree(value: Value, titles: Sequence[str] = ("Schedule A",)) -> Re def test_an_attachment_value_is_a_field_when_it_is_the_title_else_a_link_with_its_text_and_text_when_unplaced() -> None: def paragraph(value: Value, *, placed: bool = True) -> str: - # The attachments section is not written yet: a unit stands in for the bookmark it will place. - the_plan = plan(unit("u1", bookmarks=("ldatt_att_0",)) if placed else unit("u1"), - anchors={"att-0": "ldatt_att_0"}) - document = tree(para("see ", value), bullets(item("x", unit="u1")), - attachments=attachment_tree(value).attachments) + # The attachment's title is written with its bookmark; a plan that names no bookmark for it leaves the text. + the_plan = plan(anchors={"att-0": "ldatt_att_0"} if placed else {}) + document = tree(para("see ", value), attachments=attachment_tree(value).attachments) return flow(body(document, the_plan)[0]) assert paragraph(Value("attach", "Schedule A", "#att-0")) == "see {REF ldatt_att_0 \\h|Schedule A}" @@ -1058,11 +1056,11 @@ def build_everything() -> bytes: the_plan = plan( unit("s1", "section", bookmarks=("ld_a",), toc="_Toc1"), unit("s2", "section", num_id=0, ilvl=0, explicit=True, ind=(567, 567), alt=alt), - unit("u1"), unit("u2", "item", num_id=1, ilvl=0, explicit=True, bookmarks=("ld_att",)), + unit("u1"), unit("u2", "item", num_id=1, ilvl=0, explicit=True), unit("u3", "item", num_id=1, ilvl=1, explicit=True), unit("u4", "item", num_id=1, ilvl=0, explicit=True), unit("u5", "paragraph", num_id=2, ilvl=0, explicit=True), abstracts=[abstract(0, "headings", [level(0, p_style="Heading1")]), abstract(1)], nums=[Num(1, 1), Num(2, 1, ((0, 1),))], - heading=None, anchors={"def:t": "lddef_t", "att-0": "ld_att"}, refs={"s1": (field("ld_a", "1"),)}) + heading=None, anchors={"def:t": "lddef_t", "att-0": "ldatt_att_0"}, refs={"s1": (field("ld_a", "1"),)}) return DocxWriter(load_style("default")).write(document, the_plan) diff --git a/tests/test_docx_libreoffice.py b/tests/test_docx_libreoffice.py index bf77c40..85fd6b2 100644 --- a/tests/test_docx_libreoffice.py +++ b/tests/test_docx_libreoffice.py @@ -99,7 +99,7 @@ def shown() -> dict[str, Exported]: def test_the_words_come_back_in_order(shown: dict[str, Exported]) -> None: # the numbers are drawn by LibreOffice from the numbering definitions (test_docx_roundtrip.py compares them) assert shown["round_trip"].lines == [ - "Round Trip", "Sub", "Preamble ž€.", "1. First", "Body with emphasis, strong and code.", "Second line.", + "Round Trip", "Sub", "Datum účinnosti", "4. března 2026", "Preamble ž€.", "1. First", "Body with emphasis, strong and code.", "Second line.", "1.1 Second", " (a) item one", " (b) item two", "A quote", "code line 1", " code line 2", "H1", "H2", "a", "b", ""] diff --git a/tests/test_docx_matter.py b/tests/test_docx_matter.py new file mode 100644 index 0000000..49a6cf5 --- /dev/null +++ b/tests/test_docx_matter.py @@ -0,0 +1,865 @@ +"""The front and back matter of the DOCX (§13.6, §13.8, §3.6): the header rows and the parties, the table of +contents, the attachments and the signature blocks. Their XML (no tool needed), the validity of documents +that hold every part, hostile text included (marker ``ooxml_schema``), and what LibreOffice shows once its +fields and indexes are updated (marker ``libreoffice``): the table of contents lists the HTML contents' +entries with page numbers, a reference to an attachment resolves, the attachments are paged as configured, +and the document reads in the order the plain-text writer prints it.""" +from __future__ import annotations + +import random +import re +import xml.etree.ElementTree as ET +from typing import Any + +import pytest +import yaml +from conftest import DOCUMENTS +from docx_support import ( + W_NS, + Exported, + libreoffice_edit, + libreoffice_export, + parts, + repackage, + require_libreoffice, + require_schemas, + texts, + validation_errors, +) +from test_docx_escaping import ALPHABET, assert_well_formed, soup + +from legaldown_render import render +from legaldown_render.errors import InternalError +from legaldown_render.resolve.docx_plan import DocxPlan, plan_docx +from legaldown_render.style import load_style +from legaldown_render.style.docx import docx_settings +from legaldown_render.tree import plain_text +from legaldown_render.writers import DocxWriter + +W = f"{{{W_NS}}}" + +FRONT = """--- +legaldown: '0.2' +title: Master Agreement +subtitle: Every part of the matter +version: '2.1' +effective_date: 2026-03-04 +language: en +questions: + forum: + type: choice + prompt: How are disputes resolved? + choices: + courts: State courts + arbitration: Arbitration +sides: + - name: sellers + label: Seller + parties: + - name: acme + type: legal_entity + legal_name: Acme Corporation + identification_number: CZ-123 + address: 1 Main Street + representatives: + - name: John Smith + title: Chief Executive Officer + - name: Jane Roe + title: Director + - name: buyers + label: Buyer + parties: + - name: beta + type: legal_entity + legal_name: Beta Industries Inc. + - name: gamma + type: legal_entity + legal_name: Gamma Ltd. + representatives: + - name: Pat Doe + title: Proxy +attachments: + - id: schedule-a + title: "Schedule A: Services" + file: files/services.lgd + - id: price-list + title: Price List + when: forum:courts + - id: terms + title: Terms +--- + +Preamble text. + +# Services {#services} + +The Seller provides {{attach: schedule-a}}, see {{attach: price-list, label=the prices}}. + +## Delivery + +Text of the delivery. + +# Fees {#fees} + +Fees are in {{attach: terms}}. + +## Payment {#payment} + +Net 30. + +### Late payment + +Interest applies. +""" + + +def docx(source: str = FRONT, **settings: Any) -> bytes: + result = render(source, format="docx", **settings) + assert result.binary is not None + return result.binary + + +def document_xml(data: bytes) -> ET.Element: + return ET.fromstring(parts(data)["word/document.xml"]) + + +def body(data: bytes) -> list[ET.Element]: + return list(document_xml(data).find(f"{W}body")) # type: ignore[arg-type] + + +def style_of(paragraph: ET.Element) -> str | None: + node = paragraph.find(f"{W}pPr/{W}pStyle") + return None if node is None else node.get(f"{W}val") + + +def paragraphs(data: bytes, style: str) -> list[ET.Element]: + return [p for p in document_xml(data).iter(f"{W}p") if style_of(p) == style] + + +def text_of(element: ET.Element) -> str: + """The text of a paragraph (a tab as TAB), or of any other element: its runs' text.""" + if element.tag in (f"{W}p", f"{W}tc"): + return "\n".join(texts(element)) if element.tag == f"{W}tc" else texts(element)[0] + return "".join({f"{W}t": lambda n: n.text or "", f"{W}tab": lambda n: "\t"}.get(n.tag, lambda n: "")(n) + for n in element.iter()) + + +def instructions(root: ET.Element) -> list[str]: + return [(node.text or "").strip() for node in root.iter(f"{W}instrText")] + + +def bookmark_names(root: ET.Element) -> list[str]: + return [node.get(f"{W}name") for node in root.iter(f"{W}bookmarkStart")] # type: ignore[misc] + + +def tables(data: bytes) -> list[ET.Element]: + return list(document_xml(data).iter(f"{W}tbl")) + + +def contents_style(**overrides: Any) -> dict[str, Any]: + return {"overrides": {"contents.enabled": True, **overrides}} + + +# -- Order ----------------------------------------------------------------------------------------------------- + + +def test_the_matter_is_written_in_the_order_the_html_shows_it() -> None: + data = docx(**contents_style()) + styles = [style_of(c) if c.tag == f"{W}p" else "table" for c in body(data) if c.tag != f"{W}sectPr"] + order = [name for name in styles if name in ("Title", "Subtitle", "table", "PartySide", "ContentsHeading", "TOC1", + "Heading1", "AttachmentsHeading", "AttachmentTitle", + "SignatureSide")] + first = {} + for index, name in enumerate(order): + first.setdefault(name, index) + sequence = ["Title", "Subtitle", "table", "PartySide", "ContentsHeading", "TOC1", "Heading1", + "AttachmentsHeading", "AttachmentTitle"] + assert [first[name] for name in sequence] == sorted(first[name] for name in sequence) + assert order[-1] == "table" # the signature table, after the attachments + assert max(i for i, name in enumerate(order) if name == "AttachmentTitle") < len(order) - 1 + + +# -- Header rows and parties ----------------------------------------------------------------------------------- + + +def test_the_header_rows_are_a_borderless_table_of_label_and_value() -> None: + data = docx() + header = tables(data)[0] + rows = [[text_of(cell) for cell in row.iter(f"{W}tc")] for row in header.iter(f"{W}tr")] + assert rows == [["Effective date", "March 4, 2026"], ["Version", "2.1"]] or rows[0] == ["Effective date", + "March 4, 2026"] + assert header.find(f"{W}tblPr/{W}tblBorders") is None and header.find(f"{W}tblPr/{W}tblStyle") is None + label = header.find(f"{W}tr/{W}tc/{W}p/{W}r/{W}rPr/{W}rStyle") + assert label is not None and label.get(f"{W}val") == "TitleBlockLabel" + + +def test_the_parties_follow_their_side_with_their_details() -> None: + data = docx() + sides = [text_of(p) for p in paragraphs(data, "PartySide")] + assert sides == ["Seller", "Buyer"] + assert [text_of(p) for p in paragraphs(data, "PartyName")] == ["Acme Corporation", "Beta Industries Inc.", + "Gamma Ltd."] + details = tables(data)[1] + rows = [[text_of(cell) for cell in row.iter(f"{W}tc")] for row in details.iter(f"{W}tr")] + assert ["Identification No.", "CZ-123"] in rows and ["Address", "1 Main Street"] in rows + assert any(row[0] == "Represented by" and "John Smith" in row[1] for row in rows) + + +def test_no_header_rows_no_parties_means_no_table() -> None: + data = docx("---\ntitle: T\n---\n# A\n", overrides={"title_block.effective_date": False}) + assert tables(data) == [] + + +def test_the_style_can_hide_the_parties() -> None: + data = docx(overrides={"title_block.parties": False}) + assert paragraphs(data, "PartySide") == [] + + +def test_version_and_date_are_the_rows_the_title_block_shows() -> None: + data = docx(overrides={"title_block.version": True}) + rows = [[text_of(cell) for cell in row.iter(f"{W}tc")] for row in tables(data)[0].iter(f"{W}tr")] + assert rows == [["Effective date", "March 4, 2026"], ["Version", "2.1"]] + + +def test_two_tables_in_a_row_are_kept_apart() -> None: + source = "---\ntitle: T\neffective_date: 2026-01-02\n---\n\n| a | b |\n|---|---|\n| c | d |\n" + elements = [c for c in body(docx(source)) if c.tag != f"{W}sectPr"] + tags = [c.tag.removeprefix(W) for c in elements] + assert tags[:5] == ["p", "tbl", "p", "tbl", "p"] + + +# -- Table of contents ----------------------------------------------------------------------------------------- + + +def toc_paragraphs(data: bytes) -> list[ET.Element]: + return [p for p in document_xml(data).iter(f"{W}p") if (style_of(p) or "").startswith("TOC")] + + +def test_the_contents_label_is_a_style_without_numbers_or_outline() -> None: + data = docx(**contents_style()) + [label] = paragraphs(data, "ContentsHeading") + assert text_of(label) == "Contents" + assert label.find(f"{W}pPr/{W}numPr") is None and label.find(f"{W}pPr/{W}outlineLvl") is None + styles = ET.fromstring(parts(data)["word/styles.xml"]) + heading = next(s for s in styles.iter(f"{W}style") if s.get(f"{W}styleId") == "ContentsHeading") + assert heading.find(f"{W}pPr/{W}outlineLvl") is None and heading.find(f"{W}pPr/{W}numPr") is None + assert heading.find(f"{W}name").get(f"{W}val") == "Contents Heading" + + +def test_the_toc_field_is_one_complex_field_over_the_entries() -> None: + data = docx(**contents_style()) + root = document_xml(data) + toc = [i for i in instructions(root) if i.startswith("TOC")] + assert toc == ['TOC \\o "1-2" \\h \\z \\u'] + kinds = [node.get(f"{W}fldCharType") for node in root.iter(f"{W}fldChar")] + assert kinds.count("begin") == kinds.count("separate") == kinds.count("end") + # the TOC field opens in the first entry and closes in a paragraph of its own after the last + elements = list(root.find(f"{W}body")) # type: ignore[arg-type] + entries = [p for p in elements if (style_of(p) or "").startswith("TOC")] + first_kinds = [n.get(f"{W}fldCharType") for n in entries[0].iter(f"{W}fldChar")] + assert first_kinds[:2] == ["begin", "separate"] + last = elements[elements.index(entries[-1]) + 1] + assert text_of(last) == "" and [n.get(f"{W}fldCharType") for n in last.iter(f"{W}fldChar")] == ["end"] + + +@pytest.mark.parametrize("depth", [1, 2, 3, 5]) +def test_the_depth_is_the_style_s(depth: int) -> None: + data = docx(**contents_style(**{"contents.depth": depth})) + assert f'TOC \\o "1-{depth}" \\h \\z \\u' in instructions(document_xml(data)) + + +def test_the_cached_entries_are_the_planned_ones() -> None: + result = render(FRONT, format="docx", **contents_style()) + assert result.binary is not None + plan = plan_docx(result.tree)[0] + entries = toc_paragraphs(result.binary) + assert len(entries) == len(plan.toc) == len(result.tree.contents) + for paragraph, planned, entry in zip(entries, plan.toc, result.tree.contents, strict=True): + assert style_of(paragraph) == f"TOC{planned.level}" + link = paragraph.find(f"{W}hyperlink") + assert link is not None and link.get(f"{W}anchor") == planned.bookmark + shown = text_of(paragraph) + expected = (planned.number + planned.sep + plain_text(planned.title) if planned.number + else plain_text(planned.title)) + (f" {planned.condition_label}" if planned.condition_label else "") + assert shown == expected + "\t" + assert entry.condition_label == planned.condition_label + + +def test_each_entry_links_and_refers_to_a_bookmark_the_document_has() -> None: + data = docx(**contents_style()) + root = document_xml(data) + names = set(bookmark_names(root)) + entries = toc_paragraphs(data) + anchors = [p.find(f"{W}hyperlink").get(f"{W}anchor") for p in entries] # type: ignore[union-attr] + pagerefs = [i.split()[1] for i in instructions(root) if i.startswith("PAGEREF")] + assert anchors == pagerefs and all(a.startswith("_Toc") for a in anchors) + assert set(anchors) <= names and len(set(anchors)) == len(anchors) + assert all(i.endswith("\\h") for i in instructions(root) if i.startswith("PAGEREF")) + for paragraph in entries: # the cached page is empty: separate and end follow each other + kinds = [n.get(f"{W}fldCharType") for n in paragraph.iter(f"{W}fldChar")] + assert kinds[-3:] == ["begin", "separate", "end"] + assert paragraph.find(f"{W}hyperlink").find(f"{W}r/{W}fldChar[@{W}fldCharType='end']") is not None # type: ignore[union-attr] + + +def test_each_entry_has_a_dotted_right_tab_at_the_text_width() -> None: + data = docx(**contents_style()) + text_width = docx_settings(load_style("default")).page.text_width + for paragraph in toc_paragraphs(data): + right = [t for t in paragraph.findall(f"{W}pPr/{W}tabs/{W}tab") if t.get(f"{W}val") == "right"] + assert [(t.get(f"{W}leader"), t.get(f"{W}pos")) for t in right] == [("dot", str(text_width))] + styles = ET.fromstring(parts(data)["word/styles.xml"]) + toc1 = next(s for s in styles.iter(f"{W}style") if s.get(f"{W}styleId") == "TOC1") + tab = toc1.find(f"{W}pPr/{W}tabs/{W}tab") + assert (tab.get(f"{W}val"), tab.get(f"{W}leader"), tab.get(f"{W}pos")) == ("right", "dot", str(text_width)) + + +def test_a_number_is_followed_by_the_tab_the_heading_has_and_aligned_per_level() -> None: + data = docx(**contents_style()) + stops: dict[str, set[str]] = {} + for paragraph in toc_paragraphs(data): + left = [t.get(f"{W}pos") for t in paragraph.findall(f"{W}pPr/{W}tabs/{W}tab") if t.get(f"{W}val") == "left"] + link = paragraph.find(f"{W}hyperlink") + runs = list(link) # type: ignore[arg-type] + has_number = (style_of(paragraph) in ("TOC1", "TOC2")) and text_of(paragraph)[:1].isdigit() + if has_number: + assert runs[1].find(f"{W}tab") is not None and len(left) == 1 + stops.setdefault(style_of(paragraph), set()).add(left[0]) # type: ignore[arg-type] + else: + assert not left + assert all(len(stop) == 1 for stop in stops.values()) + + +def test_a_condition_label_is_a_run_after_the_title() -> None: + source = (DOCUMENTS / "template.lgd").read_text(encoding="utf-8") + data = docx(source, style="continental", overrides={"contents.enabled": True}) + result = render(source, format="text", style="continental", overrides={"contents.enabled": True}) + labelled = [e for e in result.tree.contents if e.condition_label] + assert labelled + runs = [r for p in toc_paragraphs(data) for r in p.iter(f"{W}r") + if r.find(f"{W}rPr/{W}rStyle") is not None and r.find(f"{W}rPr/{W}rStyle").get(f"{W}val") == "ConditionLabel"] # type: ignore[union-attr] + assert [text_of(r) for r in runs] == [e.condition_label for e in labelled] + + +def test_no_contents_no_field_and_no_entries() -> None: + data = docx() + root = document_xml(data) + assert not any(i.startswith(("TOC", "PAGEREF")) for i in instructions(root)) + assert toc_paragraphs(data) == [] and paragraphs(data, "ContentsHeading") == [] + assert not [n for n in bookmark_names(root) if n.startswith("_Toc")] + + +def test_headings_carry_the_toc_bookmarks_when_listed() -> None: + data = docx(**contents_style()) + root = document_xml(data) + names = bookmark_names(root) + assert len([n for n in names if n.startswith("_Toc")]) == len(toc_paragraphs(data)) + + +def test_a_contents_with_nothing_to_list_writes_nothing() -> None: + data = docx("---\ntitle: T\n---\n\nJust text.\n", **contents_style()) + assert toc_paragraphs(data) == [] and paragraphs(data, "ContentsHeading") == [] + + +def test_a_blanked_contents_label_leaves_the_field() -> None: + data = docx(**contents_style(**{"labels.contents": ""})) + assert paragraphs(data, "ContentsHeading") == [] and toc_paragraphs(data) + + +def test_the_planned_entries_and_the_tree_must_agree() -> None: + result = render(FRONT, format="docx", **contents_style()) + plan = plan_docx(result.tree)[0] + broken = DocxPlan(plan.abstract_nums, plan.nums, plan.heading_num_id, units=plan.units, anchors=plan.anchors, + refs=plan.refs, toc=plan.toc[:-1]) + with pytest.raises(InternalError): + DocxWriter(load_style("default", overrides={"contents.enabled": True})).write(result.tree, broken) + + +def test_an_entry_pointing_at_a_bookmark_the_document_lacks_is_a_bug() -> None: + result = render(FRONT, format="docx", **contents_style()) + plan = plan_docx(result.tree)[0] + units = {key: (unit if unit.toc_bookmark is None else _without_toc(unit)) for key, unit in plan.units.items()} + broken = DocxPlan(plan.abstract_nums, plan.nums, plan.heading_num_id, units=units, anchors=plan.anchors, + refs=plan.refs, toc=plan.toc) + with pytest.raises(InternalError): + DocxWriter(load_style("default", overrides={"contents.enabled": True})).write(result.tree, broken) + + +def _without_toc(unit: Any) -> Any: + from dataclasses import replace + + return replace(unit, toc_bookmark=None) + + +# -- Attachments in the contents: outline levels ---------------------------------------------------------------- + + +def outline(paragraph: ET.Element) -> str | None: + node = paragraph.find(f"{W}pPr/{W}outlineLvl") + return None if node is None else node.get(f"{W}val") + + +def test_the_attachments_are_listed_by_outline_level_not_by_style_list() -> None: + data = docx(**contents_style()) + assert not any("\\t" in i for i in instructions(document_xml(data))) + [heading] = paragraphs(data, "AttachmentsHeading") + titles = paragraphs(data, "AttachmentTitle") + assert outline(heading) == "0" and {outline(t) for t in titles} == {"1"} + styles = ET.fromstring(parts(data)["word/styles.xml"]) + by_id = {s.get(f"{W}styleId"): s for s in styles.iter(f"{W}style")} + assert by_id["AttachmentsHeading"].find(f"{W}pPr/{W}outlineLvl").get(f"{W}val") == "0" + assert by_id["AttachmentTitle"].find(f"{W}pPr/{W}outlineLvl").get(f"{W}val") == "1" + for name in ("AttachmentsHeading", "AttachmentTitle"): # no heading numbering reaches them + assert by_id[name].find(f"{W}pPr/{W}numPr") is None and by_id[name].find(f"{W}basedOn").get(f"{W}val") == "Normal" + + +def test_every_outline_level_the_contents_lists_is_what_the_html_lists() -> None: + """The ``\\o`` range and the outline levels select exactly the entries of the tree's contents.""" + for depth in (1, 2, 3): + result = render(FRONT, format="docx", **contents_style(**{"contents.depth": depth})) + assert result.binary is not None + root = document_xml(result.binary) + listed = [] + for paragraph in root.iter(f"{W}p"): + style = style_of(paragraph) or "" + level = None + if style.startswith("Heading"): + level = int(style.removeprefix("Heading")) - 1 + elif outline(paragraph) is not None: + level = int(outline(paragraph)) # type: ignore[arg-type] + if level is not None and level < depth: + listed.append(text_of(paragraph)) + expected = [" ".join(filter(None, (plain_text(e.title), e.condition_label))) for e in result.tree.contents] + assert listed == expected + + +def test_attachments_not_listed_are_body_text_for_the_contents() -> None: + data = docx(**contents_style(**{"contents.attachments": False})) + assert {outline(p) for p in paragraphs(data, "AttachmentTitle")} == {"9"} + assert outline(paragraphs(data, "AttachmentsHeading")[0]) == "9" + assert not [p for p in toc_paragraphs(data) if "Schedule A" in text_of(p)] + + +def test_a_depth_of_one_keeps_the_attachments_in_the_list() -> None: + data = docx(**contents_style(**{"contents.depth": 1})) + assert {outline(p) for p in paragraphs(data, "AttachmentTitle")} == {"0"} + assert [style_of(p) for p in toc_paragraphs(data)].count("TOC1") == len(toc_paragraphs(data)) + + +def test_a_blanked_attachments_label_lists_the_titles_at_the_first_level() -> None: + data = docx(**contents_style(**{"labels.attachments": ""})) + assert paragraphs(data, "AttachmentsHeading") == [] + assert {outline(p) for p in paragraphs(data, "AttachmentTitle")} == {"0"} + assert [text_of(p) for p in toc_paragraphs(data)][-3:] == [ + "Schedule A: Services\t", "Price List " + "Only if: forum:courts\t", "Terms\t"] + + +# -- Attachments ----------------------------------------------------------------------------------------------- + + +def page_break(paragraph: ET.Element) -> bool: + return paragraph.find(f"{W}pPr/{W}pageBreakBefore") is not None + + +def test_the_attachments_are_a_heading_and_titles_with_their_files_and_conditions() -> None: + data = docx() + [heading] = paragraphs(data, "AttachmentsHeading") + assert text_of(heading) == "Attachments" + titles = [text_of(p) for p in paragraphs(data, "AttachmentTitle")] + assert titles == ["Schedule A: Services", "Price List Only if: forum:courts", "Terms"] + assert [text_of(p) for p in paragraphs(data, "AttachmentFile")] == ["File: files/services.lgd"] + label = [r for p in paragraphs(data, "AttachmentTitle") for r in p.iter(f"{W}r") + if r.find(f"{W}rPr/{W}rStyle") is not None] + assert [text_of(r) for r in label] == ["Only if: forum:courts"] + + +def test_each_attachment_is_a_bookmark_and_an_attach_reference_resolves_to_it() -> None: + result = render(FRONT, format="docx", **contents_style()) + assert result.binary is not None + root = document_xml(result.binary) + plan = plan_docx(result.tree)[0] + names = bookmark_names(root) + for anchor in (a.anchor for a in result.tree.attachments): + assert plan.anchors[anchor] in names + assert plan.anchors[result.tree.attachments_anchor] in names + refs = [i for i in instructions(root) if i.startswith("REF ldatt_")] + assert refs and all(i.split()[1] in names for i in refs) + links = [h.get(f"{W}anchor") for h in root.iter(f"{W}hyperlink") if (h.get(f"{W}anchor") or "").startswith("ldatt_")] + assert links and all(a in names for a in links) + assert len(names) == len(set(names)) + # the bookmark holds the title but not the condition label + [paragraph] = [p for p in paragraphs(result.binary, "AttachmentTitle") if "Price List" in text_of(p)] + start = next(e for e in paragraph if e.tag == f"{W}bookmarkStart" and e.get(f"{W}name").startswith("ldatt_")) # type: ignore[union-attr] + inside = [] + for element in list(paragraph)[list(paragraph).index(start) + 1:]: + if element.tag == f"{W}bookmarkEnd" and element.get(f"{W}id") == start.get(f"{W}id"): + break + inside.append(element) + assert "".join(text_of(e) for e in inside) == "Price List" + + +def test_the_attachments_are_bookmarked_without_a_contents_too() -> None: + data = docx() + names = bookmark_names(document_xml(data)) + assert {"ldatt_schedule_a", "ldatt_price_list", "ldatt_terms", "_LdAttachments"} <= set(names) + assert not [n for n in names if n.startswith("_Toc")] + + +def test_page_break_separator_breaks_before_the_part_and_each_attachment() -> None: + data = docx() # the default style + [heading] = paragraphs(data, "AttachmentsHeading") + titles = paragraphs(data, "AttachmentTitle") + assert page_break(heading) and [page_break(t) for t in titles] == [False, True, True] + assert "w:br w:type" not in parts(data)["word/document.xml"].decode("utf-8") + + +def test_page_break_without_a_label_breaks_before_the_first_title() -> None: + data = docx(overrides={"labels.attachments": ""}) + titles = paragraphs(data, "AttachmentTitle") + assert [page_break(t) for t in titles] == [True, True, True] + + +def test_rule_separator_draws_a_line_above_each_attachment() -> None: + data = docx(overrides={"attachments.separator": "rule"}) + titles = paragraphs(data, "AttachmentTitle") + assert all(t.find(f"{W}pPr/{W}pBdr/{W}top") is not None for t in titles) + assert not any(page_break(p) for p in titles + paragraphs(data, "AttachmentsHeading")) + + +def test_no_separator_draws_nothing() -> None: + data = docx(overrides={"attachments.separator": "none"}) + for p in paragraphs(data, "AttachmentTitle") + paragraphs(data, "AttachmentsHeading"): + assert not page_break(p) and p.find(f"{W}pPr/{W}pBdr") is None + + +def test_omitted_attachments_write_nothing() -> None: + data = docx(overrides={"attachments.render": "omit"}) + assert paragraphs(data, "AttachmentsHeading") == [] and paragraphs(data, "AttachmentTitle") == [] + + +def test_the_attachments_listed_in_the_contents_carry_its_bookmarks() -> None: + result = render(FRONT, format="docx", **contents_style()) + assert result.binary is not None + plan = plan_docx(result.tree)[0] + unitless = [t.bookmark for e, t in zip(result.tree.contents, plan.toc, strict=True) if e.unit is None] + root = document_xml(result.binary) + heading, *titles = [*paragraphs(result.binary, "AttachmentsHeading"), *paragraphs(result.binary, "AttachmentTitle")] + for paragraph, name in zip([heading, *titles], unitless, strict=True): + assert name in [b.get(f"{W}name") for b in paragraph.iter(f"{W}bookmarkStart")] + assert len(set(bookmark_names(root))) == len(bookmark_names(root)) + + +# -- Signature blocks ------------------------------------------------------------------------------------------ + + +def signature_table(data: bytes) -> ET.Element: + return next(t for t in reversed(tables(data)) if any(style_of(p) == "SignatureSide" for p in t.iter(f"{W}p"))) + + +def test_the_signatures_are_a_borderless_table_two_parties_to_a_row() -> None: + data = docx() + table = signature_table(data) + assert table.find(f"{W}tblPr/{W}tblBorders") is None + rows = table.findall(f"{W}tr") + assert [len(row.findall(f"{W}tc")) for row in rows] == [2, 2] + cells = [c for row in rows for c in row.findall(f"{W}tc")] + sides = [text_of(c.find(f"{W}p")) for c in cells] # type: ignore[arg-type] + assert sides == ["Seller", "Buyer", "Buyer", ""] + assert [text_of(c.findall(f"{W}p")[1]) for c in cells[:3]] == ["Acme Corporation", "Beta Industries Inc.", "Gamma Ltd."] + + +def test_each_signatory_has_a_line_a_name_and_a_title() -> None: + data = docx() + acme = signature_table(data).find(f"{W}tr/{W}tc") + lines = [(style_of(p), text_of(p)) for p in acme.findall(f"{W}p")] # type: ignore[union-attr] + assert lines == [ + ("SignatureSide", "Seller"), ("SignatureText", "Acme Corporation"), + ("SignatureLine", ""), ("SignatureText", "Name: John Smith"), ("SignatureText", "Title: Chief Executive Officer"), + ("SignatureLine", ""), ("SignatureText", "Name: Jane Roe"), ("SignatureText", "Title: Director"), + ("SignatureText", "Date:"), ("SignatureText", "Place:")] + line = acme.find(f"{W}p[3]") # type: ignore[union-attr] + assert line.find(f"{W}pPr/{W}ind") is not None # type: ignore[union-attr] + styles = ET.fromstring(parts(data)["word/styles.xml"]) + style = next(s for s in styles.iter(f"{W}style") if s.get(f"{W}styleId") == "SignatureLine") + assert style.find(f"{W}pPr/{W}pBdr/{W}bottom") is not None + + +def test_a_party_without_representatives_has_one_blank_signing_line() -> None: + data = docx() + beta = signature_table(data).findall(f"{W}tr")[0].findall(f"{W}tc")[1] + lines = [(style_of(p), text_of(p)) for p in beta.findall(f"{W}p")] + assert lines == [("SignatureSide", "Buyer"), ("SignatureText", "Beta Industries Inc."), ("SignatureLine", ""), + ("SignatureText", "Name:"), ("SignatureText", "Title:"), ("SignatureText", "Date:"), + ("SignatureText", "Place:")] + + +def test_the_signatures_stay_together() -> None: + data = docx() + table = signature_table(data) + assert all(row.find(f"{W}trPr/{W}cantSplit") is not None for row in table.findall(f"{W}tr")) + paragraphs_in = list(table.iter(f"{W}p")) + kept = [p.find(f"{W}pPr/{W}keepNext") is not None for p in paragraphs_in] + assert all(kept[:-1]) and not kept[-1] + + +def test_one_party_is_one_cell() -> None: + source = ("---\nlegaldown: '0.2'\ntitle: T\nsides:\n - name: a\n parties:\n - name: p\n type: legal_entity\n" + " legal_name: Only One\n---\n\n# A\n") + table = signature_table(docx(source)) + assert [len(r.findall(f"{W}tc")) for r in table.findall(f"{W}tr")] == [1] + + +def test_no_signatures_when_the_style_or_the_document_says_so() -> None: + assert paragraphs(docx(overrides={"signatures.enabled": False}), "SignatureSide") == [] + off = FRONT.replace("language: en\n", "language: en\ninclude_signatures: false\n", 1) + assert paragraphs(docx(off), "SignatureSide") == [] + + +def test_a_signature_label_blanked_by_the_style_leaves_the_value() -> None: + data = docx(overrides={"labels.signature_name": "", "labels.signature_title": "", "labels.signature_place": ""}) + acme = signature_table(data).find(f"{W}tr/{W}tc") + lines = [text_of(p) for p in acme.findall(f"{W}p")] # type: ignore[union-attr] + assert "John Smith" in lines and "Chief Executive Officer" in lines and "Place:" not in lines and "Date:" in lines + + +def test_the_document_ends_with_a_paragraph_after_the_signature_table() -> None: + elements = [c for c in body(docx()) if c.tag != f"{W}sectPr"] + assert elements[-1].tag == f"{W}p" and elements[-2].tag == f"{W}tbl" + + +# -- The interim warning is gone ------------------------------------------------------------------------------- + + +def test_a_docx_render_reports_what_the_text_render_reports() -> None: + document = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + docx_rules = [d.rule for d in render(document, format="docx").diagnostics] + assert "render-docx-interim" not in docx_rules + text = [d.rule for d in render(document, format="text").diagnostics] + assert [r for r in docx_rules if not r.startswith("render-docx")] == text + + +# -- Schema validity ------------------------------------------------------------------------------------------- + + +def hostile_document(rnd: random.Random) -> str: + def party(index: int) -> dict[str, Any]: + return {"name": f"p{index}", "type": "legal_entity", "legal_name": soup(rnd, 12) + "x", + "address": soup(rnd, 10), + "representatives": [{"name": soup(rnd, 8) + "n", "title": soup(rnd, 8)} for _ in range(rnd.randint(0, 2))]} + + front = {"legaldown": "0.2", "title": soup(rnd, 20), "subtitle": soup(rnd, 20), "version": soup(rnd, 6), + "effective_date": "2026-01-02", + "sides": [{"name": f"s{i}", "label": soup(rnd, 8) + "l", "parties": [party(i * 3 + j) for j in range(2)]} + for i in range(2)], + "attachments": [{"id": f"a{i}", "title": soup(rnd, 15) + "t", "file": soup(rnd, 10) + "f"} + for i in range(3)]} + return (f"---\n{yaml.safe_dump(front, allow_unicode=True)}---\n\n{soup(rnd, 20)}\n\n# {soup(rnd, 15)} {{#h1}}\n\n" + f"{soup(rnd, 30)} {{{{attach: a1}}}}\n\n## {soup(rnd, 10)}\n\ntext\n") + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize("settings", [ + {}, {"overrides": {"contents.enabled": True}}, + {"overrides": {"contents.enabled": True, "contents.depth": 5, "attachments.separator": "rule"}}, + {"overrides": {"contents.enabled": True, "contents.attachments": False, "attachments.separator": "none"}}, + {"overrides": {"contents.enabled": True, "labels.attachments": "", "labels.contents": ""}}, + {"overrides": {"title_block.version": True, "page.size": "A4 landscape"}}, + {"style": "continental", "overrides": {"contents.enabled": True}}, + {"style": "outline", "overrides": {"contents.enabled": True, "contents.depth": 3}}, + {"style": "continental", "overrides": {"numbering.scheme": "none", "contents.enabled": True}}, +]) +def test_a_document_with_every_part_validates(settings: dict[str, Any]) -> None: + require_schemas() + assert validation_errors(docx(**settings)) == [] + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize("name", ["features", "template"]) +@pytest.mark.parametrize("style", ["default", "continental", "outline"]) +def test_the_test_documents_validate_with_a_contents(name: str, style: str) -> None: + require_schemas() + source = (DOCUMENTS / f"{name}.lgd").read_text(encoding="utf-8") + assert validation_errors(docx(source, style=style, overrides={"contents.enabled": True, "contents.depth": 5})) == [] + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize("seed", range(12)) +def test_hostile_text_in_every_part_of_the_matter_validates(seed: int) -> None: + require_schemas() + rnd = random.Random(7000 + seed) + data = docx(hostile_document(rnd), overrides={"contents.enabled": True, + "labels.contents": soup(rnd, 6), "labels.attachments": soup(rnd, 6), + "labels.colon": soup(rnd, 3) + ":"}) + assert_well_formed(data) + assert validation_errors(data) == [] + + +def test_hostile_front_matter_is_well_formed() -> None: + for seed in range(20): + rnd = random.Random(seed) + assert_well_formed(docx(hostile_document(rnd), overrides={"contents.enabled": True})) + assert ALPHABET + + +# -- LibreOffice ----------------------------------------------------------------------------------------------- + +def toc_lines(exported: Exported) -> list[str]: + """The entries of the contents LibreOffice drew, as ``number title`` (the page number and the tabs left + out), and the page numbers.""" + lines = [] + for line in exported.text.splitlines(): + match = re.fullmatch(r"(.*)\t(\d+)", line) + if match: + lines.append(" ".join(match.group(1).split())) + return lines + + +def toc_pages(exported: Exported) -> list[int]: + return [int(m.group(1)) for line in exported.text.splitlines() if (m := re.fullmatch(r".*\t(\d+)", line))] + + +def expected_entries(source: str, **settings: Any) -> list[str]: + tree = render(source, format="text", **settings).tree + return [" ".join(" ".join(filter(None, (e.label, plain_text(e.title), + e.condition_label))).split()) for e in tree.contents] + + +LONG = ("---\nlegaldown: '0.2'\ntitle: Long\nattachments:\n - id: one\n title: First\n - id: two\n title: Second\n" + " - id: three\n title: Third\n---\n\n" + "".join(f"# Heading {n}\n\nParagraph {n} " + "word " * 100 + "\n\n" + for n in range(6)) + "See {{attach: two}}.\n") + + +@pytest.fixture(scope="module") +def shown() -> dict[str, Exported]: + require_libreoffice() + features = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + template = (DOCUMENTS / "template.lgd").read_text(encoding="utf-8") + contents = {"contents.enabled": True, "contents.depth": 3} + documents = { + "master": docx(FRONT, overrides=contents), + "features": docx(features, overrides=contents), + "template": docx(template, style="continental", overrides=contents), + "long_break": docx(LONG, overrides={"contents.enabled": True}), + "long_rule": docx(LONG, overrides={"contents.enabled": True, "attachments.separator": "rule"}), + "long_none": docx(LONG, overrides={"contents.enabled": True, "attachments.separator": "none"}), + } + # The cached title of the attachment reference is wrong on purpose: LibreOffice must compute it. + wrong = repackage(docx(LONG), "word/document.xml", + lambda text: re.sub(r"(REF ldatt_two [^<]*" + r")Second", r"\1WRONG", text)) + assert b"WRONG" in parts(wrong)["word/document.xml"] + documents["wrong_cache"] = wrong + return libreoffice_export(documents) + + +@pytest.mark.libreoffice +def test_the_contents_lists_exactly_the_html_entries(shown: dict[str, Exported]) -> None: + features = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + assert toc_lines(shown["features"]) == expected_entries(features, overrides={"contents.enabled": True, + "contents.depth": 3}) + assert toc_lines(shown["master"]) == expected_entries(FRONT, overrides={"contents.enabled": True, + "contents.depth": 3}) + # An alternative that shares a number (§15.8) is numbered in sequence by LibreOffice until the planner keeps it + # on the first one's number: the titles are the contents' either way. + template = (DOCUMENTS / "template.lgd").read_text(encoding="utf-8") + drawn = [line.split(" ", 1)[1] for line in toc_lines(shown["template"])] + entries = expected_entries(template, style="continental", overrides={"contents.enabled": True, "contents.depth": 3}) + assert drawn == [line.split(" ", 1)[1] for line in entries] + + +@pytest.mark.libreoffice +def test_page_numbers_appear_after_the_update(shown: dict[str, Exported]) -> None: + pages = toc_pages(shown["master"]) + assert len(pages) == len(toc_lines(shown["master"])) and all(p >= 1 for p in pages) + assert pages == sorted(pages) + assert max(pages) <= shown["master"].pages + + +@pytest.mark.libreoffice +def test_the_contents_is_not_one_of_its_own_entries(shown: dict[str, Exported]) -> None: + assert "Contents" not in toc_lines(shown["master"]) + + +@pytest.mark.libreoffice +def test_an_attachment_reference_resolves_to_the_title_not_the_cache(shown: dict[str, Exported]) -> None: + assert "See Second." in shown["wrong_cache"].text and "WRONG" not in shown["wrong_cache"].text + assert "Schedule A: Services" in shown["master"].text # the {{attach:}} of FRONT, a field to the title + + +@pytest.mark.libreoffice +def test_a_page_break_separator_puts_each_attachment_on_its_own_page(shown: dict[str, Exported]) -> None: + pages = toc_pages(shown["long_break"]) + entries = toc_lines(shown["long_break"]) + by_title = dict(zip(entries, pages, strict=True)) + assert by_title["Attachments"] == by_title["First"] # the heading stays with the first attachment + assert by_title["First"] > by_title["6. Heading 5"] + assert by_title["First"] < by_title["Second"] < by_title["Third"] + assert shown["long_break"].pages == by_title["Third"] + + +@pytest.mark.libreoffice +def test_rule_and_none_keep_the_attachments_on_the_pages_they_fall_on(shown: dict[str, Exported]) -> None: + for name in ("long_rule", "long_none"): + by_title = dict(zip(toc_lines(shown[name]), toc_pages(shown[name]), strict=True)) + assert by_title["First"] == by_title["Second"] == by_title["Third"] + assert shown["long_break"].pages >= shown["long_rule"].pages + 2 + + +def words(text: str) -> list[str]: + """The words of *text* in order (the plain-text writer's ```` after a link left out: a hyperlink is not text).""" + return re.findall(r"[^\W_]+", re.sub(r"<(?:https?|mailto|tel):[^>]*>", "", text).lower()) + + +def lo_words(exported: Exported) -> list[str]: + lines = [] + for line in exported.text.splitlines(): + lines.append(re.sub(r"\t\d+$", "", line)) + return words("\n".join(lines)) + + +@pytest.mark.libreoffice +@pytest.mark.parametrize("name", ["master", "features"]) +def test_the_whole_document_reads_in_the_order_the_text_writer_prints_it(shown: dict[str, Exported], name: str) -> None: + contents = {"contents.enabled": True, "contents.depth": 3} + source = FRONT if name == "master" else (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + text = render(source, format="text", overrides=contents).output + assert lo_words(shown[name]) == words(text) + + +@pytest.mark.libreoffice +def test_the_template_reads_in_the_text_writers_order_but_for_its_numbering(shown: dict[str, Exported]) -> None: + source = (DOCUMENTS / "template.lgd").read_text(encoding="utf-8") + text = render(source, format="text", style="continental", overrides={"contents.enabled": True, + "contents.depth": 3}).output + expected, found = words(text), lo_words(shown["template"]) + # the numbers Word draws for an alternative that shares one are numbering details (see the plan's diagnostics) + assert [w for w in found if not w.isdigit()] == [w for w in expected if not w.isdigit()] + + +@pytest.mark.libreoffice +def test_a_heading_inserted_in_the_editor_is_listed_once_the_contents_is_updated() -> None: + """The contents is live: the heading a lawyer adds (Enter at the start of a heading) is an entry, and the numbers + after it move, as they do in the contents of the document written with it.""" + require_libreoffice() + contents = {"contents.enabled": True, "contents.depth": 3} + said = libreoffice_edit({"insert": (docx(FRONT, overrides=contents), [("Payment", "Inserted")])})["insert"] + edited = FRONT.replace("## Payment {#payment}", "## Inserted\n\n## Payment {#payment}") + lines = [" ".join(m.group(1).split()) for line in said.splitlines() if (m := re.fullmatch(r"(.*)\t(\d+)", line))] + assert lines == expected_entries(edited, overrides=contents) + assert "2.1 Inserted" in lines and "2.2 Payment" in lines + + +#: SHA-256 of the document part and the styles part of :data:`FRONT` with a contents (uncompressed: the ZIP's bytes +#: depend on the zlib). When the matter changes on purpose, run this test, review the change, and replace the digests. +MATTER_GOLDEN = { + "word/document.xml": "d67511803cd53640fbd96c3c2f6531cc5cf649158c7f56a0d0681db4da5c1b98", + "word/styles.xml": "6ccea95d23325c1d13fd3f11d953567df2ce075bbb00f0f537cac57939c6dcfc", +} + + +def test_the_matter_of_a_fixed_document_is_what_it_was() -> None: + import hashlib + + contents = parts(DocxWriter(load_style("default", overrides={"contents.enabled": True}), generator="golden") + .write(render(FRONT, format="text", overrides={"contents.enabled": True}).tree)) + digests = {name: hashlib.sha256(contents[name]).hexdigest() for name in MATTER_GOLDEN} + assert digests == MATTER_GOLDEN, "\n".join(f' "{name}": "{digest}",' for name, digest in digests.items()) diff --git a/tests/test_docx_package.py b/tests/test_docx_package.py index 3130626..01899cd 100644 --- a/tests/test_docx_package.py +++ b/tests/test_docx_package.py @@ -156,7 +156,10 @@ def test_the_styles_the_body_and_word_need_exist_under_the_names_word_knows() -> "TOC3": "toc 3", "TOC4": "toc 4", "TOC5": "toc 5", "Hyperlink": "Hyperlink", "InlineCode": "Inline Code", "DefinedTerm": "Defined Term", "TermReference": "Term Reference", "Blank": "Blank", "Choice": "Choice", "FailureMarker": "Failure Marker", "ConditionLabel": "Condition Label", - "DraftingNoteLabel": "Drafting Note Label"} + "DraftingNoteLabel": "Drafting Note Label", "TitleBlockText": "Title Block Text", + "PartySide": "Party Side", "PartyName": "Party Name", "AttachmentsHeading": "Attachments Heading", + "AttachmentTitle": "Attachment Title", "AttachmentFile": "Attachment File", + "SignatureSide": "Signature Side", "SignatureText": "Signature Text", "SignatureLine": "Signature Line"} assert by_id["Normal"].get(f"{W}default") == "1" assert by_id["TableGrid"].get(f"{W}type") == "table" assert by_id["Hyperlink"].get(f"{W}type") == "character" @@ -409,8 +412,10 @@ def test_the_body_without_a_plan_has_the_title_headings_and_text_and_no_numbers( body = xml(contents["word/document.xml"]).find(f"{W}body") styles_used = [(p.find(f"{W}pPr/{W}pStyle").get(f"{W}val") if p.find(f"{W}pPr/{W}pStyle") is not None else None, texts(p)[0]) for p in body.iter(f"{W}p")] - assert styles_used[:5] == [("Title", "Services Agreement"), ("Subtitle", "A subtitle"), (None, "Preamble text."), - ("Heading1", "First"), (None, "Body one.")] + # the header row of the title block (effective date, in Czech) comes between the subtitle and the preamble + assert styles_used[:7] == [("Title", "Services Agreement"), ("Subtitle", "A subtitle"), + ("TitleBlockText", "Datum účinnosti"), ("TitleBlockText", "4. března 2026"), + (None, "Preamble text."), ("Heading1", "First"), (None, "Body one.")] assert ("Heading2", "Nested") in styles_used assert (None, "item a") in styles_used and ("Quote", "quoted") in styles_used assert [text for style, text in styles_used if style == "Code"] == ["code 1", "code 2"] @@ -615,21 +620,17 @@ def test_render_file_renders_docx(tmp_path: Path) -> None: assert render_file(source, format="docx").binary == render(SOURCE, format="docx").binary -def test_diagnostics_are_the_other_formats_plus_the_interim_warning() -> None: +def test_diagnostics_are_the_other_formats_plus_the_planners() -> None: document = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") docx_result = render(document, format="docx").diagnostics - assert docx_result[:-1] == render(document, format="text").diagnostics - warning = docx_result[-1] - assert (warning.rule, warning.level, warning.file) == ("render-docx-interim", "warning", "") - assert warning.message.startswith("DOCX output is incomplete in this version:") + assert [d for d in docx_result if not d.rule.startswith("render-docx")] == render(document, format="text").diagnostics + assert "render-docx-interim" not in {d.rule for d in docx_result} -def test_the_interim_warning_is_one_warning_that_does_not_refuse_under_strict() -> None: +def test_a_clean_document_has_no_docx_diagnostics_and_does_not_refuse_under_strict() -> None: result = render(FRONT + "# A\n", format="docx", strict=True, filename="d.lgd") - assert [(d.rule, d.file) for d in result.diagnostics if d.rule.startswith("render-")] == [ - ("render-docx-interim", "d.lgd")] + assert [d for d in result.diagnostics if d.rule.startswith("render-docx")] == [] assert result.ok - assert not [d for d in render(FRONT + "# A\n", format="html").diagnostics if d.rule == "render-docx-interim"] def test_the_planners_findings_follow_the_renderers_in_the_order_of_their_lines_with_the_documents_file() -> None: @@ -637,7 +638,7 @@ def test_the_planners_findings_follow_the_renderers_in_the_order_of_their_lines_ result = render(source, format="docx", filename="deal.lgd") text = render(source, format="text", filename="deal.lgd").diagnostics assert [d.rule for d in result.diagnostics] == [ - *(d.rule for d in text), "render-docx-letter-overflow", "render-docx-interim"] + *(d.rule for d in text), "render-docx-letter-overflow"] assert result.diagnostics[:len(text)] == text and {d.file for d in result.diagnostics} == {"deal.lgd"} assert "ref-broken" in {d.rule for d in text} @@ -646,16 +647,8 @@ def test_the_planners_findings_are_ordered_with_the_renderers_own_by_line() -> N source = FRONT + "# A\n\n" + "- r\n" * 30 + "\nA [link](javascript:alert) here.\n" result = render(source, format="docx") found = [(d.rule, d.line) for d in result.diagnostics if d.rule.startswith("render-")] - assert [rule for rule, _ in found] == ["render-docx-letter-overflow", "render-link-refused", "render-docx-interim"] - assert found[0][1] < found[1][1] and found[2][1] is None - - -def test_the_interim_warning_names_what_is_still_missing() -> None: - warning = render(FRONT + "# A\n", format="docx").diagnostics[-1] - assert warning.rule == "render-docx-interim" - for missing in ("title block", "table of contents", "attachments", "signature blocks"): - assert missing in warning.message - assert "carry no numbers" not in warning.message + assert [rule for rule, _ in found] == ["render-docx-letter-overflow", "render-link-refused"] + assert found[0][1] < found[1][1] def test_the_writer_gets_the_plan_the_render_reported_on() -> None: @@ -853,9 +846,9 @@ def test_code_is_left_aligned_even_when_the_body_is_justified() -> None: '_rels/.rels': 'd6f21829c0081670bdb417d3e3f644b81134a121e074fb153c7686faaa8b167b', 'docProps/core.xml': '159ed0e537106ab29739fd357a5f6d89a2c0b0154daefd4c13d878bd16dff47e', 'docProps/app.xml': 'b285204d159fd58da80b6a0242cef523448f9c3a598b357b15faf4627160dcc3', - 'word/document.xml': 'fe9fef7508bd6260d6c6fb707080e5e6ef8987d26e4a0088933c8cd19ce664a4', + 'word/document.xml': '39cbe0403104ccefefb9b936e3e432b266d651e54628873ff2d71a57dde798c8', 'word/_rels/document.xml.rels': 'ada136c2e7d05a317579c9503c1f0dd8dfaa8f0cce687df86fe59e36d5fa5d57', - 'word/styles.xml': 'cdb4ca1d3be44bd27761763c040ee637516ed7aa7cf995e2cc0b60cc80d7f8ac', + 'word/styles.xml': '5c6b9461b055c8c123fbb8e57385fffb2effe991107316ba065da778854d0862', 'word/numbering.xml': '368f179b5e26a34c1f23db597966dfd96b43dc96801a9a7ded728c379db07f59', 'word/settings.xml': 'b3c72bc5720862c1a9c1449d89903673777f30df6ff38ceb23da73e21cad22e1', 'word/footer1.xml': 'e693781f92e5a7ccf747fd2ed3facb627e448f389b021882cac589d746aa709b', From 1ae4e80918fb608484aabf2fdd26fec1f47abeb7 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:35:57 +0000 Subject: [PATCH 094/118] Document the DOCX front and back matter and the manual Word checklist README, conformance notes, architecture, roadmap and the style-template reference describe the matter and drop the interim warning. PUBLISHING.md gains the checklist for opening the golden documents in Word before a release. --- .github/PUBLISHING.md | 58 +++++++++++++++++++++++++++++++++++++++++ CONFORMANCE.md | 2 +- README.md | 15 ++++++++--- docs/architecture.md | 7 ++--- docs/roadmap.md | 16 +++++++----- docs/style-templates.md | 15 +++++++++-- 6 files changed, 98 insertions(+), 15 deletions(-) diff --git a/.github/PUBLISHING.md b/.github/PUBLISHING.md index a812aaa..8c3e9a0 100644 --- a/.github/PUBLISHING.md +++ b/.github/PUBLISHING.md @@ -78,6 +78,64 @@ legaldown-render --version The extra index is needed because the dependencies (`legaldown-validator`, Babel, PyYAML, markdown-it-py) come from PyPI. +## Checking the DOCX in Word (before a release) + +CI proves the DOCX against the ISO/IEC 29500 schemas and in LibreOffice; it cannot run Word. Before a release, a maintainer +opens the golden documents in Word (the desktop application, a current version) and checks what only Word can show. Make +the documents with a table of contents, for each of the three built-in styles: + +```bash +for style in default continental outline; do + for doc in features template; do + legaldown-render tests/documents/$doc.lgd -s $style --set contents.enabled=true --set contents.depth=3 \ + -o /tmp/word-check/$doc.$style.docx + done +done +``` + +For each file: + +1. **Opens clean.** No "unreadable content" or repair prompt, and Word does not open it in Compatibility Mode. + Answer *Yes* to the prompt to update the fields (`docx.update_fields_on_open`), or press Ctrl+A, F9. +2. **Table of contents.** It lists exactly the entries of the HTML contents (numbers and titles; the condition + labels of a template after the title), indented by level, with a dotted leader and page numbers that are right. + The attachments heading and each attachment's title are listed (at the levels the HTML shows) and nothing else is: + not the "Contents" label, no quote or note. A click on an entry (Ctrl+click) goes to the heading. Right-click the + TOC, *Update field*, *Update entire table*: the same entries come back. +3. **References.** Every `{{ref:}}` reads as in the HTML before and after the update (Ctrl+A, F9). Insert a clause + (a heading, a list item, a numbered paragraph) above a referenced one, update the fields: the numbers after it move + and every reference and the contents follow. A reference to an attachment (`{{attach:}}`) shows its title and links + to it. +4. **Numbering after inserting a clause.** Press Enter at the end of a numbered item and at the start of one: the new + paragraph is numbered, the rest renumbers, nested lists restart under it. In the continental style a new + sub-item continues `a)`, `b)`. +5. **Zero-length bookmarks.** A `REF \n` to the zero-length (point) bookmark at the start of a numbered paragraph + returns the number. Press Enter at the start of that paragraph and update the fields: the references still + point at the paragraph they pointed at and read right. +6. **`none` scheme.** With `--set numbering.scheme=none`, a heading referenced with `{{ref:}}` shows its title text + (`REF \h`), and the references inside a heading that refers to itself are plain text: no "Error! Reference source + not found" and no self-reference. +7. **Front and back matter.** The title block (header rows, parties) is two aligned columns without borders; each + attachment starts on a new page with `attachments.separator: page-break` and has a rule above it with `rule`; the + signature blocks are borderless, with a signing line above each name, and stay on one page (try a longer document). +8. **The numbering corner cases:** + - `\t` with words before the number: a label such as `Article {n}` or `§ {n}` referenced with `{n}`, whose reference + drops the words (`REF \n \t`): Word shows the bare number, where LibreOffice ignores `\t` and shows the words. + - a period before the number (`No. {n}`): the reference to the item reads as the HTML reads it, or the render + reported `render-docx-reference` and the reference is fixed text. + - skipped heading levels (a `###` straight after a `#`, an Error for the validator but rendered): the numbers after + it, and the references to them, read as the plain-text output reads them, or the render reported a + `render-docx-reference` Warning and the reference is fixed text; nothing is unreadable. + - counters shared across nums: a nested list of the same kind restarts under each new parent; a list of another + kind restarts by a fixed setting (the render reported `render-docx-static-restart`). + - Roman numerals of 4000 and over, and letters past 26: Word's text (`render-docx-roman-overflow`, + `render-docx-letter-overflow`) is the one the render reported, not a different one. +9. **Page layout.** The running footer shows `n / N`; a header template with `{title}`, `{version}` and `{date}` shows + them; a landscape page is landscape when printed (print preview). + +Record what differs in the release notes. A difference from LibreOffice is expected where the documentation says Word +and LibreOffice differ (`\t`, `lvlRestart`, `SEQ \s`, `w:updateFields`). + ## Notes - **Version numbers are permanent.** PyPI does not allow re-uploading a version, even after diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 4d24853..5cbfd14 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -20,7 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | -| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference for which Word's fields cannot reproduce the designation (a heading level skipped before the target under a `{n}`-style scheme, text around the counter that Word's `\t` switch cannot drop such as `(a)`, `{n}:` or `No. {n}`) is fixed text, reported. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says, but **the file is still interim**: there is no table of contents, title block, attachments or signature blocks, alternatives that share a number (§15.8) are numbered in sequence (reported), and a label that shows the section's number in a list item (the continental `5.1`) is drawn without it (reported). The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a list continued by a fixed number, which will not follow edits), `render-docx-number-mismatch` (any other difference, including alternatives), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-static-restart` (a list that restarts by a fixed setting) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). Every DOCX render also reports the Warning `render-docx-interim` ("DOCX output is incomplete in this version") until the body is complete; it goes away then. HTML is the format this renderer claims §13.6 with | +| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference for which Word's fields cannot reproduce the designation (a heading level skipped before the target under a `{n}`-style scheme, text around the counter that Word's `\t` switch cannot drop such as `(a)`, `{n}:` or `No. {n}`) is fixed text, reported. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says, **The front and back matter are written too**, in the order the HTML shows them: the title block (header rows and parties), a live table of contents (`TOC \o "1-N" \h \z \u` with cached entries equal to the HTML contents, each a link with a `PAGEREF` page number Word fills in on update; the attachments are listed by outline level), the attachments (bookmarked so `{{attach:}}` resolves, separated by a page break or a rule per `attachments.separator`) and the signature blocks (§3.6, borderless tables kept on one page). A LibreOffice round trip shows that after its indexes are updated the contents lists exactly the HTML entries with page numbers, also after a heading is inserted in the editor, and that the document reads in the plain-text writer's order. Remaining differences are the reported ones: alternatives that share a number (§15.8) are numbered in sequence, and a label that shows the section's number in a list item (the continental `5.1`) is drawn without it (reported). The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a list continued by a fixed number, which will not follow edits), `render-docx-number-mismatch` (any other difference, including alternatives), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-static-restart` (a list that restarts by a fixed setting) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). The Word-only checks (the contents' page numbers, what `\t` does, numbering after an insertion) are a manual checklist before a release (`.github/PUBLISHING.md`). HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | diff --git a/README.md b/README.md index cf84196..5db0bb5 100644 --- a/README.md +++ b/README.md @@ -155,9 +155,18 @@ section's number in a list item, alternatives that share a number, a list restar reports it as a `render-docx-*` Warning or Info with the line, see [the conformance notes](https://github.com/ForLegalAI/legaldown-render/blob/main/CONFORMANCE.md). The body writer also lays out paragraphs, quotes, drafting notes, code, tables with a repeated header row, rules, links, blanks and -choices. **The file is still interim:** it has **no title block, table of contents, attachments or signature -blocks**, and every render reports the Warning `render-docx-interim` to say so. Do not send such a file as a -contract. Settings Word cannot express (a `%` length, say) are an error for DOCX only. See +choices. **Front and back matter are there too, in the order the HTML shows them:** the title and subtitle, the header +rows and the parties (borderless tables), the **table of contents** (a live `TOC \o "1-N" \h \z \u` field whose +cached entries are the contents the HTML shows, each a link to its heading with a `PAGEREF` page number that Word +fills in when it updates the field; with `contents.attachments` the attachments are listed by outline level, so +the field has no locale-dependent style list), the attachments (the heading and a titled placeholder each, bookmarked so +that `{{attach:}}` is a `REF` field or a link, separated by a page break or a rule as `attachments.separator` says), +and the signature blocks (borderless tables with signing lines, kept on one page). Viewers that never update +fields (Word Online, Quick Look, Google Docs import, Outlook's preview) show the contents without page numbers. A +reference whose form Word cannot compose from fields (a custom style's `{path}` item labels, say) is fixed text and +is reported as `render-docx-reference`. Before a DOCX is sent as a contract, the maintainers' manual check in Word +([the checklist](https://github.com/ForLegalAI/legaldown-render/blob/main/.github/PUBLISHING.md)) is the last +word. Settings Word cannot express (a `%` length, say) are an error for DOCX only. See [Styles and preferences](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/style-templates.md#docx) and the [roadmap](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/roadmap.md). diff --git a/docs/architecture.md b/docs/architecture.md index 6f72381..1775405 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -355,7 +355,7 @@ heading level that is skipped is the Core Error `heading-skip`; the plan only no |---|---| | `TextWriter` | Plain text (§13.6). A soft line break inside a paragraph is a space, a hard break a new line, and lists are indented with their labels, one quote stays one quote, and tables keep escaped cells and alignment. It takes the style's `attachments.separator` (default `page-break`; `rule` and `page-break` both print `---` before the attachments heading and between attachments). It is the **test oracle** for resolution | | `HtmlWriter` | A self-contained HTML5 page, or with `standalone=False` just the `
`. Semantic markup, with labels as real text rather than CSS counters, and a stylesheet generated from the style's presentation settings, including print rules | -| `DocxWriter` | A `.docx` file, as bytes (`RenderResult.binary`; `output` is empty). OOXML written directly with `zipfile` and `xml.etree` ([ADR 0008](decisions/0008-docx-word-native.md)): the parts, the styles generated from the style template, settings, page setup, running headers and footers, properties, a deterministic ZIP, and one sanitiser for every text and attribute. **The body is laid out as a `DocxPlan` says** (`resolve/docx_plan.py`, the planner's data model): `numbering.xml` from its abstract numberings and nums; each section, list item and top-level paragraph gets `w:numPr`, `w:ind`, bookmarks and, for a later alternative that shares a number (§15.8), `numId` 0 and its number as `REF` fields, as its `UnitPlan` says; a `{{ref:}}` is the literal text and `REF` fields (cached results as planned) of its `RefPlan`; a `{{term:}}` or an `{{attach:}}` is an internal hyperlink (or a `REF` for an attachment's title), a defined term a bookmark; every block (lists with their continuation blocks, quotes, drafting notes, code, tables with a repeated header row, rules) and inline (emphasis, strong, code, external links through relationships, breaks, images as alt text, blanks, choices, failure markers) has its Word form. The writer decides none of it (ADR 0003) and a field that points at a bookmark the document does not hold is a bug in the plan (`InternalError`). **Interim:** the title block, the table of contents, the attachments and the signature blocks are not written yet; the numbering, bookmarks and cross-reference fields are, from the plan `render` computes (the writer plans a tree itself when it is used directly and given none). The DOCX-only settings are converted and checked before rendering (`style/docx.py`) | +| `DocxWriter` | A `.docx` file, as bytes (`RenderResult.binary`; `output` is empty). OOXML written directly with `zipfile` and `xml.etree` ([ADR 0008](decisions/0008-docx-word-native.md)): the parts, the styles generated from the style template, settings, page setup, running headers and footers, properties, a deterministic ZIP, and one sanitiser for every text and attribute. **The body is laid out as a `DocxPlan` says** (`resolve/docx_plan.py`, the planner's data model): `numbering.xml` from its abstract numberings and nums; each section, list item and top-level paragraph gets `w:numPr`, `w:ind`, bookmarks and, for a later alternative that shares a number (§15.8), `numId` 0 and its number as `REF` fields, as its `UnitPlan` says; a `{{ref:}}` is the literal text and `REF` fields (cached results as planned) of its `RefPlan`; a `{{term:}}` or an `{{attach:}}` is an internal hyperlink (or a `REF` for an attachment's title), a defined term a bookmark; every block (lists with their continuation blocks, quotes, drafting notes, code, tables with a repeated header row, rules) and inline (emphasis, strong, code, external links through relationships, breaks, images as alt text, blanks, choices, failure markers) has its Word form. The writer decides none of it (ADR 0003) and a field that points at a bookmark the document does not hold is a bug in the plan (`InternalError`). **Front and back matter** (`writers/docx/matter.py`, in the order the HTML shows them): the header rows and the parties as borderless tables; the contents label (style `ContentsHeading`: no outline level, so never an entry) and one complex `TOC \o "1-N" \h \z \u` field (N is `contents.depth`) whose cached result is `plan.toc` as `TOC1` to `TOC5` paragraphs (a hyperlink to the `_Toc` bookmark holding number, separator, title, condition label, a right tab with a dot leader at the text width and a `PAGEREF` field with an empty cached page), opened in the first entry and closed in a paragraph of its own; the attachments heading and each attachment's title are paragraphs of styles with an outline level (not a `\t` style list, whose separator depends on the locale), bookmarked by the plan's `ldatt_` name (so `{{attach:}}` resolves) and their `_Toc` name, with `attachments.separator` as a page break (the paragraph property `pageBreakBefore`, so no empty paragraph or blank page is left) or a rule above each attachment; the signature blocks as a borderless table, two parties to a row, rows that do not split and paragraphs that keep with the next. Every bookmark is written through the same `Bookmarks` as the body, whose first, dry run tells the second which names exist. The numbering, bookmarks and cross-reference fields come from the plan `render` computes (the writer plans a tree itself when it is used directly and given none). The DOCX-only settings are converted and checked before rendering (`style/docx.py`) | | PDF | Planned ([ADR 0004](decisions/0004-html-first.md)) | ## Package layout @@ -388,7 +388,7 @@ src/legaldown_render/ └── docx/ the DOCX writer: ooxml.py (sanitiser, builders), package.py (parts, relationships, ZIP), styles.py, settings.py, headers.py, properties.py, numbering.py (numbering.xml from the plan), body.py (blocks, numbered paragraphs, tables), runs.py (inlines, fields, - hyperlinks) + hyperlinks), matter.py (title block, contents field, attachments, signature blocks) ``` ## Dependencies @@ -425,7 +425,7 @@ Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can ev `ref-not-enumerated`, `path-not-relative` and `path-outside-root` (Errors for a file reference that is not relative, or that leaves the document root, §2.3; reported here because validator 0.4 checks neither), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and `render-docx-interim` (a Warning on every DOCX render, while the DOCX body is incomplete), and the rules the DOCX planner reports on what Word draws differently from the other formats (`render-docx-section-label`, `-list-depth`, `-letter-overflow`, `-roman-overflow`, `-shared-counter`, `-number-mismatch`, `-reference`, `-reference-cycle`, as Warnings, `-static-restart` and `-reference-word-only` as Infos; see "The DOCX planner"). The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and the rules the DOCX planner reports on what Word draws differently from the other formats (`render-docx-section-label`, `-list-depth`, `-letter-overflow`, `-roman-overflow`, `-shared-counter`, `-number-mismatch`, `-reference`, `-reference-cycle`, as Warnings, `-static-restart` and `-reference-word-only` as Infos; see "The DOCX planner"). The validator's diagnostics carry their line (§16.9), and so do the renderer's own, and the CLI prints it as `file:line:`. @@ -607,5 +607,6 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_docx_escaping.py` | Well-formedness of every part for any text, with a seeded fuzz and the characters XML cannot carry | | `tests/test_docx_validation.py` | Every part of every generated document against the ISO/IEC 29500-4 schemas (marker `ooxml_schema`; needs lxml and `python tests/schemas/fetch.py`) | | `tests/test_docx_roundtrip.py` | LibreOffice as the oracle of the numbering and the cross-references, in one LibreOffice session: every document of the corpus (the tests', the specification's, stress documents with cycles and unanchored ancestors) under the default, outline, continental, mixed and `none` schemes, with every cached field result made wrong (`XXX`), every numbered paragraph marked and a line per reference added; the number LibreOffice draws, every reference it computes and every body paragraph's text equal what the file cached and the renderer wrote, modulo the gaps the plan reports (`\t` ignored, letters past 26, labels that need the section, alternatives; the table of contents is not written yet, so its entries and the `_Toc` bookmarks are not exercised); and edit-then-update (`tests/lo/uno_edit.py`): items and headings inserted before the targets as a lawyer does, fields updated, every reference and the numbers equal the renderer's for the edited source (marker `libreoffice`) | +| `tests/test_docx_matter.py` | The front and back matter: the XML of the title block, the `TOC` field and its cached entries, the attachments and the signature blocks; documents with every part (hostile text too) against the schemas; LibreOffice listing exactly the contents' entries after an update, resolving `{{attach:}}`, paging the attachments as configured, taking a heading inserted in the editor into the contents, and reading in the text writer's order | | `tests/test_docx_libreoffice.py` | The documents opened in LibreOffice through UNO (`tests/lo/uno_export.py`), fields updated: the words in order, header, footer, page count (marker `libreoffice`; skipped without `soffice` and `python3-uno`) | | `tests/conformance/` | Every example and fixture in the specification repository renders, every internal link has a target, and each invalid fixture renders its rule's failure marker. Needs `$LEGALDOWN_SPEC_DIR` or a sibling `../LegalDown` checkout | diff --git a/docs/roadmap.md b/docs/roadmap.md index 69515fd..17c2cc4 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -100,13 +100,13 @@ values appear only as the cached results of fields. Decided in - **Package core (done).** `docx` writer in the base package, with the standard library only (no `[docx]` extra): the OOXML parts, styles generated from the style template, settings (compatibility mode 15, `w:updateFields`), page setup, running headers and footers with `PAGE`/`NUMPAGES` fields, - document properties, deterministic output. `RenderResult.binary`, `-o x.docx`, `-o -`. The body is - an interim one at that point: the title, `heading N` paragraphs and plain-text paragraphs, with no numbers yet + document properties, deterministic output. `RenderResult.binary`, `-o x.docx`, `-o -`. The body at that point + was an interim one: the title, `heading N` paragraphs and plain-text paragraphs, with no numbers yet - **Behaviour changes this brings.** `-o -` now writes to standard output for every format (it used to create a file named `-`); `.docx` is a writable extension, and a DOCX with no `-o` goes to standard output when that is not a terminal; `--fragment` with DOCX is a usage error; `RenderTree` gains `version` and - `effective_date` (plain text), filled whether or not the title block shows them; `.docx` renders add - the Warning `render-docx-interim` until the body is complete + `effective_date` (plain text), filled whether or not the title block shows them; `.docx` renders added + the Warning `render-docx-interim` until the body was complete (it is gone since the front and back matter) - **Style settings (done).** `page.header` and `page.footer` (the HTML writer ignores them for now), `docx.update_fields_on_open`; settings Word cannot express are a `StyleError` for DOCX - **Verification (done).** Every part validated against the ISO/IEC 29500 schemas (fetched in CI, not @@ -120,9 +120,13 @@ values appear only as the cached results of fields. Decided in and the table of contents is planned from Word's numbers; `render` runs the planner and reports its `render-docx-*` diagnostics. A LibreOffice round trip with wrong cached results checks the numbers and references against the plain-text oracle +- **Front and back matter (done).** The title block (header rows and parties), the `TOC` field with its cached + entries and `PAGEREF` page numbers, the attachments (bookmarked, separated by a page break or a rule) and the + signature blocks; the `render-docx-interim` Warning is gone. A LibreOffice round trip checks the contents against the + HTML's entries, also after a heading is inserted in the editor; a manual Word checklist for releases is in + `.github/PUBLISHING.md` - **Next.** Qualified labels (the continental `5.1`) on the heading list, alternatives that share a number, - restarts Word would otherwise get wrong; the table of contents field, title block, attachments and signature - blocks; the corpus checked also after edits made through UNO; a manual Word checklist for releases + restarts Word would otherwise get wrong; the corpus checked also after edits made through UNO ### v0.6 — PDF diff --git a/docs/style-templates.md b/docs/style-templates.md index 7106b2c..d9e6a07 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -267,8 +267,19 @@ style in Word applies to every paragraph that uses it. |---|---| | `docx.update_fields_on_open` | `true` (the default) asks Word to update every field when it opens the file: the cross-references, the table of contents and its page numbers. Word asks the reader once ("This document contains fields that may refer to other files. Update the fields?"). The fields hold the values the renderer computed either way, so a viewer that never updates fields (Word Online, Quick Look, Outlook's preview) shows the document right, but the table of contents without page numbers. LibreOffice and Word Online ignore the setting. `false` writes no such request | -The body of the DOCX is not complete yet (it is the title, headings and plain text, with no numbers; see -the [roadmap](roadmap.md)); the settings above are what is already in place. +Settings that were already part of the template gain a DOCX meaning: + +| Setting | In the DOCX | +|---|---| +| `contents.enabled`, `contents.depth` | A live `TOC \o "1-" \h \z \u` field after the title block, with the contents' entries as its cached result (`toc 1` to `toc 5` paragraphs, a dotted right tab at the text width, a `PAGEREF` page number Word fills in when it updates the field). The contents label is a `Contents Heading` paragraph: no outline level, no number, so never an entry | +| `contents.attachments` | The attachments heading and each attachment title are paragraphs of the styles `Attachments Heading` and `Attachment Title`, which carry an outline level, so Word's TOC lists them (`\u`); without it they are body text for the contents. At `contents.depth: 1` the titles are listed with the headings (the first level), not under the attachments heading | +| `attachments.separator` | `page-break`: a page break before the attachments part and between attachments (the paragraph property "page break before", so no empty paragraph or blank page is left); `rule`: a line above each attachment; `none`: nothing | +| `signatures.enabled` | Borderless tables with a signing line, kept on one page | +| `title_block.parties`, `title_block.effective_date`, `title_block.version` | The header rows and the parties, as borderless tables | + +The styles for this matter (`Title Block Text`, `Party Side`, `Party Name`, `Attachments Heading`, `Attachment Title`, +`Attachment File`, `Signature Side`, `Signature Text`, `Signature Line`) are Word styles like the others: a lawyer who +changes one in Word changes every paragraph that uses it. ### Trusted input From 056f8ffb142c5be4422c43544667e9b08e912cad Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:38:05 +0000 Subject: [PATCH 095/118] Plan qualified labels, alternatives and resynchronised numbers for DOCX A label that needs the section's number now rides the heading list's level after the section's (continental items and numbered paragraphs), and lists nested in such items ride its reserved levels so Word keeps restarting them by structure. A later alternative that shares a number takes numId 0 and the holder's number as a field, with the sections and lists under it on a num of their own. A unit Word would count off the resolver's is restarted at a fixed value and reported. The planner no longer reports alternatives as number mismatches, and the property tests check every unit. --- src/legaldown_render/resolve/docx_plan.py | 409 ++++++++++++--- tests/test_docx_plan.py | 602 +++++++++++++++++++--- tests/test_docx_roundtrip.py | 6 +- 3 files changed, 853 insertions(+), 164 deletions(-) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index a7a4f1d..734ab27 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -429,6 +429,44 @@ class _Placed(NamedTuple): abstract: _Abstract +class _Paragraphs(NamedTuple): + """Where the numbered paragraphs of one section go: their num, the abstractNum and level it is on, and whether + their label is the foreign-free one for want of a place where Word can show the section.""" + + num: Num + abstract: _Abstract + ilvl: int + fallback: bool + + +class _Alternative: + """A later alternative that shares a number, for the sections under it: they restart in a num of their own, made + when the first of them is numbered.""" + + def __init__(self, level: int) -> None: + self.level = level + self.num: Num | None = None + + +#: The owner of the counter of the unit a level text is being made for. +_SELF = object() + +#: What the list of a nested list that restarts by a fixed setting is called (``UnitPlan.cause``), and what is reported +#: about it (information: Word's own "Restart at 1", which cannot be off when the document is opened). +_RESTARTS: dict[str, tuple[tuple[str, ...], str]] = { + "restart-family": (("restart-family",), "A list nested in a list of another kind restarts by a fixed setting (Restart " + "at 1), not by structure: an item added above it in Word will not restart it."), + "restart-section": (("restart-section",), "A list nested in an item whose number carries the section's cannot " + "restart by structure here: it restarts by a fixed setting (Restart at 1), and an item " + "added above it in Word will not restart it."), + "restart-alternative": (("restart-alternative",), "A list nested in an alternative that shares a number restarts by a " + "fixed setting (Restart at 1), not by structure: an item added above it in Word will not " + "restart it."), + "resync": (("resync",), "A list that Word would continue from another list in the same item restarts by a fixed " + "setting (Restart at 1), not by structure."), +} + + class _Reference(NamedTuple): """The ``{{ref:}}`` to one unit that the plan builds fields for: the unit, its section, what the renderer shows and the first line it stands at (None outside the body).""" @@ -480,7 +518,22 @@ def __init__(self, tree: RenderTree, file: str | None) -> None: self.parent: dict[str, str | None] = {} self.section_level: dict[str, int] = {} self.placed: dict[str, _Placed] = {} - self.paragraph_nums: dict[str, Num] = {} + self.paragraph_nums: dict[str, _Paragraphs] = {} + #: A later alternative that shares a number -> the unit it repeats and the suffix of its level; and every such + #: unit (items and paragraphs too) -> the unit that holds its number, for designations and fields. + self.alts: dict[str, tuple[str, Literal["tab", "space"]]] = {} + self.canon: dict[str, str] = {} + self.alternatives: list[_Alternative] = [] + #: The levels of the heading abstractNum a unit other than a heading has defined, and the units on it that + #: Word would number off the resolver's (reported once). + self.claimed: set[int] = set() + self.collisions: list[tuple[_Unit, str]] = [] + #: Items and paragraphs, and where the text of each on the heading list starts (twips), for its nested items. + self.item_units: set[str] = set() + self.text_left: dict[str, int] = {} + #: The level after the sections' fifth is kept for units of a level-5 section when the document has one and + #: a label that needs the section. + self.reserve5 = False self.flagged: set[str] = set() self.reported: set[tuple[str, object]] = set() self.drifts: list[_Drift] = [] @@ -702,6 +755,7 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan built[target] = made self.report_references() needed = {piece.owner for made in built.values() for piece in made.pieces if isinstance(piece, Field)} + needed |= {holder for holder, _ in self.alts.values()} #: A bookmark holds its paragraph's text only if a field returns that text; any other is a point at the start #: of the paragraph, so that no field ever stands inside the bookmark it points at. enclosing = {piece.owner for made in built.values() for piece in made.pieces @@ -739,6 +793,13 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan units[unit] = replace(units[unit], bookmarks=(name,), collapsed=unit not in enclosing) for unit, targets in cyclic.items(): units[unit] = replace(units[unit], literal_refs=targets) + for unit, (holder, suff) in self.alts.items(): + # The number of a later alternative is the number of the unit it repeats (§15.8). + shared = self.units[holder] + tail = shared.label[len(shared.word_ref):] + segments: list[Segment] = [FieldSeg(bookmark[holder], True, False, True, shared.word_ref, shared.word_ref)] + units[unit] = replace(units[unit], alt=AltNumber( + (*segments, *([Literal_(tail)] if tail else [])), suff)) for inline in iter_tree_inlines(self.tree): if isinstance(inline, DefinedTerm) and inline.defining and inline.anchor and inline.anchor not in anchors: anchors[inline.anchor] = names.anchor("lddef_", inline.anchor.removeprefix("def:")) @@ -786,6 +847,9 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan def run(self) -> tuple[DocxPlan, list[Diagnostic]]: if self.formats.headings: self.heading_num = self.new_num(self.abstract("headings")) + formats = [f for f in (*self.formats.enumerated[:1], self.formats.paragraphs) if f is not None] + self.reserve5 = any(section.level == 5 for section in self.tree.sections) and any( + shows_section(f.label, [f.ref], 0) for f in formats) for block in self.tree.preamble: self.block(block, None) for section in self.tree.sections: @@ -794,6 +858,7 @@ def run(self) -> tuple[DocxPlan, list[Diagnostic]]: self.block(block, section) for drift in self.drifts: self.report_drift(drift) + self.report_collisions() units, anchors, refs, toc = self.finish() abstracts = tuple( AbstractNum(a.id, a.kind, f"{0x4C440000 + a.id:08X}", # type: ignore[arg-type] @@ -803,28 +868,37 @@ def run(self) -> tuple[DocxPlan, list[Diagnostic]]: return (DocxPlan(abstracts, tuple(self.nums), heading_id, units=units, anchors=anchors, refs=refs, toc=toc), by_line(self.diagnostics)) - def block(self, block: object, section: Section | None) -> None: + def block(self, block: object, section: Section | None, quoted: bool = False) -> None: + """Plan *block*; *quoted*: it stands in a quote or a drafting note (§13.2), whose items are numbered like any + other but, being built on the section whatever they are nested in, never ride the heading list.""" match block: case Paragraph(unit=str()) if section is not None: self.paragraph(block, section) case List(): - self.list_(block, section) + self.list_(block, section, quoted) case Quote(blocks=blocks) | DraftingNote(blocks=blocks): for child in blocks: - self.block(child, section) + self.block(child, section, True) # -- what a unit gets ----------------------------------------------------------- def add(self, node: _Unit, kind: Literal["section", "item", "paragraph"], num: Num | None, ilvl: int | None, *, - explicit: bool, cause: tuple[str, ...] = (), step: Step | None = None) -> None: + explicit: bool, cause: tuple[str, ...] = (), step: Step | None = None, + ind: tuple[int, int] | None = None, holder: str | None = None) -> None: + """Record the plan of *node*. A later alternative (*holder*: the unit whose number it repeats) takes the + label of its holder and ``numId`` 0; its number is written as fields by :meth:`finish`.""" assert node.unit is not None self.nodes[node.unit] = node + num_id = num.id if num else None + label = word_ref = "" + count = None if step is not None: self.steps[node.unit] = step - self.units[node.unit] = UnitPlan( - node.unit, kind, num.id if num else None, ilvl, explicit, None, None, - step.label if step else "", step.word_ref if step else "", step.count if step else None, - cause, (), None) + label, word_ref, count = step.label, step.word_ref, step.count + if holder is not None: + num_id, label, word_ref = 0, self.units[holder].label, self.units[holder].word_ref + self.units[node.unit] = UnitPlan(node.unit, kind, num_id, ilvl, explicit, ind, None, label, word_ref, count, + cause, (), None) def ancestor_flagged(self, unit: str) -> bool: parent = self.parent.get(unit) @@ -906,7 +980,8 @@ def shared_counter(self, message: str, line: int | None) -> None: def section_designation(self, section: Section) -> list[Shown]: """The designation of *section* as tokens, written as the resolver writes it (§13.1): the ``ref`` of every part of its number after the one before; a part the document skipped is a :class:`Ghost` that no unit holds. Under - the ``none`` scheme it is the heading's text (§13.3), and empty when that is.""" + the ``none`` scheme it is the heading's text (§13.3), and empty when that is. A later alternative that shares + a number with another holds none of its own: its part is the counter of the unit it repeats (§15.8).""" assert section.unit is not None if self.textual: text = section.designation or "" @@ -915,7 +990,7 @@ def section_designation(self, section: Section) -> list[Shown]: del self.owners[section.level - 1:] while len(self.owners) < section.level - 1: self.owners.append((Ghost(section.unit, len(self.owners)), 1)) - self.owners.append((section.unit, section.number)) + self.owners.append((self.canon.get(section.unit, section.unit), section.number)) designation: list[Shown] = [] for part, (owner, value) in enumerate(self.owners): level = self.formats.headings[min(part, len(self.formats.headings) - 1)] @@ -929,20 +1004,32 @@ def section(self, section: Section) -> None: assert unit is not None self.parent[unit] = section.parent_unit self.section_level[unit] = section.level - self.desig[unit] = self.section_designation(section) self.heading_text[unit] = _shown(section.title) if self.heading_num is None or section.number is None: + self.desig[unit] = self.section_designation(section) self.add(section, "section", None, None, explicit=False) return + abstract = self.by_kind["headings"] ilvl = section.level - 1 parent = section.parent_unit skipped = (parent is None and section.level > 1) or ( parent is not None and self.section_level[parent] + 1 < section.level) - cause = ("skipped-level",) if skipped else () - step = self.sim.step(self.heading_num.id, ilvl, unit) - self.add(section, "section", self.heading_num, ilvl, explicit=False, cause=cause, step=step) - self.check(section, "heading", step, self.formats.headings[ilvl].counter, ("heading", section.level), cause, - (0, ilvl)) + # The sections under a later alternative restart in a num of their own: the alternative counts nothing, so + # on the headings' num they would go on from the alternative it repeats (§15.8). + while self.alternatives and self.alternatives[-1].level >= section.level: + self.alternatives.pop() + num, explicit = self.heading_num, False + if self.alternatives: + context = self.alternatives[-1] + if context.num is None: + context.num = self.new_num(abstract, ilvl, section.number) + num, explicit = context.num, True + self.numbered(section, "section", num, abstract, ilvl, counter=self.formats.headings[ilvl].counter, + group=("heading", section.level), cause=("skipped-level",) if skipped else (), + explicit=explicit) + if unit in self.alts: + self.alternatives.append(_Alternative(section.level)) + self.desig[unit] = self.section_designation(section) def paragraph(self, paragraph: Paragraph, section: Section) -> None: assert paragraph.unit is not None and section.unit is not None @@ -953,49 +1040,237 @@ def paragraph(self, paragraph: Paragraph, section: Section) -> None: self.desig[paragraph.unit] = base self.add(paragraph, "paragraph", None, None, explicit=False) return - piece = fill_tokens(fmt.ref, n=[Count(paragraph.unit, format_counter(paragraph.number, fmt.counter))], - section=base) - self.desig[paragraph.unit] = extend_tokens(base, piece, textual=self.textual) - abstract = self.abstract("paragraphs") - num = self.paragraph_nums.get(section.unit) - if num is None: - num = self.paragraph_nums[section.unit] = self.new_num(abstract, 0) + shows = shows_section(fmt.label, [fmt.ref], 0, textual=self.textual) + entry = self.paragraph_nums.get(section.unit) + if entry is None: + slot = self.qualified(fmt, section) if shows else None + abstract, ilvl = slot or (self.abstract("paragraphs"), 0) + entry = self.paragraph_nums[section.unit] = _Paragraphs( + self.new_num(abstract, ilvl), abstract, ilvl, shows and slot is None) cause: tuple[str, ...] = () - if shows_section(fmt.label, [fmt.ref], 0, textual=self.textual): + if entry.fallback: cause = ("label-fallback",) self.fallback_label(fmt, "paragraphs", 0, "paragraph", paragraph.line) - step = self.sim.step(num.id, 0, paragraph.unit) - self.add(paragraph, "paragraph", num, 0, explicit=True, cause=cause, step=step) - self.check(paragraph, "paragraph", step, fmt.counter, ("paragraphs", section.unit), cause, (abstract.id, 0)) + self.numbered(paragraph, "paragraph", entry.num, entry.abstract, entry.ilvl, counter=fmt.counter, + group=("paragraphs", section.unit), cause=cause) + piece = fill_tokens(fmt.ref, n=[self.counted(paragraph, fmt.counter)], section=base) + self.desig[paragraph.unit] = extend_tokens(base, piece, textual=self.textual) - def list_(self, lst: List, section: Section | None) -> None: + def counted(self, node: ListItem | Paragraph, counter: str) -> Count: + """The counter token of *node* in a designation: the unit's own, or, for a later alternative, the one of the + unit it repeats.""" + assert node.unit is not None and node.number is not None + return Count(self.canon.get(node.unit, node.unit), format_counter(node.number, counter)) + + # -- the heading list's free levels (§2.5) ---------------------------------------- + + def candidate(self, fmt: LevelFormat, section: Section) -> LevelParts | None: + """The level text a unit of *fmt* directly in *section* would have on the heading list: its label filled in + as the resolver does, with the counter of each part of the section's number as ``%j`` of the heading level + that holds it and its own counter at the level after the section's. None when a counter is of another + owner or the text cannot be written.""" + assert section.unit is not None + designation = self.desig[section.unit] + own = [Count(_SELF, "")] + path = extend_tokens(designation, fill_tokens(fmt.ref, n=own, section=designation), textual=False) + tokens = fill_tokens(fmt.label, n=own, path=path, section=designation) + level_of = {owner: j for j, (owner, _) in enumerate(self.owners)} + parts: list[str | int] = [] + for token in tokens: + if isinstance(token, Count): + j = section.level if token.owner is _SELF else level_of.get(token.owner) + if j is None: + return None + parts.append(j) + elif parts and isinstance(parts[-1], str): + parts[-1] += token.text + else: + parts.append(token.text) + return None if level_text_problem(tuple(parts)) is not None else tuple(parts) + + def claim(self, ilvl: int, definition: LevelDef, *, section_level: bool) -> tuple[_Abstract, int] | None: + """The heading abstractNum at *ilvl* if the level reads as *definition*: a heading level when it is the same, + a free level (levels 6-9) when nothing claimed it or it is the same, which defines it. The level after the + sections' fifth, which only a level-5 section's units can use, is kept for them when the document has one.""" + abstract = self.by_kind["headings"] + current = abstract.levels[ilvl] + if ilvl <= 4 or ilvl in self.claimed: + same = (current.lvl_text, current.num_fmt) == (definition.lvl_text, definition.num_fmt) + return (abstract, ilvl) if same else None + if ilvl == 5 and self.reserve5 and not section_level: + return None + abstract.levels[ilvl] = definition + self.claimed.add(ilvl) + return abstract, ilvl + + def qualified(self, fmt: LevelFormat, section: Section) -> tuple[_Abstract, int] | None: + """Where the units of *fmt* directly in *section* go when their label needs the section's number: the heading + list's level after the section's (``%1.%2`` after a level-1 section), where Word can show the section's + counters, if that level reads as the label does. None: the label has a foreign-free form of its own + (:meth:`fallback_label`). Under the ``none`` scheme the section's number is its heading text.""" + if self.textual or "headings" not in self.by_kind: + return None + parts = self.candidate(fmt, section) + if parts is None: + return None + levels = self.by_kind["headings"].levels + definition = _level(section.level, NUM_FMT[fmt.counter], parts, + widths={j: levels[j].num_fmt for j in range(9)}) + return self.claim(section.level, definition, section_level=True) + + def reserved(self, fmt: LevelFormat, depth: int) -> tuple[_Abstract, int] | None: + """Where a list nested *depth* levels below a unit on the heading list goes: the heading abstractNum's level + ``4 + depth``, defined on first use with the label's foreign-free text (nothing of the section in it).""" + refs = [self.formats.enumerated[j % len(self.formats.enumerated)].ref for j in range(9)] + parts = level_text(fmt.label, refs, depth, section=(), textual=False) + if any(isinstance(part, int) and part != depth for part in parts) or level_text_problem(parts) is not None: + return None + ilvl = 4 + depth + return self.claim(ilvl, _level(ilvl, NUM_FMT[fmt.counter], tuple(ilvl if isinstance(p, int) else p + for p in parts)), section_level=False) + + # -- numbering a unit --------------------------------------------------------------- + + def indent(self, abstract: _Abstract, ilvl: int, unit: str, parent: str | None, *, alternative: bool + ) -> tuple[int, int] | None: + """The indent a unit carries itself, in twips ``(left, hanging)``: the level's for a later alternative (which + has no numbering to take it from), and for a unit of the heading abstractNum that is no heading (whose level + indents the headings) the width of the heading level it stands on, one step more for each level of nesting + below it.""" + level = abstract.levels[ilvl] + if abstract.kind == "headings" and unit in self.item_units: + if parent in self.text_left: + left, hanging = self.text_left[parent] + _STEP, _STEP + else: + left, hanging = level.left, level.hanging + self.text_left[unit] = left + return left, hanging + return (level.left, level.hanging) if alternative else None + + def numbered(self, node: _Unit, kind: Literal["section", "item", "paragraph"], num: Num, abstract: _Abstract, + ilvl: int, *, counter: str, group: object, cause: tuple[str, ...] = (), explicit: bool = True) -> Num: + """Number *node* on *num* at *ilvl* of *abstract* and check it against the resolver; returns the num it is on. + + * A later alternative whose number Word would not give (§15.8) takes ``numId`` 0 and the number of the unit + it repeats, as fields. + * A unit whose number Word would give differently (it goes on where the resolver restarts) is put on a num + that starts there, a fixed setting (§2.7). + * A label that shows a number of a unit outside the unit's own ancestors, which Word shows from the levels + of its list, is marked ``label-fallback``.""" + assert node.unit is not None and node.number is not None + unit = node.unit + word = {"section": "heading", "item": "list item", "paragraph": "paragraph"}[kind] + parent = node.parent_unit + if kind != "section": + self.item_units.add(unit) + peek = self.sim.peek(num.id, ilvl) + if node.shares_number_with is not None and peek != node.number: + holder = self.canon.get(node.shares_number_with, node.shares_number_with) + self.canon[unit] = holder + self.alts[unit] = (holder, abstract.levels[ilvl].suff) + self.add(node, kind, None, ilvl, explicit=True, cause=(*cause, "alternative"), holder=holder, + ind=self.indent(abstract, ilvl, unit, parent, alternative=True)) + return num + used = num + if peek != node.number: + used = self.new_num(abstract, ilvl, node.number) + cause, explicit = (*cause, "resync"), True + if abstract.kind == "headings": + self.collisions.append((node, word)) + else: + self.shared_counter( + "Word's counter for a level of this list was also used by another list in between; the numbers " + "are kept as the renderer shows them by restarting at a fixed number, which will not follow " + "items added or removed before it.", node.line) + step = self.sim.step(used.id, ilvl, unit) + if kind != "section": + ancestors = {unit} + above = parent + while above is not None: + ancestors.add(self.canon.get(above, above)) + above = self.parent.get(above) + if "label-fallback" not in cause and any( + isinstance(t, Count) and (isinstance(t.owner, Ghost) or t.owner not in ancestors) + for t in step.tokens): + # The label shows the counter of a level of its list that no unit of its own chain holds: Word shows + # a level nothing numbers as 0, or the counter of whatever last counted it, and the resolver's label + # is that of the enclosing unit. Not left to Word's reading. + cause += ("label-fallback",) + self.report("render-docx-section-label", "warning", + f"The label of {word}s '{printable(node.label or '')}' shows the number of an enclosing " + f"item that Word cannot show here (the list is not nested directly in an item of its own kind): in " + f"Word it reads '{printable(step.label)}'.", node.line) + self.add(node, kind, used, ilvl, explicit=explicit, cause=cause, step=step, + ind=self.indent(abstract, ilvl, unit, parent, alternative=False)) + self.check(node, word, step, counter, group, cause, (abstract.id, ilvl)) + return used + + def restart_class(self, parent: str, placed: _Placed, abstract: _Abstract) -> str: + """Why a list nested in the item *parent* does not take its num: the item is a later alternative (numbered by + no list), it is on the heading list but the list cannot be, it is on another kind of list, or Word would + go on counting from another list in the same item.""" + if self.units[parent].num_id == 0: + return "restart-alternative" + if placed.abstract is not abstract: + return "restart-section" if placed.abstract.kind == "headings" else "restart-family" + return "resync" + + def report_collisions(self) -> None: + """One finding for the units of the heading list that Word would number off the resolver's because they share + a counter with others of their section (§2.7): the first, and how many more.""" + if not self.collisions: + return + node, word = self.collisions[0] + assert node.unit is not None + designation = text_of(self.desig[node.unit]) + more = f" {len(self.collisions) - 1} more units do the same." if len(self.collisions) > 1 else "" + self.shared_counter( + f"The {word} '{printable(node.label or '')}' has the designation {printable(designation)}, which another " + f"unit of its section has too (in every output format their designations collide), and in Word they " + f"share one counter. The number is kept as the renderer shows it by restarting at a fixed value, so " + f"inserting or deleting a clause before it will renumber it.{more}", node.line) + + def list_(self, lst: List, section: Section | None, quoted: bool = False) -> None: first = lst.items[0] if lst.family == "bullet": - self.bullets(lst, section) + self.bullets(lst, section, quoted) return fmt = lst.format assert fmt is not None - abstract = self.abstract(lst.family) - ilvl = min(lst.depth, 8) + depth = lst.depth # The counter of the level Word uses: past level 9 that is level 9's, not the resolver's cycle. levels = self.formats.enumerated if lst.family == "enumerated" else (fmt,) - counter = levels[min(lst.depth, 8) % len(levels)].counter + counter = levels[min(depth, 8) % len(levels)].counter parent = first.parent_unit placed = self.placed.get(parent) if parent is not None else None + parent_live = placed is not None and self.units[parent].num_id not in (None, 0) + refs = ([f.ref for f in (self.formats.enumerated[j % len(self.formats.enumerated)] for j in range(depth + 1))] + if lst.family == "enumerated" else [fmt.ref] * (depth + 1)) + shows = section is not None and shows_section(fmt.label, refs, depth, textual=self.textual) + # A label that needs the section's number goes on the heading list's level after the section's; a list nested + # in a unit that is on the heading list goes on its reserved levels, so that Word keeps restarting it by + # structure. Otherwise the family's own abstractNum, with a label Word can write. + slot = None + if lst.family == "enumerated" and section is not None and not quoted: + if shows: + slot = self.qualified(fmt, section) if depth == 0 and placed is None else None + elif placed is not None and placed.abstract.kind == "headings" and 1 <= depth <= 4: + slot = self.reserved(fmt, depth) + abstract, ilvl = slot or (self.abstract(lst.family), min(depth, 8)) cause: tuple[str, ...] = () - # Nested in an item: that item's num, if it is of the same kind and Word counts the list from where the + # Nested in an item: that item's num, if it is live, of the same kind and Word counts the list from where the # resolver does (a sibling list continues, a list in a new item restarts); otherwise a num of its own. - if (placed is not None and placed.abstract is abstract and lst.depth <= 8 + if (placed is not None and parent_live and placed.abstract is abstract and depth <= 8 and self.sim.peek(placed.num.id, ilvl) == first.number): num = placed.num else: num = self.new_num(abstract, ilvl, first.number or 1) - if lst.depth > 8: + if depth > 8: cause = ("clamped",) if not self.depth_reported: self.depth_reported = True self.report("render-docx-list-depth", "warning", - f"A list is nested {lst.depth} levels deep: Word has 9. Items from level 10 on are " + f"A list is nested {depth} levels deep: Word has 9. Items from level 10 on are " f"numbered in the format of level 9 and restart by fixed values.", first.line) elif placed is not None and first.number > 1: cause = ("resync",) @@ -1004,56 +1279,34 @@ def list_(self, lst: List, section: Section | None) -> None: "starts at a fixed number in Word: it will not follow items added or removed before it.", first.line) elif placed is not None: - kinds = placed.abstract is not abstract - cause = ("restart-family",) if kinds else ("resync",) - self.report("render-docx-static-restart", "info", - "A list nested in a list of another kind restarts by a fixed setting (Restart at 1), " - "not by structure: an item added above it in Word will not restart it." if kinds else - "A list that Word would continue from another list in the same item restarts by a fixed " - "setting (Restart at 1), not by structure.", first.line) - refs = ([f.ref for f in (self.formats.enumerated[j % len(self.formats.enumerated)] - for j in range(lst.depth + 1))] - if lst.family == "enumerated" else [fmt.ref] * (lst.depth + 1)) - qualified = section is not None and shows_section(fmt.label, refs, lst.depth, textual=self.textual) - if qualified: - self.fallback_label(fmt, lst.family, ilvl, "list item", first.line) + cause, message = _RESTARTS[self.restart_class(parent, placed, abstract)] + self.report("render-docx-static-restart", "info", message, first.line) + if slot is not None and abstract.kind == "headings" and depth > 0: + cause += ("reserved-level",) + fallback = shows and slot is None + if fallback: + self.fallback_label(fmt, lst.family, min(depth, 8), "list item", first.line) parent_des = self.desig.get(parent, []) if parent is not None else [] section_des = self.desig[section.unit] if section is not None and section.unit is not None else [] for index, item in enumerate(lst.items): assert item.unit is not None and item.number is not None self.parent[item.unit] = item.parent_unit - piece = fill_tokens(fmt.ref, n=[Count(item.unit, format_counter(item.number, fmt.counter))], - section=section_des) - self.desig[item.unit] = extend_tokens(parent_des, piece, textual=self.textual) item_cause = cause if index == 0 else () - if lst.depth > 8 and "clamped" not in item_cause: + if depth > 8 and "clamped" not in item_cause: item_cause += ("clamped",) - if qualified: - item_cause += ("label-fallback",) - # Another list used Word's counter of this level in between: restart it where the resolver is. - if index and item.shares_number_with is None and self.sim.peek(num.id, ilvl) != item.number: - num = self.new_num(abstract, ilvl, item.number) - item_cause += ("resync",) - self.shared_counter( - "Word's counter for a level of this list was also used by another list in between; the numbers " - "are kept as the renderer shows them by restarting at a fixed number, which will not follow " - "items added or removed before it.", item.line) - step = self.sim.step(num.id, ilvl, item.unit) - if any(isinstance(token, Count) and isinstance(token.owner, Ghost) for token in step.tokens): - # The label shows a level of its list that nothing numbers here: Word shows it as 0, and the - # resolver's label is that of the enclosing item of another kind. Not left to Word's reading. + if slot is not None and depth > 0 and "reserved-level" not in item_cause: + item_cause += ("reserved-level",) + if fallback: item_cause += ("label-fallback",) - self.report("render-docx-section-label", "warning", - f"The label '{printable(fmt.label)}' of list items shows the number of an enclosing item " - f"that Word cannot show here (the list is not nested in one of its own kind): in Word " - f"it reads '{printable(step.label)}'.", item.line) - self.add(item, "item", num, ilvl, explicit=True, cause=item_cause, step=step) - self.check(item, "list item", step, counter, ("list", first.unit), item_cause, (abstract.id, ilvl)) - self.placed[item.unit] = _Placed(num, abstract) + used = self.numbered(item, "item", num, abstract, ilvl, counter=counter, group=("list", first.unit), + cause=item_cause) + piece = fill_tokens(fmt.ref, n=[self.counted(item, fmt.counter)], section=section_des) + self.desig[item.unit] = extend_tokens(parent_des, piece, textual=self.textual) + self.placed[item.unit] = _Placed(used, abstract) for child in item.blocks: - self.block(child, section) + self.block(child, section, quoted) - def bullets(self, lst: List, section: Section | None) -> None: + def bullets(self, lst: List, section: Section | None, quoted: bool) -> None: """Bullets count nothing, so every bullet list of the document is numbered by one num.""" abstract = self.abstract("bullet") if self.bullet_num is None: @@ -1070,7 +1323,7 @@ def bullets(self, lst: List, section: Section | None) -> None: None, cause, (), None) self.placed[item.unit] = _Placed(self.bullet_num, abstract) for child in item.blocks: - self.block(child, section) + self.block(child, section, quoted) def plan_docx(tree: RenderTree, *, file: str | None = None) -> tuple[DocxPlan, list[Diagnostic]]: diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index a7b5b55..27763c6 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -18,12 +18,13 @@ from legaldown_render.errors import StyleError from legaldown_render.resolve.docx_bookmarks import BookmarkNames from legaldown_render.resolve.docx_cover import Field, Title, cover, suppressed, uncovered -from legaldown_render.resolve.docx_numbering import NUM_FMT, Count, Ghost, Lit, word_counter +from legaldown_render.resolve.docx_numbering import NUM_FMT, Count, Ghost, Lit, Step, word_counter from legaldown_render.resolve.docx_plan import ( DocxPlan, FieldSeg, Literal_, Segment, + _Planner, docx_style_errors, plan_docx, ) @@ -45,7 +46,11 @@ NONE = {"numbering.scheme": "none"} PARAGRAPHS = {"paragraphs.numbered": True} ALTERNATIVES = FRONT.replace("title: T", ( - "title: T\nquestions:\n forum:\n type: choice\n choices:\n courts: Courts\n arb: Arbitration")) + "title: T\nquestions:\n forum:\n type: choice\n choices:\n courts: Courts\n arb: Arbitration\n" + " expert: Expert")) +#: The answers of the question of :data:`ALTERNATIVES`: any two exclude each other, so units that have the same +#: identifier and one answer each are alternatives (§15.4), up to three ways. +FORUMS = ("courts", "arb", "expert") #: The styles every document is planned under: the built-in ones, and the overrides that change which lists exist #: and which labels need the section. @@ -426,21 +431,208 @@ def test_paragraphs_are_not_numbered_unless_the_style_numbers_them() -> None: # -- Labels that need the section ----------------------------------------------------- -def test_a_label_that_needs_the_sections_number_is_written_without_it_and_reported() -> None: - body = "# A\n\n- x\n- y\n\n## B\n\n- z\n" +# -- Labels that need the section: on the heading list (§13.2) -------------------------------------------------- + +CONTINENTAL_ITEMS = "# A\n\n- x\n - y\n - z\n - w\n- v\n\n## B\n\n- u\n" + + +def test_an_item_whose_label_needs_the_section_rides_the_heading_list_one_level_below_the_section() -> None: + tree, plan, diagnostics = planned(CONTINENTAL_ITEMS, style="continental") + x, y, z, w, v, u = items(tree) + # the heading num, at the level after the section's (%1.%2 after "1."), one num for the list with an override + assert shown(plan, x, v) == [(2, 1, "1.1"), (2, 1, "1.2")] + assert plan.nums[1].abstract_id == plan.heading_num_id - 1 == 0 and plan.nums[1].start_overrides == ((1, 1),) + assert [a.kind for a in plan.abstract_nums] == ["headings"] # no abstractNum of the items' own is made + assert plan.units[x.unit].cause == () and not plan.units[x.unit].alt + assert [x.label, v.label, u.label] == ["1.1", "1.2", "1.1.1"] + # an item under a level-2 section is on the level after that: %1.%2.%3 + assert shown(plan, u) == [(4, 2, "1.1.1")] + assert plan.abstract_nums[0].levels[2].lvl_text == "%1.%2.%3" + # the items are no headings: no heading style and a name of their own for their indent + assert plan.units[x.unit].ind == (plan.abstract_nums[0].levels[1].left,) * 2 + + +def test_the_heading_levels_an_item_stands_on_are_not_changed_by_it() -> None: + _, plain, _ = planned("# A\n\n## B\n\n### C\n", style="continental") + _, with_items, _ = planned(CONTINENTAL_ITEMS, style="continental") + assert plain.abstract_nums[0].levels[:5] == with_items.abstract_nums[0].levels[:5] + + +def test_a_sub_heading_after_the_items_of_its_section_is_restarted_where_the_resolver_is() -> None: + tree, plan, diagnostics = planned(CONTINENTAL_ITEMS, style="continental") + b = tree.sections[1] + # Word's counter of the level is shared by the sub-heading and the items: 1.3 where the resolver has 1.1 again + assert (b.label, plan.units[b.unit].label, plan.units[b.unit].cause) == ("1.1", "1.1", ("resync",)) + assert plan.nums[plan.units[b.unit].num_id - 1].start_overrides == ((1, 1),) + [found] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert found.level == "warning" and found.line == b.line + assert "every output format" in found.message and "designation 1.1" in found.message + assert "render-docx-number-mismatch" not in rules(diagnostics) and "render-docx-section-label" not in rules(diagnostics) + + +def test_a_resync_is_reported_once_with_the_number_of_places_that_do_the_same() -> None: + body = "".join(f"# S{i}\n\n- a\n\n## T{i}\n\n" for i in range(5)) + _, _, diagnostics = planned(body, style="continental") + [found] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert found.message.endswith("4 more units do the same.") + + +def test_a_list_nested_in_an_item_on_the_heading_list_rides_the_levels_reserved_for_it() -> None: + tree, plan, diagnostics = planned(CONTINENTAL_ITEMS, style="continental") + x, y, z, w, v, u = items(tree) + # depth 1 and 2: levels 6 and 7 of the heading abstractNum, the num of the parent and no override + assert shown(plan, y, z, w) == [(2, 5, "a)"), (2, 5, "b)"), (2, 6, "i)")] + assert [plan.units[i.unit].cause for i in (y, w)] == [("reserved-level",), ("reserved-level",)] + levels = plan.abstract_nums[0].levels + assert [(levels[i].num_fmt, levels[i].lvl_text) for i in (5, 6)] == [("lowerLetter", "%6)"), ("lowerRoman", "%7)")] + assert [(levels[i].num_fmt, levels[i].lvl_text) for i in (7, 8)] == [("decimal", "%8"), ("decimal", "%9")] # unused + assert len(plan.nums) == 4 and plan.nums[1].start_overrides == ((1, 1),) + # one step in from the text of the item it is nested in, as the items of any list are + assert plan.units[y.unit].ind == (plan.units[x.unit].ind[0] + 567, 567) + assert plan.units[w.unit].ind == (plan.units[x.unit].ind[0] + 1134, 567) + # the item after the nested ones goes on, and a new item's list restarts by structure + assert plan.units[v.unit].count == 2 and diagnostics[0].rule == "render-docx-shared-counter" + + +def test_a_nested_list_under_a_new_item_on_the_heading_list_restarts_by_structure() -> None: + tree, plan, diagnostics = planned("# A\n\n- a\n - x\n- b\n - y\n - z\n", style="continental") + _, x, _, y, z = items(tree) + assert shown(plan, x, y, z) == [(2, 5, "a)"), (2, 5, "a)"), (2, 5, "b)")] and diagnostics == [] + assert len(plan.nums) == 2 and plan.units[y.unit].cause == ("reserved-level",) + + +def test_siblings_nested_in_one_item_on_the_heading_list_go_on_counting() -> None: + body = "# A\n\n- a\n - x\n - y\n\n Between.\n\n - z\n" tree, plan, diagnostics = planned(body, style="continental") - assert [plan.units[i.unit].label for i in items(tree)] == ["1", "2", "1"] # the resolver wrote 1.1, 1.2, 1.1.1 - assert [u.label for u in units(tree) if isinstance(u, ListItem)] == ["1.1", "1.2", "1.1.1"] - assert plan.units[items(tree)[0].unit].cause == ("label-fallback",) - [found] = [d for d in diagnostics if d.rule == "render-docx-section-label"] - assert found.level == "warning" and "{section}.{n}" in found.message and "'1'" in found.message - assert found.line == tree.sections[0].blocks[0].items[0].line + assert [(plan.units[i.unit].label, plan.units[i.unit].num_id) for i in items(tree)[1:]] == [ + ("a)", 2), ("b)", 2), ("c)", 2)] + assert len(plan.nums) == 2 and diagnostics == [] + + +def test_a_reserved_level_is_defined_on_first_use_and_only_when_it_is_used() -> None: + _, plain, _ = planned("# A\n\n- x\n", style="continental") + assert [(level.lvl_text, level.num_fmt) for level in plain.abstract_nums[0].levels[5:]] == [ + (f"%{k}", "decimal") for k in range(6, 10)] + _, nested, _ = planned("# A\n\n- x\n - y\n - z\n", style="continental") + assert [level.lvl_text for level in nested.abstract_nums[0].levels[5:]] == ["%6)", "%7)", "%8", "%9"] + + +def test_a_nested_label_that_shows_the_section_does_not_ride_the_reserved_levels() -> None: + # depth 3 of the continental style is the first level's label again, {section}.{n} + body = "# A\n\n- x\n - y\n - z\n - w\n - v\n" + tree, plan, diagnostics = planned(body, style="continental") + x, y, z, w, v = items(tree) + assert [plan.units[i.unit].ilvl for i in (x, y, z)] == [1, 5, 6] + assert plan.units[w.unit].ilvl == 3 and plan.units[w.unit].cause == ("restart-section", "label-fallback") + assert plan.units[w.unit].label == "1" and w.label == "1.1" and plan.units[w.unit].ind is None + # the list after it is on the family's abstractNum too, under a parent that is not on the heading list + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated"] + assert plan.units[v.unit].ilvl == 4 and plan.units[v.unit].cause == () + assert [(d.rule, d.level) for d in diagnostics] == [ + ("render-docx-static-restart", "info"), ("render-docx-section-label", "warning")] + + +def test_an_ordered_list_nested_in_an_item_on_the_heading_list_keeps_its_own_numbering() -> None: + tree, plan, diagnostics = planned("# A\n\n- x\n 1. y\n 1. z\n", style="continental") + x, y, z = items(tree) + assert [a.kind for a in plan.abstract_nums] == ["headings", "renumber"] + assert shown(plan, y, z) == [(3, 1, "1."), (3, 1, "2.")] and plan.units[y.unit].cause == ("restart-section",) + assert plan.units[y.unit].ind is None + [found] = diagnostics + assert (found.rule, found.level) == ("render-docx-static-restart", "info") and "carries the section" in found.message -def test_paragraph_numbers_that_need_the_section_are_reported_too() -> None: - tree, plan, diagnostics = planned("# A\n\nOne.\n", overrides=PARAGRAPHS) +def test_the_level_after_the_fifth_is_kept_for_level_5_sections_when_the_document_has_one() -> None: + body = "# A\n\n## B\n\n### C\n\n#### D\n\n##### E\n\n- five\n - nested\n- six\n" + tree, plan, diagnostics = planned(body, style="continental") + five, nested, six = items(tree) + assert shown(plan, five, six) == [(2, 5, "1.1.1.1.1.1"), (2, 5, "1.1.1.1.1.2")] + assert plan.abstract_nums[0].levels[5].lvl_text == "%1.%2.%3.%4.%5.%6" + # the nested list falls back: its slot is the level-5 section's + assert plan.units[nested.unit].cause == ("restart-section",) and plan.units[nested.unit].ilvl == 1 + # a document without a level-5 section lets the nested list have it + _, other, _ = planned("# A\n\n- x\n - y\n", style="continental") + assert other.abstract_nums[0].levels[5].lvl_text == "%6)" + + +def test_a_level_a_different_label_has_taken_is_not_shared() -> None: + # the numbered paragraphs under a level-5 section need another text on the level after the section's than the + # items: the second kind falls back, and says so + body = "# A\n\n## B\n\n### C\n\n#### D\n\n##### E\n\n- five\n\nPara.\n" + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}-{n}", + "ref": "-{n}"}} + tree, plan, diagnostics = planned(body, style="continental", overrides=levels) [paragraph] = [u for u in units(tree) if isinstance(u, Paragraph)] - assert plan.units[paragraph.unit].label == "1" and plan.units[paragraph.unit].cause == ("label-fallback",) + assert plan.units[paragraph.unit].cause == ("label-fallback",) and plan.units[paragraph.unit].label == "-1" + assert plan.units[items(tree)[0].unit].ilvl == 5 + assert rules(diagnostics) == {"render-docx-section-label"} + + +def test_a_label_the_heading_list_cannot_show_is_written_without_the_section_and_reported() -> None: + levels = [{"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{n})", "ref": "({n})"}] + tree, plan, diagnostics = planned("# A\n\n- x\n - y\n", style="outline", overrides={"enumeration.levels": levels}) + x, y = items(tree) + assert (x.label, plan.units[x.unit].label, plan.units[x.unit].cause) == ("I.1", "1", ("label-fallback",)) + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated"] and plan.units[x.unit].ind is None + [found] = diagnostics + assert found.rule == "render-docx-section-label" and found.level == "warning" and "'1'" in found.message + assert found.line == x.line + + +def test_under_the_none_scheme_the_label_that_needs_the_section_is_written_without_it() -> None: + tree, plan, diagnostics = planned("# A\n\n- x\n", style="continental", overrides=NONE) + [x] = items(tree) + assert (x.label, plan.units[x.unit].label, plan.units[x.unit].cause) == ("A.1", "1", ("label-fallback",)) + assert rules(diagnostics) == {"render-docx-section-label"} + + +def test_the_items_of_a_quote_do_not_ride_the_heading_list() -> None: + tree, plan, diagnostics = planned("# A\n\n> - quoted\n\n- real\n", style="continental") + quoted, real = items(tree) + assert plan.units[quoted.unit].cause == ("label-fallback",) and plan.units[quoted.unit].label == "1" + assert shown(plan, real)[0][1:] == (1, "1.1") + assert rules(diagnostics) == {"render-docx-section-label"} + + +def test_numbered_paragraphs_whose_label_needs_the_section_ride_the_heading_list_too() -> None: + tree, plan, diagnostics = planned("# A\n\nOne.\n\nTwo.\n\n## B\n\nThree.\n", overrides=PARAGRAPHS) + one, two, three = [u for u in units(tree) if isinstance(u, Paragraph)] + assert [(plan.units[p.unit].num_id, plan.units[p.unit].ilvl, plan.units[p.unit].label) for p in (one, two, three)] \ + == [(2, 1, "1.1"), (2, 1, "1.2"), (4, 2, "1.1.1")] + assert [a.kind for a in plan.abstract_nums] == ["headings"] + # B's own number goes on from Word's counter of the level unless restarted + assert plan.units["s2"].cause == ("resync",) and rules(diagnostics) == {"render-docx-shared-counter"} + + +def test_items_in_the_middle_of_numbered_paragraphs_share_their_counter_and_the_paragraph_after_is_restarted() -> None: + # continental's paragraphs are (1), a list of their own; made {section}.{n} they share the level with the items + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}.{n}", + "ref": ".{n}"}} + tree, plan, diagnostics = planned("# A\n\nOne.\n\n- x\n- y\n\nTwo.\n", style="continental", overrides=levels) + one, x, y, two = units(tree)[1:] + assert [(plan.units[u.unit].label, plan.units[u.unit].cause) for u in (one, x, y, two)] == [ + ("1.1", ()), ("1.1", ()), ("1.2", ()), ("1.2", ("resync",))] # the renderer counts them apart + assert plan.nums[plan.units[two.unit].num_id - 1].start_overrides == ((1, 2),) + assert rules(diagnostics) == {"render-docx-shared-counter"} + + +def test_an_item_nested_under_a_foreign_parent_reports_a_section_label_once() -> None: + levels = [{"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}] + tree, plan, diagnostics = planned("# A\n\n1. a\n - x\n", overrides={"enumeration.levels": levels}) + _, nested = items(tree) + assert plan.units[nested.unit].cause == ("restart-family", "label-fallback") + assert [d.rule for d in diagnostics].count("render-docx-section-label") == 1 + + +def test_a_quote_inside_a_preamble_item_with_a_path_label_is_reported_not_left_unexplained() -> None: + levels = [{"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}] + tree, plan, diagnostics = planned("- a\n\n > - quoted\n\n# A\n", overrides={"enumeration.levels": levels}) + _, quoted = items(tree) + assert quoted.parent_unit is None + assert plan.units[quoted.unit].cause == ("label-fallback",) and plan.units[quoted.unit].label == "(a)(1)" assert rules(diagnostics) == {"render-docx-section-label"} @@ -488,18 +680,201 @@ def test_roman_numerals_from_4000_are_reported() -> None: assert found.line == items[3999].line and "'4000'" in found.message and "'mmmm'" in found.message -# -- Alternatives: planned in a later step; today Word's numbers differ and the finding says so --------- +# -- Alternatives that share a number (§15.8) ------------------------------------------------------------------- + +SECTIONS = ("# Intro\n\n# D {#d when=forum:courts}\n\n## Sub {#s1}\n\n## Sub two {#s2}\n\n# D {#d when=forum:arb}\n\n" + "## Sub alt\n\n- q\n\n# D {#d when=forum:expert}\n\n## Sub exp\n\n## Sub exp two\n\n## Sub exp three\n\n" + "# Last\n\n## L1\n") + +def segments_of(plan: DocxPlan, unit: str) -> str: + alt = plan.units[unit].alt + assert alt is not None + return show(plan, alt.segments) -def test_an_alternative_that_shares_a_number_is_reported_while_it_is_numbered_in_sequence() -> None: + +def test_a_later_alternative_section_has_no_number_of_its_own_and_shows_the_holders_as_a_field() -> None: body = "# Disputes {#d when=forum:courts}\n\n# Disputes {#d when=forum:arb}\n\n# Last\n" tree, plan, diagnostics = planned(body, ALTERNATIVES) assert tree.sections[1].shares_number_with == "s1" - assert [plan.units[s].label for s in ("s1", "s2", "s3")] == ["1.", "2.", "3."] - # the second section and the one after it are off by the same amount: one finding + first, second, last = (plan.units[s] for s in ("s1", "s2", "s3")) + assert (second.num_id, second.ilvl, second.explicit, second.count, second.cause) == (0, 0, True, None, ("alternative",)) + assert (second.label, second.word_ref, second.ind) == ("1.", "1", (567, 567)) + assert (first.label, last.label, last.count) == ("1.", "2.", 2) and last.num_id == first.num_id + assert segments_of(plan, "s2") == 'REF d + "."' and second.alt.suff == "tab" + segment, period = second.alt.segments + assert (segment.number, segment.hyperlink, segment.suppress_text, segment.cached) == (True, True, False, "1") + assert period == Literal_(".") and segment.bookmark == plan.units["s1"].bookmarks[0] and diagnostics == [] + + +def test_the_text_after_the_number_of_a_later_alternative_is_the_label_beyond_what_the_field_returns() -> None: + body = "# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n" + _, plan, _ = planned(body, ALTERNATIVES, overrides={"numbering.scheme": "legal-outline"}) + assert (show(plan, plan.units["s2"].alt.segments), plan.units["s2"].label) == ('REF d + "."', "I.") + # REF \n drops periods only: a label that ends otherwise is all the field's + _, plan, _ = planned(body, ALTERNATIVES, overrides={"numbering.scheme": "mixed"}) + assert (show(plan, plan.units["s2"].alt.segments), plan.units["s2"].label) == ('REF d + "."', "1.") + _, plan, _ = planned("# A\n\n- a {#i when=forum:courts}\n- b {#i when=forum:arb}\n", ALTERNATIVES) + assert (show(plan, plan.units["u2"].alt.segments), plan.units["u2"].label) == ("REF i", "(a)") + + +def test_a_reference_to_the_shared_identifier_is_the_first_alternatives() -> None: + body = "# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n\nSee {{ref: d}}.\n" + tree, plan, _ = planned(body, ALTERNATIVES) + assert list(plan.refs) == ["s1"] and show(plan, plan.refs["s1"].segments) == "REF d" + assert plan.units["s2"].bookmarks == () # the holder carries the bookmark, the alternative has none of its own + + +def test_the_holder_of_an_alternatives_number_has_a_bookmark_even_when_nothing_refers_to_it() -> None: + _, plan, _ = planned("# D\n\n- a {#i when=forum:courts}\n- b {#i when=forum:arb}\n", ALTERNATIVES) + [segment] = plan.units["u2"].alt.segments + assert plan.units["u1"].bookmarks == (segment.bookmark,) and segment.bookmark.startswith("ld_") + _, plan, _ = planned("# D\n\n- a {when=forum:courts}\n- b {when=forum:arb}\n", ALTERNATIVES) + assert plan.units["u2"].alt is None # no identifier: no alternative + _, plan, _ = planned("# D {when=forum:courts}\n\n# D2 {when=forum:arb}\n", ALTERNATIVES) + assert plan.units["s2"].alt is None + + +def test_the_sections_under_a_later_alternative_start_in_a_num_of_their_own() -> None: + tree, plan, diagnostics = planned(SECTIONS, ALTERNATIVES) + sub, sub_two, alt, sub_alt, alt_three, exp, exp_two, exp_three, last, l1 = ( + plan.units[s] for s in ("s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10", "s11", "s12")) + assert [s.label for s in (sub, sub_two, sub_alt, exp, exp_two, exp_three, last, l1)] == [ + "2.1", "2.2", "2.1", "2.1", "2.2", "2.3", "3.", "3.1"] + assert (alt.num_id, alt_three.num_id) == (0, 0) and alt.label == alt_three.label == "2." + # both holders' numbers are the first alternative's, the field points at it + assert segments_of(plan, "s5") == segments_of(plan, "s7") == 'REF d + "."' + # the first alternative's children are on the headings' num; each later one's on a fresh num that starts at its level + assert {sub.num_id, sub_two.num_id, last.num_id} == {1} and sub_alt.num_id != 1 and exp.num_id not in (1, sub_alt.num_id) + assert (sub_alt.explicit, exp.explicit, sub.explicit) == (True, True, False) + assert [plan.nums[n - 1].start_overrides for n in (sub_alt.num_id, exp.num_id)] == [((1, 1),), ((1, 1),)] + assert exp.num_id == exp_two.num_id == exp_three.num_id + # the section after the alternatives goes back to the headings' num: it goes on from the holder's counter + assert (last.count, l1.count) == (3, 1) and l1.num_id == 1 and last.explicit is False + assert diagnostics == [] or all(d.level == "info" for d in diagnostics) + + +def test_an_alternative_that_is_not_the_first_to_be_numbered_takes_no_num_for_its_children_until_they_come() -> None: + _, plan, _ = planned("# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n\n# E\n", ALTERNATIVES) + assert [n.id for n in plan.nums] == [1] # no num made for sections that never come + + +def test_an_alternative_below_a_later_alternative_whose_number_is_live_there_is_numbered_by_word() -> None: + body = ("# D {#d when=forum:courts}\n\n## E {#e when=forum:courts}\n\n### F {#f}\n\n" + "# D {#d when=forum:arb}\n\n## E {#e when=forum:arb}\n\n### F2\n\n## G\n\n### H\n") + tree, plan, _ = planned(body, ALTERNATIVES) + assert [(plan.units[s.unit].label, plan.units[s.unit].num_id, bool(plan.units[s.unit].alt)) for s in tree.sections] == [ + ("1.", 1, False), ("1.1", 1, False), ("1.1.1", 1, False), ("1.", 0, True), ("1.1", 2, False), + ("1.1.1", 2, False), ("1.2", 2, False), ("1.2.1", 2, False)] + # the alternative's first section starts the fresh num with the number it shares: no field is needed for it + assert plan.units["s5"].explicit and plan.nums[1].start_overrides == ((1, 1),) + + +def test_an_item_that_repeats_the_number_of_the_one_before_is_numbered_by_a_field() -> None: + body = "# S\n\n- a {#i when=forum:courts}\n- b {#i when=forum:arb}\n- c\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES) + a, b, c = items(tree) + assert (b.shares_number_with, plan.units[b.unit].num_id, plan.units[b.unit].label) == (a.unit, 0, "(a)") + assert (plan.units[b.unit].ilvl, plan.units[b.unit].ind, plan.units[b.unit].cause) == (0, (567, 567), ("alternative",)) + assert shown(plan, a, c) == [(2, 0, "(a)"), (2, 0, "(b)")] and plan.units[c.unit].count == 2 + assert segments_of(plan, b.unit) == "REF i" and diagnostics == [] + + +def test_the_children_of_a_later_alternative_item_get_a_num_of_their_own_and_it_is_reported_as_information() -> None: + body = "# S\n\n- a {#i when=forum:courts}\n - x\n- b {#i when=forum:arb}\n - y\n - z\n- c\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES) + _, x, _, y, z, _ = items(tree) + assert shown(plan, x, y, z) == [(2, 1, "(i)"), (3, 1, "(i)"), (3, 1, "(ii)")] + assert plan.units[y.unit].cause == ("restart-alternative",) and plan.units[z.unit].cause == () + assert plan.nums[2].start_overrides == ((1, 1),) [found] = diagnostics - assert (found.rule, found.line) == ("render-docx-number-mismatch", tree.sections[1].line) - assert "'1.'" in found.message and "'2.'" in found.message and "the following 1" in found.message + assert (found.rule, found.level, found.line) == ("render-docx-static-restart", "info", y.line) + assert "alternative" in found.message + + +def test_a_paragraph_that_repeats_the_number_of_the_one_before_is_numbered_by_a_field() -> None: + body = "# S\n\nOne. {#p when=forum:courts}\n\nOne arb. {#p when=forum:arb}\n\nTwo.\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES, overrides=PARAGRAPHS) + one, arb, two = [u for u in units(tree) if isinstance(u, Paragraph)] + assert [(plan.units[u.unit].num_id, plan.units[u.unit].label) for u in (one, arb, two)] == [ + (2, "1.1"), (0, "1.1"), (2, "1.2")] + assert plan.units[arb.unit].ilvl == 1 and plan.units[two.unit].count == 2 + assert diagnostics == [] + + +def test_alternatives_on_the_heading_list_carry_the_indent_of_the_unit_not_of_the_heading() -> None: + body = "# S\n\n- a {#i when=forum:courts}\n - n\n- b {#i when=forum:arb}\n - m\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES, style="continental") + a, n, b, m = items(tree) + heading_level = plan.abstract_nums[0].levels[1] + assert plan.units[b.unit].num_id == 0 and plan.units[b.unit].ilvl == 1 + assert plan.units[b.unit].ind == (heading_level.left, heading_level.hanging) == plan.units[a.unit].ind + # the children of the alternative restart on a fresh num of the heading list's reserved level + assert (plan.units[m.unit].ilvl, plan.units[m.unit].cause) == (5, ("restart-alternative", "reserved-level")) + assert plan.units[m.unit].num_id not in (plan.units[n.unit].num_id, 1) + assert plan.nums[plan.units[m.unit].num_id - 1].start_overrides == ((5, 1),) + + +def test_a_designation_below_an_alternative_is_made_of_the_holders_counters() -> None: + body = ("# D {#d when=forum:courts}\n\n- a {#a when=forum:courts}\n- b {#a when=forum:arb}\n - deep {#deep}\n\n" + "# D {#d when=forum:arb}\n\n- z {#z}\n\nSee {{ref: deep}} {{ref: z}}.\n") + tree, plan, diagnostics = planned(body, ALTERNATIVES, style="continental") + texts = {i.target_unit: i.text for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)} + assert [(texts[t], show(plan, plan.refs[t].segments)) for t in plan.refs] == [ + ("1.1(a)", 'REF a + "(" + REF deep'), ("1.1", "REF z")] + assert all(r.exact and not r.fixed for r in plan.refs.values()) + assert "render-docx-reference" not in rules(diagnostics) + + +def test_the_contents_show_the_number_the_alternative_shares() -> None: + body = "# D {#d when=forum:courts}\n\n## S\n\n# D {#d when=forum:arb}\n\n## S2\n\n# Last\n" + _, plan, _ = planned(body, ALTERNATIVES, overrides={"contents.enabled": True, "contents.depth": 2}) + assert [(e.unit, e.number, e.sep) for e in plan.toc] == [ + ("s1", "1.", "\t"), ("s2", "1.1", "\t"), ("s3", "1.", "\t"), ("s4", "1.1", "\t"), ("s5", "2.", "\t")] + assert plan.units["s3"].toc_bookmark == plan.toc[2].bookmark + + +def test_the_items_on_the_heading_list_are_not_in_the_contents() -> None: + body = "# A\n\n- x\n - y\n\n## B\n\n- z\n" + tree, plan, _ = planned(body, style="continental", overrides={"contents.enabled": True, "contents.depth": 5}) + assert [e.unit for e in plan.toc] == ["s1", "s2"] + assert all(plan.units[i.unit].toc_bookmark is None for i in items(tree)) + + +def test_an_alternative_that_starts_a_section_with_its_own_shared_number_repeats_it_by_a_live_counter() -> None: + # the children of a later alternative section repeat the numbers of the earlier one's: they are numbered by Word + # (on the fresh num) and are not alternatives themselves + tree, plan, _ = planned(SECTIONS, ALTERNATIVES) + for unit in ("s6", "s8", "s9"): + assert tree.sections[int(unit[1:]) - 1].shares_number_with in ("s3", "s4", None) + assert plan.units[unit].count is not None and not plan.units[unit].alt + + +def test_drift_is_reported_once_for_a_run_of_units_counted_off_by_the_same_amount() -> None: + # not a finding the planner makes for a document it can place (a unit Word would number differently is restarted + # or made a field), but the catch-all that tells of a bug in the placement: it groups runs + tree, _, _ = planned("# A\n\n" + "".join(f"- i{n}\n" for n in range(6))) + planner = _Planner(tree, None) + nodes = items(tree) + for node in nodes: + step = Step(node.number + 1, "", "", "", ()) + planner.check(node, "list item", step, "decimal", ("list", "u1"), (), (0, 0)) + for drift in planner.drifts: + planner.report_drift(drift) + [found] = planner.diagnostics + assert found.rule == "render-docx-number-mismatch" and found.line == nodes[0].line + assert "1 later than the renderer" in found.message and "the following 5" in found.message + + +def test_a_different_drift_is_a_finding_of_its_own() -> None: + tree, _, _ = planned("# A\n\n" + "".join(f"- i{n}\n" for n in range(4))) + planner = _Planner(tree, None) + for node, delta in zip(items(tree), (1, 1, 2, 2), strict=True): + planner.check(node, "list item", Step(node.number + delta, "", "", "", ()), "decimal", ("list", "u1"), (), (0, 0)) + for drift in planner.drifts: + planner.report_drift(drift) + assert len(planner.diagnostics) == 2 and "1 later" in planner.diagnostics[0].message + assert "2 later" in planner.diagnostics[1].message # -- Diagnostics: lines, files, order, repeats -------------------------------------------- @@ -649,25 +1024,6 @@ def test_a_label_that_needs_a_level_nothing_numbers_is_not_left_to_word() -> Non assert {d.rule for d in diagnostics} == {"render-docx-static-restart", "render-docx-section-label"} -def test_a_run_of_units_counted_off_by_the_same_amount_is_one_finding() -> None: - body = "".join(f"# S{i}\n\n- a\n- b\n\n" for i in range(4)) - body += "# D {#d when=forum:courts}\n\n- x\n\n# D {#d when=forum:arb}\n\n- y\n\n" - body += "".join(f"# T{i}\n\n- a\n\n## sub\n\n- q\n\n" for i in range(10)) - tree, plan, diagnostics = planned(body, ALTERNATIVES) - [found] = diagnostics - assert found.rule == "render-docx-number-mismatch" and found.line == tree.sections[5].line - assert "1 later than the renderer" in found.message and "the following 10" in found.message - - -def test_a_different_difference_is_a_finding_of_its_own() -> None: - body = ("# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n\n# Mid\n\n" - "# E {#e when=forum:courts}\n\n# E {#e when=forum:arb}\n\n# End\n") - tree, _, diagnostics = planned(body, ALTERNATIVES) - assert [(d.rule, d.line) for d in diagnostics] == [ - ("render-docx-number-mismatch", tree.sections[1].line), ("render-docx-number-mismatch", tree.sections[4].line)] - assert "1 later" in diagnostics[0].message and "2 later" in diagnostics[1].message - - # -- Word's counters against LibreOffice's, observed ----------------------------------------------------- #: What LibreOffice 24.2 writes for these values, from a document with one list per value (it follows the OOXML @@ -729,50 +1085,60 @@ def assert_planned_as_resolved(result, plan, diagnostics) -> None: # noqa: ANN0 """The property, unit by unit: Word's count is the resolver's number, and Word's label is the resolver's label, unless that unit's own counter reads differently in Word (a letter past 26, a Roman numeral from 4000: its label and those of its descendants may differ), its label cannot show what the resolver's does (the plan says so on the - unit), or its depth is past Word's. A document without alternatives has no other exception; a diagnostic is - reported exactly where there is an exception.""" + unit), or its depth is past Word's. A later alternative that shares a number has no counter of its own (``numId`` 0, + its number the holder's, as fields) and reads as the resolver's label. There is no other exception, and a + diagnostic is reported exactly where there is one.""" tree = result.tree found = rules(diagnostics) - shares = any(unit.shares_number_with for unit in units(tree)) overflows: dict[str, str] = {} # unit -> the rule its counter reads differently under (itself or an ancestor) parents = {unit.unit: unit.parent_unit for unit in units(tree)} - count_differs = False own_rules: set[str] = set() for node, counter in numbered(tree): mine = plan.units[node.unit] clamped = "clamped" in mine.cause - if not shares: + alternative = node.shares_number_with is not None and mine.num_id == 0 + if alternative: + assert mine.count is None and "alternative" in mine.cause and mine.explicit and mine.ind is not None + assert mine.ilvl is not None and mine.label.rstrip(".") == mine.word_ref + else: assert mine.count == node.number, (node.unit, mine.count, node.number) - count_differs = count_differs or mine.count != node.number own = "" if not clamped and word_counter(node.number, NUM_FMT[counter]) != format_counter(node.number, counter): own = "render-docx-roman-overflow" if counter.endswith("roman") else "render-docx-letter-overflow" inherited = overflows.get(parents[node.unit] or "", "") or "" overflows[node.unit] = own or inherited - if not shares and not (own or inherited or clamped or "label-fallback" in mine.cause): + if not (own or inherited or clamped or "label-fallback" in mine.cause): assert mine.label == node.label, (node.unit, mine.label, node.label) - for cause, rule in (("label-fallback", "render-docx-section-label"), ("clamped", "render-docx-list-depth")): + assert mine.cause.count("label-fallback") <= 1, (node.unit, mine.cause) + for cause, rule in (("label-fallback", "render-docx-section-label"), ("clamped", "render-docx-list-depth"), + ("resync", "render-docx-shared-counter")): if cause in mine.cause: assert rule in found, (node.unit, cause) own_rules.update([own] if own else []) assert found & {"render-docx-roman-overflow", "render-docx-letter-overflow"} == own_rules - if shares: - assert count_differs <= ("render-docx-number-mismatch" in found) - else: - assert "render-docx-number-mismatch" not in found, "a placement is wrong: " + "; ".join( - d.message for d in diagnostics if d.rule == "render-docx-number-mismatch") + assert "render-docx-number-mismatch" not in found, "a placement is wrong: " + "; ".join( + d.message for d in diagnostics if d.rule == "render-docx-number-mismatch") assert all(d.rule.startswith("render-docx-") for d in diagnostics) assert {u.unit for u in units(tree)} == set(plan.units) # Every num with an override is used, by a unit whose level the override names. first: dict[int, str] = {} for unit in units(tree): mine = plan.units[unit.unit] - if mine.num_id is not None: + if mine.num_id: first.setdefault(mine.num_id, unit.unit) for num in plan.nums: assert num.id in first, f"num {num.id} is never used" if num.start_overrides: assert plan.units[first[num.id]].ilvl == num.start_overrides[0][0] + # A unit carries an indent of its own where its level's serves another (a unit that is no heading on the heading + # list), and where it has no level to take it from (a later alternative). + headings = next((a.id for a in plan.abstract_nums if a.kind == "headings"), None) + nums = {n.id: n.abstract_id for n in plan.nums} + for unit in units(tree): + mine = plan.units[unit.unit] + on_headings = mine.num_id is not None and nums.get(mine.num_id) == headings and not isinstance(unit, Section) + assert (mine.ind is not None) == (on_headings or mine.num_id == 0), (unit.unit, mine.ind, mine.num_id) + assert (mine.alt is not None) == (mine.num_id == 0) @pytest.mark.parametrize("variant", VARIANTS) @@ -850,48 +1216,85 @@ def _random_document(rng: random.Random) -> str: return "\n".join(out) +def _alternative_list(rng: random.Random, index: int, indent: str, depth: int) -> list[str]: + """A list whose first item has two or three alternatives, some with lists of their own (children that differ in + number between the alternatives), among other items.""" + lines: list[str] = [] + for way in range(rng.choice([2, 3])): + lines.append(f"{indent}- item {way} {{#i{index}d{depth} when=forum:{FORUMS[way]}}}") + if depth < 3 and rng.random() < 0.6: + lines.extend(_random_list(rng, depth + 1, indent + " ", 2)) + if depth < 2 and rng.random() < 0.3: + lines.extend(_alternative_list(rng, index, indent + " ", depth + 1)) + lines.append(f"{indent}- after") + if depth < 3 and rng.random() < 0.5: + lines.extend(_random_list(rng, depth + 1, indent + " ", 2)) + return lines + [""] + + def _random_alternatives(rng: random.Random) -> str: - """Sections, items and paragraphs that share a number, among others.""" + """Sections, items and paragraphs that share a number, in twos and threes, with children of their own and with + other units around them.""" out: list[str] = [] for index in range(rng.randint(2, 6)): - out.append(f"# Section {index}\n") + out.append(f"# Section {index} {{#h{index}}}\n") if rng.random() < 0.5: - out.append(f"Para. {{#p{index} when=forum:courts}}\n\nPara arb. {{#p{index} when=forum:arb}}\n") + ways = rng.choice([2, 3]) + out.append("\n\n".join(f"Para {way}. {{#p{index} when=forum:{FORUMS[way]}}}" for way in range(ways)) + + "\n\nAfter.\n") + if rng.random() < 0.6: + out.append("\n".join(_alternative_list(rng, index, "", 0))) if rng.random() < 0.5: - out.append(f"- item {{#i{index} when=forum:courts}}\n- item arb {{#i{index} when=forum:arb}}\n - nested\n" - f"- after\n") - if rng.random() < 0.4: - out.append(f"# Disputes {{#d{index} when=forum:courts}}\n\n## Sub\n\n" - f"# Disputes {{#d{index} when=forum:arb}}\n\n## Sub alt\n") + for way in range(rng.choice([2, 3])): + out.append(f"# Disputes {{#d{index} when=forum:{FORUMS[way]}}}\n") + for sub in range(rng.randint(0, 3)): + out.append(f"## Sub {way}.{sub} {{#s{index}w{way}n{sub}}}\n") + if rng.random() < 0.4: + out.extend(_random_list(rng, 0, "", 2)) + if rng.random() < 0.3: + out.append(f"### Deeper {{#t{index}w{way}n{sub}}}\n") + if rng.random() < 0.4: + out.append("Text.\n\n- an item\n") return "\n".join(out) #: The random documents: one set for the numbering and the references. Each has an anchor on every unit and a -#: paragraph that refers to up to 40 of them, and is planned under every fourth variant (rotating, so that each variant -#: sees a different quarter), once. +#: paragraph that refers to up to 40 of them (a different part of them in each document, so that every unit is +#: referred to by some document), and each variant plans its own sample of them, once. RANDOM_DOCUMENTS = 36 -ROTATION = 4 +ALTERNATIVE_DOCUMENTS = 24 +PER_VARIANT = 9 @functools.cache def _random_sources() -> tuple[str, ...]: rng = random.Random(20261028) - return tuple(with_references(with_anchors(_random_document(rng))) for _ in range(RANDOM_DOCUMENTS)) + return tuple(with_references(with_anchors(_random_document(rng)), shift=index) for index in range(RANDOM_DOCUMENTS)) + + +@functools.cache +def _alternative_sources() -> tuple[str, ...]: + rng = random.Random(7) + return tuple(ALTERNATIVES + with_references(_random_alternatives(rng), shift=index) + for index in range(ALTERNATIVE_DOCUMENTS)) + + +def sample(variant: str, count: int, sources: tuple[str, ...]) -> list[str]: + """*count* of *sources*, a different choice for each variant.""" + return [sources[i] for i in sorted(random.Random(variant).sample(range(len(sources)), count))] @functools.cache def random_runs(variant: str) -> tuple: """(source, result, plan, diagnostics) of the random documents planned under *variant*.""" - offset = list(VARIANTS).index(variant) return tuple((source, *planned_run(FRONT + source, variant)) - for index, source in enumerate(_random_sources()) if (index + offset) % ROTATION == 0) + for source in sample(variant, PER_VARIANT, _random_sources())) @functools.cache def alternative_runs(variant: str) -> tuple: - rng = random.Random(7) - sources = [ALTERNATIVES + with_references(_random_alternatives(rng)) for _ in range(15)] - return tuple((source, *planned_run(source, variant)) for source in sources) + return tuple((source, *planned_run(source, variant)) + for source in sample(variant, PER_VARIANT, _alternative_sources())) @pytest.mark.parametrize("variant", VARIANTS) @@ -903,10 +1306,13 @@ def test_random_documents_are_planned_as_resolved(variant: str) -> None: raise AssertionError(f"{error}\n--- document ---\n{source}") from error -@pytest.mark.parametrize("variant", ["default", "continental", "paragraphs", "none"]) -def test_random_alternatives_are_planned_and_reported(variant: str) -> None: - for _, result, plan, diagnostics in alternative_runs(variant): - assert_planned_as_resolved(result, plan, diagnostics) +@pytest.mark.parametrize("variant", VARIANTS) +def test_random_alternatives_are_planned_as_resolved(variant: str) -> None: + for source, result, plan, diagnostics in alternative_runs(variant): + try: + assert_planned_as_resolved(result, plan, diagnostics) + except AssertionError as error: + raise AssertionError(f"{error}\n--- document ---\n{source}") from error def test_planning_twice_gives_the_same_plan() -> None: @@ -924,12 +1330,33 @@ def test_the_random_documents_reach_every_case_the_property_is_about() -> None: seen_rules |= rules(diagnostics) seen_causes |= {cause for unit in plan.units.values() for cause in unit.cause} depth = max(depth, max((lst.depth for lst in lists(result.tree)), default=0)) + for variant in VARIANTS: + for _, _, plan, diagnostics in alternative_runs(variant): + seen_rules |= rules(diagnostics) + seen_causes |= {cause for unit in plan.units.values() for cause in unit.cause} assert {"render-docx-letter-overflow", "render-docx-section-label", "render-docx-static-restart", "render-docx-list-depth", "render-docx-shared-counter"} <= seen_rules - assert {"restart-family", "resync", "label-fallback", "skipped-level", "clamped"} <= seen_causes + assert {"restart-family", "restart-section", "restart-alternative", "resync", "label-fallback", "skipped-level", + "clamped", "alternative", "reserved-level"} <= seen_causes assert depth >= 10 +def test_documents_of_one_size_refer_to_different_units_so_that_more_of_them_are_referred_to() -> None: + groups: dict[int, list[set[str]]] = {} + for source in _random_sources(): + stride = -(-len(re.findall(r"\{#([a-z][a-z0-9-]*)", source)) // 40) or 1 + groups.setdefault(stride, []).append(set(re.findall(r"\{\{ref: ([a-z][a-z0-9-]*)", source))) + spread = [members for stride, members in groups.items() if stride > 1 and len(members) > 1] + assert spread + for members in spread: + assert len(set().union(*members)) > max(len(refs) for refs in members) + + +def test_each_variant_plans_its_own_sample_of_the_random_documents() -> None: + samples = {tuple(sample(variant, PER_VARIANT, _random_sources())) for variant in VARIANTS} + assert len(samples) == len(VARIANTS) + + # ===================================================================================================== # Cross-references, bookmarks and the table of contents (docs/architecture.md, "The DOCX planner") # ===================================================================================================== @@ -1026,10 +1453,10 @@ def covers(variant: str, **settings: object) -> dict[str, tuple[str, str]]: ("mixed", {"overrides": {"numbering.scheme": "mixed"}}, "gamma", "1(a)(i)", "REF alpha + REF beta + REF gamma"), ("mixed", {"overrides": {"numbering.scheme": "mixed"}}, "o2", "1(a)(i)(2)", 'REF alpha + REF beta + REF gamma + "(" + REF o2 + ")"'), - # continental: 1.1 for an item (the number of the section is not part of the item's own level in Word yet), - # a(n) item nested in an item written "b)" and referred to as "(b)" - ("continental", {"style": "continental"}, "i1", "1.1", 'REF alpha + "." + REF i1'), - ("continental", {"style": "continental"}, "n2", "1.2(b)", 'REF alpha + "." + REF i2 + "(" + REF n2'), + # continental: an item rides the heading list's next level (%1.%2), so one field is its whole number; an item + # nested in an item is written "b)" and referred to as "(b)" + ("continental", {"style": "continental"}, "i1", "1.1", "REF i1"), + ("continental", {"style": "continental"}, "n2", "1.2(b)", 'REF i2 + "(" + REF n2'), ("continental", {"style": "continental"}, "p1", "1(1)", "REF alpha + REF p1"), ("continental", {"style": "continental"}, "gamma", "1.1.1", "REF gamma"), # the none scheme: the text of the heading, then the pieces after a space @@ -1469,11 +1896,13 @@ def with_anchors(source: str) -> str: return "\n".join(out) -def with_references(source: str) -> str: - """*source* with a paragraph at its end that refers to the anchors in it (at most 40, evenly spread: resolving - a reference costs more than numbering a unit).""" +def with_references(source: str, shift: int = 0) -> str: + """*source* with a paragraph at its end that refers to the anchors in it (at most 40, evenly spread, starting at + the *shift*-th: resolving a reference costs more than numbering a unit, and documents of different shifts refer + to different units).""" anchors = list(dict.fromkeys(re.findall(r"\{#([a-z][a-z0-9-]*)", source))) - anchors = anchors[::-(-len(anchors) // 40) or 1] + stride = -(-len(anchors) // 40) or 1 + anchors = anchors[shift % stride::stride] return source + "\n\nRefs: " + " ".join(f"{{{{ref: {anchor}}}}}" for anchor in anchors) + "\n" @@ -1578,10 +2007,13 @@ def test_random_documents_references_are_planned_as_resolved(variant: str) -> No raise AssertionError(f"{error}\n--- document ---\n{source}") from error -@pytest.mark.parametrize("variant", ["default", "continental", "paragraphs", "none"]) +@pytest.mark.parametrize("variant", VARIANTS) def test_random_alternatives_references_are_planned_as_resolved(variant: str) -> None: - for _, result, plan, diagnostics in alternative_runs(variant): - assert_references_as_resolved(result, plan, diagnostics) + for source, result, plan, diagnostics in alternative_runs(variant): + try: + assert_references_as_resolved(result, plan, diagnostics) + except AssertionError as error: + raise AssertionError(f"{error}\n--- document ---\n{source}") from error def test_the_random_documents_reach_every_kind_of_cover() -> None: diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index d7b49c0..f6b65ce 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -460,10 +460,14 @@ def check(case: Case) -> None: mine = case.plan.units[node.unit] if not (isinstance(node, Section) or mine.explicit): continue - number, _ = lines[node.unit] + number, text = lines[node.unit] label = node.label or "" if mine.num_id is None: assert number == "", (node.unit, number) + elif mine.num_id == 0: # a later alternative: no number of its own, the holder's as fields, which LibreOffice + # updates to what the renderer wrote + assert number == "" and text.startswith(label), (node.unit, "LibreOffice", number, text, "renderer", label) + assert text.startswith(mine.label), (node.unit, "LibreOffice", text, "plan", mine.label) elif mine.count is None: # a bullet: LibreOffice draws its own symbol assert number != "", node.unit else: From 3611768c1c2e2a9ac628bf5bf3705e7946768be1 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:46:37 +0000 Subject: [PATCH 096/118] Round-trip alternatives, qualified items and edits; find cycles by components LibreOffice now draws the numbers of alternatives (numId 0 and a field), of items on the heading list and of lists on its reserved levels as the renderer writes them, the contents leave out the items on the heading list, and clauses and nested items inserted under the continental style are numbered as the renderer numbers them. A heading's text is now made of the cached results of the fields in its title, so a reference to it reads what the runs show, and the cycles among title references are found once by strongly connected components instead of by a walk from every heading. --- src/legaldown_render/resolve/docx_plan.py | 146 +++++++--- tests/docx_support.py | 6 +- tests/lo/uno_edit.py | 19 +- tests/test_docx_plan.py | 100 +++++++ tests/test_docx_roundtrip.py | 326 +++++++++++++++++++++- 5 files changed, 546 insertions(+), 51 deletions(-) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 734ab27..cad62d1 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -22,7 +22,7 @@ from __future__ import annotations import re -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field, replace from typing import Literal, NamedTuple @@ -386,13 +386,16 @@ def _reads(level: LevelDef, levels: Sequence[LevelDef]) -> str: # --------------------------------------------------------------------------- -def _shown(inlines: tuple[Inline, ...]) -> str: +def _shown(inlines: tuple[Inline, ...], ref_text: Callable[[CrossRef], str] | None = None) -> str: """What the runs of *inlines* show in the document (``writers/docx/runs.py``): the text of each, a hard break as a line break, a blank with its prompt. A heading's bookmark returns this, which is not always the heading's - designation (white space is collapsed in that).""" + designation (white space is collapsed in that). A cross-reference shows what *ref_text* says its fields give (its + own text by default).""" out: list[str] = [] for inline in inlines: match inline: + case CrossRef() if ref_text is not None: + out.append(ref_text(inline)) case Text(text=text) | Code(text=text) | CrossRef(text=text) | TermRef(text=text) | Value(text=text) \ | FailureMarker(text=text): out.append(text) @@ -407,11 +410,59 @@ def _shown(inlines: tuple[Inline, ...]) -> str: case Image(children=children): out.append(plain_text(children)) case Link(href=href, children=children): - out.append(_shown(children) if has_visible_content(children) else displayed_url(href)) + out.append(_shown(children, ref_text) if has_visible_content(children) else displayed_url(href)) case Emphasis(children=children) | Strong(children=children) | DefinedTerm(children=children): - out.append(_shown(children)) + out.append(_shown(children, ref_text)) return "".join(out) + +def _components(successors: Mapping[str, Sequence[str]]) -> tuple[dict[str, int], list[list[str]]]: + """The strongly connected components of a directed graph (Tarjan, iterative, so that a chain of any length is + fine): the component of each node, and the components in the order they are completed, which is every node's + successors first.""" + index: dict[str, int] = {} + low: dict[str, int] = {} + on_stack: set[str] = set() + stack: list[str] = [] + component: dict[str, int] = {} + order: list[list[str]] = [] + nodes = list(dict.fromkeys([*successors, *(n for out in successors.values() for n in out)])) + for root in nodes: + if root in index: + continue + index[root] = low[root] = len(index) + stack.append(root) + on_stack.add(root) + work = [(root, iter(successors.get(root, ())))] + while work: + node, following = work[-1] + for nxt in following: + if nxt not in index: + index[nxt] = low[nxt] = len(index) + stack.append(nxt) + on_stack.add(nxt) + work.append((nxt, iter(successors.get(nxt, ())))) + break + if nxt in on_stack: + low[node] = min(low[node], index[nxt]) + else: + work.pop() + if work: + parent = work[-1][0] + low[parent] = min(low[parent], low[node]) + if low[node] == index[node]: + members = [] + while True: + member = stack.pop() + on_stack.discard(member) + component[member] = len(order) + members.append(member) + if member == node: + break + order.append(members) + return component, order + + _Unit = Section | ListItem | Paragraph @@ -697,40 +748,62 @@ def report_references(self) -> None: f"written differently there.{more}") self.report("render-docx-reference", "warning", message, first.found.line) - def cyclic_titles(self, built: dict[str, _Built]) -> dict[str, tuple[str, ...]]: - """The cross-references in a heading's title that must be text, by heading unit. Under the ``none`` scheme a - heading is designated by its text, and a field for it (``REF bookmark \\h``) returns what its bookmark holds; - a reference in the title that leads back to the heading (directly, or through the titles of other headings) - would be a field inside the bookmark it points at, which Word cannot show (a self-reference) and - LibreOffice expands without end. Such a reference is written as its text and reported.""" + def cyclic_titles(self, built: dict[str, _Built]) -> tuple[dict[str, tuple[str, ...]], list[str]]: + """The cross-references in a heading's title that must be text, by heading unit, and the headings in the + order their texts can be worked out (those a title's fields return first). Under the ``none`` scheme a heading + is designated by its text, and a field for it (``REF bookmark \\h``) returns what its bookmark holds; a + reference in the title that leads back to the heading (directly, or through the titles of other headings) + would be a field inside the bookmark it points at, which Word cannot show (a self-reference) and LibreOffice + expands without end. Such a reference is written as its text and reported. The headings are the nodes of a + graph, an edge for each heading whose text a field in another's title returns; a reference in a title is + cyclic exactly when such a field's heading is in the title's own strongly connected component (the heading + itself included).""" if not any(not p.number for made in built.values() for p in made.pieces if isinstance(p, Field)): - return {} # no field returns the text of a heading: nothing can lead back into one + return {}, [] # no field returns the text of a heading: nothing can lead back into one inside: dict[str, list[str]] = {} for section in self.tree.sections: if section.unit is not None: targets = [i.target_unit for i in iter_inlines(section.title) if isinstance(i, CrossRef) and i.target_unit in built] inside[section.unit] = list(dict.fromkeys(t for t in targets if t is not None)) - owners = {target: {p.owner for p in made.pieces if isinstance(p, Field) and not p.number} + owners = {target: tuple(dict.fromkeys(p.owner for p in made.pieces if isinstance(p, Field) and not p.number)) for target, made in built.items()} - - def leads_to(start: str, goal: str) -> bool: - seen, pending = set(), [start] - while pending: - heading = pending.pop() - if heading == goal: - return True - if heading not in seen: - seen.add(heading) - pending += [t for target in inside.get(heading, ()) for t in owners[target]] - return False - + component, completed = _components({heading: list(dict.fromkeys(o for t in targets for o in owners[t])) + for heading, targets in inside.items()}) found: dict[str, tuple[str, ...]] = {} for heading, targets in inside.items(): - cyclic = tuple(t for t in targets if any(leads_to(owner, heading) for owner in owners[t])) + cyclic = tuple(t for t in targets if any(component[o] == component[heading] for o in owners[t])) if cyclic: found[heading] = cyclic - return found + return found, [heading for members in completed for heading in members] + + def heading_texts(self, built: dict[str, _Built], cyclic: Mapping[str, tuple[str, ...]], order: list[str]) -> None: + """What a field that returns a heading's text gives, worked out from the runs the heading is written with + (§13.3): a reference in the title is the text of the fields it is written as (which return the text of other + headings, worked out first), or its own text where it is written as text.""" + sections = {section.unit: section for section in self.tree.sections if section.unit is not None} + for heading in order: + section = sections.get(heading) + if section is None: + continue + + def shown(ref: CrossRef, heading: str = heading) -> str: + made = built.get(ref.target_unit or "") + if made is None or ref.target_unit in cyclic.get(heading, ()): + return ref.text + return "".join(piece if isinstance(piece, str) else + text_of(piece.tokens) if piece.number else self.heading_text[piece.owner] + for piece in made.pieces) + + self.heading_text[heading] = _shown(section.title, shown) + + def references(self, found: Mapping[str, _Reference]) -> dict[str, _Built]: + built: dict[str, _Built] = {} + for target, reference in found.items(): + made = self.reference(reference) + if made is not None: + built[target] = made + return built def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan], tuple[TocEntryPlan, ...]]: """The bookmarks, the cross-reference fields and the table of contents, which need the numbering of every @@ -748,11 +821,14 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan known = found[inline.target_unit] if known.line is None or line < known.line: found[inline.target_unit] = known._replace(line=line) - built: dict[str, _Built] = {} - for target, reference in found.items(): - made = self.reference(reference) - if made is not None: - built[target] = made + built = self.references(found) + cyclic, order = self.cyclic_titles(built) + if order: + # a heading's text is made of the fields in its title: the first pass found which are text and in what + # order the texts can be worked out, and the references are made again from them + self.heading_texts(built, cyclic, order) + self.unreferable = [] + built = self.references(found) self.report_references() needed = {piece.owner for made in built.values() for piece in made.pieces if isinstance(piece, Field)} needed |= {holder for holder, _ in self.alts.values()} @@ -760,7 +836,6 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan #: of the paragraph, so that no field ever stands inside the bookmark it points at. enclosing = {piece.owner for made in built.values() for piece in made.pieces if isinstance(piece, Field) and not piece.number} - cyclic = self.cyclic_titles(built) word_only = [(found[target].text, made.lo_text) for target, made in built.items() if made.lo_text is not None] if word_only: first, lo = word_only[0] @@ -770,9 +845,10 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan f"switch, which LibreOffice ignores: there it reads '{printable(lo)}'{more}.", None) for section in self.tree.sections: if section.unit in cyclic: + where = f"line {section.line}" if section.line is not None else f"section {section.unit}" self.report("render-docx-reference-cycle", "warning", - f"The heading '{printable(plain_text(section.title)[:40])}' holds a cross-reference that " - f"leads back to the heading itself, which a field cannot show (the heading's text is what " + f"The heading '{printable(plain_text(section.title)[:40])}' ({where}) holds a " + f"cross-reference that leads back to the heading itself, which a field cannot show (the heading's text is what " f"the reference returns): it is written as text, so it will not follow edits made in Word.", section.line) diff --git a/tests/docx_support.py b/tests/docx_support.py index 4675ade..ea2c814 100644 --- a/tests/docx_support.py +++ b/tests/docx_support.py @@ -244,10 +244,10 @@ def libreoffice_export(documents: dict[str, bytes], *, update: bool = True) -> d return result -def libreoffice_edit(jobs: dict[str, tuple[bytes, list[tuple[str, str]]]]) -> dict[str, str]: +def libreoffice_edit(jobs: dict[str, tuple[bytes, list[tuple[str, str | None]]]]) -> dict[str, str]: """Open each document (name to DOCX bytes and edits) in one LibreOffice instance, insert a paragraph with the text - ``new`` before the first paragraph that holds ``match`` for each ``(match, new)``, update the fields and indexes, - and return the text of each (tests/lo/uno_edit.py).""" + ``new`` before the first paragraph that holds ``match`` for each ``(match, new)`` (delete that paragraph when + ``new`` is None), update the fields and indexes, and return the text of each (tests/lo/uno_edit.py).""" python = require_libreoffice() with tempfile.TemporaryDirectory(prefix="ld-lo-") as folder: out = Path(folder, "out") diff --git a/tests/lo/uno_edit.py b/tests/lo/uno_edit.py index 6fe3bee..608503c 100644 --- a/tests/lo/uno_edit.py +++ b/tests/lo/uno_edit.py @@ -1,14 +1,15 @@ #!/usr/bin/python3 """Edit DOCX files in headless LibreOffice through UNO the way a lawyer does, update the fields, and write what the document says: for each job, a paragraph is inserted before the first paragraph that holds some text (so that -it takes that paragraph's numbering and style, as pressing Enter at its start does), then every field and index is -updated and the body is exported as plain text (``JOB.txt``, UTF-8). +it takes that paragraph's numbering and style, as pressing Enter at its start does), or that paragraph is deleted, +then every field and index is updated and the body is exported as plain text (``JOB.txt``, UTF-8). Run it with the system Python that has the ``uno`` module, **not** with the project's virtual environment: /usr/bin/python3 tests/lo/uno_edit.py OUTDIR JOBS.json -``JOBS.json`` is a list of ``{"name": "job", "file": "/path/doc.docx", "edits": [["text of a paragraph", "new"], ...]}``. +``JOBS.json`` is a list of ``{"name": "job", "file": "/path/doc.docx", "edits": [["text of a paragraph", "new"], ...]}``; +an edit whose new text is ``null`` deletes the paragraph instead. One LibreOffice instance serves every job. Exit status 0 when every job was exported. """ from __future__ import annotations @@ -47,12 +48,20 @@ def _connect(pipe: str, process: subprocess.Popen[bytes]) -> object: raise SystemExit("soffice did not accept a connection in time") -def _insert_before(document: object, match: str, new: str) -> None: +def _edit(document: object, match: str, new: str | None) -> None: + """Insert a paragraph with the text *new* before the first paragraph that holds *match*, or, when *new* is None, + delete that paragraph (with its break, as Delete at its end does).""" text = document.getText() paragraphs = text.createEnumeration() while paragraphs.hasMoreElements(): paragraph = paragraphs.nextElement() if paragraph.supportsService("com.sun.star.text.Paragraph") and match in paragraph.getString(): + if new is None: + cursor = text.createTextCursorByRange(paragraph.getStart()) + cursor.gotoEndOfParagraph(True) + cursor.goRight(1, True) + cursor.setString("") + return cursor = text.createTextCursorByRange(paragraph.getStart()) text.insertControlCharacter(cursor, PARAGRAPH_BREAK, False) cursor.goLeft(1, False) @@ -79,7 +88,7 @@ def edit(out_dir: str, jobs: list[dict]) -> None: raise SystemExit(f"LibreOffice could not open {job['file']}") try: for match, new in job["edits"]: - _insert_before(document, match, new) + _edit(document, match, new) for _ in range(2): # page numbers and indexes settle after the first layout document.getTextFields().refresh() document.refresh() diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index 27763c6..1923216 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -8,6 +8,7 @@ import functools import random import re +import time from collections.abc import Iterator, Sequence from pathlib import Path @@ -24,6 +25,7 @@ FieldSeg, Literal_, Segment, + _components, _Planner, docx_style_errors, plan_docx, @@ -1667,6 +1669,104 @@ def test_a_reference_in_a_heading_to_a_unit_inside_another_heading_is_still_a_fi assert "render-docx-reference-cycle" not in rules(diagnostics) +def test_two_cyclic_headings_with_the_same_text_are_both_reported() -> None: + body = "# Same {{ref: a}} {#a}\n\n# Same {{ref: b}} {#b}\n\n# Same {{ref: c}} {#c}\n" + tree, plan, diagnostics = planned(body, overrides=NONE) + found = [d for d in diagnostics if d.rule == "render-docx-reference-cycle"] + assert [d.line for d in found] == [section.line for section in tree.sections] + assert all(f"line {d.line}" in d.message for d in found) + + +def test_the_headings_that_lead_to_each_other_are_found_by_components_not_by_walking_from_each() -> None: + # a ring of three with a tail into it and a chain out of it: only the ring is cyclic, whatever the order + successors = {"a": ["b"], "b": ["c"], "c": ["a", "d"], "t": ["a"], "d": ["e"], "e": []} + component, order = _components(successors) + assert component["a"] == component["b"] == component["c"] + assert len({component[n] for n in "abctde"}) == 4 + assert [sorted(members) for members in order if len(members) > 1] == [["a", "b", "c"]] + # each component comes after those it leads to + assert order.index(["e"]) < order.index(["d"]) < order.index(next(m for m in order if "a" in m)) + assert order.index(["t"]) == len(order) - 1 + + +def test_the_components_of_a_graph_agree_with_what_each_node_can_reach() -> None: + rng = random.Random(3) + for _ in range(200): + size = rng.randint(1, 9) + names = [f"n{i}" for i in range(size)] + graph = {n: [m for m in names if rng.random() < 0.25] for n in names} + component, _ = _components(graph) + + def reaches(start: str, goal: str, graph: dict[str, list[str]] = graph) -> bool: + seen, pending = set(), list(graph[start]) + while pending: + node = pending.pop() + if node == goal: + return True + if node not in seen: + seen.add(node) + pending += graph[node] + return False + + for a in names: + for b in names: + assert (component[a] == component[b]) == (a == b or (reaches(a, b) and reaches(b, a))) + + +def test_a_very_long_chain_of_components_does_not_hit_the_recursion_limit() -> None: + chain = {f"n{i}": [f"n{i + 1}"] for i in range(50_000)} + component, order = _components(chain) + assert len(order) == 50_001 and len(set(component.values())) == 50_001 + ring = {f"n{i}": [f"n{(i + 1) % 50_000}"] for i in range(50_000)} + _, order = _components(ring) + assert len(order) == 1 and len(order[0]) == 50_000 + + +def _chain(count: int, close: bool) -> str: + """Headings that each refer to the one before (and the first to the last, with *close*): under the none scheme + each reference is a field for the heading's text.""" + parts = [f"# H0 {{{{ref: h{count - 1}}}}} {{#h0}}\n" if close else "# H0 {#h0}\n"] + parts += [f"# H{i} see {{{{ref: h{i - 1}}}}} {{#h{i}}}\n" for i in range(1, count)] + return "\n".join(parts) + + +@pytest.mark.parametrize("close", [False, True]) +def test_the_headings_of_a_long_chain_of_references_are_planned_in_linear_time(close: bool) -> None: + def seconds(count: int) -> float: + tree = render(FRONT + _chain(count, close), format="text", overrides=NONE).tree + best = float("inf") + for _ in range(2): + started = time.perf_counter() + plan_docx(tree) + best = min(best, time.perf_counter() - started) + return best + + small, large = seconds(1000), seconds(4000) + assert large < 6 * max(small, 0.02), (small, large) # four times the headings: a walk from each would be sixteen + assert large < 2.0 + + +def test_a_ring_of_references_in_titles_is_all_text_and_a_chain_all_fields() -> None: + _, ring, _ = planned(_chain(30, True), overrides=NONE) + assert all(ring.units[f"s{i}"].literal_refs for i in range(1, 31)) + _, chain, _ = planned(_chain(30, False), overrides=NONE) + assert not any(unit.literal_refs for unit in chain.units.values()) + + +def test_the_text_a_heading_field_returns_is_made_of_the_fields_in_the_title_it_stands_for() -> None: + # A's title refers to the item under it (written as text: the item's designation starts with A's own text); B's + # refers to A, a field for A's text, which includes the text A's title has for that reference. + body = "# A {{ref: i}} {#a}\n\n- x {#i}\n\n# B {{ref: a}} {#b}\n\nRefs {{ref: a}} {{ref: b}} {{ref: i}}\n" + tree, plan, _ = planned(body, overrides=NONE) + a, b = tree.sections + assert plan.units[a.unit].literal_refs == ("u1",) and plan.units[b.unit].literal_refs == () + texts = {t: cached(plan.refs[t].segments) for t in plan.refs} + title_of_a = next(i.text for i in iter_tree_inlines(tree) if isinstance(i, CrossRef) and i.target_unit == "u1") + # B's heading text is "B " and what the field for A gives, which is A's text: "A " and the reference as text + assert texts[a.unit] == "A " + title_of_a + assert texts[b.unit] == "B " + texts[a.unit] + + # -- What the bookmark returns is what the runs show --------------------------------------------------------------- diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index f6b65ce..98793e5 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -20,8 +20,9 @@ prints), **modulo the gaps the plan documents**, each of which the plan reports with a diagnostic: * a unit whose number Word draws differently from the renderer's (a letter counter past 26, a label that needs the - section's number, a list deeper than Word's nine levels, alternatives, which the next stage of the planner - places): LibreOffice must agree with the plan (the simulator), and the plan must say so; + section's number where the heading list cannot show it, a list deeper than Word's nine levels): LibreOffice must + agree with the plan (the simulator), and the plan must say so; a later alternative that shares a number has no number + of its own (``numId`` 0) and shows the holder's as a field, which LibreOffice updates to the renderer's; * ``\\t`` is ignored by LibreOffice (it is Word's switch), so a reference built with it reads there as ``FieldSeg.cached_lo`` says; * LibreOffice fills a skipped heading level with its start value, which is what the simulator models for headings. @@ -43,7 +44,7 @@ from legaldown_render import DocumentError, render from legaldown_render.resolve.docx_plan import DocxPlan, FieldSeg, Literal_, plan_docx -from legaldown_render.tree import CrossRef, ListItem, Paragraph, RenderTree, Section, iter_tree_inlines +from legaldown_render.tree import CrossRef, ListItem, Paragraph, RenderTree, Section, iter_tree_inlines, plain_text from legaldown_render.writers.docx.ooxml import Bookmarks, p, text_run from legaldown_render.writers.docx.package import Relationships from legaldown_render.writers.docx.runs import InlineWriter @@ -196,7 +197,7 @@ ## Sub one {#s1} - courts item {#ci when=forum:courts} -- arbitration item {#ai when=forum:arb} +- arbitration item {#ci when=forum:arb} - nested under alt {#an} - common item {#cm} @@ -207,6 +208,108 @@ # Last {#last} Ref {{ref: d}} {{ref: s1}} {{ref: sa}} {{ref: ci}} {{ref: an}} {{ref: cm}} {{ref: last}} +""", + "alternatives in threes": ALTERNATIVES + """ +# Intro {#intro} + +Para one. {#pc when=forum:courts} + +Para arbitration. {#pc when=forum:arb} + +Para expert. {#pc when=forum:expert} + +Para next. {#pn} + +# Disputes {#d when=forum:courts} + +## Sub one {#s1} + +## Sub two {#s2} + +# Disputes {#d when=forum:arb} + +## Sub alt {#sa} + +- quoted item {#q} + +# Disputes {#d when=forum:expert} + +## Expert one {#e1} + +## Expert two {#e2} + +## Expert three {#e3} + +# Items {#items} + +- courts {#ci when=forum:courts} + - courts child {#cc} +- arbitration {#ci when=forum:arb} + - arbitration child one {#ac1} + - arbitration child two {#ac2} +- expert {#ci when=forum:expert} +- common {#cm} + - common child {#cmc} + +# Last {#last} + +## Last one {#l1} + +Ref {{ref: d}} {{ref: s1}} {{ref: s2}} {{ref: sa}} {{ref: q}} {{ref: e1}} {{ref: e3}} {{ref: ci}} {{ref: cc}} +{{ref: ac1}} {{ref: ac2}} {{ref: cm}} {{ref: cmc}} {{ref: pc}} {{ref: pn}} {{ref: last}} {{ref: l1}} +""", + "nested titles": FRONT + """ +# A {{ref: i}} {#a} + +- x {#i} + +# B {{ref: a}} {#b} + +# C {{ref: b}} and {{ref: a}} {#c} + +Refs {{ref: a}} {{ref: b}} {{ref: c}} {{ref: i}} +""", + "qualified": FRONT + """ +# Alpha {#alpha} + +Intro. {#p1} + +- first {#i1} + - nested one {#n1} + - nested two {#n2} + - deep {#d1} + - deeper {#d2} + - deepest {#d3} +- second {#i2} + 1. ordered {#o1} + 1. ordered two {#o2} + +Between. + +- third {#i3} + +## Beta {#beta} + +- beta item {#bi} + - beta nested {#bn} + +### Gamma {#gamma} + +#### Delta {#delta} + +##### Epsilon {#eps} + +- five item {#ei} + - five nested {#en} +- five two {#ej} + +# Second {#second} + +- x {#sx} + - y {#sy} + +Ref {{ref: i1}} {{ref: n2}} {{ref: d3}} {{ref: o2}} {{ref: i3}} {{ref: beta}} {{ref: bn}} {{ref: ei}} {{ref: en}} +{{ref: ej}} {{ref: sy}} {{ref: p1}} """, } @@ -217,11 +320,15 @@ "section-sign": {"overrides": {"numbering.levels": [ {"counter": "decimal", "label": "§ {n}", "ref": "{n}"}], "references.format": "§ {designation}"}}, "paragraphs": {"overrides": {"paragraphs.numbered": True}}, + # the items and the paragraphs of a section share the heading list's level: the numbers are restarted where the + # resolver counts them apart + "shared level": {"style": "continental", "overrides": {"paragraphs.numbered": True, "paragraphs.format": { + "counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}}}, } #: What a diagnostic of the plan has to say when Word numbers a unit differently from the renderer. -EXPLAINING = {"render-docx-number-mismatch", "render-docx-section-label", "render-docx-letter-overflow", - "render-docx-roman-overflow", "render-docx-list-depth", "render-docx-shared-counter"} +EXPLAINING = {"render-docx-section-label", "render-docx-letter-overflow", "render-docx-roman-overflow", + "render-docx-list-depth"} WRONG = "XXX" OPEN, CLOSE = "⟦", "⟧" @@ -504,10 +611,12 @@ def test_the_round_trip_reaches_numbers_references_and_every_gap() -> None: """The corpus is not vacuous: it numbers headings, items and paragraphs, it has references of every kind, and it reaches each documented gap (``\\t``, an overflowing letter counter, a label that needs the section).""" seen_rules: set[str] = set() + seen_causes: set[str] = set() references = suppressing = numbered = hidden = 0 for variant in list(VARIANTS) + list(WORDS): for case in round_trip(variant): seen_rules |= case.rules + seen_causes |= {cause for unit in case.plan.units.values() for cause in unit.cause} references += len(case.plan.refs) numbered += sum(1 for unit in case.plan.units.values() if unit.count is not None) suppressing += sum(any(isinstance(s, FieldSeg) and s.suppress_text for s in r.segments) @@ -515,7 +624,9 @@ def test_the_round_trip_reaches_numbers_references_and_every_gap() -> None: hidden += sum(isinstance(s, FieldSeg) and s.bookmark.startswith("_Ld") for r in case.plan.refs.values() for s in r.segments) assert numbered > 200 and references > 100 and suppressing > 0 and hidden > 0 - assert {"render-docx-letter-overflow", "render-docx-section-label"} <= seen_rules + assert {"render-docx-letter-overflow", "render-docx-section-label", "render-docx-shared-counter"} <= seen_rules + assert "render-docx-number-mismatch" not in seen_rules + assert {"alternative", "reserved-level", "resync", "restart-alternative", "restart-section"} <= seen_causes def test_the_cached_results_are_wrong_in_the_file_and_libreoffice_does_not_read_them() -> None: @@ -603,8 +714,207 @@ def test_the_references_follow_the_insertions_a_lawyer_makes(variant: str) -> No assert [squeeze(line) for line in before.splitlines() if "Refs:" in line] != expected # the edits moved them -@pytest.mark.parametrize("variant", ["default", "outline", "none"]) +@pytest.mark.parametrize("variant", ["default", "outline", "continental", "none"]) def test_the_numbers_follow_the_insertions_a_lawyer_makes(variant: str) -> None: said, written = edited()[variant] assert [squeeze(line) for line in said.lstrip("\ufeff").splitlines() if squeeze(line)] == [ squeeze(line) for line in written.splitlines() if squeeze(line)] + + +# -- Clauses and nested items inserted under the continental style -------------------------------------------------- + + +CLAUSES = FRONT + """ +# Alpha {#alpha} + +- clause one {#c1} + - nested of one {#n1} +- clause two {#c2} + - nested of two {#n2} + - deep of two {#d2} + +Refs: {{ref: c1}} {{ref: n1}} {{ref: c2}} {{ref: n2}} {{ref: d2}} +""" + +#: A clause and a nested item inserted before the second clause and its nested item. +CLAUSE_EDITS = [ + ("clause two", "NEW clause", "- clause two {#c2}", "- NEW clause"), + ("nested of two", "NEW nested", " - nested of two {#n2}", " - NEW nested"), +] + +#: A heading section whose sub-headings follow numbered clauses: Word's counter of the level is shared with the clauses, +#: so the first sub-heading is restarted at the number the renderer gives it (a fixed value). +RESYNCED = FRONT + """ +# Alpha {#alpha} + +- clause one {#c1} +- spare clause +- clause two {#c2} + +## Beta {#beta} + +## Gamma {#gamma} + +Refs: {{ref: c1}} {{ref: c2}} {{ref: beta}} {{ref: gamma}} +""" + + +def applied(source: str, edits: list[tuple[str, str]]) -> str: + """*source* with each new line put before the line it goes before.""" + for before, new in edits: + source = source.replace(before, f"{new}\n{before}", 1) + return source + + +@functools.cache +def clauses() -> dict[str, str]: + """What LibreOffice says about the clause document after the two insertions, under the continental style and under + the default (whose nested lists are the family's own).""" + require_libreoffice() + edits = [(match, new) for match, new, _, _ in CLAUSE_EDITS] + jobs = {style: (render(CLAUSES, format="docx", **settings).binary, edits) + for style, settings in {"continental": {"style": "continental"}, "default": {}}.items()} + return libreoffice_edit(jobs) # type: ignore[arg-type] + + +def lines_of(text: str) -> list[str]: + return [squeeze(line) for line in text.lstrip("\ufeff").splitlines() if squeeze(line)] + + +@pytest.mark.parametrize("style", ["continental", "default"]) +def test_a_clause_and_a_nested_item_inserted_in_word_are_numbered_as_the_renderer_numbers_them(style: str) -> None: + settings = {"style": "continental"} if style == "continental" else {} + after = render(applied(CLAUSES, [(before, new) for _, _, before, new in CLAUSE_EDITS]), format="text", **settings) + said = lines_of(clauses()[style]) + assert said == lines_of(after.output) + # the item inserted into a list of nested items starts it: "a)" under the continental style, "(i)" under the default + first, second = ("a)", "b)") if style == "continental" else ("(i)", "(ii)") + assert [line for line in said if line.endswith(("NEW nested", "nested of two"))] == [ + f"{first} NEW nested", f"{second} nested of two"] + + +def test_the_clause_inserted_under_the_continental_style_takes_the_next_number_of_the_heading_list() -> None: + said = lines_of(clauses()["continental"]) + assert [line for line in said if line.endswith(("NEW clause", "clause two"))] == ["1.2 NEW clause", "1.3 clause two"] + assert [line for line in said if line.startswith("(1) Refs:")] == ["(1) Refs: 1.1 1.1(a) 1.3 1.3(b) 1.3(b)(i)"] + + +# -- A sub-heading Word restarts at a fixed number: the limit of what an edit keeps live ------------------------------- + + +@functools.cache +def resynced() -> dict[str, str]: + require_libreoffice() + data = render(RESYNCED, format="docx", style="continental").binary + jobs = {"original": (data, []), "clause deleted": (data, [("spare clause", None)]), + "clause inserted": (data, [("spare clause", "NEW clause")]), + "heading deleted": (data, [("Beta", None)]), "heading inserted": (data, [("Beta", "NEWH")])} + return libreoffice_edit(jobs) # type: ignore[arg-type] + + +def test_a_sub_heading_after_clauses_is_restarted_by_a_fixed_value_and_the_plan_says_so() -> None: + result = render(RESYNCED, format="docx", style="continental") + plan, diagnostics = plan_docx(result.tree) + assert plan.units["s2"].cause == ("resync",) + [found] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert found.level == "warning" + assert lines_of(resynced()["original"]) == lines_of(render(RESYNCED, format="text", style="continental").output) + + +def test_a_clause_inserted_or_deleted_before_the_restarted_sub_heading_leaves_it_and_what_follows_right() -> None: + sources = {"clause deleted": RESYNCED.replace("- spare clause\n", ""), + "clause inserted": applied(RESYNCED, [("- spare clause", "- NEW clause")])} + for name, source in sources.items(): + assert lines_of(resynced()[name]) == lines_of(render(source, format="text", style="continental").output), name + + +def test_a_sub_heading_inserted_or_deleted_before_the_restarted_one_is_the_documented_limit() -> None: + # The restarted heading keeps its fixed value, and the counter it shared goes on where the edit left it: the + # warning of the plan is what tells the author. Only the warning is asserted, not what the drift looks like. + _, diagnostics = plan_docx(render(RESYNCED, format="docx", style="continental").tree) + assert [d.level for d in diagnostics if d.rule == "render-docx-shared-counter"] == ["warning"] + for name in ("heading deleted", "heading inserted"): + said = lines_of(resynced()[name]) + assert [line for line in said if "clause" in line and not line.startswith("(")] == [ + "1.1 clause one", "1.2 spare clause", "1.3 clause two"] # what stands before the edit is right + + +# -- The contents: the headings with the numbers Word draws, and none of the units on the heading list ------------------ + +CONTENTS = ALTERNATIVES + """ +# Alpha {#alpha} + +- item one {#i1} + - nested {#n1} +- item two {#i2} + +## Beta {#beta} + +- beta item + +# Disputes {#d when=forum:courts} + +- courts item + +## Sub + +# Disputes {#d when=forum:arb} + +## Sub alt + +# Last +""" + + +def with_contents_field(data: bytes) -> bytes: + """*data* with a ``TOC \\o "1-5" \\h \\z \\u`` field at the start of the body, with a result that is wrong, for LibreOffice to + build the index from (the writer does not write the field itself).""" + def change(document: str) -> str: + ET.register_namespace("w", W.strip("{}")) + root = ET.fromstring(document.encode("utf-8")) + body = root.find(f"{W}body") + assert body is not None + + def run(child: ET.Element) -> ET.Element: + holder = ET.Element(f"{W}r") + holder.append(child) + return holder + + paragraph = ET.Element(f"{W}p") + for kind in ("begin", "instruction", "separate", "text", "end"): + if kind == "instruction": + child = ET.Element(f"{W}instrText") + child.text = ' TOC \\o "1-5" \\h \\z \\u ' + elif kind == "text": + child = ET.Element(f"{W}t") + child.text = WRONG + else: + child = ET.Element(f"{W}fldChar") + child.set(f"{W}fldCharType", kind) + paragraph.append(run(child)) + body.insert(0, paragraph) + return ET.tostring(root, encoding="unicode") + + return repackage(data, "word/document.xml", change) + + +@functools.cache +def contents() -> tuple[DocxPlan, tuple[str, ...], str]: + require_libreoffice() + result = render(CONTENTS, format="docx", style="continental", overrides={"contents.enabled": True, + "contents.depth": 5}) + assert result.binary is not None + plan, _ = plan_docx(result.tree) + shown = libreoffice_export({"contents": with_contents_field(result.binary)})["contents"] + lines = shown.text.lstrip("\ufeff").splitlines() + return plan, tuple(lines[:len(plan.toc)]), "\n".join(lines[len(plan.toc):]) + + +def test_the_contents_list_the_headings_as_the_plan_does_and_none_of_the_items_on_the_heading_list() -> None: + plan, entries, body = contents() + expected = [squeeze(f"{entry.number} {plain_text(entry.title)} {entry.condition_label or ''}") for entry in plan.toc] + said = [re.sub(r"\s+\d+$", "", squeeze(line)) for line in entries] + assert said == expected + assert [e.number for e in plan.toc] == ["1.", "1.1", "2.", "2.1", "2.", "2.1", "3."] # the alternative's is shared + assert not any(word in line for line in entries for word in ("item", "nested")) + assert "1.1 item one" in body and "a) nested" in body # the items are in the body: on the heading list, not in it From 60ffb0b88f6857c86af0478c2869aff6c6fccdcf Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:56:31 +0000 Subject: [PATCH 097/118] Keep attachments without a usable id in the DOCX contents, and define the title block label style An attachment with no id, or an id another has, renders (the validator reports it) but has no anchor of its own, so the contents entries for the attachments are now paired with the plan's by position and no bookmark is written twice. TitleBlockLabel was used but not defined; a test now checks that every style a document uses exists. The LibreOffice tests also cover a contents depth of 1, the cached contents before any update, and known page positions. --- src/legaldown_render/writers/docx/matter.py | 42 ++++--- src/legaldown_render/writers/docx/styles.py | 2 + tests/test_docx_matter.py | 116 +++++++++++++++++++- tests/test_docx_package.py | 4 +- 4 files changed, 139 insertions(+), 25 deletions(-) diff --git a/src/legaldown_render/writers/docx/matter.py b/src/legaldown_render/writers/docx/matter.py index eb15d82..8ce5809 100644 --- a/src/legaldown_render/writers/docx/matter.py +++ b/src/legaldown_render/writers/docx/matter.py @@ -113,11 +113,16 @@ def __init__(self, tree: RenderTree, plan: DocxPlan, inlines: InlineWriter, book if tree.contents and len(tree.contents) != len(plan.toc): raise InternalError("the plan's table of contents does not hold the entries of the tree's " "(a bug in the DOCX plan)") - #: The ``_Toc`` name of the contents entries that list the attachments heading and an attachment, by anchor - #: (a section's is the plan's ``UnitPlan.toc_bookmark``, written with its heading). - self._toc_of: dict[str, str] = { - entry.anchor: planned.bookmark for entry, planned in zip(tree.contents, plan.toc, strict=False) - if entry.unit is None and entry.anchor} + #: The ``_Toc`` names of the contents entries that are not a section's (a section's is the plan's + #: ``UnitPlan.toc_bookmark``, written with its heading), in order: the attachments heading when the style + #: gives it a label, then one per attachment. Paired by position, not by anchor: an attachment with no id + #: or the same id as another (validator Errors, still rendered) has no anchor of its own. + self._toc_of = [planned.bookmark for entry, planned in zip(tree.contents, plan.toc, strict=False) + if entry.unit is None] + wanted = (1 if tree.attachments_label else 0) + len(tree.attachments) + if self._toc_of and len(self._toc_of) != wanted: + raise InternalError("the table of contents does not list the attachments heading and each attachment " + "(a bug in the DOCX plan)") # -- header rows and parties ---------------------------------------------------------- @@ -243,6 +248,7 @@ def attachments(self) -> list[ET.Element]: if not tree.attachments: return [] content: list[ET.Element] = [] + toc = list(self._toc_of) separator = self.settings.separator # The page break goes before the first thing of the part, whichever that is, and before every title after the # first: the heading stays with the first attachment. @@ -250,7 +256,7 @@ def attachments(self) -> list[ET.Element]: level = 1 if tree.attachments_label: level = 2 - names = self._names(tree.attachments_anchor) + names = self._names(tree.attachments_anchor, toc.pop(0) if toc else None) content.append(p(self.inlines.wrap(names, [text_run(tree.attachments_label)]) if names else text_run(tree.attachments_label), style=ATTACHMENTS_HEADING, @@ -258,20 +264,22 @@ def attachments(self) -> list[ET.Element]: self._outline(1)])) breaking = False for attachment in tree.attachments: - content += self._attachment(attachment, level, page_break=breaking, rule=separator == "rule") + content += self._attachment(attachment, level, toc.pop(0) if toc else None, page_break=breaking, + rule=separator == "rule") breaking = separator == "page-break" return content - def _names(self, anchor: str | None) -> list[str]: - """The bookmarks of an attachments paragraph: the plan's name for its anchor, and the contents - entry's ``_Toc``.""" - names = [self.plan.anchors[anchor]] if anchor and anchor in self.plan.anchors else [] - if anchor and anchor in self._toc_of: - names.append(self._toc_of[anchor]) - return names - - def _attachment(self, attachment: AttachmentPart, level: int, *, page_break: bool, rule: bool) -> list[ET.Element]: - names = self._names(attachment.anchor) + def _names(self, anchor: str | None, toc: str | None) -> list[str]: + """The bookmarks of an attachments paragraph: the plan's name for its *anchor*, unless the document already + holds a bookmark of that name (a second attachment with the same id, or an id that is a section's), and + the contents entry's ``_Toc`` name *toc*.""" + name = self.plan.anchors.get(anchor) if anchor else None + names = [name] if name is not None and name.lower() not in self.bookmarks.names else [] + return [*names, *([toc] if toc else [])] + + def _attachment(self, attachment: AttachmentPart, level: int, toc: str | None, *, page_break: bool, + rule: bool) -> list[ET.Element]: + names = self._names(attachment.anchor, toc) title = self.inlines.runs(attachment.title) shown = self.inlines.wrap(names, title) if names else title if attachment.condition_label: diff --git a/src/legaldown_render/writers/docx/styles.py b/src/legaldown_render/writers/docx/styles.py index fe1ab48..dae96a3 100644 --- a/src/legaldown_render/writers/docx/styles.py +++ b/src/legaldown_render/writers/docx/styles.py @@ -295,6 +295,8 @@ def half_points(factor: float) -> int: run=[el("color", {"val": _FAILURE_TEXT}), _shade(_FAILURE_FILL)]), _style(CONDITION_LABEL, "Condition Label", kind="character", priority=38, custom=True, run=[el("color", {"val": _CONDITION_TEXT}), _size(half_points(0.8)), _shade(_CONDITION_FILL)]), + _style(TITLE_BLOCK_LABEL, "Title Block Label", kind="character", priority=41, custom=True, + run=[el("i"), el("iCs")]), _style(DRAFTING_NOTE_LABEL, "Drafting Note Label", kind="character", priority=40, custom=True, run=[el("b"), el("bCs"), _size(half_points(0.9))]), ] diff --git a/tests/test_docx_matter.py b/tests/test_docx_matter.py index 49a6cf5..636427b 100644 --- a/tests/test_docx_matter.py +++ b/tests/test_docx_matter.py @@ -548,6 +548,44 @@ def test_the_attachments_listed_in_the_contents_carry_its_bookmarks() -> None: assert len(set(bookmark_names(root))) == len(bookmark_names(root)) +BAD_ATTACHMENTS = { + "no ids": " - {title: NoId, file: a.lgd}\n - {title: NoId2, file: b.lgd}\n", + "a duplicate id": " - {id: x, title: A}\n - {id: x, title: B}\n", + "an id that is a section's": " - {id: a, title: A}\n - {id: a, title: B}\n", +} + + +@pytest.mark.parametrize("which", BAD_ATTACHMENTS) +@pytest.mark.parametrize("overrides", [{}, {"labels.attachments": ""}, {"contents.depth": 1}]) +def test_attachments_without_a_usable_id_are_still_written_and_listed(which: str, overrides: dict[str, Any]) -> None: + """Attachments with no id, or an id another has (validator Errors: the document still renders), have no anchor of their + own: the contents lists each by position, and no bookmark is written twice.""" + source = ("---\nlegaldown: '0.2'\ntitle: T\nattachments:\n" + BAD_ATTACHMENTS[which] + + "---\n\n# A {#a}\n\nSee {{attach: x}}.\n") + result = render(source, format="docx", overrides={"contents.enabled": True, **overrides}) + assert result.binary is not None + root = document_xml(result.binary) + names = bookmark_names(root) + assert len(names) == len({n.lower() for n in names}) + assert len(paragraphs(result.binary, "AttachmentTitle")) == 2 + toc = [text_of(p).rstrip("\t") for p in toc_paragraphs(result.binary)] + assert toc[-2:] == [text_of(p) for p in paragraphs(result.binary, "AttachmentTitle")] + anchors = [p.find(f"{W}hyperlink").get(f"{W}anchor") for p in toc_paragraphs(result.binary)] # type: ignore[union-attr] + assert set(anchors) <= set(names) + if schema_available(): + assert validation_errors(result.binary) == [] + + +def schema_available() -> bool: + from docx_support import schema_directory + + try: + import lxml # noqa: F401 + except ImportError: + return False + return schema_directory() is not None + + # -- Signature blocks ------------------------------------------------------------------------------------------ @@ -651,7 +689,8 @@ def party(index: int) -> dict[str, Any]: "sides": [{"name": f"s{i}", "label": soup(rnd, 8) + "l", "parties": [party(i * 3 + j) for j in range(2)]} for i in range(2)], "attachments": [{"id": f"a{i}", "title": soup(rnd, 15) + "t", "file": soup(rnd, 10) + "f"} - for i in range(3)]} + for i in range(3)] + [{"title": soup(rnd, 6) + "n"}, {"id": "a1", "title": soup(rnd, 6) + "d"}, + {"id": "h1", "title": soup(rnd, 6) + "s"}]} return (f"---\n{yaml.safe_dump(front, allow_unicode=True)}---\n\n{soup(rnd, 20)}\n\n# {soup(rnd, 15)} {{#h1}}\n\n" f"{soup(rnd, 30)} {{{{attach: a1}}}}\n\n## {soup(rnd, 10)}\n\ntext\n") @@ -736,6 +775,7 @@ def shown() -> dict[str, Exported]: contents = {"contents.enabled": True, "contents.depth": 3} documents = { "master": docx(FRONT, overrides=contents), + "depth1": docx(FRONT, overrides={"contents.enabled": True, "contents.depth": 1}), "features": docx(features, overrides=contents), "template": docx(template, style="continental", overrides=contents), "long_break": docx(LONG, overrides={"contents.enabled": True}), @@ -758,12 +798,19 @@ def test_the_contents_lists_exactly_the_html_entries(shown: dict[str, Exported]) "contents.depth": 3}) assert toc_lines(shown["master"]) == expected_entries(FRONT, overrides={"contents.enabled": True, "contents.depth": 3}) + assert toc_lines(shown["depth1"]) == expected_entries(FRONT, overrides={"contents.enabled": True, + "contents.depth": 1}) # An alternative that shares a number (§15.8) is numbered in sequence by LibreOffice until the planner keeps it - # on the first one's number: the titles are the contents' either way. + # on the first one's number, and the entries after it are numbered one more: the numbers are the contents' up to + # the first alternative, the titles are the contents' throughout. template = (DOCUMENTS / "template.lgd").read_text(encoding="utf-8") - drawn = [line.split(" ", 1)[1] for line in toc_lines(shown["template"])] - entries = expected_entries(template, style="continental", overrides={"contents.enabled": True, "contents.depth": 3}) - assert drawn == [line.split(" ", 1)[1] for line in entries] + settings = {"style": "continental", "overrides": {"contents.enabled": True, "contents.depth": 3}} + result = render(template, format="text", **settings) + sharing = {section.unit for section in result.tree.sections if section.shares_number_with} + first_alternative = next(i for i, e in enumerate(result.tree.contents) if e.unit in sharing) + drawn, entries = toc_lines(shown["template"]), expected_entries(template, **settings) + assert drawn[:first_alternative] == entries[:first_alternative] + assert [line.split(" ", 1)[1] for line in drawn] == [line.split(" ", 1)[1] for line in entries] @pytest.mark.libreoffice @@ -772,6 +819,29 @@ def test_page_numbers_appear_after_the_update(shown: dict[str, Exported]) -> Non assert len(pages) == len(toc_lines(shown["master"])) and all(p >= 1 for p in pages) assert pages == sorted(pages) assert max(pages) <= shown["master"].pages + # Where the pages are known: the long document's six headings of about a page each run in order from page 1, and + # the attachments after them end the document. + long = shown["long_break"] + by_title = dict(zip(toc_lines(long), toc_pages(long), strict=True)) + headings = [by_title[f"{n + 1}. Heading {n}"] for n in range(6)] + assert headings[0] == 1 and all(0 <= b - a <= 1 for a, b in zip(headings, headings[1:], strict=False)) + assert headings[-1] > headings[0] and by_title["Attachments"] > headings[-1] + assert by_title["Third"] == long.pages == by_title["Attachments"] + 2 + + +@pytest.mark.libreoffice +def test_the_cached_contents_reads_right_before_any_update() -> None: + """The entries in the file are what the HTML shows, so a viewer that never updates fields shows them (without page + numbers, which Word computes).""" + require_libreoffice() + settings = {"overrides": {"contents.enabled": True, "contents.depth": 3}} + template = (DOCUMENTS / "features.lgd").read_text(encoding="utf-8") + for source in (FRONT, template): + cached = libreoffice_export({"cached": docx(source, **settings)}, update=False)["cached"] + lines = cached.text.splitlines() + start = lines.index("Contents") + 1 + entries = lines[start:lines.index("", start)] + assert [" ".join(line.split()) for line in entries] == expected_entries(source, **settings) @pytest.mark.libreoffice @@ -852,7 +922,7 @@ def test_a_heading_inserted_in_the_editor_is_listed_once_the_contents_is_updated #: depend on the zlib). When the matter changes on purpose, run this test, review the change, and replace the digests. MATTER_GOLDEN = { "word/document.xml": "d67511803cd53640fbd96c3c2f6531cc5cf649158c7f56a0d0681db4da5c1b98", - "word/styles.xml": "6ccea95d23325c1d13fd3f11d953567df2ce075bbb00f0f537cac57939c6dcfc", + "word/styles.xml": "458e86ca2ad045ce3a714a7e5ba7f3c714e8ab41edf35afb754611046c082d98", } @@ -863,3 +933,37 @@ def test_the_matter_of_a_fixed_document_is_what_it_was() -> None: .write(render(FRONT, format="text", overrides={"contents.enabled": True}).tree)) digests = {name: hashlib.sha256(contents[name]).hexdigest() for name in MATTER_GOLDEN} assert digests == MATTER_GOLDEN, "\n".join(f' "{name}": "{digest}",' for name, digest in digests.items()) + + +def used_styles(data: bytes) -> dict[str, set[str]]: + """The paragraph, character and table styles the parts of the document use, by kind.""" + used: dict[str, set[str]] = {"pStyle": set(), "rStyle": set(), "tblStyle": set()} + for name, content in parts(data).items(): + if name.startswith("word/") and name.endswith(".xml") and name != "word/styles.xml": + for node in ET.fromstring(content).iter(): + local = node.tag.removeprefix(W) + if local in used: + used[local].add(node.get(f"{W}val")) # type: ignore[arg-type] + return used + + +@pytest.mark.parametrize("settings", [ + {"overrides": {"contents.enabled": True, "contents.depth": 5}}, + {"style": "continental", "overrides": {"contents.enabled": True, "title_block.version": True}}, +]) +@pytest.mark.parametrize("name", ["master", "features", "template"]) +def test_every_style_a_document_uses_is_defined(name: str, settings: dict[str, Any]) -> None: + source = FRONT if name == "master" else (DOCUMENTS / f"{name}.lgd").read_text(encoding="utf-8") + data = docx(source, **settings) + defined = {s.get(f"{W}styleId"): s.get(f"{W}type") for s in ET.fromstring(parts(data)["word/styles.xml"]).iter(f"{W}style")} + used = used_styles(data) + assert used["pStyle"] and used["rStyle"] + for kind, expected in (("pStyle", "paragraph"), ("rStyle", "character"), ("tblStyle", "table")): + for style in used[kind]: + assert defined.get(style) == expected, f"{kind} {style} is not a defined {expected} style" + + +def test_the_title_block_labels_are_italic() -> None: + styles = {s.get(f"{W}styleId"): s for s in ET.fromstring(parts(docx())["word/styles.xml"]).iter(f"{W}style")} + label = styles["TitleBlockLabel"] + assert label.get(f"{W}type") == "character" and label.find(f"{W}rPr/{W}i") is not None diff --git a/tests/test_docx_package.py b/tests/test_docx_package.py index 01899cd..6175862 100644 --- a/tests/test_docx_package.py +++ b/tests/test_docx_package.py @@ -156,7 +156,7 @@ def test_the_styles_the_body_and_word_need_exist_under_the_names_word_knows() -> "TOC3": "toc 3", "TOC4": "toc 4", "TOC5": "toc 5", "Hyperlink": "Hyperlink", "InlineCode": "Inline Code", "DefinedTerm": "Defined Term", "TermReference": "Term Reference", "Blank": "Blank", "Choice": "Choice", "FailureMarker": "Failure Marker", "ConditionLabel": "Condition Label", - "DraftingNoteLabel": "Drafting Note Label", "TitleBlockText": "Title Block Text", + "DraftingNoteLabel": "Drafting Note Label", "TitleBlockText": "Title Block Text", "TitleBlockLabel": "Title Block Label", "PartySide": "Party Side", "PartyName": "Party Name", "AttachmentsHeading": "Attachments Heading", "AttachmentTitle": "Attachment Title", "AttachmentFile": "Attachment File", "SignatureSide": "Signature Side", "SignatureText": "Signature Text", "SignatureLine": "Signature Line"} @@ -848,7 +848,7 @@ def test_code_is_left_aligned_even_when_the_body_is_justified() -> None: 'docProps/app.xml': 'b285204d159fd58da80b6a0242cef523448f9c3a598b357b15faf4627160dcc3', 'word/document.xml': '39cbe0403104ccefefb9b936e3e432b266d651e54628873ff2d71a57dde798c8', 'word/_rels/document.xml.rels': 'ada136c2e7d05a317579c9503c1f0dd8dfaa8f0cce687df86fe59e36d5fa5d57', - 'word/styles.xml': '5c6b9461b055c8c123fbb8e57385fffb2effe991107316ba065da778854d0862', + 'word/styles.xml': 'eec93405688edd872f0c341deeea171d8c5b72a6c83aeea3ff8cd01646c78bc1', 'word/numbering.xml': '368f179b5e26a34c1f23db597966dfd96b43dc96801a9a7ded728c379db07f59', 'word/settings.xml': 'b3c72bc5720862c1a9c1449d89903673777f30df6ff38ceb23da73e21cad22e1', 'word/footer1.xml': 'e693781f92e5a7ccf747fd2ed3facb627e448f389b021882cac589d746aa709b', From 59ab77df3c84654e82ec2a32bdbe88a658d8ee00 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:56:31 +0000 Subject: [PATCH 098/118] Make the Word checklist platform neutral and cover the Word-only settings; tighten the DOCX matter docs --- .github/PUBLISHING.md | 58 +++++++++++++++++++++++++++++++---------- CONFORMANCE.md | 2 +- README.md | 6 ++--- docs/style-templates.md | 2 +- 4 files changed, 48 insertions(+), 20 deletions(-) diff --git a/.github/PUBLISHING.md b/.github/PUBLISHING.md index 8c3e9a0..a5ae0eb 100644 --- a/.github/PUBLISHING.md +++ b/.github/PUBLISHING.md @@ -81,18 +81,38 @@ markdown-it-py) come from PyPI. ## Checking the DOCX in Word (before a release) CI proves the DOCX against the ISO/IEC 29500 schemas and in LibreOffice; it cannot run Word. Before a release, a maintainer -opens the golden documents in Word (the desktop application, a current version) and checks what only Word can show. Make -the documents with a table of contents, for each of the three built-in styles: - -```bash -for style in default continental outline; do - for doc in features template; do - legaldown-render tests/documents/$doc.lgd -s $style --set contents.enabled=true --set contents.depth=3 \ - -o /tmp/word-check/$doc.$style.docx - done -done +opens the golden documents in Word (the desktop application, a current version) and checks what only Word can show. +Make them with the Python API, which is the same on Linux, macOS and Windows (run it from the repository root, in an +environment with the package installed): + +```python +from pathlib import Path +from legaldown_render import render + +out = Path("word-check") # any folder; created here +out.mkdir(exist_ok=True) +contents = {"contents.enabled": True, "contents.depth": 3} +for doc in ("features", "template"): + source = Path(f"tests/documents/{doc}.lgd").read_text(encoding="utf-8") + for style in ("default", "continental", "outline"): + (out / f"{doc}.{style}.docx").write_bytes(render(source, format="docx", style=style, overrides=contents).binary) + # the settings whose effect only Word shows (see item 7) + for name, extra in { + "attachments-not-listed": {"contents.attachments": False}, + "depth-1": {"contents.depth": 1}, + "no-attachments-label": {"labels.attachments": ""}, + "separator-none": {"attachments.separator": "none"}, + "separator-rule": {"attachments.separator": "rule"}, + "scheme-none": {"numbering.scheme": "none"}, + }.items(): + data = render(source, format="docx", overrides={**contents, **extra}).binary + (out / f"{doc}.{name}.docx").write_bytes(data) ``` +(On the command line, `legaldown-render tests/documents/features.lgd -s continental --set contents.enabled=true +--set contents.depth=3 -o word-check/features.continental.docx` makes one of them; create the folder first with +`mkdir -p word-check` on Linux and macOS or `mkdir word-check` on Windows.) + For each file: 1. **Opens clean.** No "unreadable content" or repair prompt, and Word does not open it in Compatibility Mode. @@ -112,12 +132,22 @@ For each file: 5. **Zero-length bookmarks.** A `REF \n` to the zero-length (point) bookmark at the start of a numbered paragraph returns the number. Press Enter at the start of that paragraph and update the fields: the references still point at the paragraph they pointed at and read right. -6. **`none` scheme.** With `--set numbering.scheme=none`, a heading referenced with `{{ref:}}` shows its title text +6. **`none` scheme** (the `scheme-none` files): a heading referenced with `{{ref:}}` shows its title text (`REF \h`), and the references inside a heading that refers to itself are plain text: no "Error! Reference source not found" and no self-reference. -7. **Front and back matter.** The title block (header rows, parties) is two aligned columns without borders; each - attachment starts on a new page with `attachments.separator: page-break` and has a rule above it with `rule`; the - signature blocks are borderless, with a signing line above each name, and stay on one page (try a longer document). +7. **Front and back matter.** The title block (header rows, parties) is two aligned columns without borders, the + labels in italics; each attachment starts on a new page with `attachments.separator: page-break` and has a rule above + it with `rule`; with `none` (`separator-none`) there is no break and no rule; the signature blocks are borderless, + with a signing line above each name, and stay on one page when one page can hold them (try a longer document). The + settings variants: + - `attachments-not-listed`: the attachments heading and titles are **absent** from the table of contents and from + the navigation pane (outline level 9, body text). + - `depth-1`: the attachments are still in the contents, as first-level entries (the HTML indents the titles one + level more, the documented difference), and no second-level heading is. + - `no-attachments-label`: there is no attachments heading, and each title is a first-level entry. + - `\u` honours the outline levels of the attachment styles and paragraphs: the attachment titles are entries, at + the level the table of contents gives them, in every variant that lists them. If Word does not list them, the + fix is a `\t "Attachment Title,2"` list in the field (locale list separator permitting); report it. 8. **The numbering corner cases:** - `\t` with words before the number: a label such as `Article {n}` or `§ {n}` referenced with `{n}`, whose reference drops the words (`REF \n \t`): Word shows the bare number, where LibreOffice ignores `\t` and shows the words. diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 5cbfd14..95e26c6 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -20,7 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | -| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference for which Word's fields cannot reproduce the designation (a heading level skipped before the target under a `{n}`-style scheme, text around the counter that Word's `\t` switch cannot drop such as `(a)`, `{n}:` or `No. {n}`) is fixed text, reported. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says, **The front and back matter are written too**, in the order the HTML shows them: the title block (header rows and parties), a live table of contents (`TOC \o "1-N" \h \z \u` with cached entries equal to the HTML contents, each a link with a `PAGEREF` page number Word fills in on update; the attachments are listed by outline level), the attachments (bookmarked so `{{attach:}}` resolves, separated by a page break or a rule per `attachments.separator`) and the signature blocks (§3.6, borderless tables kept on one page). A LibreOffice round trip shows that after its indexes are updated the contents lists exactly the HTML entries with page numbers, also after a heading is inserted in the editor, and that the document reads in the plain-text writer's order. Remaining differences are the reported ones: alternatives that share a number (§15.8) are numbered in sequence, and a label that shows the section's number in a list item (the continental `5.1`) is drawn without it (reported). The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a list continued by a fixed number, which will not follow edits), `render-docx-number-mismatch` (any other difference, including alternatives), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-static-restart` (a list that restarts by a fixed setting) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). The Word-only checks (the contents' page numbers, what `\t` does, numbering after an insertion) are a manual checklist before a release (`.github/PUBLISHING.md`). HTML is the format this renderer claims §13.6 with | +| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference for which Word's fields cannot reproduce the designation (a heading level skipped before the target under a `{n}`-style scheme, text around the counter that Word's `\t` switch cannot drop such as `(a)`, `{n}:` or `No. {n}`) is fixed text, reported. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says. **The front and back matter are written too**, in the order the HTML shows them: the title block (header rows and parties), a live table of contents (`TOC \o "1-N" \h \z \u` with cached entries equal to the HTML contents, each a link with a `PAGEREF` page number Word fills in on update; the attachments are listed by outline level, so that no locale-dependent style list is in the field; at `contents.depth: 1` the attachment titles are listed as first-level entries, where the HTML indents them one level more), the attachments (bookmarked so `{{attach:}}` resolves, separated by a page break or a rule per `attachments.separator`) and the signature blocks (§3.6, borderless tables kept together on one page as far as one page allows). A LibreOffice round trip shows that after its indexes are updated the contents lists exactly the HTML entries with page numbers, also after a heading is inserted in the editor, and that the document reads in the plain-text writer's order. Remaining differences are the reported ones: until the alternatives package lands, alternatives that share a number (§15.8) are numbered in sequence (and the contents entries after one are numbered with them), and a label that shows the section's number in a list item (the continental `5.1`) is drawn without it (reported). The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a list continued by a fixed number, which will not follow edits), `render-docx-number-mismatch` (any other difference, including alternatives), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-static-restart` (a list that restarts by a fixed setting) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). The Word-only checks (the contents' page numbers, what `\t` does, numbering after an insertion) are a manual checklist before a release (`.github/PUBLISHING.md`). HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | diff --git a/README.md b/README.md index 5db0bb5..d10af20 100644 --- a/README.md +++ b/README.md @@ -161,12 +161,10 @@ cached entries are the contents the HTML shows, each a link to its heading with fills in when it updates the field; with `contents.attachments` the attachments are listed by outline level, so the field has no locale-dependent style list), the attachments (the heading and a titled placeholder each, bookmarked so that `{{attach:}}` is a `REF` field or a link, separated by a page break or a rule as `attachments.separator` says), -and the signature blocks (borderless tables with signing lines, kept on one page). Viewers that never update +and the signature blocks (borderless tables with signing lines, kept together on one page as far as one page allows). Viewers that never update fields (Word Online, Quick Look, Google Docs import, Outlook's preview) show the contents without page numbers. A reference whose form Word cannot compose from fields (a custom style's `{path}` item labels, say) is fixed text and -is reported as `render-docx-reference`. Before a DOCX is sent as a contract, the maintainers' manual check in Word -([the checklist](https://github.com/ForLegalAI/legaldown-render/blob/main/.github/PUBLISHING.md)) is the last -word. Settings Word cannot express (a `%` length, say) are an error for DOCX only. See +is reported as `render-docx-reference`. The maintainers also check the golden documents in Word before each release ([the checklist](https://github.com/ForLegalAI/legaldown-render/blob/main/.github/PUBLISHING.md)), since CI cannot run Word. Settings Word cannot express (a `%` length, say) are an error for DOCX only. See [Styles and preferences](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/style-templates.md#docx) and the [roadmap](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/roadmap.md). diff --git a/docs/style-templates.md b/docs/style-templates.md index d9e6a07..3dbb5c6 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -274,7 +274,7 @@ Settings that were already part of the template gain a DOCX meaning: | `contents.enabled`, `contents.depth` | A live `TOC \o "1-" \h \z \u` field after the title block, with the contents' entries as its cached result (`toc 1` to `toc 5` paragraphs, a dotted right tab at the text width, a `PAGEREF` page number Word fills in when it updates the field). The contents label is a `Contents Heading` paragraph: no outline level, no number, so never an entry | | `contents.attachments` | The attachments heading and each attachment title are paragraphs of the styles `Attachments Heading` and `Attachment Title`, which carry an outline level, so Word's TOC lists them (`\u`); without it they are body text for the contents. At `contents.depth: 1` the titles are listed with the headings (the first level), not under the attachments heading | | `attachments.separator` | `page-break`: a page break before the attachments part and between attachments (the paragraph property "page break before", so no empty paragraph or blank page is left); `rule`: a line above each attachment; `none`: nothing | -| `signatures.enabled` | Borderless tables with a signing line, kept on one page | +| `signatures.enabled` | Borderless tables with a signing line, kept together on one page as far as one page allows. They follow the last attachment, as in the HTML. A name, title, date or place row whose label is blanked (`labels.signature_*`) and whose value is empty is left out, as the plain-text writer leaves it out | | `title_block.parties`, `title_block.effective_date`, `title_block.version` | The header rows and the parties, as borderless tables | The styles for this matter (`Title Block Text`, `Party Side`, `Party Name`, `Attachments Heading`, `Attachment Title`, From 59da61362295b60c448530730a2935ebd242a79d Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:57:49 +0000 Subject: [PATCH 099/118] Take an alternative's holder from the counter, and test fixed references --- src/legaldown_render/resolve/docx_plan.py | 33 +++++++++++++---------- tests/test_docx_plan.py | 15 +++++++++-- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index cad62d1..8b8c512 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -1241,12 +1241,16 @@ def numbered(self, node: _Unit, kind: Literal["section", "item", "paragraph"], n self.item_units.add(unit) peek = self.sim.peek(num.id, ilvl) if node.shares_number_with is not None and peek != node.number: - holder = self.canon.get(node.shares_number_with, node.shares_number_with) - self.canon[unit] = holder - self.alts[unit] = (holder, abstract.levels[ilvl].suff) - self.add(node, kind, None, ilvl, explicit=True, cause=(*cause, "alternative"), holder=holder, - ind=self.indent(abstract, ilvl, unit, parent, alternative=True)) - return num + # Word would count on, and the number is the one the level holds now: the unit that last counted the level + # holds it, and the alternative shows that unit's number (a unit that shares a number but finds another + # in the level, because a list of the heading list counted in between, is restarted below). + count, holder = self.sim.current(num.id, ilvl) + if count == node.number and isinstance(holder, str) and holder in self.units: + self.canon[unit] = holder + self.alts[unit] = (holder, abstract.levels[ilvl].suff) + self.add(node, kind, None, ilvl, explicit=True, cause=(*cause, "alternative"), holder=holder, + ind=self.indent(abstract, ilvl, unit, parent, alternative=True)) + return num used = num if peek != node.number: used = self.new_num(abstract, ilvl, node.number) @@ -1260,22 +1264,23 @@ def numbered(self, node: _Unit, kind: Literal["section", "item", "paragraph"], n "items added or removed before it.", node.line) step = self.sim.step(used.id, ilvl, unit) if kind != "section": - ancestors = {unit} + # The counters of the unit's own chain: the units it stands under, and every part of the number of the + # section (a skipped heading level counts, as the ghost of the heading that skipped it). + chain: set[object] = {t.owner for t in self.desig.get(parent or "", ()) if isinstance(t, Count)} above = parent while above is not None: - ancestors.add(self.canon.get(above, above)) + chain.add(self.canon.get(above, above)) above = self.parent.get(above) if "label-fallback" not in cause and any( - isinstance(t, Count) and (isinstance(t.owner, Ghost) or t.owner not in ancestors) - for t in step.tokens): + isinstance(t, Count) and t.owner != unit and t.owner not in chain for t in step.tokens): # The label shows the counter of a level of its list that no unit of its own chain holds: Word shows # a level nothing numbers as 0, or the counter of whatever last counted it, and the resolver's label # is that of the enclosing unit. Not left to Word's reading. cause += ("label-fallback",) self.report("render-docx-section-label", "warning", f"The label of {word}s '{printable(node.label or '')}' shows the number of an enclosing " - f"item that Word cannot show here (the list is not nested directly in an item of its own kind): in " - f"Word it reads '{printable(step.label)}'.", node.line) + f"item that Word cannot show here (the list is not nested directly in an item of its own " + f"kind): in Word it reads '{printable(step.label)}'.", node.line) self.add(node, kind, used, ilvl, explicit=explicit, cause=cause, step=step, ind=self.indent(abstract, ilvl, unit, parent, alternative=False)) self.check(node, word, step, counter, group, cause, (abstract.id, ilvl)) @@ -1303,8 +1308,8 @@ def report_collisions(self) -> None: self.shared_counter( f"The {word} '{printable(node.label or '')}' has the designation {printable(designation)}, which another " f"unit of its section has too (in every output format their designations collide), and in Word they " - f"share one counter. The number is kept as the renderer shows it by restarting at a fixed value, so " - f"inserting or deleting a clause before it will renumber it.{more}", node.line) + f"share one counter. The number is kept as the renderer shows it by restarting at a fixed value, which " + f"will not follow a heading or clause inserted or deleted before it in Word.{more}", node.line) def list_(self, lst: List, section: Section | None, quoted: bool = False) -> None: first = lst.items[0] diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index 1923216..dd76804 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -1354,6 +1354,17 @@ def test_documents_of_one_size_refer_to_different_units_so_that_more_of_them_are assert len(set().union(*members)) > max(len(refs) for refs in members) +@pytest.mark.parametrize("variant", ["default", "continental", "paragraphs", "mixed", "article"]) +def test_a_reference_is_fixed_text_only_where_a_heading_level_is_skipped_or_the_depth_is_past_word_s(variant: str) -> None: + """The cover of every reference exists for a document without errors and a list within Word's nine levels: what + cannot be covered are the references below a skipped heading level (a Core Error: the number of the level the + heading skips has no unit to be a field of) and below a list nested past nine levels.""" + for source, result, plan, _ in (*random_runs(variant), *alternative_runs(variant)): + skipped = any("skipped-level" in unit.cause for unit in plan.units.values()) + if not skipped and max((lst.depth for lst in lists(result.tree)), default=0) <= 8: + assert not [t for t, r in plan.refs.items() if r.fixed], source + + def test_each_variant_plans_its_own_sample_of_the_random_documents() -> None: samples = {tuple(sample(variant, PER_VARIANT, _random_sources())) for variant in VARIANTS} assert len(samples) == len(VARIANTS) @@ -1741,8 +1752,8 @@ def seconds(count: int) -> float: best = min(best, time.perf_counter() - started) return best - small, large = seconds(1000), seconds(4000) - assert large < 6 * max(small, 0.02), (small, large) # four times the headings: a walk from each would be sixteen + small, large = seconds(500), seconds(4000) + assert large < 30 * max(small, 0.01), (small, large) # eight times the headings: a walk from each would be sixty-four assert large < 2.0 From 6ca33d4316834c6a9fc8e60232a394eab543079a Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:05:37 +0000 Subject: [PATCH 100/118] Document the DOCX planner's qualified labels, alternatives and resynchronised numbers; test them in the file --- CONFORMANCE.md | 2 +- README.md | 2 +- docs/architecture.md | 72 ++++++++-- docs/decisions/0008-docx-word-native.md | 26 +++- docs/roadmap.md | 11 +- src/legaldown_render/resolve/docx_plan.py | 15 ++- tests/test_docx_plan.py | 8 ++ tests/test_docx_special.py | 152 ++++++++++++++++++++++ 8 files changed, 263 insertions(+), 25 deletions(-) create mode 100644 tests/test_docx_special.py diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 4d24853..412527e 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -20,7 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | -| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference for which Word's fields cannot reproduce the designation (a heading level skipped before the target under a `{n}`-style scheme, text around the counter that Word's `\t` switch cannot drop such as `(a)`, `{n}:` or `No. {n}`) is fixed text, reported. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says, but **the file is still interim**: there is no table of contents, title block, attachments or signature blocks, alternatives that share a number (§15.8) are numbered in sequence (reported), and a label that shows the section's number in a list item (the continental `5.1`) is drawn without it (reported). The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a list continued by a fixed number, which will not follow edits), `render-docx-number-mismatch` (any other difference, including alternatives), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-static-restart` (a list that restarts by a fixed setting) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). Every DOCX render also reports the Warning `render-docx-interim` ("DOCX output is incomplete in this version") until the body is complete; it goes away then. HTML is the format this renderer claims §13.6 with | +| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can; the continental `5.1` items and the numbered paragraphs `1.1` ride the heading list's next level, with the lists nested in them on its reserved levels, so they stay live; alternatives that share a number (§15.8) are `numId 0` paragraphs whose number is a `REF` field of the first, with the sections and lists under them on a num of their own), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference for which Word's fields cannot reproduce the designation (a heading level skipped before the target under a `{n}`-style scheme, text around the counter that Word's `\t` switch cannot drop such as `(a)`, `{n}:` or `No. {n}`) is fixed text, reported. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says, but **the file is still interim**: there is no table of contents, title block, attachments or signature blocks, a label that shows the section's number that the heading list cannot show (an outline scheme's `{section}.{n}`, the `none` scheme) is drawn without it (reported). The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a unit that shares a counter with others in Word, such as a sub-heading and the items of its section that have the same designation in every output format, and that is restarted at a fixed value, which will not follow edits), `render-docx-number-mismatch` (any other difference: a catch-all that does not occur), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-static-restart` (a list that restarts by a fixed setting: in a list of another kind, in an item on the heading list that it cannot ride, in an alternative) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). Every DOCX render also reports the Warning `render-docx-interim` ("DOCX output is incomplete in this version") until the body is complete; it goes away then. HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | diff --git a/README.md b/README.md index cf84196..cf6b257 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ numbering definitions (headings by the heading styles, nested lists restarting b `{{ref: clause}}` is `REF` fields on bookmarks whose cached results are the text the plain-text writer prints, so "1(b)(ii)" or "I.A.1" stays right when a lawyer inserts a clause and updates fields. A defined term or an attachment reference is an internal link. A reference that Word's fields cannot reproduce is written as its text (it will not follow edits) and reported. Where Word cannot draw what the renderer does (letters past 26, a label that shows the -section's number in a list item, alternatives that share a number, a list restarted by a fixed setting), the render +section's number where the heading list cannot show it, a number restarted by a fixed value, a list restarted by a fixed setting), the render reports it as a `render-docx-*` Warning or Info with the line, see [the conformance notes](https://github.com/ForLegalAI/legaldown-render/blob/main/CONFORMANCE.md). The body writer also lays out paragraphs, quotes, drafting notes, code, tables with a repeated header row, rules, links, blanks and diff --git a/docs/architecture.md b/docs/architecture.md index 6f72381..83bc46a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -271,16 +271,54 @@ ordered by line with the renderer's own) to the result's. The style check, `docx starts at that number by a fixed setting, which does not follow items added or removed before it. - A bullet counts nothing: one num serves every bullet list. - Paragraph numbering has one num per section, and a section added in Word does not restart it by itself. +- Word has no way to give two paragraphs the same number on one level, except `numId 0` and a field: a paragraph with + a counter always counts. A unit that must read the number of the one before it (an alternative, §15.8) is a + `numId 0` paragraph that starts with a `REF \n` field of that unit. - `REF \n` returns the paragraph's level text with its counters, minus trailing periods. **What is planned.** Heading numbering (one multilevel abstractNum linked to the heading styles, the label templates turned into `lvlText`, the counters into `numFmt`), list families (enumerated, renumbered, bullets) with their nesting and restarts, depth beyond Word's nine levels (clamped to level 9 in that level's format), and numbered paragraphs. -A level shows only counters of its own list. A label whose filled-in form shows the section's number (`{section}`, -or the `{path}` of an item or paragraph in a section: the continental style's `5.1`), or a number of an enclosing -item that Word cannot show (a list nested in another kind of list, or in a quote), is written without it and -reported (`render-docx-section-label`). Alternatives that share a number are not planned yet (they are numbered in -sequence and reported). `UnitPlan.ind` is the one part of the plan this leaves unset. +A level of a list shows only counters of its own list, which settles where the rest of the planner puts things: + +- **Labels that need the section** (`{section}`, or the `{path}` of an item or paragraph in a section: the continental + style's `5.1`, the default style's numbered paragraphs `1.1`). Only the heading list can show the counters of a + section, so a unit whose label needs them goes on the heading abstractNum at the level after its section's + (`%1.%2` after a level-1 section), where the level reads exactly as the label does (`candidate` equals the heading + level; the first top-level list of the section gets one num with a `startOverride`, as every top-level list does). + Such an item is no heading: it has no heading style, no outline level and no entry in the table of contents, and + carries an indent of its own (`UnitPlan.ind`; a heading level's `w:ind` serves the heading). A list nested directly + in an item that is on the heading list goes on the heading abstractNum's **reserved levels**, level `4 + d` for + nesting depth `d` of 1 to 4, defined on first use with the label's text without the section (`%6)`, `%7)`), on the + same num as its parent with no override: it restarts by structure under every new item and sibling lists go on + counting, as in any other list. A level may be shared by units whose definitions are equal and is kept for the first + that needs it otherwise; the level after the fifth heading level is kept for a level-5 section's units when the + document has one, and a nested list then falls back. What does not fit (an outline or mixed scheme, where the + level after the section's reads otherwise; the `none` scheme, where the section's number is its text; a nested + item whose label shows the section; items in a quote; ordered and bullet lists nested in such an item; depth past + 4) keeps the family's own abstractNum with the label written without the section and reported + (`render-docx-section-label`); the references still cover it. +- **Alternatives that share a number** (§15.8). A unit that shares its number with the one before it and that Word + would number differently (the live number would continue) is a `numId 0` paragraph with its own indent that starts + with a `REF \n \h` field of the unit that holds the number now, then the label's text beyond what the field returns + (the period) and a tab or space (`UnitPlan.alt`, `ind`); that unit gets a bookmark, a `{{ref:}}` to the shared + identifier is a reference to the first alternative, and the table of contents shows the shared number for the later + one. The units under a later alternative restart in a num of their own with a `startOverride` (the sections after a + later alternative section until one of its level or above; a list nested in a later alternative item), because the + alternative counts nothing and Word would go on counting from the holder's children. A later alternative whose live + number is already right (the children of a later alternative section repeat the numbers of the first's) is numbered + live. Designations below an alternative are made of the holder's counters. +- **Resynchronising.** A numbered unit that Word would number differently from the resolver for want of a place where it + can count by structure (it goes on a counter that other units of the same abstractNum have used) is put on a num with + a `startOverride` at its number, and the units after it go on counting on the original. This is a fixed value: a + heading or clause inserted or deleted in Word before it does not renumber it. It repairs a sub-heading after the + items and paragraphs of its section on the heading list (`1.1` for the sub-heading and for the first item share a + counter in Word and a designation in every output format: `render-docx-shared-counter` says so), a list that + resumes a sibling's count after another list in the same item, and a list past level 9. + +A label that shows a counter of a unit outside the unit's own chain (a quote's list inside an item of the preamble, +whose label starts from the item; a list nested in another kind of list) is written with the counters Word has there +and reported. Nothing is left to be shown by Word differently without a diagnostic. **Cross-references: the designation cover** (`resolve/docx_cover.py`, §6.3, §13.3). The renderer's text of a `{{ref:}}` is the designation of the target (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)`) inside `references.format`; the planner @@ -324,7 +362,13 @@ heading under the `none` scheme): every other bookmark is a point at the start o a self-reference that Word cannot show and LibreOffice expands without end. Under the `none` scheme a heading whose title holds a reference that leads back to the heading, directly or through the titles of other headings, would have exactly that; the plan lists those references (`UnitPlan.literal_refs`, by target) and the writer writes them as -their text, and `render-docx-reference-cycle` says so (the renderer already warns with `render-designation-cycle`). +their text, and `render-docx-reference-cycle` says so, naming the heading's line (the renderer already warns with +`render-designation-cycle`). The headings are the nodes of a graph, an edge for each heading whose text a field in +another's title returns, and a reference in a title is a cycle exactly when such a field's heading is in the title's +own strongly connected component (found once, iteratively: linear in the headings and references). The text a +heading field returns is worked out from the runs, in the order the components complete: a reference in the title is +the text of the fields it is written as (the cached results of other headings' fields), or its own text where it is +written as text. **The table of contents.** `ContentsEntry.unit` (the section's unit; None for the attachments heading and an attachment) keys each entry of `tree.contents` to its section. `plan.toc` lists the entries in order with @@ -335,15 +379,18 @@ complex `TOC` field that uses them is written with the title block. **Diagnostics** (all `render-docx-*`, with the line of the unit or of the reference, once per finding): `section-label`, `list-depth`, `letter-overflow` (Word repeats letters after 26, the renderer counts on: `aa`, `ab`), `roman-overflow` -(4000 and beyond), `shared-counter` (a list that goes on from another, or whose level another list used in between, -starts at a fixed number), `number-mismatch` (any other difference: the first unit of a run that Word counts off by the -same amount, with how many follow; it does not occur without an alternative that shares a number), `reference` (a +(4000 and beyond), `shared-counter` (a unit that goes on a counter of its abstractNum that others have used, so that it is restarted at +its number, a fixed value that will not follow edits before it: a sub-heading that shares a designation with the +items of its section, a list that goes on from another, a list whose level another used in between), `number-mismatch` (any +other difference: the first unit of a run that Word counts off by the same amount, with how many follow; it is the +catch-all of the planner and does not occur), `reference` (a cross-reference that cannot be built from fields, because a heading level is skipped before the target with a `{n}`-style scheme, or Word draws text around a counter that `\t` cannot drop (`({n})`, `{n}:`, `No. {n}`): it is written as fixed text; or one that reads differently in Word for a reason that no diagnostic of a unit already gives; references that fail alike are one finding), `reference-cycle` (a heading's title holds a reference that leads back to it: written as text), and, as information, `static-restart` (a list that restarts at 1 by a -fixed setting) and `reference-word-only` (a cover relies on `\t`, which LibreOffice ignores, once per document). A +fixed setting: nested in a list of another kind, in an item whose number is on the heading list but which the list +cannot ride, or in an alternative that shares a number) and `reference-word-only` (a cover relies on `\t`, which LibreOffice ignores, once per document). A heading level that is skipped is the Core Error `heading-skip`; the plan only notes it on the unit. ### 5. Write (`writers/`) @@ -583,7 +630,7 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_values.py` | Money, dates and durations in a locale: exact digits, plural of the value shown, currency display, CLDR fallback to English | | `tests/test_numbering.py` | Counter formats, label templates and the preamble's `{section}` | | `tests/test_numbering_structure.py` | The numbering structure in the tree (`formats`, unit ids, numbers, parents, alternatives, list families, cross-reference units): headings in each scheme and with skipped levels, nested lists, section-qualified items, numbered paragraphs, alternatives, the `none` scheme and its second pass; and that every label, designation and reference can be written again from the structure alone, over the tests' and the specification's documents | -| `tests/test_docx_plan.py` | The DOCX plan: abstractNums per style and scheme, heading levels, top-level, nested, sibling, cross-family, quoted and clamped lists, numbered paragraphs, labels that need the section, letter and Roman overflow, diagnostics (lines, file, order, repeats, runs of units counted off), style errors (also through `docx_settings`), determinism; and, unit by unit, that Word's simulated count and label are the resolver's unless the unit's own counter reads differently in Word or the plan notes why, over the tests' and the specification's documents under each style and list setting, and over random documents and alternatives. Also the cross-references: the worked table of covers for each style (default, outline, mixed, continental, `none`, `Article {n}`, `§ {n}`), the cover algorithm on its own, references to other sections and to unnumbered units, `references.format`, every reason a reference cannot be covered, bookmark names (prefixes, length, collisions, hidden names, determinism), the table of contents, the render's diagnostics, and the property that every cached reference reads as the renderer's text or a diagnostic says why not, over the same documents and random documents with an anchor on every unit | +| `tests/test_docx_plan.py` | The DOCX plan: abstractNums per style and scheme, heading levels, top-level, nested, sibling, cross-family, quoted and clamped lists, numbered paragraphs, labels that need the section and where they go (the heading list's levels, the reserved levels of lists nested in them, the fallback), alternatives that share a number (of sections, items and paragraphs, in twos and threes, with children), resynchronised units, letter and Roman overflow, diagnostics (lines, file, order, repeats, the catch-all that groups runs of units counted off), style errors (also through `docx_settings`), determinism; and, unit by unit, that Word's simulated count and label are the resolver's unless the unit's own counter reads differently in Word or the plan notes why, over the tests' and the specification's documents under each style and list setting, and over random documents and alternatives (each variant plans its own sample, and every unit is referred to by some document). Also the cross-references: the worked table of covers for each style (default, outline, mixed, continental, `none`, `Article {n}`, `§ {n}`), the cover algorithm on its own, references to other sections and to unnumbered units, `references.format`, every reason a reference cannot be covered, bookmark names (prefixes, length, collisions, hidden names, determinism), the table of contents, the components of the headings' graph of title references (and that a chain of thousands of headings is planned in linear time), the render's diagnostics, and the property that every cached reference reads as the renderer's text or a diagnostic says why not, over the same documents and random documents with an anchor on every unit | | `tests/test_docx_simulator.py` | Word's counters, label templates as tokens (equal to filling the template as text), `lvlText`, and the simulator: restarts, start overrides, shared counters, skipped levels, `peek`/`current` | | `tests/test_build.py` | Building the tree: comment-only items and quotes, emphasis next to a directive, spacing where an inline comment was dropped | | `tests/test_terms.py` | What `{{term:}}` shows, references inside terms, terms that name themselves or each other, the cap on a term text | @@ -605,7 +652,8 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_docx_package.py` | The OOXML package: parts, relationships, content types, settings order, styles, page setup, headers and footers, properties, the body of a planned document, determinism, the sanitiser and builders, the API and command line | | `tests/test_docx_body.py` | The DOCX body from hand-made plans: numbering definitions, `w:numPr`/`w:ind`/alternatives, bookmarks and `REF` fields with cached results, hyperlinks, every inline and block, escaping, schema validity (marker `ooxml_schema`) and a LibreOffice round trip of headings, nested lists and references whose cached values are wrong on purpose (marker `libreoffice`) | | `tests/test_docx_escaping.py` | Well-formedness of every part for any text, with a seeded fuzz and the characters XML cannot carry | +| `tests/test_docx_special.py` | What the planner's hard cases become in the file: the heading list's levels that hold items, the reserved levels of lists nested in them, and later alternatives as `numId 0`, `w:ind` and `REF` fields; and that documents with them validate against the schemas (marker `ooxml_schema`) | | `tests/test_docx_validation.py` | Every part of every generated document against the ISO/IEC 29500-4 schemas (marker `ooxml_schema`; needs lxml and `python tests/schemas/fetch.py`) | -| `tests/test_docx_roundtrip.py` | LibreOffice as the oracle of the numbering and the cross-references, in one LibreOffice session: every document of the corpus (the tests', the specification's, stress documents with cycles and unanchored ancestors) under the default, outline, continental, mixed and `none` schemes, with every cached field result made wrong (`XXX`), every numbered paragraph marked and a line per reference added; the number LibreOffice draws, every reference it computes and every body paragraph's text equal what the file cached and the renderer wrote, modulo the gaps the plan reports (`\t` ignored, letters past 26, labels that need the section, alternatives; the table of contents is not written yet, so its entries and the `_Toc` bookmarks are not exercised); and edit-then-update (`tests/lo/uno_edit.py`): items and headings inserted before the targets as a lawyer does, fields updated, every reference and the numbers equal the renderer's for the edited source (marker `libreoffice`) | +| `tests/test_docx_roundtrip.py` | LibreOffice as the oracle of the numbering and the cross-references, in one LibreOffice session: every document of the corpus (the tests', the specification's, stress documents with cycles and unanchored ancestors) under the default, outline, continental, mixed and `none` schemes, with every cached field result made wrong (`XXX`), every numbered paragraph marked and a line per reference added; the number LibreOffice draws, every reference it computes and every body paragraph's text equal what the file cached and the renderer wrote, modulo the gaps the plan reports (`\t` ignored, letters past 26, labels the heading list cannot show); alternatives are checked against the renderer (their `numId 0` number is the holder's, as fields), and so are the continental items on the heading list and the lists on its reserved levels; the table of contents is built by LibreOffice from a `TOC` field the test adds and must list the headings with the numbers the plan gives and none of the items; and edit-then-update (`tests/lo/uno_edit.py`, which inserts and deletes paragraphs): items, headings, a clause and a nested item inserted before the targets as a lawyer does, fields updated, every reference and the numbers equal the renderer's for the edited source (the new nested item under the continental style reads `a)`), and, for a sub-heading restarted by a fixed value, the plan's warning and what an edit before it keeps right (marker `libreoffice`) | | `tests/test_docx_libreoffice.py` | The documents opened in LibreOffice through UNO (`tests/lo/uno_export.py`), fields updated: the words in order, header, footer, page count (marker `libreoffice`; skipped without `soffice` and `python3-uno`) | | `tests/conformance/` | Every example and fixture in the specification repository renders, every internal link has a target, and each invalid fixture renders its rule's failure marker. Needs `$LEGALDOWN_SPEC_DIR` or a sibling `../LegalDown` checkout | diff --git a/docs/decisions/0008-docx-word-native.md b/docs/decisions/0008-docx-word-native.md index 8f432d5..f1b007a 100644 --- a/docs/decisions/0008-docx-word-native.md +++ b/docs/decisions/0008-docx-word-native.md @@ -88,6 +88,22 @@ ancestor, joined by the literals the reference templates give. Only when no cove planner fall back to one field on the target, with a warning. Under the `none` scheme a reference is a `REF` to the heading text. +### Numbering the hard cases + +- **Items and paragraphs whose label needs the section** ("5.1" under `{section}.{n}`) go on the heading + list's level after their section's, where the level reads as the label does (`%1.%2`). Such an item has + no heading style, no outline level and so no entry in the table of contents. A list nested in one goes on + the heading list's levels 6 to 9 (headings have five), which are reserved for that: the level text is the + label's without the section (`%6)`), on the parent's num with no override, so the list restarts under every + new item and sibling lists go on counting, as in any list. Where the level after the section's reads + otherwise (the outline and mixed schemes, `none`), the label is written without the section and reported. +- **Alternatives that share a number** (§15.8). Word cannot give two paragraphs one number on one level, except + by `numId 0` and a field: the later alternative is such a paragraph, with a `REF \n` field of the unit that + holds the number as its number. The units under it restart in a num of their own with a `startOverride`. +- **A unit that Word would number off the resolver's** because it shares a counter with others of its + abstractNum (a sub-heading after the items of its section) is restarted at a fixed value and reported, so the + file is right when it is opened and a heading inserted or deleted before it in Word does not renumber it. + ### Rejected alternatives - **python-docx** (0004's choice), for the reasons in point 2. @@ -97,8 +113,9 @@ a `REF` to the heading text. produce "2.(a)" for an item of section 2 and "I.A.1.a.(i)" for an outline, not the designation the style asks for. The renderer composes the designation from `\n` fields instead. - **List items that need the section number ("5.1", "5.2" under `{section}`) on reserved deeper - levels of the heading list.** Word's levels are one hierarchy, so reserving levels 6 to 9 for items - changes how the headings above them count. + levels of the heading list.** The items would count on a level that is not the one after their + section's, and could not show the section's counters. Only the lists nested in such items use the reserved + levels 6 to 9, which no heading uses. - **`STYLEREF "Heading 1" \n` with a `SEQ` field** for the section prefix of those items. It works only in Word, cannot be tested in LibreOffice, and gives no number when the lawyer presses Enter in the middle of a list. The items instead go on the heading list's next level, which numbers them @@ -110,8 +127,9 @@ a `REF` to the heading text. - A DOCX lawyers can edit: insert a clause, update fields (Word does it on open by default), and the numbers, references, table of contents and page numbers follow. - Some restarts cannot be kept live in Word and are documented and warned about, not prevented: a - nested list of another family than its parent, list items under `{section}` labels, and paragraph - numbering in a section the user adds. A lawyer who adds a parent item restarts such a list by hand. + nested list of another family than its parent, a list nested in an alternative, a unit restarted at a fixed + value because it shares a counter, and paragraph numbering in a section the user adds. A lawyer who adds a + parent item restarts such a list by hand. - Viewers that never update fields (Word Online, Quick Look, the Google Docs import, Outlook's preview) show the cached results, which are right, but the table of contents without page numbers. - Verification is part of the design: every part is validated against the schemas, the documents are diff --git a/docs/roadmap.md b/docs/roadmap.md index 69515fd..7381c5a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -120,9 +120,14 @@ values appear only as the cached results of fields. Decided in and the table of contents is planned from Word's numbers; `render` runs the planner and reports its `render-docx-*` diagnostics. A LibreOffice round trip with wrong cached results checks the numbers and references against the plain-text oracle -- **Next.** Qualified labels (the continental `5.1`) on the heading list, alternatives that share a number, - restarts Word would otherwise get wrong; the table of contents field, title block, attachments and signature - blocks; the corpus checked also after edits made through UNO; a manual Word checklist for releases +- **Planner: qualified labels, alternatives and resynchronised numbers (done).** The continental `5.1` items and the + numbered paragraphs ride the heading list's level after their section's, and lists nested in them its reserved + levels, so they restart by structure; alternatives that share a number (§15.8) are `numId 0` paragraphs whose + number is a `REF` field of the first, with the units under them on a num of their own; a unit Word would count + off the resolver's is restarted at a fixed value and reported. Clauses and nested items inserted through UNO are + numbered as the renderer numbers them +- **Next.** The table of contents field, title block, attachments and signature blocks; a manual Word checklist for + releases ### v0.6 — PDF diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 8b8c512..7a6e055 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -12,8 +12,11 @@ renderer's designation of the target covered by ``REF`` fields and the delimiters between them, with Word's value as each field's cached result; :mod:`.docx_cover`) and the entries of the table of contents (``toc``, ``UnitPlan.toc_bookmark``), with the diagnostics for what Word will not show as the renderer does. -``UnitPlan.ind`` is never set. :func:`docx_style_errors` is the part that runs on every DOCX render, before it -(ADR 0006). +A label that needs the section's number is numbered on the heading abstractNum's level after the section's, and a list +nested in such an item on its reserved levels (§13.2); a later alternative that shares a number (§15.8) is a ``numId 0`` +paragraph whose number is a field of the unit that holds it (``UnitPlan.alt``); a unit Word would count off the +resolver's is restarted at a fixed value (``UnitPlan.ind`` is set for the units that carry an indent of their own). +:func:`docx_style_errors` is the part that runs on every DOCX render, before it (ADR 0006). **Data model.** Frozen data only; the planner assigns bookmark *names*, never ids (the writer allocates those); names are valid Word bookmark names of at most 40 characters and unique ignoring case. All sequences are tuples, all @@ -188,8 +191,12 @@ class UnitPlan: word_ref: str #: Word's counter value (None for an alternative, a bullet and an unnumbered unit). count: int | None - #: Why the unit is not on the plain path: ``resync``, ``alternative``, ``reserved-level``, ``restart-family``, - #: ``restart-section``, ``restart-alternative``, ``label-fallback``, ``clamped``, ``skipped-level``. + #: Why the unit is not on the plain path: ``resync`` (restarted at a fixed value), ``alternative`` (``numId`` 0), + #: ``reserved-level`` (a list nested in an item on the heading list, on one of its reserved levels), + #: ``restart-family`` / ``restart-section`` / ``restart-alternative`` (a nested list in a list of another kind, in + #: an item whose number carries the section, in an alternative: a num of its own that restarts by a fixed setting), + #: ``label-fallback`` (the label is written without a number Word cannot show there), ``clamped``, + #: ``skipped-level``. cause: tuple[str, ...] #: Names to wrap around the paragraph's title/first paragraph content: the bookmark of the unit's anchor when it #: has one, else a hidden one when a reference field points at it, else none. diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index dd76804..3706540 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -519,6 +519,13 @@ def test_a_reserved_level_is_defined_on_first_use_and_only_when_it_is_used() -> assert [level.lvl_text for level in nested.abstract_nums[0].levels[5:]] == ["%6)", "%7)", "%8", "%9"] +def test_a_reserved_level_is_shared_by_every_list_that_reads_as_it_does() -> None: + tree, plan, diagnostics = planned("# A\n\n- x\n - y\n\n# B\n\n- z\n - w\n", style="continental") + _, y, _, w = items(tree) + assert (plan.units[y.unit].ilvl, plan.units[w.unit].ilvl) == (5, 5) and diagnostics == [] + assert [level.lvl_text for level in plan.abstract_nums[0].levels[5:7]] == ["%6)", "%7"] + + def test_a_nested_label_that_shows_the_section_does_not_ride_the_reserved_levels() -> None: # depth 3 of the continental style is the first level's label again, {section}.{n} body = "# A\n\n- x\n - y\n - z\n - w\n - v\n" @@ -1470,6 +1477,7 @@ def covers(variant: str, **settings: object) -> dict[str, tuple[str, str]]: # nested in an item is written "b)" and referred to as "(b)" ("continental", {"style": "continental"}, "i1", "1.1", "REF i1"), ("continental", {"style": "continental"}, "n2", "1.2(b)", 'REF i2 + "(" + REF n2'), + ("continental", {"style": "continental"}, "d1", "1.2(b)(i)", 'REF i2 + "(" + REF n2 + "(" + REF d1'), ("continental", {"style": "continental"}, "p1", "1(1)", "REF alpha + REF p1"), ("continental", {"style": "continental"}, "gamma", "1.1.1", "REF gamma"), # the none scheme: the text of the heading, then the pieces after a space diff --git a/tests/test_docx_special.py b/tests/test_docx_special.py new file mode 100644 index 0000000..a911d2b --- /dev/null +++ b/tests/test_docx_special.py @@ -0,0 +1,152 @@ +"""What the planner's hard cases become in the file (docs/architecture.md, "The DOCX planner"): the heading list's levels +that hold items, the reserved levels of the lists nested in them, and the later alternatives that share a number, as +numbering definitions, ``w:numPr``, ``w:ind`` and ``REF`` fields; and that every document that has them validates +against the ISO/IEC 29500-4 schemas (marker ``ooxml_schema``). The numbers as Word's simulator and LibreOffice draw them +are checked in test_docx_plan.py and test_docx_roundtrip.py.""" +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET + +import pytest +from conftest import FRONT +from docx_support import W, parts, require_schemas, validation_errors +from test_docx_plan import ALTERNATIVES +from test_docx_roundtrip import STRESS + +from legaldown_render import render + + +def document(source: str, **settings: object) -> dict[str, ET.Element]: + result = render(source, format="docx", **settings) + assert result.binary is not None + return {name: ET.fromstring(content) for name, content in parts(result.binary).items() + if name in ("word/document.xml", "word/numbering.xml")} + + +def body_paragraphs(root: ET.Element) -> list[ET.Element]: + body = root.find(f"{W}body") + assert body is not None + return body.findall(f"{W}p") + + +def text_of(paragraph: ET.Element) -> str: + return "".join(node.text or "" for node in paragraph.iter(f"{W}t")) + + +def paragraph_with(root: ET.Element, text: str) -> ET.Element: + [found] = [p for p in body_paragraphs(root) if text_of(p).endswith(text)] + return found + + +def numbering_of(paragraph: ET.Element) -> tuple[int | None, int | None]: + ilvl = paragraph.find(f"{W}pPr/{W}numPr/{W}ilvl") + num = paragraph.find(f"{W}pPr/{W}numPr/{W}numId") + return (None if ilvl is None else int(ilvl.get(f"{W}val")), # type: ignore[arg-type] + None if num is None else int(num.get(f"{W}val"))) # type: ignore[arg-type] + + +def indent(paragraph: ET.Element) -> tuple[int, int] | None: + found = paragraph.find(f"{W}pPr/{W}ind") + return None if found is None else (int(found.get(f"{W}left")), int(found.get(f"{W}hanging", "0"))) # type: ignore[arg-type] + + +def levels(numbering: ET.Element, abstract_id: int) -> list[ET.Element]: + [abstract] = [a for a in numbering.findall(f"{W}abstractNum") if a.get(f"{W}abstractNumId") == str(abstract_id)] + return abstract.findall(f"{W}lvl") + + +def instructions(paragraph: ET.Element) -> list[str]: + return [n.text.strip() for n in paragraph.iter(f"{W}instrText") if n.text] + + +CONTINENTAL = FRONT + "# Alpha\n\n- clause one {#c1}\n - nested a\n - nested b\n - deeper\n- clause two\n" + + +def test_an_item_on_the_heading_list_is_numbered_at_the_level_after_its_section_with_an_indent_of_its_own() -> None: + parts_ = document(CONTINENTAL, style="continental") + one, two = paragraph_with(parts_["word/document.xml"], "clause one"), paragraph_with( + parts_["word/document.xml"], "clause two") + assert numbering_of(one) == numbering_of(two) == (1, 2) + assert one.find(f"{W}pPr/{W}pStyle") is None # no heading style: it is no heading + assert indent(one) == indent(two) is not None + level = levels(parts_["word/numbering.xml"], 0)[1] + assert level.find(f"{W}lvlText").get(f"{W}val") == "%1.%2" and level.find(f"{W}pStyle").get(f"{W}val") == "Heading2" + # the num of the list restarts the level at one, as every top-level list does + [num] = [n for n in parts_["word/numbering.xml"].findall(f"{W}num") if n.get(f"{W}numId") == "2"] + override = num.find(f"{W}lvlOverride") + assert override.get(f"{W}ilvl") == "1" and override.find(f"{W}startOverride").get(f"{W}val") == "1" # type: ignore[union-attr] + + +def test_a_nested_item_uses_a_reserved_level_of_the_same_num_with_the_indent_one_step_in() -> None: + parts_ = document(CONTINENTAL, style="continental") + root = parts_["word/document.xml"] + one, a, b, deeper = (paragraph_with(root, t) for t in ("clause one", "nested a", "nested b", "deeper")) + assert [numbering_of(p) for p in (a, b, deeper)] == [(5, 2), (5, 2), (6, 2)] # the parent's num, no override of its own + assert indent(a) == (indent(one)[0] + 567, 567) and indent(deeper) == (indent(one)[0] + 1134, 567) # type: ignore[index] + levels_ = levels(parts_["word/numbering.xml"], 0) + assert [(lvl.find(f"{W}numFmt").get(f"{W}val"), lvl.find(f"{W}lvlText").get(f"{W}val")) for lvl in levels_[5:7]] == [ + ("lowerLetter", "%6)"), ("lowerRoman", "%7)")] + assert [n.get(f"{W}numId") for n in parts_["word/numbering.xml"].findall(f"{W}num")] == ["1", "2"] + + +SHARED = ALTERNATIVES + """ +# Disputes {#d when=forum:courts} + +## Sub one {#s1} + +# Disputes {#d when=forum:arb} + +## Sub alt {#sa} + +- courts item {#ci when=forum:courts} +- arbitration item {#ci when=forum:arb} + - nested under it {#an} +- after +""" + + +def test_a_later_alternative_has_numid_zero_an_indent_of_its_own_and_the_number_of_the_holder_as_a_field() -> None: + parts_ = document(SHARED) + root = parts_["word/document.xml"] + heading = [p for p in body_paragraphs(root) if text_of(p).endswith("Disputes Only if: forum:arb")][0] + assert numbering_of(heading) == (0, 0) and indent(heading) == (567, 567) + assert heading.find(f"{W}pPr/{W}pStyle").get(f"{W}val") == "Heading1" # type: ignore[union-attr] + # the number: a REF field for the holder (with \n and \h), the period as text, then a tab, then the title + fields = instructions(heading) + assert len(fields) == 1 and re.fullmatch(r"REF \S+ \\n \\h", fields[0]) + assert text_of(heading).startswith("1.") # the cached result "1" and the period + runs = [r for r in heading.findall(f"{W}r")] + assert any(r.find(f"{W}tab") is not None for r in runs) + # the holder carries the bookmark the field points at + name = fields[0].split()[1] + assert name in {b.get(f"{W}name") for b in root.iter(f"{W}bookmarkStart")} + item = paragraph_with(root, "arbitration item") + [field] = instructions(item) + assert numbering_of(item) == (0, 0) and re.fullmatch(r"REF \S+ \\n \\h", field) + assert field.split()[1] in {b.get(f"{W}name") for b in root.iter(f"{W}bookmarkStart")} + # the first alternative and the section that follows are numbered by the list as usual + assert numbering_of(paragraph_with(root, "courts item")) == (0, 3) + assert numbering_of(paragraph_with(root, "after")) == (0, 3) + + +def test_the_children_of_a_later_alternative_are_numbered_by_a_num_of_their_own() -> None: + parts_ = document(SHARED) + root, numbering = parts_["word/document.xml"], parts_["word/numbering.xml"] + sub_alt, nested = paragraph_with(root, "Sub alt"), paragraph_with(root, "nested under it") + (_, sub_num), (_, nested_num) = numbering_of(sub_alt), numbering_of(nested) + assert sub_num == 2 and nested_num == 4 # the sections after the alternative; the list nested under the alternative item + for num_id, level in ((sub_num, "1"), (nested_num, "1")): + [num] = [n for n in numbering.findall(f"{W}num") if n.get(f"{W}numId") == str(num_id)] + assert num.find(f"{W}lvlOverride").get(f"{W}ilvl") == level # type: ignore[union-attr] + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize("name", ["alternatives", "alternatives in threes", "qualified", "level five", "nested titles"]) +@pytest.mark.parametrize("settings", [{"style": "continental"}, {}, {"overrides": {"numbering.scheme": "none"}}, + {"style": "continental", "overrides": {"paragraphs.numbered": True}}]) +def test_documents_with_alternatives_and_items_on_the_heading_list_validate(name: str, settings: dict) -> None: + require_schemas() + result = render(STRESS[name], format="docx", **settings) + assert result.binary is not None + assert validation_errors(result.binary) == [] From c00826156428a538e21847bca6a17bb3fb9cc49b Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:01:25 +0000 Subject: [PATCH 101/118] Review fixes: alternative information, precise restart messages, indents of nested lists, Word checklist The render reports one information when alternatives share a number as fields of the first one, which breaks if it is deleted in Word. The message about a restarted unit says what an edit does to it, and the round trip pins the drift. A list nested in an item with an indent of its own is indented one step in. New tests cover four reserved levels and a level-5 section after a nested list; the property tests exempt only documents that skip a heading level. A paragraph deleted through UNO must match its text exactly. --- .github/PUBLISHING.md | 26 +++++- CONFORMANCE.md | 2 +- docs/architecture.md | 20 +++-- docs/decisions/0008-docx-word-native.md | 18 +++- src/legaldown_render/resolve/docx_plan.py | 62 +++++++++---- tests/lo/uno_edit.py | 27 +++--- tests/test_docx_plan.py | 104 +++++++++++++++++++--- tests/test_docx_roundtrip.py | 73 +++++++++++++-- 8 files changed, 271 insertions(+), 61 deletions(-) diff --git a/.github/PUBLISHING.md b/.github/PUBLISHING.md index a5ae0eb..edb0d8c 100644 --- a/.github/PUBLISHING.md +++ b/.github/PUBLISHING.md @@ -104,6 +104,7 @@ for doc in ("features", "template"): "separator-none": {"attachments.separator": "none"}, "separator-rule": {"attachments.separator": "rule"}, "scheme-none": {"numbering.scheme": "none"}, + "paragraphs-numbered": {"paragraphs.numbered": True}, # `1.1` paragraphs on the heading list (item 9) }.items(): data = render(source, format="docx", overrides={**contents, **extra}).binary (out / f"{doc}.{name}.docx").write_bytes(data) @@ -160,7 +161,30 @@ For each file: kind restarts by a fixed setting (the render reported `render-docx-static-restart`). - Roman numerals of 4000 and over, and letters past 26: Word's text (`render-docx-roman-overflow`, `render-docx-letter-overflow`) is the one the render reported, not a different one. -9. **Page layout.** The running footer shows `n / N`; a header template with `{title}`, `{version}` and `{date}` shows +9. **Numbering that shares a list or a counter** (`template.continental.docx`, the continental `features` file, and + the `paragraphs-numbered` files; the planner models these from Word's documented behaviour and LibreOffice, and they are + what only Word can confirm). After Ctrl+A, F9: + - Two `w:num` of one abstract list share the counter, and a `startOverride` restarts only its own level: where a section with clauses `1.1`, `1.2` is followed by a sub-heading + (add one to the file if the golden documents have none), the sub-heading reads `1.1` for the + sub-heading (restarted at a fixed value; the render reported `render-docx-shared-counter`), and the next heading + after it reads `1.2`, not `1.3`. Deleting that sub-heading makes the headings after it read higher (the documented + limit). + - Clauses in the continental style (`5.1`, `5.2`) are not in the table of contents and not in the navigation pane + (they have no heading style). Under each clause the nested list reads `a)`, `b)`, and a new clause (Enter at the end + of a clause, typing) restarts it at `a)` under that clause; so does a sub-sub-list (`i)`). A list nested three + levels under a clause (`1.1.1`) is a separate list that restarts by a fixed setting (reported `render-docx-static-restart`). + - In `template.continental.docx` a later alternative (a paragraph, clause or heading that shares a number: the + second "Disputes", say) shows the first one's number: a `numId 0` paragraph that starts with a `REF \n \h` field, + with the same hanging indent as a numbered one and its text after a tab. Update the field: it still reads the + holder's number. Its entry in the table of contents reads the shared number, a tab and the title (`2.Title`). + The numbers after the alternatives continue as the renderer's. Press Enter at the start and at the end of a + later alternative: the new paragraph has no number (it copies `numId 0`; expected, apply a list style by hand). **Delete + the first alternative:** every later alternative, their children, and every reference to the shared identifier read + "Error! Reference source not found" (expected: the render reported `render-docx-alternative`, and the final document + is made by rendering with the condition values). + - In the `paragraphs-numbered` files the paragraphs read `1.1`, `1.2` on the heading list; a sub-heading after a + list of clauses in the same section is restarted (as above). +10. **Page layout.** The running footer shows `n / N`; a header template with `{title}`, `{version}` and `{date}` shows them; a landscape page is landscape when printed (print preview). Record what differs in the release notes. A difference from LibreOffice is expected where the documentation says Word diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 5ee4f96..8d75c95 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -20,7 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | -| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can; the continental `5.1` items and the numbered paragraphs `1.1` ride the heading list's next level, with the lists nested in them on its reserved levels, so they stay live; alternatives that share a number (§15.8) are `numId 0` paragraphs whose number is a `REF` field of the first, with the sections and lists under them on a num of their own), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference for which Word's fields cannot reproduce the designation (a heading level skipped before the target under a `{n}`-style scheme, text around the counter that Word's `\t` switch cannot drop such as `(a)`, `{n}:` or `No. {n}`) is fixed text, reported. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says. **The front and back matter are written too**, in the order the HTML shows them: the title block (header rows and parties), a live table of contents (`TOC \o "1-N" \h \z \u` with cached entries equal to the HTML contents, each a link with a `PAGEREF` page number Word fills in on update; the attachments are listed by outline level, so that no locale-dependent style list is in the field; at `contents.depth: 1` the attachment titles are listed as first-level entries, where the HTML indents them one level more), the attachments (bookmarked so `{{attach:}}` resolves, separated by a page break or a rule per `attachments.separator`) and the signature blocks (§3.6, borderless tables kept together on one page as far as one page allows). A LibreOffice round trip shows that after its indexes are updated the contents lists exactly the HTML entries with page numbers, also after a heading is inserted in the editor, and that the document reads in the plain-text writer's order. Remaining differences are the reported ones: a label that shows the section's number that the heading list cannot show (an outline scheme's `{section}.{n}`, the `none` scheme) is drawn without it. The contents show the shared number for a later alternative. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a unit that shares a counter with others in Word, such as a sub-heading and the items of its section that have the same designation in every output format, and that is restarted at a fixed value, which will not follow edits), `render-docx-number-mismatch` (any other difference: a catch-all that does not occur), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-static-restart` (a list that restarts by a fixed setting: in a list of another kind, in an item on the heading list that it cannot ride, in an alternative) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). The Word-only checks (the contents' page numbers, what `\t` does, numbering after an insertion) are a manual checklist before a release (`.github/PUBLISHING.md`). HTML is the format this renderer claims §13.6 with | +| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can; the continental `5.1` items and the numbered paragraphs `1.1` ride the heading list's next level, with the lists nested in them on its reserved levels, so they stay live; alternatives that share a number (§15.8) are `numId 0` paragraphs whose number is a `REF` field of the first, with the sections and lists under them on a num of their own: live for edits elsewhere, but **deleting the first alternative in Word breaks them and every reference to the shared identifier** (reported; render with the condition values for a final document); the built-in continental style uses the heading list's levels 6 and 7 for lists nested in its clauses, and a third level of nesting under a clause restarts by a fixed setting, reported), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference for which Word's fields cannot reproduce the designation (a heading level skipped before the target under a `{n}`-style scheme, text around the counter that Word's `\t` switch cannot drop such as `(a)`, `{n}:` or `No. {n}`) is fixed text, reported. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says. **The front and back matter are written too**, in the order the HTML shows them: the title block (header rows and parties), a live table of contents (`TOC \o "1-N" \h \z \u` with cached entries equal to the HTML contents, each a link with a `PAGEREF` page number Word fills in on update; the attachments are listed by outline level, so that no locale-dependent style list is in the field; at `contents.depth: 1` the attachment titles are listed as first-level entries, where the HTML indents them one level more), the attachments (bookmarked so `{{attach:}}` resolves, separated by a page break or a rule per `attachments.separator`) and the signature blocks (§3.6, borderless tables kept together on one page as far as one page allows). A LibreOffice round trip shows that after its indexes are updated the contents lists exactly the HTML entries with page numbers, also after a heading is inserted in the editor, and that the document reads in the plain-text writer's order. Remaining differences are the reported ones: a label that shows the section's number that the heading list cannot show (an outline scheme's `{section}.{n}`, the `none` scheme) is drawn without it. The contents show the shared number for a later alternative. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a unit that shares a counter with others in Word, such as a sub-heading and the items of its section that have the same designation in every output format, and that is restarted at a fixed value, which will not follow edits), `render-docx-number-mismatch` (any other difference: a catch-all that does not occur outside documents that skip a heading level), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-alternative` (alternatives share a number as fields of the first, which must not be deleted in Word), `render-docx-static-restart` (a list that restarts by a fixed setting: in a list of another kind, in an item on the heading list that it cannot ride, in an alternative) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). The Word-only checks (the contents' page numbers, what `\t` does, numbering after an insertion) are a manual checklist before a release (`.github/PUBLISHING.md`). HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | diff --git a/docs/architecture.md b/docs/architecture.md index 0c99781..7a58a0c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -293,7 +293,7 @@ A level of a list shows only counters of its own list, which settles where the r same num as its parent with no override: it restarts by structure under every new item and sibling lists go on counting, as in any other list. A level may be shared by units whose definitions are equal and is kept for the first that needs it otherwise; the level after the fifth heading level is kept for a level-5 section's units when the - document has one, and a nested list then falls back. What does not fit (an outline or mixed scheme, where the + document has one, and a nested list then falls back. The built-in continental style uses levels 6 and 7 (`a)`, `i)`); its third level of nesting under a clause shows the section's number and falls back (static, reported). What does not fit (an outline or mixed scheme, where the level after the section's reads otherwise; the `none` scheme, where the section's number is its text; a nested item whose label shows the section; items in a quote; ordered and bullet lists nested in such an item; depth past 4) keeps the family's own abstractNum with the label written without the section and reported @@ -307,16 +307,22 @@ A level of a list shows only counters of its own list, which settles where the r later alternative section until one of its level or above; a list nested in a later alternative item), because the alternative counts nothing and Word would go on counting from the holder's children. A later alternative whose live number is already right (the children of a later alternative section repeat the numbers of the first's) is numbered - live. Designations below an alternative are made of the holder's counters. + live. Designations below an alternative are made of the holder's counters. This is live for edits elsewhere but not + for the first alternative: deleting it in Word breaks every later alternative and every reference to the shared + identifier, and Enter at the start or end of a later alternative gives an unnumbered paragraph, so the render reports + one information per document (`render-docx-alternative`) and a final document is made by rendering with the + condition values. - **Resynchronising.** A numbered unit that Word would number differently from the resolver for want of a place where it can count by structure (it goes on a counter that other units of the same abstractNum have used) is put on a num with a `startOverride` at its number, and the units after it go on counting on the original. This is a fixed value: a - heading or clause inserted or deleted in Word before it does not renumber it. It repairs a sub-heading after the + restarted sub-heading keeps it when clauses or headings are inserted before it, and the headings after it read + higher if it is deleted; a restarted clause or paragraph does not follow what is inserted or deleted before it. It repairs a sub-heading after the items and paragraphs of its section on the heading list (`1.1` for the sub-heading and for the first item share a counter in Word and a designation in every output format: `render-docx-shared-counter` says so), a list that resumes a sibling's count after another list in the same item, and a list past level 9. -A label that shows a counter of a unit outside the unit's own chain (a quote's list inside an item of the preamble, +A list nested in an item that has an indent of its own (`UnitPlan.ind`) is indented one step in from it, on whatever +list it is. A label that shows a counter of a unit outside the unit's own chain (a quote's list inside an item of the preamble, whose label starts from the item; a list nested in another kind of list) is written with the counters Word has there and reported. Nothing is left to be shown by Word differently without a diagnostic. @@ -383,12 +389,12 @@ complex `TOC` field that uses them is written with the title block. its number, a fixed value that will not follow edits before it: a sub-heading that shares a designation with the items of its section, a list that goes on from another, a list whose level another used in between), `number-mismatch` (any other difference: the first unit of a run that Word counts off by the same amount, with how many follow; it is the -catch-all of the planner and does not occur), `reference` (a +catch-all of the planner and does not occur outside documents that skip a heading level), `reference` (a cross-reference that cannot be built from fields, because a heading level is skipped before the target with a `{n}`-style scheme, or Word draws text around a counter that `\t` cannot drop (`({n})`, `{n}:`, `No. {n}`): it is written as fixed text; or one that reads differently in Word for a reason that no diagnostic of a unit already gives; references that fail alike are one finding), `reference-cycle` (a heading's title holds a reference that -leads back to it: written as text), and, as information, `static-restart` (a list that restarts at 1 by a +leads back to it: written as text), and, as information, `alternative` (alternatives share a number as fields of the first, which deleting in Word breaks), `static-restart` (a list that restarts at 1 by a fixed setting: nested in a list of another kind, in an item whose number is on the heading list but which the list cannot ride, or in an alternative that shares a number) and `reference-word-only` (a cover relies on `\t`, which LibreOffice ignores, once per document). A heading level that is skipped is the Core Error `heading-skip`; the plan only notes it on the unit. @@ -472,7 +478,7 @@ Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can ev `ref-not-enumerated`, `path-not-relative` and `path-outside-root` (Errors for a file reference that is not relative, or that leaves the document root, §2.3; reported here because validator 0.4 checks neither), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and the rules the DOCX planner reports on what Word draws differently from the other formats (`render-docx-section-label`, `-list-depth`, `-letter-overflow`, `-roman-overflow`, `-shared-counter`, `-number-mismatch`, `-reference`, `-reference-cycle`, as Warnings, `-static-restart` and `-reference-word-only` as Infos; see "The DOCX planner"). The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and the rules the DOCX planner reports on what Word draws differently from the other formats (`render-docx-section-label`, `-list-depth`, `-letter-overflow`, `-roman-overflow`, `-shared-counter`, `-number-mismatch`, `-reference`, `-reference-cycle`, as Warnings, `-alternative`, `-static-restart` and `-reference-word-only` as Infos; see "The DOCX planner"). The validator's diagnostics carry their line (§16.9), and so do the renderer's own, and the CLI prints it as `file:line:`. diff --git a/docs/decisions/0008-docx-word-native.md b/docs/decisions/0008-docx-word-native.md index f1b007a..95f8390 100644 --- a/docs/decisions/0008-docx-word-native.md +++ b/docs/decisions/0008-docx-word-native.md @@ -99,10 +99,19 @@ a `REF` to the heading text. otherwise (the outline and mixed schemes, `none`), the label is written without the section and reported. - **Alternatives that share a number** (§15.8). Word cannot give two paragraphs one number on one level, except by `numId 0` and a field: the later alternative is such a paragraph, with a `REF \n` field of the unit that - holds the number as its number. The units under it restart in a num of their own with a `startOverride`. + holds the number as its number. The units under it restart in a num of their own with a `startOverride`. This is + live for edits elsewhere (inserting a clause before the alternatives moves their numbers and every reference), but + **not for the first alternative itself**: deleting it in Word makes every later alternative, their children and + every reference to the shared identifier read "Error! Reference source not found", and pressing Enter at the start + or end of a later alternative gives a new paragraph with no number (it copies `numId 0`). There is no live option + that avoids this, so the render reports one information per document (`render-docx-alternative`), and the way to a + final document is to render it with the condition values, which resolves the alternatives. - **A unit that Word would number off the resolver's** because it shares a counter with others of its - abstractNum (a sub-heading after the items of its section) is restarted at a fixed value and reported, so the - file is right when it is opened and a heading inserted or deleted before it in Word does not renumber it. + abstractNum (a sub-heading after the clauses of its section, a paragraph after clauses on the same level) is + restarted at a fixed value and reported (`render-docx-shared-counter`), so the file is right when it is opened. The + value is fixed: a restarted sub-heading keeps it when clauses or headings are inserted before it, but deleting it + makes the headings after it read higher; a restarted clause or paragraph does not follow what is inserted or + deleted before it. ### Rejected alternatives @@ -128,7 +137,8 @@ a `REF` to the heading text. numbers, references, table of contents and page numbers follow. - Some restarts cannot be kept live in Word and are documented and warned about, not prevented: a nested list of another family than its parent, a list nested in an alternative, a unit restarted at a fixed - value because it shares a counter, and paragraph numbering in a section the user adds. A lawyer who adds a + value because it shares a counter, and paragraph numbering in a section the user adds; and the numbers of the + alternatives that share one live only as long as the first alternative stays. A lawyer who adds a parent item restarts such a list by hand. - Viewers that never update fields (Word Online, Quick Look, the Google Docs import, Outlook's preview) show the cached results, which are right, but the table of contents without page numbers. diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 7a6e055..72171cf 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -876,13 +876,14 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan units[unit] = replace(units[unit], bookmarks=(name,), collapsed=unit not in enclosing) for unit, targets in cyclic.items(): units[unit] = replace(units[unit], literal_refs=targets) - for unit, (holder, suff) in self.alts.items(): + for unit, (holder, alt_suff) in self.alts.items(): # The number of a later alternative is the number of the unit it repeats (§15.8). shared = self.units[holder] tail = shared.label[len(shared.word_ref):] - segments: list[Segment] = [FieldSeg(bookmark[holder], True, False, True, shared.word_ref, shared.word_ref)] + alt_segments: list[Segment] = [ + FieldSeg(bookmark[holder], True, False, True, shared.word_ref, shared.word_ref)] units[unit] = replace(units[unit], alt=AltNumber( - (*segments, *([Literal_(tail)] if tail else [])), suff)) + (*alt_segments, *([Literal_(tail)] if tail else [])), alt_suff)) for inline in iter_tree_inlines(self.tree): if isinstance(inline, DefinedTerm) and inline.defining and inline.anchor and inline.anchor not in anchors: anchors[inline.anchor] = names.anchor("lddef_", inline.anchor.removeprefix("def:")) @@ -942,6 +943,7 @@ def run(self) -> tuple[DocxPlan, list[Diagnostic]]: for drift in self.drifts: self.report_drift(drift) self.report_collisions() + self.report_alternatives() units, anchors, refs, toc = self.finish() abstracts = tuple( AbstractNum(a.id, a.kind, f"{0x4C440000 + a.id:08X}", # type: ignore[arg-type] @@ -1216,19 +1218,20 @@ def reserved(self, fmt: LevelFormat, depth: int) -> tuple[_Abstract, int] | None def indent(self, abstract: _Abstract, ilvl: int, unit: str, parent: str | None, *, alternative: bool ) -> tuple[int, int] | None: - """The indent a unit carries itself, in twips ``(left, hanging)``: the level's for a later alternative (which - has no numbering to take it from), and for a unit of the heading abstractNum that is no heading (whose level - indents the headings) the width of the heading level it stands on, one step more for each level of nesting - below it.""" + """The indent a unit carries itself, in twips ``(left, hanging)``. An item or paragraph on the heading abstractNum + (whose levels indent the headings) takes the width of the heading level it stands on; one that stands in an + item with an indent of its own, on any list, is one step in from the text of that item; a later alternative, + which has no numbering to take an indent from, takes its level's. Any other unit takes its level's, and the + plan sets nothing.""" level = abstract.levels[ilvl] - if abstract.kind == "headings" and unit in self.item_units: - if parent in self.text_left: - left, hanging = self.text_left[parent] + _STEP, _STEP - else: - left, hanging = level.left, level.hanging - self.text_left[unit] = left - return left, hanging - return (level.left, level.hanging) if alternative else None + if unit in self.item_units and parent in self.text_left: + left, hanging = self.text_left[parent] + _STEP, _STEP + elif unit in self.item_units and abstract.kind == "headings": + left, hanging = level.left, level.hanging + else: + return (level.left, level.hanging) if alternative else None + self.text_left[unit] = left + return left, hanging def numbered(self, node: _Unit, kind: Literal["section", "item", "paragraph"], num: Num, abstract: _Abstract, ilvl: int, *, counter: str, group: object, cause: tuple[str, ...] = (), explicit: bool = True) -> Num: @@ -1305,18 +1308,41 @@ def restart_class(self, parent: str, placed: _Placed, abstract: _Abstract) -> st def report_collisions(self) -> None: """One finding for the units of the heading list that Word would number off the resolver's because they share - a counter with others of their section (§2.7): the first, and how many more.""" + a counter with others of their section (§2.7): the first, and how many more. What an edit in Word does to + the fixed value depends on the unit: a sub-heading restarted after the clauses of its section keeps it, and + the headings after it read higher if it is deleted; a clause or paragraph does not follow what is inserted or + deleted before it.""" if not self.collisions: return node, word = self.collisions[0] assert node.unit is not None designation = text_of(self.desig[node.unit]) more = f" {len(self.collisions) - 1} more units do the same." if len(self.collisions) > 1 else "" + if word == "heading": + effect = ("The heading keeps that value when clauses or headings are inserted before it, and if it is " + "deleted in Word the headings after it read higher than the renderer's.") + else: + effect = ("The unit keeps that value: a clause or paragraph inserted or deleted before it in Word does not " + "move it.") self.shared_counter( f"The {word} '{printable(node.label or '')}' has the designation {printable(designation)}, which another " f"unit of its section has too (in every output format their designations collide), and in Word they " - f"share one counter. The number is kept as the renderer shows it by restarting at a fixed value, which " - f"will not follow a heading or clause inserted or deleted before it in Word.{more}", node.line) + f"share one counter. The number is kept as the renderer shows it by restarting at a fixed value. " + f"{effect}{more}", node.line) + + def report_alternatives(self) -> None: + """One information for the document's alternatives that share a number: their numbers are fields of the first + alternative, which a lawyer can delete in Word (§15.8).""" + if not self.alts: + return + holder = next(iter(self.alts.values()))[0] + node = self.nodes[holder] + where = f", line {node.line}" if node.line is not None else "" + self.report("render-docx-alternative", "info", + f"{len(self.alts)} alternative(s) that share a number show it as a field of the first alternative " + f"('{printable(self.units[holder].label)}'{where}); deleting that paragraph in Word " + f"breaks their numbers and every reference to their shared identifier. Render the document " + f"with the condition values to produce the final one.", node.line) def list_(self, lst: List, section: Section | None, quoted: bool = False) -> None: first = lst.items[0] diff --git a/tests/lo/uno_edit.py b/tests/lo/uno_edit.py index 9825fb9..d104cc6 100644 --- a/tests/lo/uno_edit.py +++ b/tests/lo/uno_edit.py @@ -10,7 +10,7 @@ /usr/bin/python3 tests/lo/uno_edit.py OUTDIR JOBS.json ``JOBS.json`` is a list of ``{"name": "job", "file": "/path/doc.docx", "edits": [["text of a paragraph", "new"], ...]}``; -an edit whose new text is ``null`` deletes the paragraph instead. +an edit whose new text is ``null`` deletes the paragraph whose text is exactly that instead. One LibreOffice instance serves every job. Exit status 0 when every job was exported. """ from __future__ import annotations @@ -51,26 +51,33 @@ def _connect(pipe: str, process: subprocess.Popen[bytes]) -> object: def _edit(document: object, match: str, new: str | None) -> None: """Insert a paragraph with the text *new* before the first paragraph that holds *match*, or, when *new* is None, - delete that paragraph (with its break, as Delete at its end does).""" + delete the first paragraph whose text is exactly *match* (with its break, as Delete at its end does; the last + paragraph of the document with the break before it, so that no empty paragraph is left).""" text = document.getText() paragraphs = text.createEnumeration() while paragraphs.hasMoreElements(): paragraph = paragraphs.nextElement() # A line of a table of contents holds the heading's words too: only the document's own paragraphs count. - if (paragraph.supportsService("com.sun.star.text.Paragraph") and match in paragraph.getString() - and not paragraph.ParaStyleName.startswith("Contents")): - if new is None: + if not paragraph.supportsService("com.sun.star.text.Paragraph") or paragraph.ParaStyleName.startswith("Contents"): + continue + if new is None: + if paragraph.getString() != match: + continue + cursor = text.createTextCursorByRange(paragraph.getStart()) + cursor.gotoEndOfParagraph(True) + if not cursor.goRight(1, True): # the last paragraph: take the break before it instead cursor = text.createTextCursorByRange(paragraph.getStart()) - cursor.gotoEndOfParagraph(True) - cursor.goRight(1, True) - cursor.setString("") - return + cursor.goLeft(1, False) + cursor.gotoEnd(True) + cursor.setString("") + return + if match in paragraph.getString(): cursor = text.createTextCursorByRange(paragraph.getStart()) text.insertControlCharacter(cursor, PARAGRAPH_BREAK, False) cursor.goLeft(1, False) cursor.setString(new) return - raise SystemExit(f"no paragraph holds {match!r}") + raise SystemExit(f"no paragraph {'is' if new is None else 'holds'} {match!r}") def edit(out_dir: str, jobs: list[dict]) -> None: diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index 3706540..fbde434 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -533,7 +533,9 @@ def test_a_nested_label_that_shows_the_section_does_not_ride_the_reserved_levels x, y, z, w, v = items(tree) assert [plan.units[i.unit].ilvl for i in (x, y, z)] == [1, 5, 6] assert plan.units[w.unit].ilvl == 3 and plan.units[w.unit].cause == ("restart-section", "label-fallback") - assert plan.units[w.unit].label == "1" and w.label == "1.1" and plan.units[w.unit].ind is None + assert plan.units[w.unit].label == "1" and w.label == "1.1" + # it is one step in from the item it is nested in, on whatever list it is + assert plan.units[w.unit].ind == (plan.units[z.unit].ind[0] + 567, 567) # the list after it is on the family's abstractNum too, under a parent that is not on the heading list assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated"] assert plan.units[v.unit].ilvl == 4 and plan.units[v.unit].cause == () @@ -541,12 +543,54 @@ def test_a_nested_label_that_shows_the_section_does_not_ride_the_reserved_levels ("render-docx-static-restart", "info"), ("render-docx-section-label", "warning")] +FIVE_LEVELS = {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{n})", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n})", "ref": "({n})"}, + {"counter": "upper-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "[{n}]", "ref": "[{n}]"}, + {"counter": "lower-alpha", "label": "{n}.", "ref": "{n}"}]} + + +def test_a_chain_of_four_lists_in_a_clause_rides_all_four_reserved_levels_and_the_fifth_falls_back() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(6)) + tree, plan, diagnostics = planned(body, style="continental", overrides=FIVE_LEVELS) + found = items(tree) + assert [plan.units[i.unit].ilvl for i in found[:5]] == [1, 5, 6, 7, 8] + assert [plan.units[i.unit].label for i in found[:5]] == ["1.1", "a)", "i)", "(A)", "[1]"] + assert [i.label for i in found[:5]] == ["1.1", "a)", "i)", "(A)", "[1]"] + assert len({plan.units[i.unit].num_id for i in found[:5]}) == 1 # the clause's num, no override of their own + assert [level.lvl_text for level in plan.abstract_nums[0].levels[5:]] == ["%6)", "%7)", "(%8)", "[%9]"] + assert [plan.units[i.unit].cause for i in found[1:5]] == [("reserved-level",)] * 4 + # depth 5 is past the reserved levels: the family's own abstractNum, restarted by a fixed setting + assert plan.units[found[5].unit].cause == ("restart-section",) and plan.units[found[5].unit].ilvl == 5 + assert rules(diagnostics) == {"render-docx-static-restart"} + + +def test_a_nested_list_before_a_level_5_section_falls_back_without_a_section_label_finding() -> None: + # The level after the fifth heading level is kept for the units of the level-5 section, so the nested list of an + # earlier section cannot have it: it keeps the family's abstractNum, restarted by a fixed setting (information), + # and its label, which has no section in it, is not reported. + body = "# A\n\n- x\n - y\n - z\n\n## B\n\n### C\n\n#### D\n\n##### E\n\n- five\n" + tree, plan, diagnostics = planned(body, style="continental") + x, y, z, five = items(tree) + assert plan.units[x.unit].ilvl == 1 and plan.units[five.unit].ilvl == 5 + assert plan.units[y.unit].cause == ("restart-section",) and plan.units[y.unit].ilvl == 1 + assert plan.units[z.unit].cause == () and plan.units[z.unit].ilvl == 2 + assert rules(diagnostics) == {"render-docx-static-restart", "render-docx-shared-counter"} # B follows the clause + assert plan.abstract_nums[0].levels[5].lvl_text == "%1.%2.%3.%4.%5.%6" + # without the level-5 section the same lists ride the reserved levels + tree, plan, diagnostics = planned("# A\n\n- x\n - y\n - z\n", style="continental") + assert [plan.units[i.unit].ilvl for i in items(tree)] == [1, 5, 6] and diagnostics == [] + + + def test_an_ordered_list_nested_in_an_item_on_the_heading_list_keeps_its_own_numbering() -> None: tree, plan, diagnostics = planned("# A\n\n- x\n 1. y\n 1. z\n", style="continental") x, y, z = items(tree) assert [a.kind for a in plan.abstract_nums] == ["headings", "renumber"] assert shown(plan, y, z) == [(3, 1, "1."), (3, 1, "2.")] and plan.units[y.unit].cause == ("restart-section",) - assert plan.units[y.unit].ind is None + assert plan.units[y.unit].ind == (plan.units[x.unit].ind[0] + 567, 567) [found] = diagnostics assert (found.rule, found.level) == ("render-docx-static-restart", "info") and "carries the section" in found.message @@ -713,7 +757,8 @@ def test_a_later_alternative_section_has_no_number_of_its_own_and_shows_the_hold assert segments_of(plan, "s2") == 'REF d + "."' and second.alt.suff == "tab" segment, period = second.alt.segments assert (segment.number, segment.hyperlink, segment.suppress_text, segment.cached) == (True, True, False, "1") - assert period == Literal_(".") and segment.bookmark == plan.units["s1"].bookmarks[0] and diagnostics == [] + assert period == Literal_(".") and segment.bookmark == plan.units["s1"].bookmarks[0] + assert rules(diagnostics) == {"render-docx-alternative"} def test_the_text_after_the_number_of_a_later_alternative_is_the_label_beyond_what_the_field_returns() -> None: @@ -760,7 +805,7 @@ def test_the_sections_under_a_later_alternative_start_in_a_num_of_their_own() -> assert exp.num_id == exp_two.num_id == exp_three.num_id # the section after the alternatives goes back to the headings' num: it goes on from the holder's counter assert (last.count, l1.count) == (3, 1) and l1.num_id == 1 and last.explicit is False - assert diagnostics == [] or all(d.level == "info" for d in diagnostics) + assert all(d.level == "info" for d in diagnostics) def test_an_alternative_that_is_not_the_first_to_be_numbered_takes_no_num_for_its_children_until_they_come() -> None: @@ -786,7 +831,7 @@ def test_an_item_that_repeats_the_number_of_the_one_before_is_numbered_by_a_fiel assert (b.shares_number_with, plan.units[b.unit].num_id, plan.units[b.unit].label) == (a.unit, 0, "(a)") assert (plan.units[b.unit].ilvl, plan.units[b.unit].ind, plan.units[b.unit].cause) == (0, (567, 567), ("alternative",)) assert shown(plan, a, c) == [(2, 0, "(a)"), (2, 0, "(b)")] and plan.units[c.unit].count == 2 - assert segments_of(plan, b.unit) == "REF i" and diagnostics == [] + assert segments_of(plan, b.unit) == "REF i" and rules(diagnostics) == {"render-docx-alternative"} def test_the_children_of_a_later_alternative_item_get_a_num_of_their_own_and_it_is_reported_as_information() -> None: @@ -796,9 +841,9 @@ def test_the_children_of_a_later_alternative_item_get_a_num_of_their_own_and_it_ assert shown(plan, x, y, z) == [(2, 1, "(i)"), (3, 1, "(i)"), (3, 1, "(ii)")] assert plan.units[y.unit].cause == ("restart-alternative",) and plan.units[z.unit].cause == () assert plan.nums[2].start_overrides == ((1, 1),) - [found] = diagnostics + found = [d for d in diagnostics if d.rule == "render-docx-static-restart"][0] assert (found.rule, found.level, found.line) == ("render-docx-static-restart", "info", y.line) - assert "alternative" in found.message + assert "alternative" in found.message and rules(diagnostics) == {"render-docx-static-restart", "render-docx-alternative"} def test_a_paragraph_that_repeats_the_number_of_the_one_before_is_numbered_by_a_field() -> None: @@ -808,7 +853,7 @@ def test_a_paragraph_that_repeats_the_number_of_the_one_before_is_numbered_by_a_ assert [(plan.units[u.unit].num_id, plan.units[u.unit].label) for u in (one, arb, two)] == [ (2, "1.1"), (0, "1.1"), (2, "1.2")] assert plan.units[arb.unit].ilvl == 1 and plan.units[two.unit].count == 2 - assert diagnostics == [] + assert rules(diagnostics) == {"render-docx-alternative"} def test_alternatives_on_the_heading_list_carry_the_indent_of_the_unit_not_of_the_heading() -> None: @@ -835,6 +880,29 @@ def test_a_designation_below_an_alternative_is_made_of_the_holders_counters() -> assert "render-docx-reference" not in rules(diagnostics) +def test_the_alternatives_of_a_document_are_one_information_naming_the_first() -> None: + body = ("# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n\n# E {#e when=forum:courts}\n\n" + "# E {#e when=forum:arb}\n\n# E {#e when=forum:expert}\n") + tree, plan, diagnostics = planned(body, ALTERNATIVES) + [found] = [d for d in diagnostics if d.rule == "render-docx-alternative"] + assert found.level == "info" and found.line == tree.sections[0].line + assert "3 alternative(s)" in found.message and "'1.'" in found.message and "deleting that paragraph" in found.message + assert f"line {tree.sections[0].line}" in found.message and "condition values" in found.message + _, _, diagnostics = planned("# A\n\n# B\n", ALTERNATIVES) + assert "render-docx-alternative" not in rules(diagnostics) + + +def test_the_message_about_a_restarted_unit_says_what_an_edit_does_to_it() -> None: + _, _, diagnostics = planned(CONTINENTAL_ITEMS, style="continental") + [heading] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert "if it is deleted in Word the headings after it read higher" in heading.message + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}.{n}", + "ref": ".{n}"}} + _, _, diagnostics = planned("# A\n\nOne.\n\n- x\n- y\n\nTwo.\n", style="continental", overrides=levels) + [paragraph] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert "inserted or deleted before it in Word does not move it" in paragraph.message + + def test_the_contents_show_the_number_the_alternative_shares() -> None: body = "# D {#d when=forum:courts}\n\n## S\n\n# D {#d when=forum:arb}\n\n## S2\n\n# Last\n" _, plan, _ = planned(body, ALTERNATIVES, overrides={"contents.enabled": True, "contents.depth": 2}) @@ -1099,6 +1167,10 @@ def assert_planned_as_resolved(result, plan, diagnostics) -> None: # noqa: ANN0 diagnostic is reported exactly where there is one.""" tree = result.tree found = rules(diagnostics) + # A document that skips a heading level (a Core Error, which the renderer still draws) is the one exception: Word + # is expected to read the level that was skipped as 0 where the renderer and LibreOffice count it as 1, and the + # catch-all ``render-docx-number-mismatch`` may then report units Word counts off. + skips = any("skipped-level" in unit.cause for unit in plan.units.values()) overflows: dict[str, str] = {} # unit -> the rule its counter reads differently under (itself or an ancestor) parents = {unit.unit: unit.parent_unit for unit in units(tree)} own_rules: set[str] = set() @@ -1109,7 +1181,7 @@ def assert_planned_as_resolved(result, plan, diagnostics) -> None: # noqa: ANN0 if alternative: assert mine.count is None and "alternative" in mine.cause and mine.explicit and mine.ind is not None assert mine.ilvl is not None and mine.label.rstrip(".") == mine.word_ref - else: + elif not skips: assert mine.count == node.number, (node.unit, mine.count, node.number) own = "" if not clamped and word_counter(node.number, NUM_FMT[counter]) != format_counter(node.number, counter): @@ -1125,8 +1197,9 @@ def assert_planned_as_resolved(result, plan, diagnostics) -> None: # noqa: ANN0 assert rule in found, (node.unit, cause) own_rules.update([own] if own else []) assert found & {"render-docx-roman-overflow", "render-docx-letter-overflow"} == own_rules - assert "render-docx-number-mismatch" not in found, "a placement is wrong: " + "; ".join( + assert skips or "render-docx-number-mismatch" not in found, "a placement is wrong: " + "; ".join( d.message for d in diagnostics if d.rule == "render-docx-number-mismatch") + assert ("render-docx-alternative" in found) == any(u.alt is not None for u in plan.units.values()) assert all(d.rule.startswith("render-docx-") for d in diagnostics) assert {u.unit for u in units(tree)} == set(plan.units) # Every num with an override is used, by a unit whose level the override names. @@ -1140,13 +1213,20 @@ def assert_planned_as_resolved(result, plan, diagnostics) -> None: # noqa: ANN0 if num.start_overrides: assert plan.units[first[num.id]].ilvl == num.start_overrides[0][0] # A unit carries an indent of its own where its level's serves another (a unit that is no heading on the heading - # list), and where it has no level to take it from (a later alternative). + # list), where it has no level to take it from (a later alternative), and where it is nested in a unit that does + # (one step in from it, whatever list it is on). headings = next((a.id for a in plan.abstract_nums if a.kind == "headings"), None) nums = {n.id: n.abstract_id for n in plan.nums} + by_bookmark = {name: u.unit for u in plan.units.values() for name in u.bookmarks} + indented: set[str] = set() # the units that stand where an indent of their own is the one their text starts at for unit in units(tree): mine = plan.units[unit.unit] on_headings = mine.num_id is not None and nums.get(mine.num_id) == headings and not isinstance(unit, Section) - assert (mine.ind is not None) == (on_headings or mine.num_id == 0), (unit.unit, mine.ind, mine.num_id) + if mine.alt is not None: # a later alternative stands where the unit it repeats does + on_headings = by_bookmark[mine.alt.segments[0].bookmark] in indented + if not isinstance(unit, Section) and (on_headings or unit.parent_unit in indented): + indented.add(unit.unit) + assert (mine.ind is not None) == (unit.unit in indented or mine.num_id == 0), (unit.unit, mine.ind, mine.num_id) assert (mine.alt is not None) == (mine.num_id == 0) diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index 80b351b..bf5b73c 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -313,6 +313,14 @@ """, } +#: The stress documents that are shown under some variants only (each costs LibreOffice time): the continental style, where +#: the items ride the heading list, the default, and the default with numbered paragraphs. +FEW_VARIANTS: dict[str, tuple[str, ...]] = { + name: ("continental", "default", "paragraphs") for name in ("alternatives in threes", "nested titles", "qualified")} + +#: The variants that show some stress documents only. +ONLY_DOCS: dict[str, tuple[str, ...]] = {"five levels": ("qualified",), "shared level": ("qualified", "nested")} + #: The settings that give the numbering words and symbols, which Word's ``\t`` switch drops from a reference. WORDS: dict[str, dict] = { "article": {"overrides": {"numbering.levels": [ @@ -322,6 +330,15 @@ "paragraphs": {"overrides": {"paragraphs.numbered": True}}, # the items and the paragraphs of a section share the heading list's level: the numbers are restarted where the # resolver counts them apart + # six levels of enumeration with no section in them but the first: a chain of four lists nested in a clause rides all + # four reserved levels of the heading list + "five levels": {"style": "continental", "overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{n})", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n})", "ref": "({n})"}, + {"counter": "upper-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "[{n}]", "ref": "[{n}]"}, + {"counter": "lower-alpha", "label": "{n}.", "ref": "{n}"}]}}, "shared level": {"style": "continental", "overrides": {"paragraphs.numbered": True, "paragraphs.format": { "counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}}}, } @@ -465,6 +482,11 @@ def round_trips() -> dict[str, tuple[Case, ...]]: for index, (name, source) in enumerate(corpus().items()): if variant in WORDS and not name.startswith("stress/"): continue + if variant in ONLY_DOCS and name.removeprefix("stress/") not in ONLY_DOCS[variant]: + continue + if (variant not in ONLY_DOCS and name.removeprefix("stress/") in FEW_VARIANTS + and variant not in FEW_VARIANTS[name.removeprefix("stress/")]): + continue if name.startswith(("examples/", "fixtures/")) and (index + offset) % 2: continue # the specification's documents take turns under the schemes: each is shown under half case = make_case(variant, name, source) @@ -828,15 +850,50 @@ def test_a_clause_inserted_or_deleted_before_the_restarted_sub_heading_leaves_it assert lines_of(resynced()[name]) == lines_of(render(source, format="text", style="continental").output), name -def test_a_sub_heading_inserted_or_deleted_before_the_restarted_one_is_the_documented_limit() -> None: - # The restarted heading keeps its fixed value, and the counter it shared goes on where the edit left it: the - # warning of the plan is what tells the author. Only the warning is asserted, not what the drift looks like. +def test_a_sub_heading_inserted_before_the_restarted_one_takes_its_num_and_everything_follows() -> None: + source = applied(RESYNCED, [("## Beta", "## NEWH\n")]) + assert lines_of(resynced()["heading inserted"]) == lines_of( + render(source, format="text", style="continental").output) + + +def test_deleting_the_restarted_sub_heading_is_the_documented_limit() -> None: + """The heading that holds the fixed value is gone: the headings after it go on from the counter the clauses of the + section used, so Gamma reads 1.4 where the renderer's document says 1.1 (the plan's warning says so). The + clauses before it are right and the broken reference to the deleted heading is Word's.""" _, diagnostics = plan_docx(render(RESYNCED, format="docx", style="continental").tree) - assert [d.level for d in diagnostics if d.rule == "render-docx-shared-counter"] == ["warning"] - for name in ("heading deleted", "heading inserted"): - said = lines_of(resynced()[name]) - assert [line for line in said if "clause" in line and not line.startswith("(")] == [ - "1.1 clause one", "1.2 spare clause", "1.3 clause two"] # what stands before the edit is right + [found] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert "if it is deleted in Word the headings after it read higher" in found.message + said = lines_of(resynced()["heading deleted"]) + assert [line for line in said if "clause" in line and not line.startswith("(")] == [ + "1.1 clause one", "1.2 spare clause", "1.3 clause two"] + assert "1.4 Gamma" in said and "1.1 Gamma" in lines_of(render( + RESYNCED.replace("## Beta {#beta}\n\n", ""), format="text", style="continental").output) + + +@functools.cache +def shared_level() -> dict[str, str]: + require_libreoffice() + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}.{n}", + "ref": ".{n}"}} + data = render(PARAGRAPHS_AND_CLAUSES, format="docx", style="continental", overrides=levels).binary + return libreoffice_edit({"original": (data, []), "inserted": (data, [("One.", "NEWP")])}) # type: ignore[arg-type] + + +PARAGRAPHS_AND_CLAUSES = FRONT + "# A\n\nOne.\n\n- x\n- y\n\nTwo.\n" + + +def test_a_paragraph_restarted_after_clauses_does_not_follow_a_paragraph_inserted_before_it() -> None: + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}.{n}", + "ref": ".{n}"}} + assert lines_of(shared_level()["original"]) == lines_of(render( + PARAGRAPHS_AND_CLAUSES, format="text", style="continental", overrides=levels).output) + plan, diagnostics = plan_docx(render(PARAGRAPHS_AND_CLAUSES, format="docx", style="continental", + overrides=levels).tree) + [found] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] + assert "inserted or deleted before it in Word does not move it" in found.message + # the renderer would number the new paragraph 1.1, One 1.2 and Two 1.3; Two holds its fixed value + said = lines_of(shared_level()["inserted"]) + assert [line for line in said if line.endswith(("NEWP", "One.", "Two."))] == ["1.1 NEWP", "1.2 One.", "1.2 Two."] # -- The contents: the headings with the numbers Word draws, and none of the units on the heading list ------------------ From 84fbb4f4aa5877ecd77a6198edeb0e41ae9451c9 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:16:21 +0000 Subject: [PATCH 102/118] Pin the holder, the count check and the explicit num of a restarted heading in tests --- tests/test_docx_plan.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index fbde434..bc6077f 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -465,6 +465,8 @@ def test_a_sub_heading_after_the_items_of_its_section_is_restarted_where_the_res b = tree.sections[1] # Word's counter of the level is shared by the sub-heading and the items: 1.3 where the resolver has 1.1 again assert (b.label, plan.units[b.unit].label, plan.units[b.unit].cause) == ("1.1", "1.1", ("resync",)) + assert plan.units[b.unit].explicit # a num of its own: the paragraph has to carry it + assert plan.units[b.unit].explicit # a num of its own: the paragraph must carry it assert plan.nums[plan.units[b.unit].num_id - 1].start_overrides == ((1, 1),) [found] = [d for d in diagnostics if d.rule == "render-docx-shared-counter"] assert found.level == "warning" and found.line == b.line @@ -880,6 +882,36 @@ def test_a_designation_below_an_alternative_is_made_of_the_holders_counters() -> assert "render-docx-reference" not in rules(diagnostics) +def test_a_unit_that_shares_a_number_but_finds_another_count_in_the_level_is_restarted_not_made_an_alternative() -> None: + # Two clauses of the later alternative's sub-section counted the level of the heading list between: the number the + # sub-sub-section shares is not the one the level holds, so Word's counter is restarted at it instead of a field + # standing for a unit that is not where it is. + body = ("# D {#d when=forum:courts}\n\n## Sub {#s}\n\n### Deeper {#t}\n\n# D {#d when=forum:arb}\n\n" + "## Sub alt\n\n- clause one\n- clause two\n\n### Deeper alt\n") + tree, plan, diagnostics = planned(body, ALTERNATIVES, style="continental") + deeper = tree.sections[5] + assert deeper.shares_number_with == "s3" + mine = plan.units[deeper.unit] + assert (mine.alt, mine.cause, mine.label, mine.count, mine.explicit) == (None, ("resync",), "1.1.1", 1, True) + assert mine.num_id not in (0, None) and plan.nums[mine.num_id - 1].start_overrides == ((2, 1),) + assert "render-docx-shared-counter" in rules(diagnostics) + + +def test_the_holder_is_the_unit_that_holds_the_level_now_not_always_the_first_alternative() -> None: + # A clause on the same level of the heading list counted between the two paragraphs that share a number: the level + # holds the clause's 1.1 now, and the later alternative shows that (the designations below it are made of the + # counters the level holds), so it refers to the clause. Nothing is off: the text is the same. + levels = {"paragraphs.numbered": True, "paragraphs.format": {"counter": "decimal", "label": "{section}.{n}", + "ref": ".{n}"}} + body = "# S\n\nPara. {#p when=forum:courts}\n\n- clause {#i}\n\nPara arb. {#p when=forum:arb}\n\nAfter.\n" + tree, plan, diagnostics = planned(body, ALTERNATIVES, style="continental", overrides=levels) + first, clause, later, after = [u for u in units(tree)][1:] + assert plan.units[later.unit].num_id == 0 and plan.units[later.unit].label == "1.1" + assert plan.units[later.unit].alt.segments[0].bookmark == plan.units[clause.unit].bookmarks[0] + assert plan.units[first.unit].bookmarks == ("ld_p",) and plan.units[after.unit].label == "1.2" + assert "render-docx-number-mismatch" not in rules(diagnostics) + + def test_the_alternatives_of_a_document_are_one_information_naming_the_first() -> None: body = ("# D {#d when=forum:courts}\n\n# D {#d when=forum:arb}\n\n# E {#e when=forum:courts}\n\n" "# E {#e when=forum:arb}\n\n# E {#e when=forum:expert}\n") From 9e3b6dd9cba8ce38433a7bd0c7e4a1c07623215b Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:40:11 +0000 Subject: [PATCH 103/118] Cover the whole reference text with fields, so a field may stand for the words of references.format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reference is now covered as the text the renderer shows (the designation inside references.format), so a plain REF \n that returns the same words as the format ('Article I' for 'Article {designation}', '§ 3' for '§ {designation}', 'Section 1.1' for 'Section {designation}') is the whole reference: no \t switch, nothing for LibreOffice to read differently, and no fixed text where the switch could not drop the label's words. The test helper that reads the literal text of a cover accepts a field that stands for the format's words. --- src/legaldown_render/resolve/docx_cover.py | 6 ++- src/legaldown_render/resolve/docx_plan.py | 25 ++++++----- tests/test_docx_plan.py | 51 ++++++++++++++++++---- tests/test_docx_roundtrip.py | 6 ++- 4 files changed, 65 insertions(+), 23 deletions(-) diff --git a/src/legaldown_render/resolve/docx_cover.py b/src/legaldown_render/resolve/docx_cover.py index f2d17e9..be978d2 100644 --- a/src/legaldown_render/resolve/docx_cover.py +++ b/src/legaldown_render/resolve/docx_cover.py @@ -7,8 +7,10 @@ nothing else Word could know may be written as literal text. This module finds, for one designation, the combination of such fields and literal delimiters that reads exactly like it. -Everything here works on token strings (:mod:`.docx_numbering`): the designation the renderer wrote, and what each -available field returns, are lists of :class:`~.docx_numbering.Lit` (text that is not a counter) and +Everything here works on token strings (:mod:`.docx_numbering`): the text of a reference as the renderer wrote it (the +designation inside the text of ``references.format``, so that a field may stand for the words of the format when the +number it returns holds the same words: ``Article {designation}`` and the label ``Article {n}`` are one plain ``REF``), +and what each available field returns, are lists of :class:`~.docx_numbering.Lit` (text that is not a counter) and :class:`~.docx_numbering.Count` (the counter of one owner, as formatted); a :class:`Title` is the text of a heading under the ``none`` scheme (§13.3), the one thing besides counters a designation can hold that Word keeps up to date (``REF \\h`` returns the bookmarked text). The designation is cut into **atoms** (one per character of diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 72171cf..b977989 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -702,8 +702,10 @@ def why_uncovered(self, designation: Sequence[Shown], fields: Sequence[Field]) - return "no combination of Word's fields reads like it" def reference(self, found: _Reference) -> _Built | None: - """The fields of the reference *found*: its designation covered by fields and delimiters, inside the text of - ``references.format``; None for a target the plan knows nothing of. A designation that cannot be covered is + """The fields of the reference *found*: the whole text it shows (the designation inside the text of + ``references.format``) covered by fields and delimiters, so that a field may stand for the words of the format + when the number it returns holds the same words (``Article {designation}`` and a label ``Article {n}``: one + plain ``REF``, no ``\\t``); None for a target the plan knows nothing of. A designation that cannot be covered is written as the renderer's text, which is right at open and not a field (``fixed``), and noted for the report, as is a cover that reads differently in Word for a reason no diagnostic of a unit already gives.""" node = self.nodes.get(found.target) @@ -712,18 +714,17 @@ def reference(self, found: _Reference) -> _Built | None: numbered = isinstance(node, Section) or node.number is not None designation = self.desig[found.target] if numbered else self.desig.get(found.section or "", []) fields = self.fields(designation) - covered = cover(designation, fields) - if covered is None: - self.unreferable.append(_Finding("fixed", self.why_uncovered(designation, fields), found, found.text)) - return _Built([found.text] if found.text else [], True, None, True) - pieces: list[str | Field] = [] + whole: list[Shown] = [] parts = self.formats.reference.split("{designation}") for index, literal in enumerate(parts): - for piece in ([literal] if literal else []) + (covered if index < len(parts) - 1 else []): - if pieces and isinstance(piece, str) and isinstance(pieces[-1], str): - pieces[-1] += piece - else: - pieces.append(piece) + if literal: + whole.append(Lit(literal)) + if index < len(parts) - 1: + whole.extend(designation) + pieces = cover(whole, fields) + if pieces is None: + self.unreferable.append(_Finding("fixed", self.why_uncovered(designation, fields), found, found.text)) + return _Built([found.text] if found.text else [], True, None, True) shown = "".join(piece if isinstance(piece, str) else text_of(piece.tokens) for piece in pieces) if shown != found.text: chain = {t.owner for t in designation if not isinstance(t, Lit) and isinstance(t.owner, str)} diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index bc6077f..a30cf26 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -1548,6 +1548,12 @@ def cached(segments: Sequence[Segment]) -> str: ARTICLE = {"numbering.levels": [{"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}]} SECTION_SIGN = {"numbering.levels": [{"counter": "decimal", "label": "§ {n}", "ref": "{n}"}], "references.format": "§ {designation}"} +CL_PERIOD = {"numbering.levels": [ + {"counter": "decimal", "label": "Čl. {n}", "ref": "{n}"}, {"counter": "decimal", "label": "{path}", "ref": ".{n}"}], + "references.format": "Čl. {designation}"} +SECTION_WORD = {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}, + {"counter": "decimal", "label": "Section {path}", "ref": ".{n}"}], "references.format": "Section {designation}"} def covers(variant: str, **settings: object) -> dict[str, tuple[str, str]]: @@ -1598,12 +1604,22 @@ def covers(variant: str, **settings: object) -> dict[str, tuple[str, str]]: 'REF alpha \\h + " " + REF i2 + " " + REF n2 + " " + REF d1'), ("none", {"overrides": {"numbering.scheme": "none"}}, "o2", "Gamma (2)", 'REF gamma \\h + " (" + REF o2 + ")"'), # words and symbols in a label that the reference leaves out: \t drops them, the format puts them back + ("article", {"overrides": {**ARTICLE, "references.format": "Article {designation}"}}, "alpha", "Article I", "REF alpha"), ("article", {"overrides": {**ARTICLE, "references.format": "Article {designation}"}}, "i1", "Article I(a)", - '"Article " + REF alpha \\t + REF i1'), + "REF alpha + REF i1"), ("article", {"overrides": {**ARTICLE, "references.format": "Article {designation}"}}, "beta", "Article I.1", '"Article " + REF beta'), - ("section-sign", {"overrides": SECTION_SIGN}, "d1", "§ 1(b)(ii)(A)", - '"§ " + REF alpha \\t + REF i2 + REF n2 + REF d1'), + ("section-sign", {"overrides": SECTION_SIGN}, "d1", "§ 1(b)(ii)(A)", "REF alpha + REF i2 + REF n2 + REF d1"), + ("section-sign", {"overrides": SECTION_SIGN}, "gamma", "§ 1.1.1", '"§ " + REF gamma'), + # the words of the format are the number's own words: a field for the whole number stands for them too + ("cl-period", {"overrides": CL_PERIOD}, "i1", "Čl. 1(a)", "REF alpha + REF i1"), + ("cl-period", {"overrides": CL_PERIOD}, "beta", "Čl. 1.1", '"Čl. " + REF beta'), + ("section-word", {"overrides": SECTION_WORD}, "beta", "Section I.1", "REF beta"), + ("section-word", {"overrides": SECTION_WORD}, "gamma", "Section I.1.1", '"Section " + REF gamma'), + # a label that says another word than the format: the words are left out of the number with \t and the format's + # put back + ("section-word", {"overrides": SECTION_WORD}, "alpha", "Section I", '"Section " + REF alpha \\t'), + ("section-word", {"overrides": SECTION_WORD}, "i1", "Section I(a)", '"Section " + REF alpha \\t + REF i1'), ] @@ -1739,17 +1755,31 @@ def test_a_reference_whose_text_the_renderer_cut_is_reported() -> None: def test_a_cover_with_the_switch_word_ignores_in_libreoffice_is_an_information_for_the_document() -> None: body = "# A {#a}\n\n# B {#b}\n\nSee {{ref: a}} {{ref: b}}.\n" - settings = {"overrides": {**ARTICLE, "references.format": "Article {designation}"}} + settings = {"overrides": {**ARTICLE, "references.format": "Art. {designation}"}} tree, plan, diagnostics = planned(body, **settings) first, second = (plan.refs[i.target_unit].segments for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)) assert [(s.suppress_text, s.cached, s.cached_lo) for s in first if isinstance(s, FieldSeg)] == [(True, "I", "Article I")] assert [(s.suppress_text, s.cached, s.cached_lo) for s in second if isinstance(s, FieldSeg)] == [(True, "II", "Article II")] [found] = [d for d in diagnostics if d.rule == "render-docx-reference-word-only"] - assert found.level == "info" and found.line is None and "'Article I'" in found.message - assert "reads 'Article Article I'" in found.message and "so do 1 more" in found.message + assert found.level == "info" and found.line is None and "'Art. I'" in found.message + assert "reads 'Art. Article I'" in found.message and "so do 1 more" in found.message assert all(plan.refs[i].exact for i in plan.refs) +def test_the_words_of_the_format_that_the_number_holds_too_need_no_switch() -> None: + """``Article {designation}`` and a label ``Article {n}``: the plain ``REF`` returns ``Article I``, which is the + whole reference, so there is no ``\\t`` and nothing for LibreOffice to read differently.""" + body = "# A {#a}\n\n# B {#b}\n\nSee {{ref: a}} {{ref: b}}.\n" + tree, plan, diagnostics = planned(body, overrides={**ARTICLE, "references.format": "Article {designation}"}) + for reference in (plan.refs[i.target_unit] for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)): + [segment] = reference.segments + assert isinstance(segment, FieldSeg) and not segment.suppress_text and segment.cached == segment.cached_lo + assert reference.exact + assert [segment.cached for i in iter_tree_inlines(tree) if isinstance(i, CrossRef) + for segment in plan.refs[i.target_unit].segments] == ["Article I", "Article II"] + assert diagnostics == [] + + def test_reference_findings_name_the_file_and_the_first_line_of_a_repeated_reference() -> None: body = "# A {#a}\n\n### B {#b}\n\nFirst {{ref: b}}.\n\nSecond {{ref: b}}.\n" result = render(FRONT + body, format="text", style="outline") @@ -2138,10 +2168,15 @@ def with_references(source: str, shift: int = 0) -> str: def literal_text_of_a_cover(tree: RenderTree, segments: Sequence[Segment]) -> str: - """The literal text of *segments* without the text of ``references.format`` around the designation.""" + """The literal text of *segments* without the text of ``references.format`` around the designation. A field may + stand for that text (a number that holds the same words: ``Article I`` for ``Article {designation}``): it is then + in the cached text of the fields.""" text = "".join(s.text for s in segments if isinstance(s, Literal_)) + fields = "".join(s.cached for s in segments if isinstance(s, FieldSeg)) for part in tree.formats.reference.split("{designation}"): - assert part in text or not part, (part, text) + if part and part not in text: + assert part in fields, (part, text, fields) + continue text = text.replace(part, "", 1) return text diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index bf5b73c..8dfcb22 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -321,12 +321,16 @@ #: The variants that show some stress documents only. ONLY_DOCS: dict[str, tuple[str, ...]] = {"five levels": ("qualified",), "shared level": ("qualified", "nested")} -#: The settings that give the numbering words and symbols, which Word's ``\t`` switch drops from a reference. +#: The settings that give the numbering words and symbols: a field that returns the same words as the reference's format +#: stands for them, otherwise Word's ``\t`` switch drops them from the number. WORDS: dict[str, dict] = { "article": {"overrides": {"numbering.levels": [ {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}], "references.format": "Article {designation}"}}, "section-sign": {"overrides": {"numbering.levels": [ {"counter": "decimal", "label": "§ {n}", "ref": "{n}"}], "references.format": "§ {designation}"}}, + # the format says a word the label does not: Word's \t leaves the label's out and the format's is put back + "article-short": {"overrides": {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}], "references.format": "Art. {designation}"}}, "paragraphs": {"overrides": {"paragraphs.numbered": True}}, # the items and the paragraphs of a section share the heading list's level: the numbers are restarted where the # resolver counts them apart From 1186474be51fe12fda1ddfde7ab039e07fba823c Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:02:03 +0000 Subject: [PATCH 104/118] Keep path labels of nested lists live: reserved heading levels with the path in them, detached levels as the fallback A list nested in an item on the heading list now takes the reserved level whose text is the label as the resolver writes it, with the section's counters and those of the items on the way as levels of the heading list (%1.%2(%6)): the label and every reference to it are live, and the list restarts by structure under a new item. Where that level is taken by another text, the nesting is deeper than four, or the chain leaves the heading list (an ordered item), the list takes a detached abstractNum that shows each level's own counter alone, only when the enumerated level would have to show the counter of a unit that is not on it; it restarts at a fixed value under a new parent, and the label that Word draws shorter is reported with the units whose counters it leaves out. References to such items are fields of their own counters. The edit helper can put an inserted paragraph on another item's list and level, as pasting a copy does. --- src/legaldown_render/resolve/docx_plan.py | 201 ++++++++++++++++++---- tests/docx_support.py | 5 +- tests/lo/uno_edit.py | 33 +++- tests/test_docx_plan.py | 164 +++++++++++++++++- tests/test_docx_roundtrip.py | 158 ++++++++++++++++- 5 files changed, 507 insertions(+), 54 deletions(-) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index b977989..83172fb 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -75,6 +75,7 @@ LevelParts, Lit, Step, + Token, WordNumbering, extend_tokens, fill_tokens, @@ -114,7 +115,9 @@ class LevelDef: @dataclass(frozen=True, slots=True) class AbstractNum: id: int # 0.., in order of first use (headings first) - kind: Literal["headings", "enumerated", "renumber", "paragraphs", "bullet"] + #: ``detached``: the enumerated levels with each level's own counter alone, for a list nested where the enumerated list + #: would have to show the counter of a unit that is not on it (§13.2). + kind: Literal["headings", "enumerated", "detached", "renumber", "paragraphs", "bullet"] #: Eight hex digits, deterministic (``0x4C440000 + id``). Equal in every LegalDown document, so Word may #: merge lists on paste between two of them (cosmetic). nsid: str @@ -265,8 +268,8 @@ class DocxPlan: _AMBIGUOUS_PERCENT = re.compile(r"%(?=[0-9{])") #: The setting a kind of abstractNum is made from, for a message. -_SETTING = {"headings": "numbering.levels", "enumerated": "enumeration.levels", "renumber": "enumeration.ordered", - "paragraphs": "paragraphs.format"} +_SETTING = {"headings": "numbering.levels", "enumerated": "enumeration.levels", "detached": "enumeration.levels", + "renumber": "enumeration.ordered", "paragraphs": "paragraphs.format"} def docx_style_errors(style: Style) -> list[str]: @@ -367,6 +370,14 @@ def _definitions(formats: NumberingFormats, unwritable: list[tuple[str, int]] | level_text(formats.enumerated[d % size].label, refs, d, section=(), textual=textual), unwritable=unwritable, kind="enumerated") for d in range(9)] + # The same levels with nothing but their own counter (a ``{path}`` shows its own ``ref`` alone): where a list is + # nested in a unit that is not on the enumerated list, Word has no counter of the enclosing units to show. + out["detached"] = [ + _level(d, NUM_FMT[formats.enumerated[d % size].counter], + level_text(formats.enumerated[d % size].label, [""] * d + [refs[d]], d, section=(), + textual=textual), + unwritable=unwritable, kind="detached") + for d in range(9)] if formats.renumber is not None: f = formats.renumber out["renumber"] = [_level(d, NUM_FMT[f.counter], @@ -1058,6 +1069,68 @@ def fallback_label(self, level: LevelFormat, family: str, ilvl: int, kind: str, f"which Word can show only for headings: in Word it reads " f"'{printable(_reads(levels[ilvl], levels))}'.", line) + def detaches(self, depth: int, parent: str | None, placed: _Placed | None) -> bool: + """True if a list of the enumerated family at *depth* in *parent* goes on the detached abstractNum: it is nested + in an item on that abstractNum (its structure restarts it), or the enumerated level it would take names a + level that no unit of its enclosing chain holds, so that Word could show the counter of nothing, or of what + another list counted last. (The same condition as :meth:`numbered` reports a label whose counter Word cannot + show; here it is met before the list is placed.)""" + if placed is not None and placed.abstract.kind == "detached": + return True + named = {part for part in split_level_text(self.definitions["enumerated"][depth].lvl_text) + if isinstance(part, int) and part != depth} + enumerated = self.by_kind.get("enumerated") + held: set[int] = set() + above = parent + while above is not None: + unit = self.canon.get(above, above) + on = self.placed.get(unit) + if enumerated is not None and on is not None and on.abstract is enumerated \ + and self.units[unit].num_id not in (None, 0): + held.add(self.units[unit].ilvl) # type: ignore[arg-type] + above = self.parent.get(above) + return not named <= held + + def lost_counters(self, fmt: LevelFormat, section: Section | None, parent: str | None) -> list[object]: + """The owners of the counters the resolver's label of a unit of *fmt* shows besides its own, in order: what the + detached abstractNum leaves out.""" + owners = [t.owner for t in self.label_tokens(fmt, section, parent) + if isinstance(t, Count) and t.owner is not _SELF] + return list(dict.fromkeys(owners)) + + def describe(self, owners: Sequence[object]) -> str: + """What the units that hold *owners* are, for a message: the section, an enclosing item, an enclosing ordered + item.""" + names: list[str] = [] + if any(isinstance(owner, Ghost) or owner not in self.units or self.units[owner].kind == "section" # type: ignore[index] + for owner in owners): + names.append("section") + kinds = [self.placed[owner].abstract.kind for owner in owners if owner in self.placed] # type: ignore[index] + if "renumber" in kinds: + names.append("ordered item") + if any(kind != "renumber" for kind in kinds): + names.append("item") + if not names: + return "an enclosing unit" + return "the enclosing " + (", ".join(names[:-1]) + " and " + names[-1] if len(names) > 1 else names[0]) + + def detached_label(self, fmt: LevelFormat, ilvl: int, kind: str, lost: Sequence[object], + placed: _Placed | None, first: ListItem) -> None: + """Report a list that is on the detached abstractNum although the resolver's label shows the number of an + enclosing unit: Word shows the unit's own counter alone, and a reference to it is still made of fields.""" + levels = self.definitions["detached"] + if placed is None: + why = "the list stands outside the unit it belongs to" + elif placed.abstract.kind == "headings": + why = "the levels of the heading list kept for nested lists are used up or read otherwise" + else: + why = "the list is nested in a unit that is on another list" + self.report("render-docx-section-label", "warning", + f"The label '{printable(fmt.label)}' of {kind}s shows the number of {self.describe(lost)}, " + f"which Word cannot show in this list ({why}): in Word it reads " + f"'{printable(_reads(levels[ilvl], levels))}'. A reference to such an item is still made of " + f"fields.", first.line) + def shared_counter(self, message: str, line: int | None) -> None: self.report("render-docx-shared-counter", "warning", message, line) @@ -1150,21 +1223,54 @@ def counted(self, node: ListItem | Paragraph, counter: str) -> Count: # -- the heading list's free levels (§2.5) ---------------------------------------- - def candidate(self, fmt: LevelFormat, section: Section) -> LevelParts | None: - """The level text a unit of *fmt* directly in *section* would have on the heading list: its label filled in - as the resolver does, with the counter of each part of the section's number as ``%j`` of the heading level - that holds it and its own counter at the level after the section's. None when a counter is of another - owner or the text cannot be written.""" - assert section.unit is not None - designation = self.desig[section.unit] + def label_tokens(self, fmt: LevelFormat, section: Section | None, parent: str | None, *, designation: bool = False + ) -> list[Token]: + """The label of a unit of *fmt* directly in *parent* (a section or an item; None in the preamble), filled in as + the resolver does (§13.2), as tokens: the counter of the unit itself is :data:`_SELF`'s, every other counter is + that of the unit that holds it (the parts of the section's number and the items on the way). With + *designation*, the unit's designation (§13.1) in the same form.""" + section_des = self.desig[section.unit] if section is not None and section.unit is not None else [] own = [Count(_SELF, "")] - path = extend_tokens(designation, fill_tokens(fmt.ref, n=own, section=designation), textual=False) - tokens = fill_tokens(fmt.label, n=own, path=path, section=designation) + path = extend_tokens(self.desig.get(parent or "", []), fill_tokens(fmt.ref, n=own, section=section_des), + textual=self.textual) + return path if designation else fill_tokens(fmt.label, n=own, path=path, section=section_des) + + def reads_as_designation(self, fmt: LevelFormat, section: Section, parent: str | None) -> bool: + """True if what Word draws for a unit of *fmt* in *parent*, the label with its trailing periods dropped (what + ``REF \\n`` returns), is the end of the unit's designation: a field for the unit then stands for that part of + every reference to it and to what is nested in it, with the counters of the enclosing units in it.""" + shown = self.label_tokens(fmt, section, parent) + while shown and isinstance(shown[-1], Lit): + stripped = shown[-1].text.rstrip(".") + if stripped: + shown[-1] = Lit(stripped) + break + shown.pop() + wanted, mine = atoms(self.label_tokens(fmt, section, parent, designation=True)), atoms(shown) + return len(mine) <= len(wanted) and wanted[len(wanted) - len(mine):] == mine + + def heading_level(self, owner: object) -> int | None: + """The heading list's level the counter of *owner* is drawn at, when it is a unit of the open section that is + numbered on the heading list (an item or a paragraph that rides it, or a nested item on one of its reserved + levels); None for any other owner.""" + found = self.placed.get(owner) if isinstance(owner, str) else None + if found is None or found.abstract.kind != "headings": + return None + unit = self.units[owner] # type: ignore[index] + return unit.ilvl if unit.num_id not in (None, 0) else None + + def candidate(self, fmt: LevelFormat, section: Section, parent: str | None, ilvl: int) -> LevelParts | None: + """The level text a unit of *fmt* in *parent* would have at level *ilvl* of the heading list: its label filled + in as the resolver does, with the counter of each part of the section's number as ``%j`` of the heading level + that holds it, the counter of an item on the way as ``%j`` of the level that item is on, and its own counter + at *ilvl*. None when a counter is of an owner that is not on the heading list or the text cannot be + written.""" + assert section.unit is not None level_of = {owner: j for j, (owner, _) in enumerate(self.owners)} parts: list[str | int] = [] - for token in tokens: + for token in self.label_tokens(fmt, section, parent): if isinstance(token, Count): - j = section.level if token.owner is _SELF else level_of.get(token.owner) + j = ilvl if token.owner is _SELF else level_of.get(token.owner, self.heading_level(token.owner)) if j is None: return None parts.append(j) @@ -1189,6 +1295,24 @@ def claim(self, ilvl: int, definition: LevelDef, *, section_level: bool) -> tupl self.claimed.add(ilvl) return abstract, ilvl + def heading_slot(self, fmt: LevelFormat, section: Section, parent: str | None, ilvl: int, *, section_level: bool + ) -> tuple[_Abstract, int] | None: + """The heading list's level *ilvl* for the units of *fmt* in *parent*, if the label can be written there with + counters of the heading list alone and the level is free or reads the same. A level after a section's is as wide + as the widest of its labels, as a heading's is; a nested one steps in a centimetre from the item it is in, as + every nested list does (a label with the path in it is longer than that, and Word moves the text to its next tab + stop).""" + parts = self.candidate(fmt, section, parent, ilvl) + if parts is None: + return None + if not section_level and any(isinstance(part, int) and part != ilvl for part in parts) \ + and not self.reads_as_designation(fmt, section, parent): + return None # the counters it shows are not the ones its designation ends with: no field could stand for it + levels = self.by_kind["headings"].levels + definition = _level(ilvl, NUM_FMT[fmt.counter], parts, + widths={j: levels[j].num_fmt for j in range(9)} if section_level else None) + return self.claim(ilvl, definition, section_level=section_level) + def qualified(self, fmt: LevelFormat, section: Section) -> tuple[_Abstract, int] | None: """Where the units of *fmt* directly in *section* go when their label needs the section's number: the heading list's level after the section's (``%1.%2`` after a level-1 section), where Word can show the section's @@ -1196,24 +1320,15 @@ def qualified(self, fmt: LevelFormat, section: Section) -> tuple[_Abstract, int] (:meth:`fallback_label`). Under the ``none`` scheme the section's number is its heading text.""" if self.textual or "headings" not in self.by_kind: return None - parts = self.candidate(fmt, section) - if parts is None: - return None - levels = self.by_kind["headings"].levels - definition = _level(section.level, NUM_FMT[fmt.counter], parts, - widths={j: levels[j].num_fmt for j in range(9)}) - return self.claim(section.level, definition, section_level=True) + return self.heading_slot(fmt, section, section.unit, section.level, section_level=True) - def reserved(self, fmt: LevelFormat, depth: int) -> tuple[_Abstract, int] | None: + def reserved(self, fmt: LevelFormat, depth: int, section: Section, parent: str) -> tuple[_Abstract, int] | None: """Where a list nested *depth* levels below a unit on the heading list goes: the heading abstractNum's level - ``4 + depth``, defined on first use with the label's foreign-free text (nothing of the section in it).""" - refs = [self.formats.enumerated[j % len(self.formats.enumerated)].ref for j in range(9)] - parts = level_text(fmt.label, refs, depth, section=(), textual=False) - if any(isinstance(part, int) and part != depth for part in parts) or level_text_problem(parts) is not None: - return None - ilvl = 4 + depth - return self.claim(ilvl, _level(ilvl, NUM_FMT[fmt.counter], tuple(ilvl if isinstance(p, int) else p - for p in parts)), section_level=False) + ``4 + depth``, defined on first use. Its text is the label as the resolver writes it, with every counter it + shows at the level of the heading list that holds it (the section's number, the item it is nested in, the + items between): a ``{path}`` label is live, and the list restarts by structure under every new item. None when + a counter it shows is not on the heading list, or the level reads otherwise for another list.""" + return self.heading_slot(fmt, section, parent, 4 + depth, section_level=False) # -- numbering a unit --------------------------------------------------------------- @@ -1364,14 +1479,23 @@ def list_(self, lst: List, section: Section | None, quoted: bool = False) -> Non shows = section is not None and shows_section(fmt.label, refs, depth, textual=self.textual) # A label that needs the section's number goes on the heading list's level after the section's; a list nested # in a unit that is on the heading list goes on its reserved levels, so that Word keeps restarting it by - # structure. Otherwise the family's own abstractNum, with a label Word can write. + # structure, with the label's counters of the section and of the items on the way as levels of that list. + # Otherwise the family's own abstractNum, with a label Word can write; and where that abstractNum would have to + # show a counter of a unit that is not on it, the detached one, which shows each level's own counter alone. slot = None if lst.family == "enumerated" and section is not None and not quoted: - if shows: - slot = self.qualified(fmt, section) if depth == 0 and placed is None else None + if depth == 0 and placed is None: + slot = self.qualified(fmt, section) if shows else None elif placed is not None and placed.abstract.kind == "headings" and 1 <= depth <= 4: - slot = self.reserved(fmt, depth) - abstract, ilvl = slot or (self.abstract(lst.family), min(depth, 8)) + assert parent is not None + slot = self.reserved(fmt, depth, section, parent) + detached = slot is None and lst.family == "enumerated" and depth <= 8 and self.detaches(depth, parent, placed) + if slot is not None: + abstract, ilvl = slot + elif detached: + abstract, ilvl = self.abstract("detached"), depth + else: + abstract, ilvl = self.abstract(lst.family), min(depth, 8) cause: tuple[str, ...] = () # Nested in an item: that item's num, if it is live, of the same kind and Word counts the list from where the # resolver does (a sibling list continues, a list in a new item restarts); otherwise a num of its own. @@ -1398,8 +1522,11 @@ def list_(self, lst: List, section: Section | None, quoted: bool = False) -> Non self.report("render-docx-static-restart", "info", message, first.line) if slot is not None and abstract.kind == "headings" and depth > 0: cause += ("reserved-level",) - fallback = shows and slot is None - if fallback: + lost = self.lost_counters(fmt, section, parent) if detached else [] + fallback = bool(lost) if detached else shows and slot is None + if detached and lost: + self.detached_label(fmt, ilvl, "list item", lost, placed, first) + elif fallback: self.fallback_label(fmt, lst.family, min(depth, 8), "list item", first.line) parent_des = self.desig.get(parent, []) if parent is not None else [] section_des = self.desig[section.unit] if section is not None and section.unit is not None else [] diff --git a/tests/docx_support.py b/tests/docx_support.py index ea2c814..4ea86e6 100644 --- a/tests/docx_support.py +++ b/tests/docx_support.py @@ -244,10 +244,11 @@ def libreoffice_export(documents: dict[str, bytes], *, update: bool = True) -> d return result -def libreoffice_edit(jobs: dict[str, tuple[bytes, list[tuple[str, str | None]]]]) -> dict[str, str]: +def libreoffice_edit(jobs: dict[str, tuple[bytes, list[tuple[str, ...]]]]) -> dict[str, str]: """Open each document (name to DOCX bytes and edits) in one LibreOffice instance, insert a paragraph with the text ``new`` before the first paragraph that holds ``match`` for each ``(match, new)`` (delete that paragraph when - ``new`` is None), update the fields and indexes, and return the text of each (tests/lo/uno_edit.py).""" + ``new`` is None; put it on the list and level of the paragraph that holds ``like`` for ``(match, new, like)``), + update the fields and indexes, and return the text of each (tests/lo/uno_edit.py).""" python = require_libreoffice() with tempfile.TemporaryDirectory(prefix="ld-lo-") as folder: out = Path(folder, "out") diff --git a/tests/lo/uno_edit.py b/tests/lo/uno_edit.py index d104cc6..8be179c 100644 --- a/tests/lo/uno_edit.py +++ b/tests/lo/uno_edit.py @@ -10,7 +10,10 @@ /usr/bin/python3 tests/lo/uno_edit.py OUTDIR JOBS.json ``JOBS.json`` is a list of ``{"name": "job", "file": "/path/doc.docx", "edits": [["text of a paragraph", "new"], ...]}``; -an edit whose new text is ``null`` deletes the paragraph whose text is exactly that instead. +an edit whose new text is ``null`` deletes the paragraph whose text is exactly that instead. An edit with a third entry, +``["text of a paragraph", "new", "text of another"]``, puts the new paragraph on the list and the level of the paragraph that +holds the third text, as pasting a copy of that item at this place does (a new paragraph otherwise takes the numbering of +the one it is inserted before). One LibreOffice instance serves every job. Exit status 0 when every job was exported. """ from __future__ import annotations @@ -49,10 +52,22 @@ def _connect(pipe: str, process: subprocess.Popen[bytes]) -> object: raise SystemExit("soffice did not accept a connection in time") -def _edit(document: object, match: str, new: str | None) -> None: - """Insert a paragraph with the text *new* before the first paragraph that holds *match*, or, when *new* is None, - delete the first paragraph whose text is exactly *match* (with its break, as Delete at its end does; the last - paragraph of the document with the break before it, so that no empty paragraph is left).""" +def _paragraph(document: object, match: str) -> object: + """The first paragraph of the body that holds *match*.""" + paragraphs = document.getText().createEnumeration() + while paragraphs.hasMoreElements(): + paragraph = paragraphs.nextElement() + if (paragraph.supportsService("com.sun.star.text.Paragraph") and not paragraph.ParaStyleName.startswith("Contents") + and match in paragraph.getString()): + return paragraph + raise SystemExit(f"no paragraph holds {match!r}") + + +def _edit(document: object, match: str, new: str | None, like: str | None = None) -> None: + """Insert a paragraph with the text *new* before the first paragraph that holds *match*, on the list and level of the + paragraph that holds *like* when that is given, or, when *new* is None, delete the first paragraph whose text is + exactly *match* (with its break, as Delete at its end does; the last paragraph of the document with the break before + it, so that no empty paragraph is left).""" text = document.getText() paragraphs = text.createEnumeration() while paragraphs.hasMoreElements(): @@ -76,6 +91,10 @@ def _edit(document: object, match: str, new: str | None) -> None: text.insertControlCharacter(cursor, PARAGRAPH_BREAK, False) cursor.goLeft(1, False) cursor.setString(new) + if like is not None: + copied = _paragraph(document, like) + for name in ("NumberingRules", "ListId", "NumberingLevel"): + cursor.setPropertyValue(name, copied.getPropertyValue(name)) return raise SystemExit(f"no paragraph {'is' if new is None else 'holds'} {match!r}") @@ -97,8 +116,8 @@ def edit(out_dir: str, jobs: list[dict]) -> None: if document is None: raise SystemExit(f"LibreOffice could not open {job['file']}") try: - for match, new in job["edits"]: - _edit(document, match, new) + for match, new, *like in job["edits"]: + _edit(document, match, new, *like) for _ in range(2): # page numbers and indexes settle after the first layout indexes = document.getDocumentIndexes() for position in range(indexes.getCount()): diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index a30cf26..d61cdce 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -681,14 +681,17 @@ def test_an_item_nested_under_a_foreign_parent_reports_a_section_label_once() -> assert [d.rule for d in diagnostics].count("render-docx-section-label") == 1 -def test_a_quote_inside_a_preamble_item_with_a_path_label_is_reported_not_left_unexplained() -> None: +def test_a_quote_inside_a_preamble_item_with_a_path_label_does_not_show_the_counter_of_a_list_it_is_not_in() -> None: + # the quote is built on no item, so its path is its own counter; the enumerated level would show the counter of + # whatever the level above counted last ((a)(1)), so the list takes the detached levels, which draw it as it is levels = [{"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, {"counter": "decimal", "label": "{path}", "ref": "({n})"}] tree, plan, diagnostics = planned("- a\n\n > - quoted\n\n# A\n", overrides={"enumeration.levels": levels}) _, quoted = items(tree) - assert quoted.parent_unit is None - assert plan.units[quoted.unit].cause == ("label-fallback",) and plan.units[quoted.unit].label == "(a)(1)" - assert rules(diagnostics) == {"render-docx-section-label"} + assert quoted.parent_unit is None and quoted.label == "(1)" + assert plan.units[quoted.unit].cause == () and plan.units[quoted.unit].label == "(1)" + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated", "detached"] + assert diagnostics == [] def test_a_preamble_item_needs_no_section_and_is_not_reported() -> None: @@ -696,6 +699,156 @@ def test_a_preamble_item_needs_no_section_and_is_not_reported() -> None: assert plan.units["u1"].label == "1" and diagnostics == [] +# -- Labels that show the path: the reserved levels of the heading list, and the detached levels ---------------- + +PATH_ITEMS = {"style": "continental", "overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{path}", "ref": "({n})"}, + {"counter": "upper-alpha", "label": "{path}", "ref": "({n})"}, + {"counter": "decimal", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-alpha", "label": "{path}", "ref": "({n})"}]}} + + +def refs_of(tree: RenderTree, plan: DocxPlan) -> list[tuple[str, str]]: + """Each reference's text and its fields, as ``show`` writes them.""" + return [(i.text, show(plan, plan.refs[i.target_unit].segments)) + for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] + + +def test_a_path_label_nested_in_a_clause_rides_the_reserved_levels_with_the_counters_of_its_chain() -> None: + body = ("# A {#a}\n\n- one {#i1}\n - two {#i2}\n - three {#i3}\n- uno {#j1}\n - dos {#j2}\n\n" + "Refs: {{ref: i1}} {{ref: i2}} {{ref: i3}} {{ref: j2}}\n") + tree, plan, diagnostics = planned(body, **PATH_ITEMS) + one, two, three, uno, dos = items(tree)[:5] + # the label as the renderer writes it, drawn by Word: the section's counter and the clause's are levels of the list + assert [(plan.units[i.unit].label, i.label) for i in (one, two, three, uno, dos)] == [ + (label, label) for label in ("1.1", "1.1(a)", "1.1(a)(i)", "1.2", "1.2(a)")] + assert [plan.units[i.unit].ilvl for i in (one, two, three)] == [1, 5, 6] + assert [level.lvl_text for level in plan.abstract_nums[0].levels[5:7]] == ["%1.%2(%6)", "%1.%2(%6)(%7)"] + assert [plan.units[i.unit].cause for i in (two, three, dos)] == [("reserved-level",)] * 3 + # the clause's num: no override for the nested lists, which restart by structure under every new clause + assert len({plan.units[i.unit].num_id for i in (one, two, three, uno, dos)}) == 1 + assert [n for n in plan.nums if n.abstract_id == 0] == [plan.nums[0], plan.nums[1]] # the heading num, the clause's + assert diagnostics == [] + # a field for the label is the whole of the reference + assert refs_of(tree, plan) == [("1.1", "REF i1"), ("1.1(a)", "REF i2"), ("1.1(a)(i)", "REF i3"), + ("1.2(a)", "REF j2")] + + +def test_a_path_list_in_a_section_of_another_level_cannot_share_the_reserved_level_and_is_detached() -> None: + # under a level-2 section the clause is on the level after it: the reserved level would read %1.%2.%3(%6), which + # the first list has taken as %1.%2(%6) + body = ("# A {#a}\n\n- one {#i1}\n - two {#i2}\n\n## B {#b}\n\n- bee {#k1}\n - kay {#k2}\n - kay two {#k3}\n\n" + "Refs: {{ref: i2}} {{ref: k2}} {{ref: k3}}\n") + tree, plan, diagnostics = planned(body, **PATH_ITEMS) + _, two, _, kay, kay_two = [i for i in items(tree) if i.number is not None][:5] + assert plan.units[two.unit].cause == ("reserved-level",) and plan.units[two.unit].label == "1.1(a)" + assert [(plan.units[i.unit].label, i.label) for i in (kay, kay_two)] == [("(a)", "1.1.1(a)"), ("(b)", "1.1.1(b)")] + assert plan.units[kay.unit].cause == ("restart-section", "label-fallback") + assert [a.kind for a in plan.abstract_nums if a.kind == "detached"] == ["detached"] + assert plan.units[kay.unit].ilvl == 1 and plan.abstract_nums[plan.nums[plan.units[kay.unit].num_id - 1].abstract_id].kind == "detached" + # the label is shorter in Word, and the references to it are fields all the same + assert refs_of(tree, plan) == [("1.1(a)", "REF i2"), ("1.1.1(a)", "REF k1 + REF k2"), + ("1.1.1(b)", "REF k1 + REF k3")] + [restart] = [d for d in diagnostics if d.rule == "render-docx-static-restart"] + [lost] = [d for d in diagnostics if d.rule == "render-docx-section-label"] + assert restart.level == "info" and "carries the section" in restart.message + assert lost.level == "warning" and lost.line == kay.line + assert "shows the number of the enclosing section and item" in lost.message and "it reads '(a)'" in lost.message + assert "kept for nested lists are used up or read otherwise" in lost.message + + +def test_a_path_list_nested_past_the_reserved_levels_is_detached() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i} {{#l{i}}}\n" for i in range(7)) + body += "\nRefs: " + " ".join(f"{{{{ref: l{i}}}}}" for i in range(7)) + "\n" + tree, plan, diagnostics = planned(body, **PATH_ITEMS) + found = items(tree)[:7] + assert [plan.units[i.unit].ilvl for i in found] == [1, 5, 6, 7, 8, 5, 6] + assert [plan.units[i.unit].label for i in found[:5]] == [i.label for i in found[:5]] + assert [level.lvl_text for level in plan.abstract_nums[0].levels[5:]] == [ + "%1.%2(%6)", "%1.%2(%6)(%7)", "%1.%2(%6)(%7)(%8)", "%1.%2(%6)(%7)(%8)(%9)"] + # depth 5 and the list under it: the detached levels, one num for both, restarted by structure inside + assert [plan.units[i.unit].cause for i in found[5:]] == [("restart-section", "label-fallback"), ("label-fallback",)] + assert plan.units[found[5].unit].num_id == plan.units[found[6].unit].num_id + assert [plan.units[i.unit].label for i in found[5:]] == ["(a)", "1"] + assert [text for text, _ in refs_of(tree, plan)][-2:] == ["1.1(a)(i)(A)(1)(a)", "1.1(a)(i)(A)(1)(a).1"] + assert all(plan.refs[i.target_unit].exact and not plan.refs[i.target_unit].fixed + for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)) + assert [d.rule for d in diagnostics] == ["render-docx-static-restart", "render-docx-section-label", + "render-docx-section-label"] + + +ORDERED_PARENT = {"overrides": {"enumeration.levels": [ + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "decimal", "label": "({n})", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n}", "ref": "({n})"}, + {"counter": "upper-alpha", "label": "{path}", "ref": "({n})"}]}} + + +def test_a_path_list_below_an_ordered_item_is_detached_and_its_references_are_fields() -> None: + body = ("# A {#a}\n\n- one {#i1}\n 1. ordered {#o}\n - three {#i3}\n - four {#i4}\n - four b {#i5}\n" + " - three b {#i6}\n - four c {#i7}\n\nRefs: {{ref: i3}} {{ref: i4}} {{ref: i5}} {{ref: i7}}\n") + tree, plan, diagnostics = planned(body, **ORDERED_PARENT) + found = {i.unit: i for i in items(tree)} + i4, i5, i7 = (found[u] for u in ("u4", "u5", "u7")) + assert [i.label for i in (i4, i5, i7)] == ["1(a)(1)(i)(A)", "1(a)(1)(i)(B)", "1(a)(1)(ii)(A)"] + # Word draws the level's own counter: the enumerated level would read the (never used) levels above as 0 or as + # the counter of another list + assert [plan.units[i.unit].label for i in (i4, i5, i7)] == ["(A)", "(B)", "(A)"] + assert [plan.units[i.unit].cause for i in (i4, i5, i7)] == [ + ("restart-family", "label-fallback"), ("label-fallback",), ("restart-family", "label-fallback")] + assert plan.units[i4.unit].num_id == plan.units[i5.unit].num_id != plan.units[i7.unit].num_id + assert [(a.kind, a.levels[3].lvl_text) for a in plan.abstract_nums][-1] == ("detached", "(%4)") + assert [n.start_overrides for n in plan.nums if n.abstract_id == plan.abstract_nums[-1].id] == [((3, 1),)] * 2 + assert refs_of(tree, plan) == [ + ("1(a)(1)(i)", 'REF a + REF i1 + "(" + REF o + ")(" + REF i3 + ")"'), + ("1(a)(1)(i)(A)", 'REF a + REF i1 + "(" + REF o + ")(" + REF i3 + ")" + REF i4'), + ("1(a)(1)(i)(B)", 'REF a + REF i1 + "(" + REF o + ")(" + REF i3 + ")" + REF i5'), + ("1(a)(1)(ii)(A)", 'REF a + REF i1 + "(" + REF o + ")(" + REF i6 + ")" + REF i7')] + [lost] = [d for d in diagnostics if d.rule == "render-docx-section-label"] + assert "shows the number of the enclosing section, ordered item and item" in lost.message + assert "it reads '(A)'" in lost.message and "the list is nested in a unit that is on another list" in lost.message + assert "section's number" not in lost.message + + +def test_lists_nested_in_a_detached_item_stay_on_the_detached_levels_and_restart_by_structure() -> None: + body = ("# A\n\n- one\n 1. ordered\n - three\n - four\n - five\n - five b\n" + " - four b\n - five c\n") + tree, plan, diagnostics = planned(body, **ORDERED_PARENT) + four, five, five_b, four_b, five_c = items(tree)[3:8] + kinds = {a.id: a.kind for a in plan.abstract_nums} + assert [kinds[plan.nums[plan.units[i.unit].num_id - 1].abstract_id] for i in (four, five, five_b, four_b, five_c)] \ + == ["detached"] * 5 + assert len({plan.units[i.unit].num_id for i in (four, five, five_b, four_b, five_c)}) == 1 # one num, no override + assert [(plan.units[i.unit].ilvl, plan.units[i.unit].label) for i in (five, five_b, five_c)] == [ + (4, "(a)"), (4, "(b)"), (4, "(a)")] # the list under the second "four" starts again, by structure + assert [plan.units[i.unit].cause for i in (five, five_b, five_c)] == [()] * 3 + + +def test_the_detached_levels_hold_each_levels_own_counter_alone_and_exist_only_when_used() -> None: + _, plain, _ = planned("# A\n\n- x\n - y\n") + assert "detached" not in [a.kind for a in plain.abstract_nums] + _, plan, _ = planned("# A\n\n- x\n 1. y\n - z\n - w\n", **ORDERED_PARENT) + [detached] = [a for a in plan.abstract_nums if a.kind == "detached"] + enumerated = next(a for a in plan.abstract_nums if a.kind == "enumerated") + assert [(level.num_fmt, level.lvl_text) for level in detached.levels[:4]] == [ + ("lowerLetter", "(%1)"), ("decimal", "(%2)"), ("lowerRoman", "%3"), ("upperLetter", "(%4)")] + assert enumerated.levels[3].lvl_text == "(%1)(%2)(%3)(%4)" # the enumerated list shows the whole path + assert detached.multi_level == "hybridMultilevel" and len(detached.levels) == 9 + + +def test_a_label_whose_counters_are_not_the_end_of_its_designation_keeps_the_level_that_shows_less() -> None: + # {section}.{n} at depth 3 of the continental style would show the section's counter before the item's, but the + # designation of the item ends with ".1": no field for the label could stand for it, so the label is written + # without the section (reported) and the reference is made of fields + body = "# A\n\n- x {#x}\n - y {#y}\n - z {#z}\n - w {#w}\n\nRef {{ref: w}}\n" + tree, plan, diagnostics = planned(body, style="continental") + [(text, fields)] = refs_of(tree, plan) + assert text == "1.1(a)(i).1" and fields == 'REF x + "(" + REF y + "(" + REF z + "." + REF w' + assert plan.units["u4"].cause == ("restart-section", "label-fallback") and plan.units["u4"].label == "1" + + # -- Counters Word writes differently -------------------------------------------------- @@ -1129,7 +1282,8 @@ def test_a_label_that_needs_a_level_nothing_numbers_is_not_left_to_word() -> Non tree, plan, diagnostics = planned("1. a\n - x\n\n# A\n", overrides={"enumeration.levels": levels}) _, nested = items(tree) assert plan.units[nested.unit].cause == ("restart-family", "label-fallback") - assert plan.units[nested.unit].label == "()(1)" # the level above reads as nothing, not as "(a)" + assert plan.units[nested.unit].label == "(1)" # not "()(1)": the detached levels draw their own counter alone + assert [a.kind for a in plan.abstract_nums] == ["headings", "renumber", "detached"] assert {d.rule for d in diagnostics} == {"render-docx-static-restart", "render-docx-section-label"} diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index 8dfcb22..e9d1ac8 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -40,7 +40,7 @@ import pytest from conftest import DOCUMENTS, FRONT, spec_root from docx_support import Exported, W, libreoffice_edit, libreoffice_export, parts, repackage, require_libreoffice -from test_docx_plan import ALTERNATIVES, units +from test_docx_plan import ALTERNATIVES, ORDERED_PARENT, PATH_ITEMS, units from legaldown_render import DocumentError, render from legaldown_render.resolve.docx_plan import DocxPlan, FieldSeg, Literal_, plan_docx @@ -313,13 +313,72 @@ """, } +#: Labels that show the path of an item (`{path}`): in a clause (the reserved levels of the heading list carry the clause's +#: and the section's counters), in a section of another level (the reserved level reads otherwise: detached levels), past +#: the reserved levels, and below an ordered item. +STRESS["paths"] = FRONT + """ +# Alpha {#alpha} + +- one {#i1} + - two {#i2} + - three {#i3} + - four {#i4} + - five {#i5} + - six {#i6} + - seven {#i7} +- uno {#j1} + - dos {#j2} + - tres {#j3} + +## Beta {#beta} + +- bee {#k1} + - kay {#k2} + - kay two {#k3} +- bee two {#k4} + - kay three {#k5} + +# Second {#second} + +- x {#x1} + - y {#y1} + +Refs: {{ref: i1}} {{ref: i2}} {{ref: i3}} {{ref: i4}} {{ref: i5}} {{ref: i6}} {{ref: i7}} {{ref: j2}} {{ref: j3}} +{{ref: k2}} {{ref: k3}} {{ref: k5}} {{ref: y1}} +""" +STRESS["ordered paths"] = FRONT + """ +# Alpha {#alpha} + +- one {#i1} + 1. ordered {#o} + - three {#i3} + - four {#i4} + - five {#i5} + - five b {#i6} + - four b {#i7} + - three b {#i8} + - four c {#i9} + 1. ordered two {#o2} + - x {#x} + - y {#y} +- two {#t} + 1. o3 {#o3} + - z {#z} + - w {#w} + +Refs: {{ref: i3}} {{ref: i4}} {{ref: i5}} {{ref: i6}} {{ref: i7}} {{ref: i9}} {{ref: y}} {{ref: w}} +""" + #: The stress documents that are shown under some variants only (each costs LibreOffice time): the continental style, where #: the items ride the heading list, the default, and the default with numbered paragraphs. FEW_VARIANTS: dict[str, tuple[str, ...]] = { - name: ("continental", "default", "paragraphs") for name in ("alternatives in threes", "nested titles", "qualified")} + **{name: ("continental", "default", "paragraphs") for name in ("alternatives in threes", "nested titles", "qualified")}, + "paths": ("continental", "default"), "ordered paths": ("default",)} #: The variants that show some stress documents only. -ONLY_DOCS: dict[str, tuple[str, ...]] = {"five levels": ("qualified",), "shared level": ("qualified", "nested")} +ONLY_DOCS: dict[str, tuple[str, ...]] = { + "five levels": ("qualified",), "shared level": ("qualified", "nested"), "path items": ("paths", "lists"), + "ordered parent": ("ordered paths", "lists")} #: The settings that give the numbering words and symbols: a field that returns the same words as the reference's format #: stands for them, otherwise Word's ``\t`` switch drops them from the number. @@ -345,6 +404,8 @@ {"counter": "lower-alpha", "label": "{n}.", "ref": "{n}"}]}}, "shared level": {"style": "continental", "overrides": {"paragraphs.numbered": True, "paragraphs.format": { "counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}}}, + "path items": PATH_ITEMS, + "ordered parent": ORDERED_PARENT, } #: What a diagnostic of the plan has to say when Word numbers a unit differently from the renderer. @@ -950,3 +1011,94 @@ def test_the_contents_list_the_headings_as_the_plan_does_and_none_of_the_items_o assert [e.number for e in plan.toc] == ["1.", "1.1", "2.", "2.1", "2.", "2.1", "3."] # the alternative's is shared assert not any(word in line for line in entries for word in ("item", "nested")) assert "1.1 item one" in body and "a) nested" in body # the items are in the body: on the heading list, not in it + + +# -- Labels that show the path: live on the reserved levels, detached where they cannot be --------------------------------- + +PATH_CLAUSES = FRONT + """ +# Alpha {#alpha} + +- clause one {#c1} + - nested of one {#n1} + - deep of one {#d1} +- clause two {#c2} + - nested of two {#n2} + - deep of two {#d2} + +Refs: {{ref: c1}} {{ref: n1}} {{ref: d1}} {{ref: c2}} {{ref: n2}} {{ref: d2}} +""" + +#: A clause, a nested item and a deeper one inserted before the second clause and what is under it. +PATH_EDITS = [ + ("clause two", "NEW clause", "- clause two {#c2}", "- NEW clause"), + ("nested of two", "NEW nested", " - nested of two {#n2}", " - NEW nested"), + ("deep of two", "NEW deep", " - deep of two {#d2}", " - NEW deep"), +] + + +@functools.cache +def path_clauses() -> str: + require_libreoffice() + data = render(PATH_CLAUSES, format="docx", **PATH_ITEMS).binary + return libreoffice_edit({"paths": (data, [(match, new) for match, new, _, _ in PATH_EDITS])})["paths"] # type: ignore[arg-type] + + +def test_a_path_label_on_the_reserved_levels_follows_the_items_a_lawyer_inserts_and_so_do_its_references() -> None: + """The label and every reference to it are live: after a clause, a nested item and a deeper one are inserted in + the editor and the fields are updated, the numbers (with the section's and the clause's counters in them) and the + references read as the renderer writes them for the edited source.""" + source = applied(PATH_CLAUSES, [(before, new) for _, _, before, new in PATH_EDITS]) + said = lines_of(path_clauses()) + assert said == lines_of(render(source, format="text", **PATH_ITEMS).output) + assert {"1.3(a) NEW nested", "1.3(b) nested of two", "1.3(b)(i) NEW deep", "1.3(b)(ii) deep of two"} <= set(said) + assert [line for line in said if line.startswith("(1) Refs")] == [ + "(1) Refs: 1.1 1.1(a) 1.1(a)(i) 1.3 1.3(b) 1.3(b)(ii)"] + + +ORDERED_PATHS = FRONT + """ +# Alpha {#alpha} + +- one {#i1} + 1. ordered {#o} + - three {#i3} + - four {#i4} + - four b {#i5} + - three b {#i6} + - four c {#i7} + +Refs: {{ref: i4}} {{ref: i5}} {{ref: i7}} +""" + + +@functools.cache +def ordered_paths() -> dict[str, str]: + """The ordered-parent document as it is, with a nested item inserted in the detached list, and with a new parent whose + nested item is a copy of one of the detached list's (Word's paste keeps the list).""" + require_libreoffice() + data = render(ORDERED_PATHS, format="docx", **ORDERED_PARENT).binary + return libreoffice_edit({ # type: ignore[arg-type] + "original": (data, []), + "nested inserted": (data, [("four b", "NEW four")]), + "new parent": (data, [("three b", "NEW three"), ("three b", "NEW copy", "four b")])}) + + +def test_a_detached_list_draws_its_own_counter_and_its_references_are_live() -> None: + said = lines_of(ordered_paths()["original"]) + assert [line for line in said if line.endswith(("four", "four b", "four c"))] == ["(A) four", "(B) four b", "(A) four c"] + assert "Refs: 1(a)(1)(i)(A) 1(a)(1)(i)(B) 1(a)(1)(ii)(A)" in said[-1] + # an item inserted in the detached list renumbers what follows, in the list and in the references to it + after = lines_of(ordered_paths()["nested inserted"]) + assert [line for line in after if line.endswith(("four", "NEW four", "four b", "four c"))] == [ + "(A) four", "(B) NEW four", "(C) four b", "(A) four c"] + assert "Refs: 1(a)(1)(i)(A) 1(a)(1)(i)(C) 1(a)(1)(ii)(A)" in after[-1] + + +def test_a_new_parent_whose_nested_item_is_copied_from_a_detached_list_goes_on_counting_there() -> None: + """The documented limit of the detached levels: a nested list restarts at a fixed value under the parent that has it, + so a list that is new in Word, made by copying an item of another, goes on from that one (the renderer would start + it again at ``(A)``), while the list that was restarted keeps its start.""" + said = lines_of(ordered_paths()["new parent"]) + assert [line for line in said if line.endswith(("four b", "NEW copy", "four c"))] == [ + "(B) four b", "(C) NEW copy", "(A) four c"] + assert "ii NEW three" in said and "iii three b" in said + assert "Refs: 1(a)(1)(i)(A) 1(a)(1)(i)(B) 1(a)(1)(iii)(A)" in said[-1] From 6e7c748ad22f391aeed55df587373fd9964af24f Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:14:37 +0000 Subject: [PATCH 105/118] Go on past Word's nine list levels in further abstractNums instead of clamping A list nested deeper than nine levels now takes the next abstractNum of its family (enumerated+1 to enumerated+3, renumber+1 to renumber+3; 36 levels in all), whose level j is the family's level of depth 9k+j with its own counter alone. The items keep their numbers and labels, restart by structure inside each abstractNum, and the list that begins a chunk restarts at a fixed value like any list nested in a list of another kind (reported as such). Only a nesting past 36 levels is still clamped and reported with render-docx-list-depth. Items stop stepping in at thirteen centimetres so that very deep lists stay on the page. --- src/legaldown_render/resolve/docx_plan.py | 129 ++++++++++++++-------- tests/test_docx_plan.py | 95 ++++++++++++---- tests/test_docx_roundtrip.py | 42 ++++++- 3 files changed, 195 insertions(+), 71 deletions(-) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 83172fb..83a6ecc 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -116,8 +116,11 @@ class LevelDef: class AbstractNum: id: int # 0.., in order of first use (headings first) #: ``detached``: the enumerated levels with each level's own counter alone, for a list nested where the enumerated list - #: would have to show the counter of a unit that is not on it (§13.2). - kind: Literal["headings", "enumerated", "detached", "renumber", "paragraphs", "bullet"] + #: would have to show the counter of a unit that is not on it (§13.2). ``enumerated+1`` to ``enumerated+3`` (and + #: ``renumber+k``): Word has nine levels, so a list nested deeper takes the next abstractNum of its family, whose level + #: *j* is the family's level of depth ``9k + j`` with its own counter alone. + kind: Literal["headings", "enumerated", "detached", "renumber", "paragraphs", "bullet", + "enumerated+1", "enumerated+2", "enumerated+3", "renumber+1", "renumber+2", "renumber+3"] #: Eight hex digits, deterministic (``0x4C440000 + id``). Equal in every LegalDown document, so Word may #: merge lists on paste between two of them (cosmetic). nsid: str @@ -196,10 +199,10 @@ class UnitPlan: count: int | None #: Why the unit is not on the plain path: ``resync`` (restarted at a fixed value), ``alternative`` (``numId`` 0), #: ``reserved-level`` (a list nested in an item on the heading list, on one of its reserved levels), - #: ``restart-family`` / ``restart-section`` / ``restart-alternative`` (a nested list in a list of another kind, in - #: an item whose number carries the section, in an alternative: a num of its own that restarts by a fixed setting), - #: ``label-fallback`` (the label is written without a number Word cannot show there), ``clamped``, - #: ``skipped-level``. + #: ``restart-family`` / ``restart-section`` / ``restart-alternative`` / ``restart-depth`` (a nested list in a list of + #: another kind, in an item whose number carries the section, in an alternative, past Word's nine levels: a num of + #: its own that restarts by a fixed setting), ``label-fallback`` (the label is written without a number Word cannot + #: show there), ``clamped`` (nested past :data:`MAX_DEPTH`, or a bullet past Word's nine levels), ``skipped-level``. cause: tuple[str, ...] #: Names to wrap around the paragraph's title/first paragraph content: the bookmark of the unit's anchor when it #: has one, else a hidden one when a reference field points at it, else none. @@ -307,7 +310,7 @@ def docx_style_errors(style: Style) -> list[str]: for kind, levels in definitions.items(): for level in levels: if len(level.lvl_text) > MAX_LVL_TEXT: - problems.append(f"{_SETTING.get(kind, kind)}: the numbering level {level.ilvl + 1} is longer than " + problems.append(f"{_SETTING.get(kind.split('+')[0], kind)}: the numbering level {level.ilvl + 1} is longer than " f"{MAX_LVL_TEXT} characters in Word (DOCX): shorten its label or the refs it shows") return problems @@ -321,23 +324,32 @@ def docx_style_errors(style: Style) -> list[str]: _STEP = 567 #: Twips per character of a heading's widest label. _CHARACTER = 113 +#: Word's levels in one list, and how many lists of them a nesting of items can use: a list nested deeper than nine +#: levels goes on in the next abstractNum of its family (``enumerated+1`` ...), up to this many levels in all; past them +#: the last level is used again and the lists restart by fixed values. +_LEVELS = 9 +MAX_DEPTH = 4 * _LEVELS +#: Twips: the indent items stop stepping at (thirteen centimetres), so that a list nested very deeply stays on the page. +_MAX_LEFT = 13 * _STEP def _level(ilvl: int, num_fmt: str, parts: LevelParts, *, p_style: str | None = None, widths: Mapping[int, str] | None = None, unwritable: list[tuple[str, int]] | None = None, - kind: str = "") -> LevelDef: + kind: str = "", depth: int | None = None) -> LevelDef: """One level. Headings (*widths*: the ``numFmt`` of each level a ``%k`` names) get an indent wide enough for - their widest label; items step one centimetre per level. A label with a word in it (``Article {n}``) is + their widest label; items step one centimetre per level of nesting (*depth*, the level's own by default: a level of + a list that goes on past Word's nine levels is nested deeper). A label with a word in it (``Article {n}``) is followed by a space, any other by a tab. With *unwritable*, parts a level text cannot write are noted there (and written as nothing) instead of raising.""" words = any(isinstance(part, str) and any(c.isalpha() for c in part) for part in parts) + depth = ilvl if depth is None else depth if widths is None: - left, hanging = _STEP * (ilvl + 1), _STEP + left, hanging = min(_STEP * (depth + 1), _MAX_LEFT), _STEP else: sample = sum(len(part) if isinstance(part, str) else len(SAMPLE_COUNTER[widths[part]]) for part in parts) left = hanging = max(_STEP, _CHARACTER * (sample + 2)) if unwritable is not None and level_text_problem(parts) is not None: - unwritable.append((kind, ilvl)) + unwritable.append((kind, depth)) parts = () return LevelDef(ilvl, num_fmt, lvl_text(parts), p_style=p_style, suff="space" if words else "tab", left=left, hanging=hanging) @@ -348,6 +360,18 @@ def _neutral(ilvl: int) -> LevelDef: return LevelDef(ilvl, "decimal", f"%{ilvl + 1}", left=_STEP * (ilvl + 1), hanging=_STEP) +def _own(formats: NumberingFormats, ilvl: int, depth: int, textual: bool, unwritable: list[tuple[str, int]] | None + ) -> LevelDef: + """Level *ilvl* of an enumerated list nested *depth* levels deep, with its own counter alone (a ``{path}`` shows + its own ``ref`` and nothing of the units it is nested in): the levels of the detached list and of the lists that go + on past Word's nine levels, where no unit of the list holds the levels above.""" + levels = formats.enumerated + here = levels[depth % len(levels)] + return _level(ilvl, NUM_FMT[here.counter], + level_text(here.label, [""] * ilvl + [here.ref], ilvl, section=(), textual=textual), + unwritable=unwritable, kind="enumerated", depth=depth) + + def _definitions(formats: NumberingFormats, unwritable: list[tuple[str, int]] | None = None ) -> dict[str, list[LevelDef]]: """The nine levels of every abstractNum *formats* needs, by kind. A label that needs a number of the section @@ -372,17 +396,15 @@ def _definitions(formats: NumberingFormats, unwritable: list[tuple[str, int]] | for d in range(9)] # The same levels with nothing but their own counter (a ``{path}`` shows its own ``ref`` alone): where a list is # nested in a unit that is not on the enumerated list, Word has no counter of the enclosing units to show. - out["detached"] = [ - _level(d, NUM_FMT[formats.enumerated[d % size].counter], - level_text(formats.enumerated[d % size].label, [""] * d + [refs[d]], d, section=(), - textual=textual), - unwritable=unwritable, kind="detached") - for d in range(9)] + out["detached"] = [_own(formats, d, d, textual, unwritable) for d in range(_LEVELS)] + for k in range(1, MAX_DEPTH // _LEVELS): + out[f"enumerated+{k}"] = [_own(formats, d, _LEVELS * k + d, textual, unwritable) for d in range(_LEVELS)] if formats.renumber is not None: f = formats.renumber - out["renumber"] = [_level(d, NUM_FMT[f.counter], - level_text(f.label, [f.ref] * 9, d, section=(), textual=textual), - unwritable=unwritable, kind="renumber") for d in range(9)] + for k in range(MAX_DEPTH // _LEVELS): + out["renumber" if k == 0 else f"renumber+{k}"] = [ + _level(d, NUM_FMT[f.counter], level_text(f.label, [f.ref] * 9, d, section=(), textual=textual), + unwritable=unwritable, kind="renumber", depth=_LEVELS * k + d) for d in range(_LEVELS)] if formats.paragraphs is not None: f = formats.paragraphs out["paragraphs"] = [ @@ -533,6 +555,9 @@ def __init__(self, level: int) -> None: "restart it."), "resync": (("resync",), "A list that Word would continue from another list in the same item restarts by a fixed " "setting (Restart at 1), not by structure."), + "restart-depth": (("restart-depth",), "A list nested past Word's nine levels goes on in a list of its own, which " + "restarts by a fixed setting (Restart at 1), not by structure: an item added above it in Word will " + "not restart it."), } @@ -1095,7 +1120,7 @@ def lost_counters(self, fmt: LevelFormat, section: Section | None, parent: str | """The owners of the counters the resolver's label of a unit of *fmt* shows besides its own, in order: what the detached abstractNum leaves out.""" owners = [t.owner for t in self.label_tokens(fmt, section, parent) - if isinstance(t, Count) and t.owner is not _SELF] + if isinstance(t, Count | Title) and t.owner is not _SELF] # a Title: the heading's text, under ``none`` return list(dict.fromkeys(owners)) def describe(self, owners: Sequence[object]) -> str: @@ -1105,7 +1130,8 @@ def describe(self, owners: Sequence[object]) -> str: if any(isinstance(owner, Ghost) or owner not in self.units or self.units[owner].kind == "section" # type: ignore[index] for owner in owners): names.append("section") - kinds = [self.placed[owner].abstract.kind for owner in owners if owner in self.placed] # type: ignore[index] + kinds = [self.placed[owner].abstract.kind.split("+")[0] # type: ignore[index] + for owner in owners if owner in self.placed] if "renumber" in kinds: names.append("ordered item") if any(kind != "renumber" for kind in kinds): @@ -1114,12 +1140,15 @@ def describe(self, owners: Sequence[object]) -> str: return "an enclosing unit" return "the enclosing " + (", ".join(names[:-1]) + " and " + names[-1] if len(names) > 1 else names[0]) - def detached_label(self, fmt: LevelFormat, ilvl: int, kind: str, lost: Sequence[object], + def detached_label(self, fmt: LevelFormat, abstract: _Abstract, ilvl: int, kind: str, lost: Sequence[object], placed: _Placed | None, first: ListItem) -> None: - """Report a list that is on the detached abstractNum although the resolver's label shows the number of an - enclosing unit: Word shows the unit's own counter alone, and a reference to it is still made of fields.""" - levels = self.definitions["detached"] - if placed is None: + """Report a list on an abstractNum that shows each level's own counter alone (the detached one, or the next of + its family past Word's nine levels) although the resolver's label shows the number of an enclosing unit: Word + draws the unit's own counter, and a reference to it is still made of fields.""" + levels = abstract.levels + if abstract.kind != "detached": + why = "the list is nested past Word's nine levels and goes on in a list of its own" + elif placed is None: why = "the list stands outside the unit it belongs to" elif placed.abstract.kind == "headings": why = "the levels of the heading list kept for nested lists are used up or read otherwise" @@ -1269,6 +1298,8 @@ def candidate(self, fmt: LevelFormat, section: Section, parent: str | None, ilvl level_of = {owner: j for j, (owner, _) in enumerate(self.owners)} parts: list[str | int] = [] for token in self.label_tokens(fmt, section, parent): + if isinstance(token, Title): + return None if isinstance(token, Count): j = ilvl if token.owner is _SELF else level_of.get(token.owner, self.heading_level(token.owner)) if j is None: @@ -1341,7 +1372,7 @@ def indent(self, abstract: _Abstract, ilvl: int, unit: str, parent: str | None, plan sets nothing.""" level = abstract.levels[ilvl] if unit in self.item_units and parent in self.text_left: - left, hanging = self.text_left[parent] + _STEP, _STEP + left, hanging = min(self.text_left[parent] + _STEP, _MAX_LEFT), _STEP elif unit in self.item_units and abstract.kind == "headings": left, hanging = level.left, level.hanging else: @@ -1412,12 +1443,14 @@ def numbered(self, node: _Unit, kind: Literal["section", "item", "paragraph"], n self.check(node, word, step, counter, group, cause, (abstract.id, ilvl)) return used - def restart_class(self, parent: str, placed: _Placed, abstract: _Abstract) -> str: + def restart_class(self, parent: str, placed: _Placed, abstract: _Abstract, *, deeper: bool = False) -> str: """Why a list nested in the item *parent* does not take its num: the item is a later alternative (numbered by - no list), it is on the heading list but the list cannot be, it is on another kind of list, or Word would - go on counting from another list in the same item.""" + no list), the list goes on past Word's nine levels (*deeper*), the item is on the heading list but the list + cannot be, it is on another kind of list, or Word would go on counting from another list in the same item.""" if self.units[parent].num_id == 0: return "restart-alternative" + if deeper: + return "restart-depth" if placed.abstract is not abstract: return "restart-section" if placed.abstract.kind == "headings" else "restart-family" return "resync" @@ -1468,9 +1501,13 @@ def list_(self, lst: List, section: Section | None, quoted: bool = False) -> Non fmt = lst.format assert fmt is not None depth = lst.depth - # The counter of the level Word uses: past level 9 that is level 9's, not the resolver's cycle. + # Word has nine levels in a list: a list nested deeper goes on in the next abstractNum of its family (levels + # 10-18 in ``enumerated+1``, and so on up to MAX_DEPTH), so that it keeps restarting by structure inside that + # abstractNum; past MAX_DEPTH the last level is used again. + reached = min(depth, MAX_DEPTH - 1) + chunk, rem = divmod(reached, _LEVELS) levels = self.formats.enumerated if lst.family == "enumerated" else (fmt,) - counter = levels[min(depth, 8) % len(levels)].counter + counter = levels[reached % len(levels)].counter parent = first.parent_unit placed = self.placed.get(parent) if parent is not None else None parent_live = placed is not None and self.units[parent].num_id not in (None, 0) @@ -1489,28 +1526,29 @@ def list_(self, lst: List, section: Section | None, quoted: bool = False) -> Non elif placed is not None and placed.abstract.kind == "headings" and 1 <= depth <= 4: assert parent is not None slot = self.reserved(fmt, depth, section, parent) - detached = slot is None and lst.family == "enumerated" and depth <= 8 and self.detaches(depth, parent, placed) + detached = slot is None and lst.family == "enumerated" and chunk == 0 and self.detaches(depth, parent, placed) if slot is not None: abstract, ilvl = slot elif detached: abstract, ilvl = self.abstract("detached"), depth else: - abstract, ilvl = self.abstract(lst.family), min(depth, 8) + abstract, ilvl = self.abstract(lst.family if chunk == 0 else f"{lst.family}+{chunk}"), rem cause: tuple[str, ...] = () # Nested in an item: that item's num, if it is live, of the same kind and Word counts the list from where the # resolver does (a sibling list continues, a list in a new item restarts); otherwise a num of its own. - if (placed is not None and parent_live and placed.abstract is abstract and depth <= 8 + if (placed is not None and parent_live and placed.abstract is abstract and depth < MAX_DEPTH and self.sim.peek(placed.num.id, ilvl) == first.number): num = placed.num else: num = self.new_num(abstract, ilvl, first.number or 1) - if depth > 8: + if depth >= MAX_DEPTH: cause = ("clamped",) if not self.depth_reported: self.depth_reported = True self.report("render-docx-list-depth", "warning", - f"A list is nested {depth} levels deep: Word has 9. Items from level 10 on are " - f"numbered in the format of level 9 and restart by fixed values.", first.line) + f"A list is nested {depth} levels deep: the plan has {MAX_DEPTH} levels of Word lists. " + f"Items from level {MAX_DEPTH + 1} on are numbered in the format of level {MAX_DEPTH} " + f"and restart by fixed values.", first.line) elif placed is not None and first.number > 1: cause = ("resync",) self.shared_counter( @@ -1518,14 +1556,15 @@ def list_(self, lst: List, section: Section | None, quoted: bool = False) -> Non "starts at a fixed number in Word: it will not follow items added or removed before it.", first.line) elif placed is not None: - cause, message = _RESTARTS[self.restart_class(parent, placed, abstract)] + cause, message = _RESTARTS[self.restart_class(parent, placed, abstract, deeper=chunk > 0 and rem == 0)] self.report("render-docx-static-restart", "info", message, first.line) if slot is not None and abstract.kind == "headings" and depth > 0: cause += ("reserved-level",) - lost = self.lost_counters(fmt, section, parent) if detached else [] - fallback = bool(lost) if detached else shows and slot is None - if detached and lost: - self.detached_label(fmt, ilvl, "list item", lost, placed, first) + own_only = detached or chunk > 0 # the levels of these abstractNums show each level's own counter alone + lost = self.lost_counters(fmt, section, parent) if own_only else [] + fallback = bool(lost) if own_only else shows and slot is None + if own_only and lost: + self.detached_label(fmt, abstract, ilvl, "list item", lost, placed, first) elif fallback: self.fallback_label(fmt, lst.family, min(depth, 8), "list item", first.line) parent_des = self.desig.get(parent, []) if parent is not None else [] @@ -1534,7 +1573,7 @@ def list_(self, lst: List, section: Section | None, quoted: bool = False) -> Non assert item.unit is not None and item.number is not None self.parent[item.unit] = item.parent_unit item_cause = cause if index == 0 else () - if depth > 8 and "clamped" not in item_cause: + if depth >= MAX_DEPTH and "clamped" not in item_cause: item_cause += ("clamped",) if slot is not None and depth > 0 and "reserved-level" not in item_cause: item_cause += ("reserved-level",) diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index d61cdce..2fbf6b7 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -368,21 +368,25 @@ def test_items_in_a_quote_or_a_drafting_note_are_numbered_like_any_item() -> Non assert [(n, label) for n, _, label in shown(plan, *items(tree))] == [(2, "(a)"), (3, "(a)")] -def test_a_list_deeper_than_word_has_levels_is_clamped_and_reported_once() -> None: +def test_a_list_deeper_than_word_has_levels_goes_on_in_the_next_abstract_num_and_restarts_by_a_fixed_setting() -> None: body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(11)) tree, plan, diagnostics = planned(body) deep = items(tree) - assert [plan.units[i.unit].ilvl for i in deep] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8] - assert [plan.units[i.unit].cause for i in deep[8:]] == [(), ("clamped",), ("clamped",)] - # each clamped list restarts on a num of its own: Word's level 9 would count on otherwise - assert len({plan.units[i.unit].num_id for i in deep[8:]}) == 3 - assert all(plan.units[i.unit].count == 1 for i in deep[8:]) - [found] = [d for d in diagnostics if d.rule == "render-docx-list-depth"] - assert found.level == "warning" and found.line == deep[9].line and "9" in found.message - assert rules(diagnostics) == {"render-docx-list-depth"} + assert [plan.units[i.unit].ilvl for i in deep] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1] + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated", "enumerated+1"] + # the resolver's cycle goes on, and Word draws it: level 10 is the first level of the next list, in its format + assert [plan.units[i.unit].label for i in deep] == [i.label for i in deep] + assert [plan.units[i.unit].cause for i in deep[8:]] == [(), ("restart-depth",), ()] + # the list at level 10 is numbered by a num of its own, and the list in it by structure + assert plan.units[deep[9].unit].num_id != plan.units[deep[8].unit].num_id == 2 + assert plan.units[deep[10].unit].num_id == plan.units[deep[9].unit].num_id + assert plan.nums[plan.units[deep[9].unit].num_id - 1].start_overrides == ((0, 1),) + [found] = diagnostics + assert (found.rule, found.level, found.line) == ("render-docx-static-restart", "info", deep[9].line) + assert "past Word's nine levels" in found.message and "Restart at 1" in found.message -def test_items_of_lists_past_level_nine_keep_their_numbers_and_a_sibling_at_level_nine_is_restarted() -> None: +def test_items_of_lists_past_level_nine_keep_their_numbers_and_the_list_at_level_nine_goes_on() -> None: body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(9)) body += "".join(" " * 9 + f"- nine {k}\n" for k in "abc") body += "".join(" " * 10 + f"- ten {k}\n" for k in "ab") @@ -390,19 +394,62 @@ def test_items_of_lists_past_level_nine_keep_their_numbers_and_a_sibling_at_leve tree, plan, diagnostics = planned(body) found = items(tree) nine, ten, again = found[9:12], found[12:14], found[14] - # every item of a clamped list is Word's level 9 in the format of level 9, with the resolver's count - assert [plan.units[i.unit].cause for i in (*nine, *ten)] == [("clamped",)] * 5 + # each item has Word's counter of its own level and the resolver's label assert [(plan.units[i.unit].ilvl, plan.units[i.unit].count) for i in (*nine, *ten)] == [ - (8, 1), (8, 2), (8, 3), (8, 1), (8, 2)] - assert [plan.units[i.unit].label for i in (*nine, *ten)] == ["(A)", "(B)", "(C)", "(A)", "(B)"] - assert [i.label for i in (*nine, *ten)] == ["(a)", "(b)", "(c)", "(i)", "(ii)"] # the resolver's cycle - assert len({plan.units[i.unit].num_id for i in (nine[0], ten[0])}) == 2 - # Word's counter of level 9 was used by the clamped lists: the item after them is restarted where it is + (0, 1), (0, 2), (0, 3), (1, 1), (1, 2)] + assert [plan.units[i.unit].label for i in (*nine, *ten)] == ["(a)", "(b)", "(c)", "(i)", "(ii)"] + assert [i.label for i in (*nine, *ten)] == ["(a)", "(b)", "(c)", "(i)", "(ii)"] + # the sibling at level nine is on the list of level nine: the lists past it have their own, so it needs no restart again_plan = plan.units[again.unit] - assert (again.number, again_plan.count, again_plan.cause) == (2, 2, ("resync",)) - assert plan.nums[again_plan.num_id - 1].start_overrides == ((8, 2),) - assert rules(diagnostics) == {"render-docx-list-depth", "render-docx-shared-counter"} - assert all(d.level == "warning" for d in diagnostics) + assert (again.number, again_plan.count, again_plan.cause) == (2, 2, ()) and again_plan.ilvl == 8 + assert plan.units[again.unit].num_id == plan.units[found[8].unit].num_id + assert rules(diagnostics) == {"render-docx-static-restart"} + + +def test_the_abstract_nums_past_nine_levels_have_the_families_formats_of_the_depth_they_stand_for() -> None: + levels = [{"counter": "decimal", "label": "{n}.", "ref": "{n}"}, + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{n})", "ref": "({n})"}, + {"counter": "upper-alpha", "label": "[{n}]", "ref": "[{n}]"}] + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(20)) + _, plan, _ = planned(body, overrides={"enumeration.levels": levels}) + by_kind = {a.kind: a for a in plan.abstract_nums} + assert list(by_kind) == ["headings", "enumerated", "enumerated+1", "enumerated+2"] + # depth 9 is the second level of the cycle of four, depth 10 the third, ...; each level shows its own counter alone + assert [(level.num_fmt, level.lvl_text) for level in by_kind["enumerated+1"].levels[:5]] == [ + ("lowerLetter", "(%1)"), ("lowerRoman", "%2)"), ("upperLetter", "[%3]"), ("decimal", "%4."), + ("lowerLetter", "(%5)")] + assert [(level.num_fmt, level.lvl_text) for level in by_kind["enumerated+2"].levels[:3]] == [ + ("lowerRoman", "%1)"), ("upperLetter", "[%2]"), ("decimal", "%3.")] # depth 18, 19, 20 + # the indent goes on stepping a centimetre per level of nesting, up to thirteen centimetres + assert [level.left for level in by_kind["enumerated+1"].levels[:5]] == [567 * k for k in (10, 11, 12, 13, 13)] + assert all(len(a.levels) == 9 for a in plan.abstract_nums) + + +def test_ordered_lists_go_on_past_nine_levels_too() -> None: + body = "# A\n\n" + "".join(" " * i + f"{1}. level {i}\n" for i in range(11)) + tree, plan, diagnostics = planned(body) + deep = items(tree) + assert [a.kind for a in plan.abstract_nums] == ["headings", "renumber", "renumber+1"] + assert [plan.units[i.unit].ilvl for i in deep[8:]] == [8, 0, 1] + assert [plan.units[i.unit].label for i in deep] == [i.label for i in deep] + assert [plan.units[i.unit].cause for i in deep[8:]] == [(), ("restart-depth",), ()] + assert rules(diagnostics) == {"render-docx-static-restart"} + + +def test_a_list_nested_past_the_levels_the_plan_has_is_clamped_and_reported_once() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(38)) + tree, plan, diagnostics = planned(body) + deep = items(tree) + assert [a.kind for a in plan.abstract_nums] == ["headings", "enumerated", "enumerated+1", "enumerated+2", + "enumerated+3"] + assert [plan.units[i.unit].ilvl for i in deep[34:]] == [7, 8, 8, 8] + assert [plan.units[i.unit].cause for i in deep[35:]] == [(), ("clamped",), ("clamped",)] + # each clamped list restarts on a num of its own: Word's level 9 of the last list would count on otherwise + assert len({plan.units[i.unit].num_id for i in deep[35:]}) == 3 + assert all(plan.units[i.unit].count == 1 for i in deep[35:]) + [found] = [d for d in diagnostics if d.rule == "render-docx-list-depth"] + assert found.level == "warning" and found.line == deep[36].line and "36" in found.message def test_a_list_that_goes_on_from_a_sibling_with_a_fixed_start_is_a_warning_not_a_restart() -> None: @@ -1610,9 +1657,9 @@ def test_the_random_documents_reach_every_case_the_property_is_about() -> None: seen_rules |= rules(diagnostics) seen_causes |= {cause for unit in plan.units.values() for cause in unit.cause} assert {"render-docx-letter-overflow", "render-docx-section-label", "render-docx-static-restart", - "render-docx-list-depth", "render-docx-shared-counter"} <= seen_rules - assert {"restart-family", "restart-section", "restart-alternative", "resync", "label-fallback", "skipped-level", - "clamped", "alternative", "reserved-level"} <= seen_causes + "render-docx-shared-counter"} <= seen_rules + assert {"restart-family", "restart-section", "restart-alternative", "restart-depth", "resync", "label-fallback", + "skipped-level", "alternative", "reserved-level"} <= seen_causes assert depth >= 10 diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index e9d1ac8..231590f 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -369,15 +369,22 @@ Refs: {{ref: i3}} {{ref: i4}} {{ref: i5}} {{ref: i6}} {{ref: i7}} {{ref: i9}} {{ref: y}} {{ref: w}} """ +#: Lists nested past Word's nine levels: each nine levels are a list of their own, restarted at a fixed value where they +#: begin, and numbered by structure inside. +STRESS["deep"] = FRONT + "# A {#a}\n\n" + "".join( + " " * d + f"- level {d} {{#l{d}}}\n" for d in range(12)) + " " * 8 + "- level 8 again {#l8b}\n" + ( + " " * 9 + "- level 9 again {#l9b}\n") + "\nRefs: " + " ".join( + f"{{{{ref: {anchor}}}}}" for anchor in ("l0", "l7", "l8", "l9", "l10", "l11", "l8b", "l9b")) + "\n" + #: The stress documents that are shown under some variants only (each costs LibreOffice time): the continental style, where #: the items ride the heading list, the default, and the default with numbered paragraphs. FEW_VARIANTS: dict[str, tuple[str, ...]] = { **{name: ("continental", "default", "paragraphs") for name in ("alternatives in threes", "nested titles", "qualified")}, - "paths": ("continental", "default"), "ordered paths": ("default",)} + "paths": ("continental", "default"), "ordered paths": ("default",), "deep": ("continental", "default", "outline")} #: The variants that show some stress documents only. ONLY_DOCS: dict[str, tuple[str, ...]] = { - "five levels": ("qualified",), "shared level": ("qualified", "nested"), "path items": ("paths", "lists"), + "five levels": ("qualified",), "shared level": ("qualified", "nested"), "path items": ("paths", "lists", "deep"), "ordered parent": ("ordered paths", "lists")} #: The settings that give the numbering words and symbols: a field that returns the same words as the reference's format @@ -1102,3 +1109,34 @@ def test_a_new_parent_whose_nested_item_is_copied_from_a_detached_list_goes_on_c "(B) four b", "(C) NEW copy", "(A) four c"] assert "ii NEW three" in said and "iii three b" in said assert "Refs: 1(a)(1)(i)(A) 1(a)(1)(i)(B) 1(a)(1)(iii)(A)" in said[-1] + + +# -- Lists nested past Word's nine levels --------------------------------------------------------------------------------- + +DEEP = FRONT + "# A {#a}\n\n" + "".join(" " * d + f"- level {d} {{#l{d}}}\n" for d in range(12)) + ( + "\nRefs: {{ref: l8}} {{ref: l9}} {{ref: l10}} {{ref: l11}}\n") + +#: An item inserted before the first of the list at level 9, before the one at level 10, and before the one at level 8. +DEEP_EDITS = [ + ("level 9", "NEW nine", " - level 9 {#l9}", " - NEW nine"), + ("level 10", "NEW ten", " - level 10 {#l10}", " - NEW ten"), + ("level 8", "NEW eight", " - level 8 {#l8}", " - NEW eight"), +] + + +@functools.cache +def deep_lists() -> str: + require_libreoffice() + data = render(DEEP, format="docx").binary + return libreoffice_edit({"deep": (data, [(match, new) for match, new, _, _ in DEEP_EDITS])})["deep"] # type: ignore[arg-type] + + +def test_items_nested_past_nine_levels_are_numbered_by_the_list_they_go_on_in_and_follow_edits() -> None: + """The lists nested past Word's nine levels keep restarting by structure inside the list they go on in, so an item + inserted at level 8, 9 or 10 renumbers what follows and every reference to it, as the renderer does.""" + source = applied(DEEP, [(before, new) for _, _, before, new in DEEP_EDITS]) + assert lines_of(deep_lists()) == lines_of(render(source, format="text").output) + said = lines_of(deep_lists()) + assert [line for line in said if line.endswith(("NEW eight", "level 8", "NEW nine", "level 9", "NEW ten", + "level 10", "level 11"))] == [ + "(A) NEW eight", "(B) level 8", "(a) NEW nine", "(b) level 9", "(i) NEW ten", "(ii) level 10", "(A) level 11"] From 28fed31edf3fea487cdf43174a865fe4ab4f4131 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:29:02 +0000 Subject: [PATCH 106/118] Add docx.reference_text_switch: what a cross-reference may expect of Word's \t switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit off never uses the switch; documented (the default) uses it where Microsoft's documentation settles the result (the words before the first counter and after the last go, a delimiter between counters stays: 'Section 1.01' gives '1.01'); guess also drops a period after the letters that lead a number ('No. 1', 'Čl. 1'). A reference that needs the switch where the setting does not allow it is fixed text, reported. The setting is validated like the other enumerations, reaches the planner through the DOCX settings, and is documented with the style template. --- docs/style-templates.md | 2 + src/legaldown_render/api.py | 2 +- src/legaldown_render/resolve/docx_cover.py | 56 ++++++++--- src/legaldown_render/resolve/docx_plan.py | 21 ++-- src/legaldown_render/style/docx.py | 6 +- src/legaldown_render/style/model.py | 7 ++ src/legaldown_render/writers/docx/__init__.py | 2 +- tests/test_docx_plan.py | 99 +++++++++++++++++-- tests/test_docx_roundtrip.py | 13 ++- tests/test_docx_style.py | 25 +++++ 10 files changed, 197 insertions(+), 36 deletions(-) diff --git a/docs/style-templates.md b/docs/style-templates.md index 3dbb5c6..e9fcb30 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -146,6 +146,7 @@ html: docx: # the DOCX writer only (see "DOCX") update_fields_on_open: true + reference_text_switch: documented # off | documented | guess: what a cross-reference may expect of Word's \t switch ``` ### Level formats @@ -266,6 +267,7 @@ style in Word applies to every paragraph that uses it. | Setting | Meaning | |---|---| | `docx.update_fields_on_open` | `true` (the default) asks Word to update every field when it opens the file: the cross-references, the table of contents and its page numbers. Word asks the reader once ("This document contains fields that may refer to other files. Update the fields?"). The fields hold the values the renderer computed either way, so a viewer that never updates fields (Word Online, Quick Look, Outlook's preview) shows the document right, but the table of contents without page numbers. LibreOffice and Word Online ignore the setting. `false` writes no such request | +| `docx.reference_text_switch` | What a cross-reference may expect of Word's `\t` switch (`REF bookmark \n \t`), which leaves the words of a number out. A reference is built from `REF` fields so that it stays right when a clause is inserted; where the reference shows less than the number Word draws (`Section {designation}` for a number `Article I`, `čl. {designation}` for `Čl. 1`), the switch leaves the words out and the `references.format` text goes back in as literal text. LibreOffice ignores the switch (and shows the whole number there, which the render reports once as `render-docx-reference-word-only`). Microsoft documents it with one example (`Section 1.01` gives `1.01`: the words around the number go, a delimiter between its counters stays); what Word does with a delimiter in the text around the counters is not documented. **`documented`** (the default) uses the switch only where that example settles the result: words before the first counter or after the last go, a delimiter between counters stays, and a label with a delimiter around its number (`(a)`, `a)`, `1:`, `No. 1`) is not asked for it. **`off`** never uses it: a reference that needs it is fixed text, reported (`render-docx-reference`). **`guess`** also expects Word to leave out a period after the letters that lead a number (`No. 1`, `Čl. 1`): the checklist in `.github/PUBLISHING.md` says how to check what your Word does, and a reference that is wrong in Word is fixed by setting `documented` | Settings that were already part of the template gain a DOCX meaning: diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index 9eff8d6..c4e55ab 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -175,7 +175,7 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any if isinstance(writer, DocxWriter): # What Word will draw differently from the other formats (docs/decisions/0008) is the planner's to report; its # findings are the renderer's, and are ordered with them. - plan, planned = plan_docx(tree, file=document.filename) + plan, planned = plan_docx(tree, file=document.filename, text_switch=writer.settings.reference_text_switch) extra = [*extra, *planned] # The renderer's findings that the validator's or the assembly's already # name (the same rule, level and message), wherever they are, are not repeated; diff --git a/src/legaldown_render/resolve/docx_cover.py b/src/legaldown_render/resolve/docx_cover.py index be978d2..d868297 100644 --- a/src/legaldown_render/resolve/docx_cover.py +++ b/src/legaldown_render/resolve/docx_cover.py @@ -21,22 +21,27 @@ covers the one with the fewest fields wins, then the fewest fields using ``\\t``, then the fewest literal characters; of equal ones the first in the order the fields were given. -``\\t`` (Word only; LibreOffice ignores it) suppresses the text around the counters of a number. It is modelled only -where its result is certain: when every literal of the field is text to drop (``Article I`` gives ``I``, -``§ 3`` gives ``3``), and not when one of them is a delimiter (``(a)``, ``a)``, ``1.2``), where what Word does -is not known (:func:`suppressed`). +``\\t`` (Word only; LibreOffice ignores it) suppresses the text of a number that is not a counter or a delimiter. +Microsoft documents it with one example (``Section 1.01`` gives ``1.01``): the words around the number go, the delimiter +between its counters stays. What it returns where the text around the counters holds a delimiter (``(a)``, ``a)``, +``No. 1``) is not documented, so it is modelled by the style's ``docx.reference_text_switch`` (:func:`suppressed`): +``off`` never, ``documented`` only for what the documentation settles, ``guess`` also for a period after the +letters that lead a number. """ from __future__ import annotations from collections.abc import Sequence -from typing import NamedTuple +from typing import Literal, NamedTuple from .docx_numbering import Count, Lit, Token -#: The characters Word keeps when ``\t`` drops the text of a number: a field whose literal text holds one of them -#: has no ``\t`` form (what Word would make of it is not known). +#: The characters Word keeps when ``\t`` drops the text of a number. Between two counters they stay; in the text before +#: the first counter or after the last, what Word does with them is not documented (:func:`suppressed`). DELIMITERS = frozenset(".,;:()[]/-–") +#: What ``docx.reference_text_switch`` can say of ``\t``. +TextSwitch = Literal["off", "documented", "guess"] + class Title(NamedTuple): """The text of the heading of *owner* (a unit id), as the ``none`` scheme designates a section with it.""" @@ -77,18 +82,37 @@ def atoms(tokens: Sequence[Shown]) -> list[tuple[str, object]]: return out -def suppressed(tokens: Sequence[Shown]) -> tuple[Shown, ...] | None: - """What ``REF \\n \\t`` returns for a field that returns *tokens* with ``\\n``: the counters alone, when every - literal text between them is text Word drops; None when one holds a delimiter or there is no counter, since what - Word keeps is then not known.""" +def suppressed(tokens: Sequence[Shown], mode: TextSwitch = "documented") -> tuple[Shown, ...] | None: + """What ``REF \\n \\t`` returns for a field that returns *tokens* with ``\\n``, or None where the switch is not used + (the *mode* is ``off``, there is no counter, or what Word returns is not known). + + * Text before the first counter and after the last that holds no delimiter is dropped (``Article I`` gives ``I``, + ``§ 3`` gives ``3``). With a delimiter in it (``(a)``, ``a)``, ``1:``) Word's result is not documented: None. + Mode ``guess`` also drops leading text that has a period after letters in it (``No. 1``, ``Čl. 1`` give ``1``). + * Text between two counters that is nothing but delimiters stays (Microsoft's example: ``Section 1.01`` gives + ``1.01``); text between them with no delimiter in it is dropped; a mixture is None. + """ + if mode == "off": + return None + counters = [index for index, token in enumerate(tokens) if not isinstance(token, Lit)] + if not counters: + return None kept: list[Shown] = [] - for token in tokens: - if isinstance(token, Lit): - if any(character in DELIMITERS for character in token.text): + for index, token in enumerate(tokens): + if not isinstance(token, Lit): + kept.append(token) + continue + found = {character for character in token.text if character in DELIMITERS} + if index < counters[0] or index > counters[-1]: + period_after_letters = (mode == "guess" and index < counters[0] and found == {"."} + and any(character.isalpha() for character in token.text)) + if found and not period_after_letters: + return None + elif found: + if not all(character in DELIMITERS for character in token.text): return None - else: kept.append(token) - return tuple(kept) if kept else None + return tuple(kept) def _fits(wanted: Sequence[tuple[str, object]], at: int, field: Sequence[tuple[str, object]]) -> bool: diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 83a6ecc..250b855 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -66,7 +66,7 @@ ) from ..urls import displayed_url, printable from .docx_bookmarks import ATTACHMENTS_HEADING, BookmarkNames -from .docx_cover import Field, Shown, Title, atoms, cover, suppressed, text_of, uncovered +from .docx_cover import Field, Shown, TextSwitch, Title, atoms, cover, suppressed, text_of, uncovered from .docx_numbering import ( NUM_FMT, SAMPLE_COUNTER, @@ -599,8 +599,8 @@ def __init__(self, node: _Unit, kind: str, step: Step, delta: int) -> None: class _Planner: - def __init__(self, tree: RenderTree, file: str | None) -> None: - self.tree, self.file = tree, file or "" + def __init__(self, tree: RenderTree, file: str | None, text_switch: TextSwitch = "documented") -> None: + self.tree, self.file, self.text_switch = tree, file or "", text_switch self.formats = tree.formats self.textual = tree.formats.scheme == "none" self.diagnostics: list[Diagnostic] = [] @@ -719,7 +719,7 @@ def fields(self, designation: Sequence[Shown]) -> list[Field]: elif owner in self.steps: words = self.words(owner) fields.append(Field(owner, True, False, words)) - bare = suppressed(words) + bare = suppressed(words, self.text_switch) if bare is not None and atoms(bare) != atoms(words): fields.append(Field(owner, True, True, bare)) return fields @@ -1607,9 +1607,12 @@ def bullets(self, lst: List, section: Section | None, quoted: bool) -> None: self.block(child, section, quoted) -def plan_docx(tree: RenderTree, *, file: str | None = None) -> tuple[DocxPlan, list[Diagnostic]]: +def plan_docx(tree: RenderTree, *, file: str | None = None, text_switch: TextSwitch = "documented" + ) -> tuple[DocxPlan, list[Diagnostic]]: """The numbering plan of *tree* for Word, and the diagnostics only planning can produce (all - ``render-docx-*``, in the order of their lines, naming *file*). Pure: the same tree gives an equal plan, and - ``tree.formats`` carries the style's numbering. A style Word's numbering cannot express (:func:`docx_style_errors`) - is refused before rendering (ADR 0006), so it never reaches here.""" - return _Planner(tree, file).run() + ``render-docx-*``, in the order of their lines, naming *file*). Pure: the same tree and *text_switch* (the style's + ``docx.reference_text_switch``: what a reference may expect of Word's ``\\t`` switch, see + :func:`.docx_cover.suppressed`) give an equal plan, and ``tree.formats`` carries the style's numbering. A style + Word's numbering cannot express (:func:`docx_style_errors`) is refused before rendering (ADR 0006), so it never + reaches here.""" + return _Planner(tree, file, text_switch).run() diff --git a/src/legaldown_render/style/docx.py b/src/legaldown_render/style/docx.py index d4b32cf..d3bc135 100644 --- a/src/legaldown_render/style/docx.py +++ b/src/legaldown_render/style/docx.py @@ -31,6 +31,7 @@ import math import re from dataclasses import dataclass +from typing import Literal from ..errors import StyleError from ..resolve.docx_plan import docx_style_errors @@ -153,6 +154,8 @@ class DocxSettings: justify: bool headings: dict[int, HeadingFormat] update_fields_on_open: bool + #: ``off``, ``documented`` or ``guess``: what a cross-reference may expect of Word's ``\t`` switch. + reference_text_switch: Literal["off", "documented", "guess"] @property def points(self) -> float: @@ -207,7 +210,8 @@ def docx_settings(style: Style, *, trusted: bool = True) -> DocxSettings: return DocxSettings( page=page, font=_family(typography.font_family), half_points=half_points, line=line, line_rule=rule, color=color, link_color=link_color, justify=typography.justify, headings=headings, - update_fields_on_open=style.docx.update_fields_on_open) + update_fields_on_open=style.docx.update_fields_on_open, + reference_text_switch=style.docx.reference_text_switch) # -- Lengths ---------------------------------------------------------------------- diff --git a/src/legaldown_render/style/model.py b/src/legaldown_render/style/model.py index 711578d..f1973c1 100644 --- a/src/legaldown_render/style/model.py +++ b/src/legaldown_render/style/model.py @@ -355,6 +355,13 @@ class Docx: #: shows the document correctly, only without the page numbers of the table #: of contents. update_fields_on_open: bool = True + #: What Word's ``\t`` switch (``REF \n \t``, which leaves the words of a number out) is taken to do when a + #: cross-reference is built from fields (docs/style-templates.md, "DOCX"). ``off``: never used, so a reference that + #: needs it is fixed text. ``documented``: only where Microsoft's documentation says what Word returns (words around + #: the number are left out, a delimiter between its counters stays). ``guess``: also where Word is expected to do + #: what it is not documented to (a period after the letters that lead a number: ``No. 1``, ``Čl. 1``); to be checked + #: in Word. + reference_text_switch: Literal["off", "documented", "guess"] = "documented" @dataclass(frozen=True, slots=True) diff --git a/src/legaldown_render/writers/docx/__init__.py b/src/legaldown_render/writers/docx/__init__.py index f31838b..fc48e5f 100644 --- a/src/legaldown_render/writers/docx/__init__.py +++ b/src/legaldown_render/writers/docx/__init__.py @@ -61,7 +61,7 @@ def write(self, tree: RenderTree, plan: DocxPlan | None = None) -> bytes: one the tree is planned here (:func:`~legaldown_render.resolve.docx_plan.plan_docx`), whose diagnostics are then lost: :func:`~legaldown_render.render` plans first and reports them.""" if plan is None: - plan = plan_docx(tree)[0] + plan = plan_docx(tree, text_switch=self.settings.reference_text_switch)[0] settings = self.settings language = language_tag(tree.language, self.style.locale) title = document_title(tree) diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index 2fbf6b7..6d9fc3f 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -6,9 +6,11 @@ from __future__ import annotations import functools +import io import random import re import time +import zipfile from collections.abc import Iterator, Sequence from pathlib import Path @@ -44,6 +46,7 @@ Section, iter_tree_inlines, ) +from legaldown_render.writers.docx import DocxWriter NONE = {"numbering.scheme": "none"} PARAGRAPHS = {"paragraphs.numbered": True} @@ -1757,10 +1760,10 @@ def cached(segments: Sequence[Segment]) -> str: {"counter": "decimal", "label": "Section {path}", "ref": ".{n}"}], "references.format": "Section {designation}"} -def covers(variant: str, **settings: object) -> dict[str, tuple[str, str]]: +def covers(variant: str, text_switch: str = "documented", **settings: object) -> dict[str, tuple[str, str]]: """Anchor of each target to (what the renderer shows, the plan's cover), for the document of :data:`TARGETS`.""" result = render(FRONT + TARGETS + REFERENCES, format="text", **settings) - plan, _ = plan_docx(result.tree) + plan, _ = plan_docx(result.tree, text_switch=text_switch) # type: ignore[arg-type] units_of = {n.unit: n for n in units(result.tree)} found = {} for inline in iter_tree_inlines(result.tree): @@ -1981,6 +1984,68 @@ def test_the_words_of_the_format_that_the_number_holds_too_need_no_switch() -> N assert diagnostics == [] +#: A label that says another word than the format (the words go with ``\t``, the number between its counters stays) and a +#: label that leads with an abbreviation that has a period. +SECTION_SHORT = {"overrides": {**SECTION_WORD, "references.format": "Sec. {designation}"}} +CL_LOWER = {"overrides": {**CL_PERIOD, "references.format": "čl. {designation}"}} + + +@pytest.mark.parametrize(("switch", "settings", "anchor", "expected"), [ + # Microsoft's example, "Section 1.01" gives "1.01": words go, the delimiter between the counters stays + ("off", SECTION_SHORT, "beta", ("Sec. I.1", '"Sec. I.1"')), + ("documented", SECTION_SHORT, "beta", ("Sec. I.1", '"Sec. " + REF beta \\t')), + ("guess", SECTION_SHORT, "beta", ("Sec. I.1", '"Sec. " + REF beta \\t')), + ("documented", SECTION_SHORT, "i1", ("Sec. I(a)", '"Sec. " + REF alpha \\t + REF i1')), + # a period after letters ("Čl. 1") is not documented: only the setting that guesses uses it + ("off", CL_LOWER, "alpha", ("čl. 1", '"čl. 1"')), + ("documented", CL_LOWER, "alpha", ("čl. 1", '"čl. 1"')), + ("guess", CL_LOWER, "alpha", ("čl. 1", '"čl. " + REF alpha \\t')), + ("guess", CL_LOWER, "i1", ("čl. 1(a)", '"čl. " + REF alpha \\t + REF i1')), + # what the cover does without the switch does not depend on the setting + ("off", CL_LOWER, "beta", ("čl. 1.1", '"čl. " + REF beta')), + ("guess", CL_LOWER, "beta", ("čl. 1.1", '"čl. " + REF beta')), +]) +def test_the_text_switch_setting_decides_which_references_use_the_switch( + switch: str, settings: dict, anchor: str, expected: tuple[str, str]) -> None: + assert covers("x", switch, **settings)[anchor] == expected + + +def test_a_reference_that_needs_a_switch_the_setting_does_not_allow_is_fixed_text_and_reported() -> None: + body = "# A {#a}\n\nSee {{ref: a}}.\n" + for switch, fixed in (("off", True), ("documented", True), ("guess", False)): + result = render(FRONT + body, format="text", **CL_LOWER) + plan, diagnostics = plan_docx(result.tree, text_switch=switch) # type: ignore[arg-type] + assert plan.refs["s1"].fixed is fixed + assert any(d.rule == "render-docx-reference" and "not a field" in d.message for d in diagnostics) is fixed + assert any(d.rule == "render-docx-reference-word-only" for d in diagnostics) is not fixed + # off: no field of the document has the switch, whatever the labels + result = render(FRONT + TARGETS + REFERENCES, format="text", **SECTION_SHORT) + plan, diagnostics = plan_docx(result.tree, text_switch="off") + assert not any(isinstance(s, FieldSeg) and s.suppress_text for r in plan.refs.values() for s in r.segments) + assert "render-docx-reference-word-only" not in rules(diagnostics) + assert any(r.fixed for r in plan.refs.values()) + + +def test_the_rendered_file_has_the_switch_only_where_the_style_allows_it() -> None: + body = "# A {#a}\n\nSee {{ref: a}}.\n" + texts = {} + for switch in ("off", "documented", "guess"): + result = render(FRONT + body, format="docx", overrides={**CL_LOWER["overrides"], "docx.reference_text_switch": switch}) + assert result.binary is not None + texts[switch] = zipfile.ZipFile(io.BytesIO(result.binary)).read("word/document.xml").decode("utf-8") + assert ["\\t" in texts[switch] for switch in texts] == [False, False, True] + + +def test_a_writer_that_plans_the_tree_itself_plans_it_with_the_styles_switch() -> None: + result = render(FRONT + "# A {#a}\n\nSee {{ref: a}}.\n", format="text", + overrides={**CL_LOWER["overrides"], "docx.reference_text_switch": "guess"}) + data = DocxWriter(result.style).write(result.tree) + assert "\\t" in zipfile.ZipFile(io.BytesIO(data)).read("word/document.xml").decode("utf-8") + other = render(FRONT + "# A {#a}\n\nSee {{ref: a}}.\n", format="text", **CL_LOWER) + data = DocxWriter(other.style).write(other.tree) + assert "\\t" not in zipfile.ZipFile(io.BytesIO(data)).read("word/document.xml").decode("utf-8") + + def test_reference_findings_name_the_file_and_the_first_line_of_a_repeated_reference() -> None: body = "# A {#a}\n\n### B {#b}\n\nFirst {{ref: b}}.\n\nSecond {{ref: b}}.\n" result = render(FRONT + body, format="text", style="outline") @@ -2205,13 +2270,35 @@ def test_the_atom_nothing_can_stand_for_is_found() -> None: assert uncovered([Count("a", "I")], fields) is None -def test_the_switch_that_drops_text_is_modelled_only_where_it_leaves_bare_counters() -> None: +def test_the_switch_that_drops_text_is_modelled_only_where_its_result_is_known() -> None: + one, two = Count("a", "1"), Count("b", "2") + # the words around a number go (Microsoft: "Section 1.01" gives "1.01"), the delimiter between its counters stays assert suppressed([Lit("Article "), Count("a", "I")]) == (Count("a", "I"),) assert suppressed([Lit("§ "), Count("a", "3")]) == (Count("a", "3"),) + assert suppressed([Lit("Section "), one, Lit("."), two]) == (one, Lit("."), two) + assert suppressed([one, Lit("."), two]) == (one, Lit("."), two) # nothing to drop: no field of its own is needed + assert suppressed([Lit("Part "), one, Lit(" of "), two]) == (one, two) # words between counters go too + # a delimiter around the counters, or mixed with words between them, is not documented assert suppressed([Lit("("), Count("a", "a"), Lit(")")]) is None - assert suppressed([Count("a", "1"), Lit("."), Count("b", "2")]) is None - assert suppressed([Lit("No. "), Count("a", "1")]) is None - assert suppressed([Lit("Article")]) is None + assert suppressed([Count("a", "a"), Lit(")")]) is None + assert suppressed([one, Lit(":")]) is None + assert suppressed([Lit("No. "), one]) is None + assert suppressed([one, Lit(" to."), two]) is None + assert suppressed([Lit("Article")]) is None and suppressed([]) is None + + +def test_the_switch_setting_says_what_word_may_be_expected_to_do() -> None: + one, two = Count("a", "1"), Count("b", "2") + # off: never + assert suppressed([Lit("Article "), Count("a", "I")], "off") is None + assert suppressed([Lit("Section "), one, Lit("."), two], "off") is None + # guess: a period after the letters that lead a number goes too; no other delimiter does, and only in leading text + assert suppressed([Lit("No. "), one], "guess") == (one,) + assert suppressed([Lit("Čl. "), one], "guess") == (one,) + assert suppressed([Lit("Art. "), one, Lit("."), two], "guess") == (one, Lit("."), two) + assert suppressed([Lit("("), one], "guess") is None and suppressed([Lit("1. "), one], "guess") is None + assert suppressed([one, Lit(" Art.")], "guess") is None and suppressed([Lit("No: "), one], "guess") is None + assert suppressed([Lit("Article "), Count("a", "I")], "guess") == (Count("a", "I"),) # -- Bookmarks ---------------------------------------------------------------------------------------------- diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index 231590f..fcb860c 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -385,7 +385,8 @@ #: The variants that show some stress documents only. ONLY_DOCS: dict[str, tuple[str, ...]] = { "five levels": ("qualified",), "shared level": ("qualified", "nested"), "path items": ("paths", "lists", "deep"), - "ordered parent": ("ordered paths", "lists")} + "ordered parent": ("ordered paths", "lists"), "section-short": ("nested", "qualified"), + "cl-guess": ("nested", "qualified")} #: The settings that give the numbering words and symbols: a field that returns the same words as the reference's format #: stands for them, otherwise Word's ``\t`` switch drops them from the number. @@ -397,6 +398,14 @@ # the format says a word the label does not: Word's \t leaves the label's out and the format's is put back "article-short": {"overrides": {"numbering.levels": [ {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}], "references.format": "Art. {designation}"}}, + # a number with a delimiter between its counters ("Section 1.1") that the format words differently: Word's \t keeps it + "section-short": {"overrides": {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}, + {"counter": "decimal", "label": "Section {path}", "ref": ".{n}"}], "references.format": "Sec. {designation}"}}, + # a period after letters ("Čl. 1") that only the setting that guesses lets the switch drop + "cl-guess": {"overrides": {"numbering.levels": [ + {"counter": "decimal", "label": "Čl. {n}", "ref": "{n}"}, {"counter": "decimal", "label": "{path}", "ref": ".{n}"}], + "references.format": "čl. {designation}", "docx.reference_text_switch": "guess"}}, "paragraphs": {"overrides": {"paragraphs.numbered": True}}, # the items and the paragraphs of a section share the heading list's level: the numbers are restarted where the # resolver counts them apart @@ -539,7 +548,7 @@ def make_case(variant: str, name: str, source: str) -> Case | None: except DocumentError: return None assert result.binary is not None - plan, diagnostics = plan_docx(result.tree) + plan, diagnostics = plan_docx(result.tree, text_switch=result.style.docx.reference_text_switch) return Case(variant, name, result.tree, plan, {d.rule for d in diagnostics}, result.binary, prepared(result.binary, result.tree, plan)) diff --git a/tests/test_docx_style.py b/tests/test_docx_style.py index 5c8d139..3ed1904 100644 --- a/tests/test_docx_style.py +++ b/tests/test_docx_style.py @@ -93,6 +93,31 @@ def test_fields_are_updated_on_open_by_default_and_the_setting_is_a_boolean() -> load_style("default", overrides={"docx.update_fields_on_open": "maybe"}) +# -- docx.reference_text_switch ----------------------------------------------------------- + + +def test_the_text_switch_is_documented_by_default_and_the_setting_is_one_of_three_words() -> None: + assert load_style("default").docx == Docx(update_fields_on_open=True, reference_text_switch="documented") + for word in ("off", "documented", "guess"): + style = load_style("default", overrides={"docx.reference_text_switch": word}) + assert style.docx.reference_text_switch == word + assert docx_settings(style).reference_text_switch == word + assert parse_override("docx.reference_text_switch", word) == word + with pytest.raises(StyleError, match="docx.reference_text_switch: must be one of off, documented, guess"): + load_style("default", overrides={"docx.reference_text_switch": "always"}) + with pytest.raises(StyleError, match="docx.reference_text_switch: must be one of"): + load_style("default", overrides={"docx.reference_text_switch": True}) + assert "reference_text_switch: documented" in dump_style(load_style("default")) + assert load_style(dump_style(load_style("default", overrides={"docx.reference_text_switch": "guess"})) + ).docx.reference_text_switch == "guess" + + +def test_an_untrusted_style_does_not_quote_the_value_of_the_text_switch() -> None: + with pytest.raises(StyleError) as caught: + load_style("default", overrides={"docx.reference_text_switch": "secret"}, trusted=False) + assert "secret" not in str(caught.value) and "must be one of off, documented, guess" in str(caught.value) + + def test_an_unknown_docx_key_is_a_style_error_with_a_suggestion() -> None: with pytest.raises(StyleError, match=r"docx.update_field_on_open: unknown key \(did you mean 'update_fields_on_open'"): load_style("default", overrides={"docx.update_field_on_open": True}) From d86479f19a921d43d2d54c9214d61cee2518f48b Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:39:07 +0000 Subject: [PATCH 107/118] Write the number of a skipped heading level as text so the rest of a reference stays live MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A document that skips a heading level (heading-skip, §4.1, a Core error the renderer still numbers) has a part of a number that no unit holds, so no field returns it. The cover now writes that part (and only that: the counter of a Ghost, never a stale counter of another level) as literal text between the fields of the units that do hold their counters, instead of making the whole reference fixed text. The reference is reported once per run of such references and says which part is text. The property test of the references allows letters and digits as literal text only for those parts, in documents that skip a level. --- src/legaldown_render/resolve/docx_cover.py | 12 +++-- src/legaldown_render/resolve/docx_plan.py | 43 ++++++++++++++--- tests/test_docx_plan.py | 55 +++++++++++++++------- tests/test_docx_roundtrip.py | 42 +++++++++++++++++ 4 files changed, 123 insertions(+), 29 deletions(-) diff --git a/src/legaldown_render/resolve/docx_cover.py b/src/legaldown_render/resolve/docx_cover.py index d868297..251bddf 100644 --- a/src/legaldown_render/resolve/docx_cover.py +++ b/src/legaldown_render/resolve/docx_cover.py @@ -4,8 +4,9 @@ with the ``ref`` templates of every unit on the way (``1.1.1``, ``I.A.1.a(i)``, ``1(b)(ii)``). In Word every part of it that Word can compute must be a field, so that it stays right when a clause is inserted: ``REF \\n`` returns the number Word draws for the bookmarked paragraph (its whole ``lvlText``, trailing periods dropped), and -nothing else Word could know may be written as literal text. This module finds, for one designation, the -combination of such fields and literal delimiters that reads exactly like it. +nothing else Word could know may be written as literal text, but for the part of a number that belongs to a heading +level the document skips (§4.1: a Core error the renderer still numbers), which no unit holds. This module finds, for +one designation, the combination of such fields and literal delimiters that reads exactly like it. Everything here works on token strings (:mod:`.docx_numbering`): the text of a reference as the renderer wrote it (the designation inside the text of ``references.format``, so that a field may stand for the words of the format when the @@ -33,7 +34,7 @@ from collections.abc import Sequence from typing import Literal, NamedTuple -from .docx_numbering import Count, Lit, Token +from .docx_numbering import Count, Ghost, Lit, Token #: The characters Word keeps when ``\t`` drops the text of a number. Between two counters they stay; in the text before #: the first counter or after the last, what Word does with them is not documented (:func:`suppressed`). @@ -70,10 +71,11 @@ def text_of(tokens: Sequence[Shown]) -> str: def atoms(tokens: Sequence[Shown]) -> list[tuple[str, object]]: """*tokens* as atoms: ``("d", character)`` for each character of literal text, ``("c", owner)`` for a counter and - ``("t", owner)`` for a title.""" + ``("t", owner)`` for a title. The counter of a heading level the document skips (a :class:`~.docx_numbering.Ghost`: + no unit holds it, so no field can return it) is literal text, which is all that can stand for it.""" out: list[tuple[str, object]] = [] for token in tokens: - if isinstance(token, Lit): + if isinstance(token, Lit) or (isinstance(token, Count) and isinstance(token.owner, Ghost)): out.extend(("d", character) for character in token.text) elif isinstance(token, Count): out.append(("c", token.owner)) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 250b855..4cc5b85 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -228,6 +228,10 @@ class RefPlan: #: No cover exists, so the reference is the renderer's text with no field: right when the file is opened, but #: fixed (an edit in Word does not change it). The plan reports each (``render-docx-reference``). fixed: bool = False + #: The cover writes the number of a heading level the document skips (a Core error, ``heading-skip``, §4.1) as text, + #: since no unit holds that level's counter and so no field returns it; the rest is fields. Reported like a fixed + #: reference (``render-docx-reference``). + ghost: bool = False @dataclass(frozen=True, slots=True) @@ -580,10 +584,12 @@ class _Built(NamedTuple): exact: bool lo_text: str | None fixed: bool + ghost: bool = False class _Finding(NamedTuple): - """A reference that is fixed text (*kind* ``fixed``, *reason* why) or reads differently in Word (``differs``).""" + """A reference that is fixed text (*kind* ``fixed``, *reason* why), has the number of a skipped heading level as + fixed text (``ghost``) or reads differently in Word (``differs``).""" kind: str reason: str @@ -727,9 +733,7 @@ def fields(self, designation: Sequence[Shown]) -> list[Field]: def why_uncovered(self, designation: Sequence[Shown], fields: Sequence[Field]) -> str: """Why no combination of *fields* and delimiters reads like *designation* (§13.1).""" wanted = atoms(designation) - kind, owner = wanted[uncovered(designation, fields) or 0] - if kind == "c" and isinstance(owner, Ghost): - return "a heading level is skipped before it, and no field stands for the level that is missing" + _, owner = wanted[uncovered(designation, fields) or 0] for candidate in fields: if candidate.number and any(isinstance(t, Count) and t.owner == owner for t in candidate.tokens): return (f"Word draws its number as '{printable(text_of(candidate.tokens))}', with text or punctuation " @@ -761,6 +765,9 @@ def reference(self, found: _Reference) -> _Built | None: if pieces is None: self.unreferable.append(_Finding("fixed", self.why_uncovered(designation, fields), found, found.text)) return _Built([found.text] if found.text else [], True, None, True) + ghost = self.skipped_part(whole, pieces) + if ghost: + self.unreferable.append(_Finding("ghost", "", found, ghost)) shown = "".join(piece if isinstance(piece, str) else text_of(piece.tokens) for piece in pieces) if shown != found.text: chain = {t.owner for t in designation if not isinstance(t, Lit) and isinstance(t.owner, str)} @@ -770,7 +777,23 @@ def reference(self, found: _Reference) -> _Built | None: if any(isinstance(piece, Field) and piece.suppress_text for piece in pieces): lo_text = "".join(piece if isinstance(piece, str) else text_of(self.words(piece.owner) if piece.number else piece.tokens) for piece in pieces) - return _Built(pieces, shown == found.text, lo_text, False) + return _Built(pieces, shown == found.text, lo_text, False, bool(ghost)) + + @staticmethod + def skipped_part(whole: Sequence[Shown], pieces: Sequence[str | Field]) -> str: + """The text of the counters of heading levels the document skips (a :class:`Ghost`: nothing holds them) that the + cover writes as literal text, in order; empty when it writes none (a field returns them, or the reference has + none).""" + ghosts: list[bool] = [] + for token in whole: + ghosts += [isinstance(token, Count) and isinstance(token.owner, Ghost)] * len(atoms([token])) + text, at = "", 0 + for piece in pieces: + size = len(piece) if isinstance(piece, str) else len(atoms(piece.tokens)) + if isinstance(piece, str): + text += "".join(character for character, ghost in zip(piece, ghosts[at:at + size], strict=True) if ghost) + at += size + return text def report_references(self) -> None: """One finding for the references that fail in the same way (the first, by line, and how many more), as @@ -783,7 +806,13 @@ def report_references(self) -> None: first = members[0] text, shown = printable(first.found.text), printable(first.shown) more = f" {len(members) - 1} more cross-references fail in the same way." if len(members) > 1 else "" - if kind == "fixed": + if kind == "ghost": + more = f" {len(members) - 1} more cross-references are written the same way." if len(members) > 1 else "" + message = (f"The cross-reference '{text}' holds the number of a heading level that the document skips " + f"(a heading level must follow the one above, §4.1), which no Word number returns: " + f"'{shown}' is written as text there and will not follow edits made in Word; the rest is " + f"made of fields.{more}") + elif kind == "fixed": message = (f"The cross-reference '{text}' cannot be built from Word's numbering fields: {reason}. It is " f"written as the text '{text}', which is right now but is not a field: it will not follow " f"edits made in Word.{more}") @@ -941,7 +970,7 @@ def finish(self) -> tuple[dict[str, UnitPlan], dict[str, str], dict[str, RefPlan plain = text_of(self.words(piece.owner)) if piece.number else cached segments.append(FieldSeg(bookmark[piece.owner], piece.number, piece.suppress_text, True, cached, plain if piece.suppress_text else cached)) - refs[target] = RefPlan(tuple(segments), made.exact, made.fixed) + refs[target] = RefPlan(tuple(segments), made.exact, made.fixed, made.ghost) toc: list[TocEntryPlan] = [] for entry in self.tree.contents: diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index 6d9fc3f..038266f 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -1911,19 +1911,20 @@ def references_of(tree: RenderTree, plan: DocxPlan) -> list[tuple[str, str, bool for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)] -def test_a_skipped_heading_level_leaves_a_part_of_the_designation_no_field_stands_for() -> None: +def test_a_skipped_heading_level_is_the_one_part_of_a_reference_that_is_written_as_text() -> None: body = "# A {#a}\n\n### B {#b}\n\nSee {{ref: b}}.\n" tree, plan, diagnostics = planned(body, style="outline") - # no field can stand for the missing level: the reference is the renderer's text, right now and not a field - assert references_of(tree, plan) == [("I.A.1", '"I.A.1"', True)] and plan.refs["s2"].fixed + # no unit holds the number of the level the document skips (an error, §4.1), so it is text between two fields + assert references_of(tree, plan) == [("I.A.1", 'REF a + ".A." + REF b', True)] + assert plan.refs["s2"].ghost and not plan.refs["s2"].fixed [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] assert found.level == "warning" and found.line == tree.sections[1].blocks[0].line - assert found.message.startswith("The cross-reference 'I.A.1' cannot be built from Word's numbering fields: a " - "heading level is skipped before it") - assert "written as the text 'I.A.1'" in found.message and "not a field" in found.message + assert found.message.startswith("The cross-reference 'I.A.1' holds the number of a heading level that the document skips") + assert "'A' is written as text there and will not follow edits made in Word; the rest is made of fields" in found.message # the decimal scheme writes the whole number in the heading's own field, so nothing is missing there tree, plan, diagnostics = planned(body) - assert references_of(tree, plan) == [("1.1.1", "REF b", True)] and not diagnostics and not plan.refs["s2"].fixed + assert references_of(tree, plan) == [("1.1.1", "REF b", True)] and not diagnostics + assert not plan.refs["s2"].fixed and not plan.refs["s2"].ghost def test_text_around_a_counter_that_word_cannot_drop_leaves_the_reference_as_one_field() -> None: @@ -2222,9 +2223,9 @@ def test_references_that_fail_in_the_same_way_are_one_finding_naming_the_first_a body = "# A {#a}\n\n### B {#b}\n\n### C {#c}\n\n# D {#d}\n\n### E {#e}\n\n{{ref: b}} {{ref: c}} {{ref: e}}\n" tree, plan, diagnostics = planned(body, style="outline") found = [d for d in diagnostics if d.rule == "render-docx-reference"] - assert len(found) == 1 and found[0].message.startswith("The cross-reference 'I.A.1' cannot be built") - assert found[0].message.endswith(" 2 more cross-references fail in the same way.") - assert all(plan.refs[t].fixed for t in ("s2", "s3", "s5")) + assert len(found) == 1 and found[0].message.startswith("The cross-reference 'I.A.1' holds the number") + assert found[0].message.endswith(" 2 more cross-references are written the same way.") + assert all(plan.refs[t].ghost for t in ("s2", "s3", "s5")) assert found[0].line == tree.sections[-1].blocks[-1].line # the lowest line of the three: all in one paragraph @@ -2260,16 +2261,28 @@ def test_a_field_fits_where_the_counters_belong_to_the_same_units_whatever_their designation = [Count("a", "ab")] assert cover(designation, [field("a", Count("a", "bb"))]) == [field("a", Count("a", "bb"))] assert cover(designation, [field("a", Count("other", "ab"))]) is None - assert cover([Count(Ghost("s2", 1), "1")], [field("s2", Count("s2", "1"))]) is None def test_the_atom_nothing_can_stand_for_is_found() -> None: - designation = [Count("a", "I"), Lit("."), Count(Ghost("s2", 1), "1"), Lit("."), Count("b", "1")] + designation = [Count("a", "I"), Lit("."), Count("c", "1"), Lit("."), Count("b", "1")] fields = [field("a", Count("a", "I")), field("b", Count("b", "1"))] assert cover(designation, fields) is None and uncovered(designation, fields) == 2 assert uncovered([Count("a", "I")], fields) is None +def test_the_counter_of_a_skipped_heading_level_is_literal_text_since_no_unit_holds_it() -> None: + ghost = Count(Ghost("s2", 1), "A") + designation = [Count("a", "I"), Lit("."), ghost, Lit("."), Count("b", "1")] + fields = [field("a", Count("a", "I")), field("b", Count("b", "1"))] + assert cover(designation, fields) == [fields[0], ".A.", fields[1]] + # a field that returns the ghost's text covers it too (the decimal scheme's number holds every level's counter) + whole = field("b", Count("a", "1"), Lit("."), Count(Ghost("s2", 1), "1"), Lit("."), Count("b", "1")) + assert cover([Count("a", "1"), Lit("."), Count(Ghost("s2", 1), "1"), Lit("."), Count("b", "1")], [whole]) == [whole] + assert uncovered(designation, fields) is None + # any other counter that no field returns is still not literal text + assert cover([Count("a", "I"), Lit("."), Count("c", "A")], fields) is None + + def test_the_switch_that_drops_text_is_modelled_only_where_its_result_is_known() -> None: one, two = Count("a", "1"), Count("b", "2") # the words around a number go (Microsoft: "Section 1.01" gives "1.01"), the delimiter between its counters stays @@ -2488,8 +2501,12 @@ def assert_references_as_resolved(result, plan, diagnostics) -> None: # noqa: A templates = [f.ref for f in (*tree.formats.headings, *tree.formats.enumerated, tree.formats.renumber, tree.formats.paragraphs) if f is not None] delimiters_only = not any(c.isalnum() for t in templates for c in re.sub(r"\{(n|path|section)\}", "", t)) + # The one counter a cover may write as text is that of a heading level the document skips (a Core error), which no + # unit holds: it reads as the first counter of that level. + skipping = any("skipped-level" in u.cause for u in plan.units.values()) + ghost_text = {c for level in tree.formats.headings for c in format_counter(1, level.counter)} needing_a_finding = 0 - fixed = 0 + fixed = ghosts = 0 for target, text in texts.items(): reference = plan.refs[target] assert all(s.bookmark in marked for s in reference.segments if isinstance(s, FieldSeg)) @@ -2502,9 +2519,11 @@ def assert_references_as_resolved(result, plan, diagnostics) -> None: # noqa: A fixed += 1 assert reference.segments == ((Literal_(text),) if text else ()) and reference.exact continue + assert not reference.ghost or skipping, (target, text) if delimiters_only: - assert not any(c.isalnum() for c in literal_text_of_a_cover(tree, reference.segments)), ( - target, text, show(plan, reference.segments)) + written = {c for c in literal_text_of_a_cover(tree, reference.segments) if c.isalnum()} + assert written <= (ghost_text if reference.ghost else set()), (target, text, show(plan, reference.segments)) + ghosts += reference.ghost assert any(isinstance(s, FieldSeg) for s in reference.segments) or not text, (target, text) if reference.exact: continue @@ -2523,11 +2542,13 @@ def assert_references_as_resolved(result, plan, diagnostics) -> None: # noqa: A # Every fixed text and every other difference is in a finding, named or counted among "N more". counted = 0 for finding in reported: - more = re.search(r" (\d+) more cross-references fail in the same way\.$", finding.message) + more = re.search(r" (\d+) more cross-references (?:fail in|are written) the same way\.$", finding.message) counted += 1 + (int(more.group(1)) if more else 0) - assert needing_a_finding + fixed <= counted <= len(texts) + assert needing_a_finding + fixed + ghosts <= counted <= len(texts) if fixed: assert any("not a field" in d.message for d in reported) + if ghosts: + assert any("heading level that the document skips" in d.message for d in reported) @pytest.mark.parametrize("variant", VARIANTS) diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index fcb860c..7c1b1a3 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -1149,3 +1149,45 @@ def test_items_nested_past_nine_levels_are_numbered_by_the_list_they_go_on_in_an assert [line for line in said if line.endswith(("NEW eight", "level 8", "NEW nine", "level 9", "NEW ten", "level 10", "level 11"))] == [ "(A) NEW eight", "(B) level 8", "(a) NEW nine", "(b) level 9", "(i) NEW ten", "(ii) level 10", "(A) level 11"] + + +# -- A skipped heading level: the one part of a reference that is text ---------------------------------------------------- + +SKIPPED = FRONT + """ +# One {#one} + +### Deep {#deep} + +- item {#i} + +# Two {#two} + +### Deeper {#deeper} + +Refs: {{ref: one}} {{ref: deep}} {{ref: i}} {{ref: two}} {{ref: deeper}} +""" + + +@functools.cache +def skipped() -> tuple[str, str]: + """What LibreOffice says about the document that skips a heading level (outline numbering) after a heading is inserted + before its first section, and what the text writer prints for the edited source.""" + require_libreoffice() + data = render(SKIPPED, format="docx", style="outline").binary + said = libreoffice_edit({"skipped": (data, [("One", "NEWH")])})["skipped"] # type: ignore[arg-type] + edited_source = applied(SKIPPED, [("# One {#one}", "# NEWH\n")]) + return said, render(edited_source, format="text", style="outline").output + + +def test_the_live_parts_of_a_reference_below_a_skipped_level_follow_a_heading_inserted_before_it() -> None: + """The skipped level has no number of its own, so its part of the reference is text (``A``) between two fields; the + fields are updated by the edit, so ``I.A.1`` becomes ``II.A.1`` with the text kept, as the renderer writes it.""" + said, written = skipped() + assert lines_of(said) == lines_of(written) + assert [line for line in lines_of(said) if line.startswith("Refs")] == ["Refs: II II.A.1 II.A.1(a) III III.A.1"] + result = render(SKIPPED, format="docx", style="outline") + plan, diagnostics = plan_docx(result.tree) + assert [r.ghost for r in plan.refs.values()] == [False, True, True, False, True] + assert not any(r.fixed for r in plan.refs.values()) + assert any(d.rule == "render-docx-reference" and "heading level that the document skips" in d.message + for d in diagnostics) From ae5917a949c3e565f9e13355ee82f63c720af0cb Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:52:57 +0000 Subject: [PATCH 108/118] Say why a reference stays fixed text, and document the live-reference classes; ship the Word check document generator A reference that is still fixed text says which class it falls in: delimiters around the counter that the reference lacks (with what the style's docx.reference_text_switch could change), or the counters of other units that the reference writes itself (one finding for all of them, since the cure is the label). References that fail alike are one finding, with the right plural. Over every test variant a reference is fixed text only in a document that skips a heading level, and over label forms the built-in styles do not have only for those two reasons. The architecture notes, ADR 0008, CONFORMANCE.md and the README describe the whole-text cover, the reserved levels with the path in them, the detached levels, the lists past nine levels, the switch setting and the skipped level's text. The Word checklist gains the rows for the switch on each label form, the reserved level %1.%2(%6) with unused levels around it, detached lists, deep lists and a skipped level, and a generator (tests/word_check.py) makes the document for them with the planner's assumption beside each row. --- .github/PUBLISHING.md | 40 +++- CONFORMANCE.md | 2 +- README.md | 5 +- docs/architecture.md | 118 ++++++++---- docs/decisions/0008-docx-word-native.md | 45 ++++- src/legaldown_render/resolve/docx_plan.py | 58 ++++-- tests/test_docx_plan.py | 144 +++++++++++++- tests/test_docx_special.py | 58 +++++- tests/test_docx_wordcheck.py | 53 ++++++ tests/word_check.py | 218 ++++++++++++++++++++++ 10 files changed, 660 insertions(+), 81 deletions(-) create mode 100644 tests/test_docx_wordcheck.py create mode 100644 tests/word_check.py diff --git a/.github/PUBLISHING.md b/.github/PUBLISHING.md index edb0d8c..fa59f46 100644 --- a/.github/PUBLISHING.md +++ b/.github/PUBLISHING.md @@ -149,14 +149,27 @@ For each file: - `\u` honours the outline levels of the attachment styles and paragraphs: the attachment titles are entries, at the level the table of contents gives them, in every variant that lists them. If Word does not list them, the fix is a `\t "Attachment Title,2"` list in the field (locale list separator permitting); report it. -8. **The numbering corner cases:** - - `\t` with words before the number: a label such as `Article {n}` or `§ {n}` referenced with `{n}`, whose reference - drops the words (`REF \n \t`): Word shows the bare number, where LibreOffice ignores `\t` and shows the words. - - a period before the number (`No. {n}`): the reference to the item reads as the HTML reads it, or the render - reported `render-docx-reference` and the reference is fixed text. - - skipped heading levels (a `###` straight after a `#`, an Error for the validator but rendered): the numbers after - it, and the references to them, read as the plain-text output reads them, or the render reported a - `render-docx-reference` Warning and the reference is fixed text; nothing is unreadable. +8. **The numbering corner cases.** Make the check document with `python tests/word_check.py + word-check/numbering-assumptions.docx` (from the repository root, in the project's environment): one row for each label + form the planner relies on, each with the planner's assumption beside it, taken from the planner's own code, so that + a change to the model shows in the file. Ctrl+A, F9, then compare each row's `REF \n` and `REF \n \t` with the last + column: + - `\t` with words before the number (rows 1 to 4: `Article %1`, `§ %1`, Microsoft's own `Section %1.%2` with a + decimal and with a roman `%1`): Word shows the bare number (`I`, `1`, `1.1`, `I.1`) where LibreOffice ignores `\t` + and shows the words. This is what the default `docx.reference_text_switch: documented` relies on. + - a period after letters before the number (rows 5 and 6: `Čl. %1`, `No. %1`): the planner does not use `\t` for + them under `documented`, and takes it to give the bare number under `guess`. If Word gives `1`, a style with such a + label may set `docx.reference_text_switch: guess`; if it gives anything else, report it. + - delimiters around the counter (rows 7 to 10: `(%1)`, `%1)`, `%1:`, `%1.%2(%3)`): the planner never uses `\t` for + them (what Word does is not documented). Note what Word returns: if it keeps the delimiter, `\t` gives the plain + number and the planner's refusal is only cautious. + - the reference to such an item reads as the HTML reads it, or the render reported `render-docx-reference` and the + reference is fixed text (with the class it falls in and the setting that could change it). + - skipped heading levels (a `###` straight after a `#`, an Error for the validator but rendered; part C of the check + document): the numbers after it, and the references to them, read as the plain-text output reads them, but the + planner caches the skipped level as 1, as LibreOffice and the renderer count it, and writes that level's part of a + reference as text (`render-docx-reference` says so). Word is expected to draw 0 for the level a heading skips: record + what it draws. The rest of such a reference is fields and follows edits. - counters shared across nums: a nested list of the same kind restarts under each new parent; a list of another kind restarts by a fixed setting (the render reported `render-docx-static-restart`). - Roman numerals of 4000 and over, and letters past 26: Word's text (`render-docx-roman-overflow`, @@ -173,6 +186,17 @@ For each file: (they have no heading style). Under each clause the nested list reads `a)`, `b)`, and a new clause (Enter at the end of a clause, typing) restarts it at `a)` under that clause; so does a sub-sub-list (`i)`). A list nested three levels under a clause (`1.1.1`) is a separate list that restarts by a fixed setting (reported `render-docx-static-restart`). + - A level of the heading list whose text names levels around ones nothing numbers (part B of the check document: levels + 3 to 5 unused, the nested level `%1.%2(%6)`): the nested items read `1.1(a)`, `1.1(b)`, and `1.2(a)` under the next + item, never `1.1.0.0.0(a)` and never continuing after a new item. This is the layout of a `{path}` label under a + clause. + - A list in an ordered item that shows the path of its item (the style `enumeration.levels` with `{path}` labels) is on + detached levels: Word draws the item's own counter alone (`(A)`, not `1(a)(1)(i)(A)`; the render reported + `render-docx-section-label` with what Word draws), and the references read as the HTML reads them. A second + such list under a new parent restarts at `(A)` by a fixed setting; a list made new in Word by copying an item + of another goes on counting (the documented limit). + - Items nested past nine levels continue in a further list: the tenth level restarts at `(a)` by a fixed setting and its + own items restart by structure; the references to them read as the HTML reads them. - In `template.continental.docx` a later alternative (a paragraph, clause or heading that shares a number: the second "Disputes", say) shows the first one's number: a `numId 0` paragraph that starts with a `REF \n \h` field, with the same hanging indent as a numbered one and its text after a tab. Update the field: it still reads the diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 8d75c95..2258cc7 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -20,7 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | -| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can; the continental `5.1` items and the numbered paragraphs `1.1` ride the heading list's next level, with the lists nested in them on its reserved levels, so they stay live; alternatives that share a number (§15.8) are `numId 0` paragraphs whose number is a `REF` field of the first, with the sections and lists under them on a num of their own: live for edits elsewhere, but **deleting the first alternative in Word breaks them and every reference to the shared identifier** (reported; render with the condition values for a final document); the built-in continental style uses the heading list's levels 6 and 7 for lists nested in its clauses, and a third level of nesting under a clause restarts by a fixed setting, reported), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference for which Word's fields cannot reproduce the designation (a heading level skipped before the target under a `{n}`-style scheme, text around the counter that Word's `\t` switch cannot drop such as `(a)`, `{n}:` or `No. {n}`) is fixed text, reported. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says. **The front and back matter are written too**, in the order the HTML shows them: the title block (header rows and parties), a live table of contents (`TOC \o "1-N" \h \z \u` with cached entries equal to the HTML contents, each a link with a `PAGEREF` page number Word fills in on update; the attachments are listed by outline level, so that no locale-dependent style list is in the field; at `contents.depth: 1` the attachment titles are listed as first-level entries, where the HTML indents them one level more), the attachments (bookmarked so `{{attach:}}` resolves, separated by a page break or a rule per `attachments.separator`) and the signature blocks (§3.6, borderless tables kept together on one page as far as one page allows). A LibreOffice round trip shows that after its indexes are updated the contents lists exactly the HTML entries with page numbers, also after a heading is inserted in the editor, and that the document reads in the plain-text writer's order. Remaining differences are the reported ones: a label that shows the section's number that the heading list cannot show (an outline scheme's `{section}.{n}`, the `none` scheme) is drawn without it. The contents show the shared number for a later alternative. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that needs a number Word cannot show there), `render-docx-list-depth` (nested past Word's nine levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a unit that shares a counter with others in Word, such as a sub-heading and the items of its section that have the same designation in every output format, and that is restarted at a fixed value, which will not follow edits), `render-docx-number-mismatch` (any other difference: a catch-all that does not occur outside documents that skip a heading level), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-alternative` (alternatives share a number as fields of the first, which must not be deleted in Word), `render-docx-static-restart` (a list that restarts by a fixed setting: in a list of another kind, in an item on the heading list that it cannot ride, in an alternative) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). The Word-only checks (the contents' page numbers, what `\t` does, numbering after an insertion) are a manual checklist before a release (`.github/PUBLISHING.md`). HTML is the format this renderer claims §13.6 with | +| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can; the continental `5.1` items and the numbered paragraphs `1.1` ride the heading list's next level, with the lists nested in them on its reserved levels, so they stay live; alternatives that share a number (§15.8) are `numId 0` paragraphs whose number is a `REF` field of the first, with the sections and lists under them on a num of their own: live for edits elsewhere, but **deleting the first alternative in Word breaks them and every reference to the shared identifier** (reported; render with the condition values for a final document); the built-in continental style uses the heading list's levels 6 and 7 for lists nested in its clauses, a label that shows the path of an item keeps its section's, clause's and own counters on those levels, and a list nested where Word cannot show the counters above it, or in a level the heading list has no room for, is drawn with its own counter alone on a detached list and restarts by a fixed setting, reported; a list nested past Word's nine levels goes on in further lists, up to 36 levels), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference is fixed text, reported, only where Word's fields cannot reproduce the designation: a number with text or punctuation around its counter that the reference does not have (`(a)`, `{n}:`, `No. {n}`) where Word's `\t` switch is not documented to leave it out (`docx.reference_text_switch`: `off`, `documented`, `guess`), and a number that holds the counters of other units the reference writes itself (`{section}.{n}` under a ref `{n}`); the counter of a heading level that the document skips (`heading-skip`, an Error) is text between the fields of the units that hold theirs; the words of `references.format` that a number repeats (`Article I`) are returned by the field itself. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says. **The front and back matter are written too**, in the order the HTML shows them: the title block (header rows and parties), a live table of contents (`TOC \o "1-N" \h \z \u` with cached entries equal to the HTML contents, each a link with a `PAGEREF` page number Word fills in on update; the attachments are listed by outline level, so that no locale-dependent style list is in the field; at `contents.depth: 1` the attachment titles are listed as first-level entries, where the HTML indents them one level more), the attachments (bookmarked so `{{attach:}}` resolves, separated by a page break or a rule per `attachments.separator`) and the signature blocks (§3.6, borderless tables kept together on one page as far as one page allows). A LibreOffice round trip shows that after its indexes are updated the contents lists exactly the HTML entries with page numbers, also after a heading is inserted in the editor, and that the document reads in the plain-text writer's order. Remaining differences are the reported ones: a label that shows the section's number that the heading list cannot show (an outline scheme's `{section}.{n}`, the `none` scheme) is drawn without it. The contents show the shared number for a later alternative. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that shows a number Word cannot show there: the section's, or the enclosing items' on a list that cannot carry them), `render-docx-list-depth` (nested past 36 levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a unit that shares a counter with others in Word, such as a sub-heading and the items of its section that have the same designation in every output format, and that is restarted at a fixed value, which will not follow edits), `render-docx-number-mismatch` (any other difference: a catch-all that does not occur outside documents that skip a heading level), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits, with the class it falls in; one that has the number of a skipped heading level as text; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-alternative` (alternatives share a number as fields of the first, which must not be deleted in Word), `render-docx-static-restart` (a list that restarts by a fixed setting: in a list of another kind, in an item on the heading list that it cannot ride, in an alternative, or the one that begins the next list past nine levels) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). The Word-only checks (the contents' page numbers, what `\t` does, numbering after an insertion) are a manual checklist before a release (`.github/PUBLISHING.md`). HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | diff --git a/README.md b/README.md index 3f9fb76..e0b0d24 100644 --- a/README.md +++ b/README.md @@ -150,8 +150,9 @@ output. **Numbering and cross-references are live:** headings, list items and nu numbering definitions (headings by the heading styles, nested lists restarting by structure), and a `{{ref: clause}}` is `REF` fields on bookmarks whose cached results are the text the plain-text writer prints, so "1(b)(ii)" or "I.A.1" stays right when a lawyer inserts a clause and updates fields. A defined term or an attachment -reference is an internal link. A reference that Word's fields cannot reproduce is written as its text (it will not follow edits) and reported. Where Word cannot draw what the renderer does (letters past 26, a label that shows the -section's number where the heading list cannot show it, a number restarted by a fixed value, a list restarted by a fixed setting), the render +reference is an internal link. A reference is written as its text (it will not follow edits, and the render reports it) only where Word's fields cannot reproduce it: +a number with punctuation around its counter or with the counters of other units that the reference writes itself, and the number of a heading level the document skips (an Error); how far Word's `\t` switch is trusted to leave the words of a number out is the style's `docx.reference_text_switch`. Where Word cannot draw what the renderer does (letters past 26, a label that shows the +section's number or the counters of enclosing items where Word's list cannot show them, a number restarted by a fixed value, a list restarted by a fixed setting), the render reports it as a `render-docx-*` Warning or Info with the line, see [the conformance notes](https://github.com/ForLegalAI/legaldown-render/blob/main/CONFORMANCE.md). The body writer also lays out paragraphs, quotes, drafting notes, code, tables with a repeated header row, rules, links, blanks and diff --git a/docs/architecture.md b/docs/architecture.md index 7a58a0c..8b115b1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -278,7 +278,7 @@ ordered by line with the renderer's own) to the result's. The style check, `docx **What is planned.** Heading numbering (one multilevel abstractNum linked to the heading styles, the label templates turned into `lvlText`, the counters into `numFmt`), list families (enumerated, renumbered, bullets) with their nesting -and restarts, depth beyond Word's nine levels (clamped to level 9 in that level's format), and numbered paragraphs. +and restarts, depth beyond Word's nine levels (further abstractNums of the family, up to 36 levels), and numbered paragraphs. A level of a list shows only counters of its own list, which settles where the rest of the planner puts things: - **Labels that need the section** (`{section}`, or the `{path}` of an item or paragraph in a section: the continental @@ -289,15 +289,38 @@ A level of a list shows only counters of its own list, which settles where the r Such an item is no heading: it has no heading style, no outline level and no entry in the table of contents, and carries an indent of its own (`UnitPlan.ind`; a heading level's `w:ind` serves the heading). A list nested directly in an item that is on the heading list goes on the heading abstractNum's **reserved levels**, level `4 + d` for - nesting depth `d` of 1 to 4, defined on first use with the label's text without the section (`%6)`, `%7)`), on the - same num as its parent with no override: it restarts by structure under every new item and sibling lists go on - counting, as in any other list. A level may be shared by units whose definitions are equal and is kept for the first - that needs it otherwise; the level after the fifth heading level is kept for a level-5 section's units when the - document has one, and a nested list then falls back. The built-in continental style uses levels 6 and 7 (`a)`, `i)`); its third level of nesting under a clause shows the section's number and falls back (static, reported). What does not fit (an outline or mixed scheme, where the - level after the section's reads otherwise; the `none` scheme, where the section's number is its text; a nested - item whose label shows the section; items in a quote; ordered and bullet lists nested in such an item; depth past - 4) keeps the family's own abstractNum with the label written without the section and reported - (`render-docx-section-label`); the references still cover it. + nesting depth `d` of 1 to 4, defined on first use with the label as the resolver writes it, every counter in it + being a level of the heading list (`candidate`): a label that is only its own counter reads `%6)`, `%7)`; a `{path}` + label reads `%1.%2(%6)` (the section's counter at heading level 0, the clause at level 1 where it is, the list's own + at level 5), and the items between at their own reserved levels (`%1.%2(%6)(%7)`). The list is on the same num as its + parent with no override: it restarts by structure under every new item and sibling lists go on counting, as in any + other list, and the label and every reference to it are live. A level may be shared by units whose definitions are + equal and is kept for the first that needs it otherwise (the level after the fifth heading level is kept for a level-5 + section's units when the document has one); a label that shows counters which are not the end of its own designation + (`{section}.{n}` at depth 3 of the continental style, under a ref `.{n}`: the designation ends in `.1`, the number + begins with the section's counter) does not take a reserved level, since no field could stand for it. What does not + fit (an outline or mixed scheme, where the level after the section's reads otherwise; the `none` scheme, where the + section's number is its text; a label whose reserved level another text has taken, depth past 4, items in a quote, + ordered and bullet lists nested in such an item) keeps the family's own abstractNum, or the **detached** one when the + enumerated level would have to show a counter of a unit that is not on it (below). A label that shows a section's + number and has no place for it is written without it and reported (`render-docx-section-label`); the references + still cover it. +- **Detached lists.** Where the enumerated level a nested list would take names a level that no unit of its chain holds + (an enumerated item below an ordered item: its `{path}` label names the ordered item's counter, which is on another + abstractNum; a quote's list in a preamble item; a path label whose reserved level is taken or past depth 4), Word + would draw `0`, or the counter some other list counted last. The list goes on the **detached** abstractNum instead: + the enumerated levels with each level's own counter alone (`{path}` shows its own `ref`), used for exactly that + case, and for every list nested in one of its items (so those restart by structure). The label Word draws is shorter + than the renderer's (`(A)` for `1(a)(1)(i)(A)`), `render-docx-section-label` names the units whose counters it + leaves out and what Word draws, and a reference is still made of fields, each unit's own counter. A detached list + restarts under a new parent at a fixed value, like any list nested in a list of another kind (`render-docx-static-restart`): + a list made new in Word by copying an item of another goes on counting where the renderer would start again. +- **Lists nested past Word's nine levels.** Word has nine levels in a list. A list nested deeper goes on in the next + abstractNum of its family (`enumerated+1` to `enumerated+3`, `renumber+1` to `renumber+3`: 36 levels in all), whose + level `j` is the family's level of depth `9k + j` (format and label with its own counter alone, indent stepping a + centimetre per level up to thirteen). Inside such an abstractNum the lists restart by structure; the list that + begins one restarts at a fixed value (`restart-depth`, reported as information). Only a nesting past 36 levels + uses the last level again (`clamped`, `render-docx-list-depth`). - **Alternatives that share a number** (§15.8). A unit that shares its number with the one before it and that Word would number differently (the live number would continue) is a `numId 0` paragraph with its own indent that starts with a `REF \n \h` field of the unit that holds the number now, then the label's text beyond what the field returns @@ -319,12 +342,12 @@ A level of a list shows only counters of its own list, which settles where the r higher if it is deleted; a restarted clause or paragraph does not follow what is inserted or deleted before it. It repairs a sub-heading after the items and paragraphs of its section on the heading list (`1.1` for the sub-heading and for the first item share a counter in Word and a designation in every output format: `render-docx-shared-counter` says so), a list that - resumes a sibling's count after another list in the same item, and a list past level 9. + resumes a sibling's count after another list in the same item. A list nested in an item that has an indent of its own (`UnitPlan.ind`) is indented one step in from it, on whatever -list it is. A label that shows a counter of a unit outside the unit's own chain (a quote's list inside an item of the preamble, -whose label starts from the item; a list nested in another kind of list) is written with the counters Word has there -and reported. Nothing is left to be shown by Word differently without a diagnostic. +list it is. A label that shows a counter of a unit outside the unit's own chain (a list nested in another kind of +list; a quote's list) is written with the counters Word has there (the detached levels, above) and reported where it +differs from the renderer's. Nothing is left to be shown by Word differently without a diagnostic. **Cross-references: the designation cover** (`resolve/docx_cover.py`, §6.3, §13.3). The renderer's text of a `{{ref:}}` is the designation of the target (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)`) inside `references.format`; the planner @@ -337,23 +360,40 @@ simulator gives as `Step.word_ref`), and fits where its counters belong to the s `{path}` label (`1.1.1`) is covered by one field on the target, a `{n}` label (`I.A.1`) by one field per level with literal delimiters between (`I` `.` `A` `.` `1`), a nested item (`(b)(ii)`) by one field per level, an ordered list in a section by `REF section` + `(` + `REF item` + `)`. Literal text may stand only for delimiters, never for a -counter. The cover with the fewest fields wins, then the fewest using `\t`, then the fewest literal characters; -the text of `references.format` is literal around it, and with two `{designation}` the same cover is written twice. -`\t` (Word's switch that suppresses the words around a counter; LibreOffice ignores it) is used only where it leaves -the bare counter (`Article I` gives `I`, `§ 3` gives `3`), never where the field's text holds a delimiter -(`(a)`, `a)`, `1.2`, and also a period: `No. 1`, so `No. {n}` has no `\t` form; this is conservative, because -what Word does with a delimiter is not known, and is on the manual Word checklist). Each field's cached result is -**Word's** value (the simulator's), and `cached_lo` what LibreOffice shows. A unit that a field points at has a +counter. What is covered is the **whole text of the reference**, the designation inside the text of `references.format` +(with two `{designation}` the designation is there twice): a field may stand for the words of the format when the number +it returns holds the same words, so `Article {designation}` over a label `Article {n}` is one plain `REF` (`Article I`), +`§ {designation}` over `§ {n}` another, and no `\t` is needed. The cover with the fewest fields wins, then the fewest +using `\t`, then the fewest literal characters. `\t` (Word's switch that suppresses the words around a counter; +LibreOffice ignores it) is used where the number holds words the reference does not (`Article I` for a format `Art. ` +gives `I`, `§ 3` gives `3`; Microsoft's own example is `Section 1.01`, which gives `1.01`), as far as the style's +`docx.reference_text_switch` says Word may be expected to do it: `documented` (the default) drops the words before the +first counter and after the last, keeps a delimiter between counters, and does not use the switch for a number with a +delimiter around its counters (`(a)`, `a)`, `1:`, `No. 1`), because what Word does with it is not documented; `off` +never uses it; `guess` also drops a period after the letters that lead a number (`No. 1`, `Čl. 1`). What Word does is +on the manual Word checklist (`.github/PUBLISHING.md`). Each field's cached result is **Word's** value (the simulator's), +and `cached_lo` what LibreOffice shows. A unit that a field points at has a bookmark (its anchor's, or a hidden one); a reference to an item or paragraph the style does not number is the designation of its section (`ref-not-enumerated`), covered as one. Under the `none` scheme a heading is a field for the bookmarked text (`REF bookmark \h`) and the item's pieces follow after a space; what the field returns is what the heading's runs show (`_shown`), which can differ from the renderer's designation by white space, and then the reference is reported. A `RefPlan` is made once per target, keyed by `CrossRef.target_unit`; `exact` says that the -cached results read as the renderer's text. **When no cover exists** (a heading level skipped before the target under -a `{n}`-style scheme, text around a counter that `\t` cannot drop) the reference is the renderer's text with no field -(`RefPlan.fixed`): it reads right when the file is opened and does not follow edits, and `render-docx-reference` says -so (one finding for the references that fail alike, naming the first and how many more). Nothing that Word could -compute is written as literal text otherwise. +cached results read as the renderer's text. **What stays text.** A reference is fixed text (`RefPlan.fixed`: it reads right when the file is opened and does not +follow edits, and `render-docx-reference` says so, one finding for the references that fail alike, naming the first and +how many more) only where Word cannot express it, and the finding says which of the classes it is, so that the style +can be changed if the lawyer wants the reference live: **delimiters** (the number Word draws has text or punctuation +around its counter that the reference does not have, such as a label `({n})` under a ref `{n}`, `{n}:` or `No. {n}`, +and `\t` is off or not documented to drop it; `docx.reference_text_switch: guess` takes a period after letters), and +**counters** (the number holds the counters of other units that the reference writes itself before it, such as a label +`{section}.{n}` under a ref `{n}`: a field returns the whole number or none of it; one finding for them all, since the +cure is the style's label). Besides, **the counter of a heading level the document skips** (`heading-skip`, §4.1: an +Error the renderer still numbers) is held by no unit and no field returns it, so that part of the reference, and only +that part, is literal text (`RefPlan.ghost`, `render-docx-reference` names it); the rest is fields. The cover treats a +`Ghost` counter, never a stale counter of another level, as literal, and the Word value assumed for the cached results +there is LibreOffice's reading of a skipped level (its start value); Word itself is expected to read it as one less +(the checklist). A heading's title that refers back to itself under the `none` scheme is the third case +(`render-docx-reference-cycle`). Nothing else that Word could compute is written as literal text; the built-in styles +have no fixed reference in a document without a skipped heading level, whatever its nesting. **Bookmarks** (`resolve/docx_bookmarks.py`). The planner names bookmarks and the writer gives them ids. Every anchor of the tree has one, referenced or not: `ld_` for a section, list item or paragraph, `lddef_` for a @@ -383,20 +423,23 @@ a rendering of the renderer's), the separator the level has (`suff`: tab or spac title, the condition label, the `_Toc` bookmark (also `UnitPlan.toc_bookmark`) and an empty cached page number. The complex `TOC` field that uses them is written with the title block. -**Diagnostics** (all `render-docx-*`, with the line of the unit or of the reference, once per finding): `section-label`, -`list-depth`, `letter-overflow` (Word repeats letters after 26, the renderer counts on: `aa`, `ab`), `roman-overflow` +**Diagnostics** (all `render-docx-*`, with the line of the unit or of the reference, once per finding): `section-label` +(a label that shows the number of a unit Word cannot show in that list: the section's on a list that is not the heading +list's, or, for a list on the detached levels or past nine levels, the counters of the enclosing units, named, with what +Word draws), `list-depth` (a nesting past 36 levels), `letter-overflow` (Word repeats letters after 26, the renderer counts on: `aa`, `ab`), `roman-overflow` (4000 and beyond), `shared-counter` (a unit that goes on a counter of its abstractNum that others have used, so that it is restarted at its number, a fixed value that will not follow edits before it: a sub-heading that shares a designation with the items of its section, a list that goes on from another, a list whose level another used in between), `number-mismatch` (any other difference: the first unit of a run that Word counts off by the same amount, with how many follow; it is the catch-all of the planner and does not occur outside documents that skip a heading level), `reference` (a -cross-reference that cannot be built from fields, because a heading level is skipped before the target with a -`{n}`-style scheme, or Word draws text around a counter that `\t` cannot drop (`({n})`, `{n}:`, `No. {n}`): it is -written as fixed text; or one that reads differently in Word for a reason that no diagnostic of a unit already -gives; references that fail alike are one finding), `reference-cycle` (a heading's title holds a reference that +cross-reference that is fixed text, because Word draws delimiters around a counter that the reference does not have +and `\t` is not allowed to drop them, or the number holds counters the reference writes itself; one that has the number +of a skipped heading level as text; or one that reads differently in Word for a reason that no diagnostic of a unit +already gives; references that fail alike are one finding), `reference-cycle` (a heading's title holds a reference that leads back to it: written as text), and, as information, `alternative` (alternatives share a number as fields of the first, which deleting in Word breaks), `static-restart` (a list that restarts at 1 by a fixed setting: nested in a list of another kind, in an item whose number is on the heading list but which the list -cannot ride, or in an alternative that shares a number) and `reference-word-only` (a cover relies on `\t`, which LibreOffice ignores, once per document). A +cannot ride, in an alternative that shares a number, or the list that begins the next abstractNum past Word's nine +levels) and `reference-word-only` (a cover relies on `\t`, which LibreOffice ignores, once per document). A heading level that is skipped is the Core Error `heading-skip`; the plan only notes it on the unit. ### 5. Write (`writers/`) @@ -636,7 +679,7 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_values.py` | Money, dates and durations in a locale: exact digits, plural of the value shown, currency display, CLDR fallback to English | | `tests/test_numbering.py` | Counter formats, label templates and the preamble's `{section}` | | `tests/test_numbering_structure.py` | The numbering structure in the tree (`formats`, unit ids, numbers, parents, alternatives, list families, cross-reference units): headings in each scheme and with skipped levels, nested lists, section-qualified items, numbered paragraphs, alternatives, the `none` scheme and its second pass; and that every label, designation and reference can be written again from the structure alone, over the tests' and the specification's documents | -| `tests/test_docx_plan.py` | The DOCX plan: abstractNums per style and scheme, heading levels, top-level, nested, sibling, cross-family, quoted and clamped lists, numbered paragraphs, labels that need the section and where they go (the heading list's levels, the reserved levels of lists nested in them, the fallback), alternatives that share a number (of sections, items and paragraphs, in twos and threes, with children), resynchronised units, letter and Roman overflow, diagnostics (lines, file, order, repeats, the catch-all that groups runs of units counted off), style errors (also through `docx_settings`), determinism; and, unit by unit, that Word's simulated count and label are the resolver's unless the unit's own counter reads differently in Word or the plan notes why, over the tests' and the specification's documents under each style and list setting, and over random documents and alternatives (each variant plans its own sample, and every unit is referred to by some document). Also the cross-references: the worked table of covers for each style (default, outline, mixed, continental, `none`, `Article {n}`, `§ {n}`), the cover algorithm on its own, references to other sections and to unnumbered units, `references.format`, every reason a reference cannot be covered, bookmark names (prefixes, length, collisions, hidden names, determinism), the table of contents, the components of the headings' graph of title references (and that a chain of thousands of headings is planned in linear time), the render's diagnostics, and the property that every cached reference reads as the renderer's text or a diagnostic says why not, over the same documents and random documents with an anchor on every unit | +| `tests/test_docx_plan.py` | The DOCX plan: abstractNums per style and scheme, heading levels, top-level, nested, sibling, cross-family, quoted lists, lists past Word's nine levels (the next abstractNums, the cap at 36 levels), numbered paragraphs, labels that need the section and where they go (the heading list's levels, the reserved levels of lists nested in them with the path in their text, the detached levels and their reports, the fallback), alternatives that share a number (of sections, items and paragraphs, in twos and threes, with children), resynchronised units, letter and Roman overflow, diagnostics (lines, file, order, repeats, the catch-all that groups runs of units counted off), style errors (also through `docx_settings`), determinism; and, unit by unit, that Word's simulated count and label are the resolver's unless the unit's own counter reads differently in Word or the plan notes why, over the tests' and the specification's documents under each style and list setting, and over random documents and alternatives (each variant plans its own sample, and every unit is referred to by some document). Also the cross-references: the worked table of covers for each style (default, outline, mixed, continental, `none`, `Article {n}`, `§ {n}`, `Čl. {n}`, `Section {path}`: a field standing for the words of `references.format`), the cover algorithm on its own (the literal counter of a skipped heading level, what `\t` is taken to do under each `docx.reference_text_switch`), references to other sections and to unnumbered units, `references.format`, every reason a reference stays fixed text (delimiters, counters of other units, a skipped heading level) and what the message tells the style, bookmark names (prefixes, length, collisions, hidden names, determinism), the table of contents, the components of the headings' graph of title references (and that a chain of thousands of headings is planned in linear time), the render's diagnostics, and the property that every cached reference reads as the renderer's text or a diagnostic says why not, over the same documents and random documents with an anchor on every unit; and that over every variant a reference is fixed text only in a document that skips a heading level, and, over label forms the built-in styles do not have, only for the two reasons the style can change | | `tests/test_docx_simulator.py` | Word's counters, label templates as tokens (equal to filling the template as text), `lvlText`, and the simulator: restarts, start overrides, shared counters, skipped levels, `peek`/`current` | | `tests/test_build.py` | Building the tree: comment-only items and quotes, emphasis next to a directive, spacing where an inline comment was dropped | | `tests/test_terms.py` | What `{{term:}}` shows, references inside terms, terms that name themselves or each other, the cap on a term text | @@ -654,13 +697,14 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_cli_io.py` | Encodings of the streams, input that is not UTF-8, unwritable output, LF line endings, the extension rule of `-o` | | `tests/test_errors.py` | The exception hierarchy and pickling of errors | | `tests/test_escaping.py` | Control characters from a document or the command line never reach a diagnostic or standard error unescaped | -| `tests/test_docx_style.py` | `page.header`, `page.footer` and `docx.*`: validation, and how CSS lengths, colours and fonts become Word's units, with the StyleErrors only DOCX has | +| `tests/test_docx_style.py` | `page.header`, `page.footer` and `docx.*` (`update_fields_on_open`, `reference_text_switch`): validation, and how CSS lengths, colours and fonts become Word's units, with the StyleErrors only DOCX has | | `tests/test_docx_package.py` | The OOXML package: parts, relationships, content types, settings order, styles, page setup, headers and footers, properties, the body of a planned document, determinism, the sanitiser and builders, the API and command line | | `tests/test_docx_body.py` | The DOCX body from hand-made plans: numbering definitions, `w:numPr`/`w:ind`/alternatives, bookmarks and `REF` fields with cached results, hyperlinks, every inline and block, escaping, schema validity (marker `ooxml_schema`) and a LibreOffice round trip of headings, nested lists and references whose cached values are wrong on purpose (marker `libreoffice`) | | `tests/test_docx_escaping.py` | Well-formedness of every part for any text, with a seeded fuzz and the characters XML cannot carry | -| `tests/test_docx_special.py` | What the planner's hard cases become in the file: the heading list's levels that hold items, the reserved levels of lists nested in them, and later alternatives as `numId 0`, `w:ind` and `REF` fields; and that documents with them validate against the schemas (marker `ooxml_schema`) | +| `tests/test_docx_special.py` | What the planner's hard cases become in the file: the heading list's levels that hold items, the reserved levels of lists nested in them, the detached and the next-nine-levels abstractNums, and later alternatives as `numId 0`, `w:ind` and `REF` fields; and that documents with them validate against the schemas (marker `ooxml_schema`) | | `tests/test_docx_validation.py` | Every part of every generated document against the ISO/IEC 29500-4 schemas (marker `ooxml_schema`; needs lxml and `python tests/schemas/fetch.py`) | -| `tests/test_docx_roundtrip.py` | LibreOffice as the oracle of the numbering and the cross-references, in one LibreOffice session: every document of the corpus (the tests', the specification's, stress documents with cycles and unanchored ancestors) under the default, outline, continental, mixed and `none` schemes, with every cached field result made wrong (`XXX`), every numbered paragraph marked and a line per reference added; the number LibreOffice draws, every reference it computes and every body paragraph's text equal what the file cached and the renderer wrote, modulo the gaps the plan reports (`\t` ignored, letters past 26, labels the heading list cannot show); alternatives are checked against the renderer (their `numId 0` number is the holder's, as fields), and so are the continental items on the heading list and the lists on its reserved levels; the table of contents is built by LibreOffice from a `TOC` field the test adds and must list the headings with the numbers the plan gives and none of the items; and edit-then-update (`tests/lo/uno_edit.py`, which inserts and deletes paragraphs): items, headings, a clause and a nested item inserted before the targets as a lawyer does, fields updated, every reference and the numbers equal the renderer's for the edited source (the new nested item under the continental style reads `a)`), and, for a sub-heading restarted by a fixed value, the plan's warning and what an edit before it keeps right (marker `libreoffice`) | +| `tests/test_docx_roundtrip.py` | LibreOffice as the oracle of the numbering and the cross-references, in one LibreOffice session: every document of the corpus (the tests', the specification's, stress documents with cycles and unanchored ancestors) under the default, outline, continental, mixed and `none` schemes, with every cached field result made wrong (`XXX`), every numbered paragraph marked and a line per reference added; the number LibreOffice draws, every reference it computes and every body paragraph's text equal what the file cached and the renderer wrote, modulo the gaps the plan reports (`\t` ignored, letters past 26, labels the heading list cannot show); alternatives are checked against the renderer (their `numId 0` number is the holder's, as fields), and so are the continental items on the heading list and the lists on its reserved levels; the table of contents is built by LibreOffice from a `TOC` field the test adds and must list the headings with the numbers the plan gives and none of the items; and edit-then-update (`tests/lo/uno_edit.py`, which inserts and deletes paragraphs, and can put an inserted paragraph on another item's list as pasting a copy does): path labels on the reserved levels, in a detached list (and the documented continuation of a list copied under a new parent) and past nine levels, a heading inserted before a document that skips a level, items, headings, a clause and a nested item inserted before the targets as a lawyer does, fields updated, every reference and the numbers equal the renderer's for the edited source (the new nested item under the continental style reads `a)`), and, for a sub-heading restarted by a fixed value, the plan's warning and what an edit before it keeps right (marker `libreoffice`) | | `tests/test_docx_matter.py` | The front and back matter: the XML of the title block, the `TOC` field and its cached entries, the attachments and the signature blocks; documents with every part (hostile text too) against the schemas; LibreOffice listing exactly the contents' entries after an update, resolving `{{attach:}}`, paging the attachments as configured, taking a heading inserted in the editor into the contents, and reading in the text writer's order | | `tests/test_docx_libreoffice.py` | The documents opened in LibreOffice through UNO (`tests/lo/uno_export.py`), fields updated: the words in order, header, footer, page count (marker `libreoffice`; skipped without `soffice` and `python3-uno`) | +| `tests/test_docx_wordcheck.py`, `tests/word_check.py` | The generator of the document for the manual Word checks of what the planner assumes (`python tests/word_check.py OUT.docx`; `.github/PUBLISHING.md` items 8 and 9): one row for each label form the plan relies on (`\t` on `Article %1`, `§ %1`, `Section %1.%2`, `Čl. %1`, `No. %1`, `(%1)`, `%1)`, `%1:`), the heading list's reserved level `%1.%2(%6)` with levels 3 to 5 unused, and a skipped heading level, each with the planner's assumption beside it taken from the planner's own code; that the package is valid (marker `ooxml_schema`) and LibreOffice computes the plain fields as the planner caches them (marker `libreoffice`) | | `tests/conformance/` | Every example and fixture in the specification repository renders, every internal link has a target, and each invalid fixture renders its rule's failure marker. Needs `$LEGALDOWN_SPEC_DIR` or a sibling `../LegalDown` checkout | diff --git a/docs/decisions/0008-docx-word-native.md b/docs/decisions/0008-docx-word-native.md index 95f8390..8ff68ee 100644 --- a/docs/decisions/0008-docx-word-native.md +++ b/docs/decisions/0008-docx-word-native.md @@ -84,9 +84,24 @@ A cross-reference is a `REF` field to a bookmark on the target, with the switch number, no trailing period) and `\h` (a link). The renderer's designation ("2.3(b)", "I.A.1.a(i)") is covered by segments: literals, and one such field for each numbered unit on the target's path whose label contributes. A `{path}` label needs one field on the target. An `{n}` label needs one field per -ancestor, joined by the literals the reference templates give. Only when no cover exists does the -planner fall back to one field on the target, with a warning. Under the `none` scheme a reference is -a `REF` to the heading text. +ancestor, joined by the literals the reference templates give. What is covered is the whole text the +reader sees (the designation inside `references.format`), so a field that returns the words of the format +(`Article I` under `Article {designation}`) stands for them, and Word's `\t` switch, which leaves the words of +a number out, is needed only where the number holds words the reference does not. What Word does with `\t` +is documented for one example only (`Section 1.01` gives `1.01`), so the style says how far to trust it +(`docx.reference_text_switch`: `off`, `documented`, `guess`). + +**A reference is fixed text only where Word cannot express it.** Each such reference is reported +(`render-docx-reference`) with the class it falls in, so that the style can be changed if it matters: +a number with **delimiters around its counter** that the reference does not have (a label `({n})` under a +ref `{n}`, `{n}:`, `No. {n}`) where the switch is off or not documented to drop them; a number with **the +counters of other units** that the reference writes itself (a label `{section}.{n}` under a ref `{n}`), +which a field returns whole or not at all; and, in a document that skips a heading level (`heading-skip`, an +Error), **the counter of the skipped level**, held by no unit, which is text between the fields of the units that +do hold theirs. A heading's title that refers back to itself under the `none` scheme is the other case +(`render-docx-reference-cycle`). The built-in styles have none of these in a document without errors, and a +reference to an item in a list nested as deep as the document likes is made of fields. +Under the `none` scheme a reference is a `REF` to the heading text. ### Numbering the hard cases @@ -95,8 +110,21 @@ a `REF` to the heading text. no heading style, no outline level and so no entry in the table of contents. A list nested in one goes on the heading list's levels 6 to 9 (headings have five), which are reserved for that: the level text is the label's without the section (`%6)`), on the parent's num with no override, so the list restarts under every - new item and sibling lists go on counting, as in any list. Where the level after the section's reads - otherwise (the outline and mixed schemes, `none`), the label is written without the section and reported. + new item and sibling lists go on counting, as in any list. When a nested list's label shows more than its + own counter (a `{path}`), the level text carries each counter at the level of the heading list that holds it + (`%1.%2(%6)`: the section's, the item's, its own), so the label and every reference to it stay live. Where the level after the + section's reads otherwise (the outline and mixed schemes, `none`), the label is written without the section and + reported. +- **Lists nested where Word cannot show the counters above them** (a list in an ordered item, a path label whose + reserved level another text has taken or that is nested past four levels, a quote's list) go on a **detached** + abstractNum: each level shows its own counter alone. The label Word draws is shorter than the renderer's + (`(A)` for `1(a)(1)(i)(A)`) and is reported with the units whose counters it leaves out; references are still + fields of each unit's own counter. The list restarts at a fixed value under a new parent (as any list nested + in a list of another kind does), and lists nested in its items restart by structure. +- **Lists nested past Word's nine levels** go on in the next abstractNum of their family (36 levels in all), whose levels are + the family's levels of those depths with their own counters, instead of repeating level 9. The list that begins + one restarts at a fixed value; inside it the lists restart by structure. Beyond 36 levels the last is used again + and reported (`render-docx-list-depth`). - **Alternatives that share a number** (§15.8). Word cannot give two paragraphs one number on one level, except by `numId 0` and a field: the later alternative is such a paragraph, with a `REF \n` field of the unit that holds the number as its number. The units under it restart in a num of their own with a `startOverride`. This is @@ -136,7 +164,8 @@ a `REF` to the heading text. - A DOCX lawyers can edit: insert a clause, update fields (Word does it on open by default), and the numbers, references, table of contents and page numbers follow. - Some restarts cannot be kept live in Word and are documented and warned about, not prevented: a - nested list of another family than its parent, a list nested in an alternative, a unit restarted at a fixed + nested list of another family than its parent (and any detached list), a list nested in an alternative, the list + that begins a further abstractNum past nine levels, a unit restarted at a fixed value because it shares a counter, and paragraph numbering in a section the user adds; and the numbers of the alternatives that share one live only as long as the first alternative stays. A lawyer who adds a parent item restarts such a list by hand. @@ -148,5 +177,5 @@ a `REF` to the heading text. `lvlRestart`, `SEQ \s` and `w:updateFields`, so those differences are normalised in the comparison. - Cost: an OOXML writer of our own, with its schema checks, instead of a library. The numbering planner is the bulk of the work. -- Style template additions: `page.header`, `page.footer` and `docx.update_fields_on_open` - ([style-templates.md](../style-templates.md)). The HTML writer does not read the first two yet. +- Style template additions: `page.header`, `page.footer`, `docx.update_fields_on_open` and + `docx.reference_text_switch` ([style-templates.md](../style-templates.md)). The HTML writer does not read the first two yet. diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 4cc5b85..0f83e96 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -66,7 +66,7 @@ ) from ..urls import displayed_url, printable from .docx_bookmarks import ATTACHMENTS_HEADING, BookmarkNames -from .docx_cover import Field, Shown, TextSwitch, Title, atoms, cover, suppressed, text_of, uncovered +from .docx_cover import DELIMITERS, Field, Shown, TextSwitch, Title, atoms, cover, suppressed, text_of, uncovered from .docx_numbering import ( NUM_FMT, SAMPLE_COUNTER, @@ -589,12 +589,14 @@ class _Built(NamedTuple): class _Finding(NamedTuple): """A reference that is fixed text (*kind* ``fixed``, *reason* why), has the number of a skipped heading level as - fixed text (``ghost``) or reads differently in Word (``differs``).""" + fixed text (``ghost``) or reads differently in Word (``differs``). References with the same *kind* and *group* + are one finding.""" kind: str reason: str found: _Reference shown: str + group: str = "" class _Drift: @@ -730,16 +732,40 @@ def fields(self, designation: Sequence[Shown]) -> list[Field]: fields.append(Field(owner, True, True, bare)) return fields - def why_uncovered(self, designation: Sequence[Shown], fields: Sequence[Field]) -> str: - """Why no combination of *fields* and delimiters reads like *designation* (§13.1).""" + def why_uncovered(self, designation: Sequence[Shown], fields: Sequence[Field]) -> tuple[str, str]: + """Why no combination of *fields* and delimiters reads like *designation* (§13.1), and which references fail for + the same reason (so that they are one finding): every reference whose number carries the counters of other + units, and those whose number is drawn with the same shape (its counters as ``#``) around the counter. Two + reasons are told apart, since the style can answer each: + + * the number carries **delimiters** the reference leaves out (a label ``({n})`` under a ref ``{n}``, a prefix + ``No. ``): Word's ``\\t`` switch could drop them, but the style's ``docx.reference_text_switch`` does not take it + to (it is ``off``, or what Word does with them is not documented); + * the number carries the **counters of other units** that the reference writes itself before it (a label + ``{section}.{n}`` under a ref ``{n}``): Word returns the whole number or none of it.""" wanted = atoms(designation) _, owner = wanted[uncovered(designation, fields) or 0] for candidate in fields: - if candidate.number and any(isinstance(t, Count) and t.owner == owner for t in candidate.tokens): - return (f"Word draws its number as '{printable(text_of(candidate.tokens))}', with text or punctuation " - f"around the counter that the designation does not have and that Word's \\t switch cannot " - f"leave out") - return "no combination of Word's fields reads like it" + plain = candidate.number and not candidate.suppress_text + if not (plain and any(isinstance(t, Count) and t.owner == owner for t in candidate.tokens)): + continue + drawn = printable(text_of(candidate.tokens)) + shape = "".join("#" if isinstance(t, Count) else t.text for t in candidate.tokens) + if any(isinstance(t, Count) and t.owner != owner for t in candidate.tokens): + return (f"Word draws its number as '{drawn}', which holds the counters of other units that the reference " + f"writes itself before it (the section's, the enclosing items'), and a field returns the whole " + f"number or none of it", "counters") + around = "".join(t.text for t in candidate.tokens if isinstance(t, Lit)) + if self.text_switch == "off": + how = "Word's \\t switch, which could leave it out, is off (docx.reference_text_switch)" + elif {c for c in around if c in DELIMITERS} == {"."}: + how = ("Word is not documented to leave a period after letters out with its \\t switch " + "(docx.reference_text_switch: guess expects it to)") + else: + how = "Word is not documented to leave such punctuation out with its \\t switch" + return (f"Word draws its number as '{drawn}', with text or punctuation around the counter that the " + f"reference does not have, and {how}", f"delimiters {shape}") + return "no combination of Word's fields reads like it", "other" def reference(self, found: _Reference) -> _Built | None: """The fields of the reference *found*: the whole text it shows (the designation inside the text of @@ -763,7 +789,8 @@ def reference(self, found: _Reference) -> _Built | None: whole.extend(designation) pieces = cover(whole, fields) if pieces is None: - self.unreferable.append(_Finding("fixed", self.why_uncovered(designation, fields), found, found.text)) + reason, group = self.why_uncovered(designation, fields) + self.unreferable.append(_Finding("fixed", reason, found, found.text, group)) return _Built([found.text] if found.text else [], True, None, True) ghost = self.skipped_part(whole, pieces) if ghost: @@ -800,14 +827,17 @@ def report_references(self) -> None: for a run of units counted off by the same amount.""" groups: dict[tuple[str, str], list[_Finding]] = {} for finding in self.unreferable: - groups.setdefault((finding.kind, finding.reason), []).append(finding) - for (kind, reason), members in groups.items(): + groups.setdefault((finding.kind, finding.group or finding.reason), []).append(finding) + for (kind, _), members in groups.items(): members.sort(key=lambda m: (m.found.line is None, m.found.line or 0)) first = members[0] + reason = first.reason text, shown = printable(first.found.text), printable(first.shown) - more = f" {len(members) - 1} more cross-references fail in the same way." if len(members) > 1 else "" + count = len(members) - 1 + many = f"{count} more cross-references" if count > 1 else "1 more cross-reference" + more = f" {many} {'fail' if count > 1 else 'fails'} in the same way." if count else "" if kind == "ghost": - more = f" {len(members) - 1} more cross-references are written the same way." if len(members) > 1 else "" + more = f" {many} {'are' if count > 1 else 'is'} written the same way." if count else "" message = (f"The cross-reference '{text}' holds the number of a heading level that the document skips " f"(a heading level must follow the one above, §4.1), which no Word number returns: " f"'{shown}' is written as text there and will not follow edits made in Word; the rest is " diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index 038266f..3f7cb8c 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -1677,15 +1677,90 @@ def test_documents_of_one_size_refer_to_different_units_so_that_more_of_them_are assert len(set().union(*members)) > max(len(refs) for refs in members) -@pytest.mark.parametrize("variant", ["default", "continental", "paragraphs", "mixed", "article"]) -def test_a_reference_is_fixed_text_only_where_a_heading_level_is_skipped_or_the_depth_is_past_word_s(variant: str) -> None: - """The cover of every reference exists for a document without errors and a list within Word's nine levels: what - cannot be covered are the references below a skipped heading level (a Core Error: the number of the level the - heading skips has no unit to be a field of) and below a list nested past nine levels.""" - for source, result, plan, _ in (*random_runs(variant), *alternative_runs(variant)): +@pytest.mark.parametrize("variant", VARIANTS) +def test_a_reference_is_fixed_text_only_in_a_document_that_skips_a_heading_level(variant: str) -> None: + """Every reference of a document without errors is made of fields and the delimiters between them, whatever the + numbering style and the depth of its lists: the nesting past Word's nine levels has levels of its own, a path label + is live, and the words of a label that the format repeats are one field. What cannot be a field is the number of a + heading level the document skips (a Core error: no unit holds it): it is text, reported, and in such a document + the counters of the units Word draws differently are text too.""" + for source, _, plan, _ in (*random_runs(variant), *alternative_runs(variant)): skipped = any("skipped-level" in unit.cause for unit in plan.units.values()) - if not skipped and max((lst.depth for lst in lists(result.tree)), default=0) <= 8: - assert not [t for t, r in plan.refs.items() if r.fixed], source + if not skipped: + assert not [t for t, r in plan.refs.items() if r.fixed or r.ghost], source + + +#: Label forms a style can write that the built-in ones do not (and the reason a reference to them may stay fixed text: +#: ``delimiters`` around the counter that Word's ``\\t`` switch is not documented to leave out, ``counters`` of other +#: units that the label shows and the reference writes itself, ``period`` a prefix like ``Čl. `` that only the switch's +#: ``guess`` setting leaves out; None: all live). +EXTRA_VARIANTS: dict[str, tuple[dict, str | None]] = { + "section-path-word": ({"overrides": {"numbering.levels": [ + {"counter": "upper-roman", "label": "Article {n}", "ref": "{n}"}, + {"counter": "decimal", "label": "Section {path}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "({n})", "ref": "({n})"}], "references.format": "Section {designation}"}}, None), + "path-all": ({"overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{path}", "ref": ".{n}"}, {"counter": "lower-alpha", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{path}", "ref": "({n})"}]}}, None), + "items-path-continental-sec": ({"style": "continental", "overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{path}", "ref": "({n})"}, + {"counter": "lower-roman", "label": "{path}", "ref": "({n})"}]}}, None), + "outline-items-with-section": ({"style": "outline", "overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": ".{n}"}, + {"counter": "lower-alpha", "label": "{n})", "ref": "({n})"}]}}, None), + "heading-path-slash": ({"overrides": {"numbering.levels": [ + {"counter": "decimal", "label": "{n}.", "ref": "{n}"}, {"counter": "decimal", "label": "{path}", "ref": "/{n}"}]}}, None), + "outline-paragraph-path": ({"style": "outline", "overrides": {"paragraphs.numbered": True, "paragraphs.format": { + "counter": "decimal", "label": "{path}", "ref": ".{n}"}}}, None), + "continental-paragraph-dash": ({"style": "continental", "overrides": {"paragraphs.numbered": True, "paragraphs.format": { + "counter": "decimal", "label": "{section}-{n}", "ref": "-{n}"}}}, None), + "parenthesis-label-bare-ref": ({"overrides": {"enumeration.levels": [ + {"counter": "lower-alpha", "label": "({n})", "ref": "{n}"}, {"counter": "lower-roman", "label": "({n})", "ref": "{n}"}, + {"counter": "upper-alpha", "label": "({n})", "ref": "{n}"}]}}, "delimiters"), + "colon-label": ({"overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{n}:", "ref": "{n}"}, {"counter": "lower-alpha", "label": "{n}:", "ref": "{n}"}]}}, + "delimiters"), + "section-in-label-not-in-ref": ({"overrides": {"enumeration.levels": [ + {"counter": "decimal", "label": "{section}.{n}", "ref": "{n}"}, {"counter": "lower-alpha", "label": "({n})", + "ref": "({n})"}]}}, "counters"), + "abbreviation-with-a-period": ({"overrides": {"numbering.levels": [ + {"counter": "decimal", "label": "Čl. {n}", "ref": "{n}"}, {"counter": "decimal", "label": "{path}", "ref": ".{n}"}], + "references.format": "čl. {designation}"}}, "period"), +} + + +@functools.cache +def extra_runs(name: str, switch: str) -> tuple: + """(source, plan, diagnostics) of a sample of the random documents without a skipped heading level, under the label + forms of :data:`EXTRA_VARIANTS` and a setting of the switch.""" + runs = [] + for source in (*sample(name, 4, _random_sources()), *sample(name, 2, _alternative_sources())): + prefix = "" if source.startswith("---") else FRONT + result = render(prefix + source, format="text", **EXTRA_VARIANTS[name][0]) + plan, diagnostics = plan_docx(result.tree, text_switch=switch) # type: ignore[arg-type] + if not any("skipped-level" in unit.cause for unit in plan.units.values()): + runs.append((source, plan, diagnostics)) + return tuple(runs) + + +@pytest.mark.parametrize("name", EXTRA_VARIANTS) +def test_every_label_form_is_live_except_where_the_reason_a_reference_stays_fixed_says_so(name: str) -> None: + reason = EXTRA_VARIANTS[name][1] + runs = extra_runs(name, "documented") + assert runs + fixed = sum(r.fixed for _, plan, _ in runs for r in plan.refs.values()) + if reason is None: + assert fixed == 0 + return + assert fixed > 0 + findings = [d.message for _, _, ds in runs for d in ds if d.rule == "render-docx-reference" and "not a field" in d.message] + phrase = {"delimiters": "text or punctuation around the counter", "counters": "holds the counters", + "period": "a period after letters"}[reason] + assert findings and all(phrase in m for m in findings) + # the setting that guesses is the one that lets a prefix with a period be left out, and no other reason is its business + guessed = sum(r.fixed for _, plan, _ in extra_runs(name, "guess") for r in plan.refs.values()) + assert (guessed == 0) == (reason == "period") def test_each_variant_plans_its_own_sample_of_the_random_documents() -> None: @@ -2229,6 +2304,56 @@ def test_references_that_fail_in_the_same_way_are_one_finding_naming_the_first_a assert found[0].line == tree.sections[-1].blocks[-1].line # the lowest line of the three: all in one paragraph +def test_references_that_are_fixed_for_the_same_reason_are_one_finding() -> None: + levels = [{"counter": "lower-alpha", "label": "({n})", "ref": "{n}"}] + tree, plan, diagnostics = planned("# A\n\n- x {#x}\n- y {#y}\n- z {#z}\n\n{{ref: x}} {{ref: y}} {{ref: z}}\n", + overrides={"enumeration.levels": levels}) + [only] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert only.message.endswith(" 2 more cross-references fail in the same way.") + assert all(r.fixed for r in plan.refs.values()) and not any(r.ghost for r in plan.refs.values()) + tree, plan, diagnostics = planned("# A\n\n- x {#x}\n- y {#y}\n\n{{ref: x}} {{ref: y}}\n", + overrides={"enumeration.levels": levels}) + [only] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert only.message.endswith(" 1 more cross-reference fails in the same way.") + + +@pytest.mark.parametrize(("levels", "switch", "phrases"), [ + # delimiters in the label that the reference leaves out + ([{"counter": "lower-alpha", "label": "({n})", "ref": "{n}"}], "documented", + ["Word draws its number as '(a)'", "text or punctuation around the counter that the reference does not have", + "Word is not documented to leave such punctuation out with its \\t switch"]), + ([{"counter": "decimal", "label": "{n}:", "ref": "{n}"}], "guess", + ["Word draws its number as '1:'", "not documented to leave such punctuation out"]), + # an abbreviation with a period: the setting that guesses leaves it out + ([{"counter": "decimal", "label": "No. {n}", "ref": "{n}"}], "documented", + ["Word draws its number as 'No. 1'", "not documented to leave a period after letters out", + "docx.reference_text_switch: guess expects it to"]), + ([{"counter": "decimal", "label": "No. {n}", "ref": "{n}"}], "off", + ["Word draws its number as 'No. 1'", "Word's \\t switch, which could leave it out, is off (docx.reference_text_switch)"]), + # a label that carries the counter of the section, which the reference writes itself + ([{"counter": "decimal", "label": "{section}.{n}", "ref": "{n}"}], "documented", + ["Word draws its number as '1.1'", "holds the counters of other units that the reference writes itself before it", + "a field returns the whole number or none of it"]), +]) +def test_the_reason_a_reference_is_fixed_names_what_the_style_could_change( + levels: list, switch: str, phrases: list[str]) -> None: + result = render(FRONT + "# A\n\n- x {#x}\n\nSee {{ref: x}}.\n", format="text", overrides={"enumeration.levels": levels}) + plan, diagnostics = plan_docx(result.tree, text_switch=switch) # type: ignore[arg-type] + [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert all(phrase in found.message for phrase in phrases), found.message + assert "written as the text" in found.message and "not a field" in found.message + assert all(r.fixed for r in plan.refs.values()) + + +def test_every_reference_that_carries_the_counters_of_other_units_is_one_finding_whatever_its_shape() -> None: + levels = [{"counter": "decimal", "label": "{section}.{n}", "ref": "{n}"}] + body = "# A\n\n- x {#x}\n\n## B\n\n- y {#y}\n\nSee {{ref: x}} {{ref: y}}.\n" + _, plan, diagnostics = planned(body, overrides={"enumeration.levels": levels}) + found = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert len(found) == 1 and found[0].message.endswith(" 1 more cross-reference fails in the same way.") + assert all(r.fixed for r in plan.refs.values()) + + # -- The cover on its own ---------------------------------------------------------------------------------- @@ -2542,7 +2667,8 @@ def assert_references_as_resolved(result, plan, diagnostics) -> None: # noqa: A # Every fixed text and every other difference is in a finding, named or counted among "N more". counted = 0 for finding in reported: - more = re.search(r" (\d+) more cross-references (?:fail in|are written) the same way\.$", finding.message) + more = re.search(r" (\d+) more cross-references? (?:fails? in|are written|is written) the same way\.$", + finding.message) counted += 1 + (int(more.group(1)) if more else 0) assert needing_a_finding + fixed + ghosts <= counted <= len(texts) if fixed: diff --git a/tests/test_docx_special.py b/tests/test_docx_special.py index a911d2b..c2f85ed 100644 --- a/tests/test_docx_special.py +++ b/tests/test_docx_special.py @@ -1,5 +1,6 @@ """What the planner's hard cases become in the file (docs/architecture.md, "The DOCX planner"): the heading list's levels -that hold items, the reserved levels of the lists nested in them, and the later alternatives that share a number, as +that hold items, the reserved levels of the lists nested in them (with the path of the item in their text), the detached +levels, the abstractNums that go on past Word's nine levels, and the later alternatives that share a number, as numbering definitions, ``w:numPr``, ``w:ind`` and ``REF`` fields; and that every document that has them validates against the ISO/IEC 29500-4 schemas (marker ``ooxml_schema``). The numbers as Word's simulator and LibreOffice draw them are checked in test_docx_plan.py and test_docx_roundtrip.py.""" @@ -11,7 +12,7 @@ import pytest from conftest import FRONT from docx_support import W, parts, require_schemas, validation_errors -from test_docx_plan import ALTERNATIVES +from test_docx_plan import ALTERNATIVES, ORDERED_PARENT, PATH_ITEMS from test_docx_roundtrip import STRESS from legaldown_render import render @@ -150,3 +151,56 @@ def test_documents_with_alternatives_and_items_on_the_heading_list_validate(name result = render(STRESS[name], format="docx", **settings) assert result.binary is not None assert validation_errors(result.binary) == [] + + +def lvl_texts(numbering: ET.Element, abstract_id: int) -> list[str]: + return [lvl.find(f"{W}lvlText").get(f"{W}val") for lvl in levels(numbering, abstract_id)] # type: ignore[misc, union-attr] + + +def test_a_path_label_on_the_reserved_levels_is_a_level_text_with_the_counters_of_its_chain() -> None: + source = FRONT + "# A\n\n- one\n - two {#two}\n - three {#three}\n- uno\n - dos\n" + parts_ = document(source, **PATH_ITEMS) + root, numbering = parts_["word/document.xml"], parts_["word/numbering.xml"] + assert lvl_texts(numbering, 0)[5:7] == ["%1.%2(%6)", "%1.%2(%6)(%7)"] + assert [numbering_of(paragraph_with(root, t)) for t in ("two", "three", "dos")] == [(5, 2), (6, 2), (5, 2)] + # the parent's num, no override of its own, and no abstractNum besides the heading list's + assert len(numbering.findall(f"{W}abstractNum")) == 1 + assert [n.get(f"{W}numId") for n in numbering.findall(f"{W}num")] == ["1", "2"] + + +def test_a_detached_abstract_num_holds_each_levels_own_counter_and_a_list_in_it_has_a_start_override() -> None: + source = FRONT + "# A\n\n- one\n 1. ordered\n - three\n - four {#four}\n - four b\n" + parts_ = document(source, **ORDERED_PARENT) + root, numbering = parts_["word/document.xml"], parts_["word/numbering.xml"] + abstracts = numbering.findall(f"{W}abstractNum") + assert len(abstracts) == 4 # headings, enumerated, renumber, detached + assert lvl_texts(numbering, 1)[3] == "(%1)(%2)(%3)(%4)" and lvl_texts(numbering, 3)[:4] == ["(%1)", "(%2)", "%3", "(%4)"] + four = paragraph_with(root, "four") + ilvl, num_id = numbering_of(four) + assert ilvl == 3 + [num] = [n for n in numbering.findall(f"{W}num") if n.get(f"{W}numId") == str(num_id)] + assert num.find(f"{W}abstractNumId").get(f"{W}val") == "3" # type: ignore[union-attr] + assert num.find(f"{W}lvlOverride/{W}startOverride").get(f"{W}val") == "1" # type: ignore[union-attr] + assert numbering_of(paragraph_with(root, "four b"))[1] == num_id + + +def test_a_list_nested_past_nine_levels_goes_on_in_the_next_abstract_num() -> None: + source = FRONT + "# A\n\n" + "".join(" " * d + f"- level {d}\n" for d in range(11)) + parts_ = document(source) + root, numbering = parts_["word/document.xml"], parts_["word/numbering.xml"] + kinds = len(numbering.findall(f"{W}abstractNum")) + assert kinds == 3 # headings, enumerated, enumerated+1 + assert [numbering_of(paragraph_with(root, f"level {d}")) for d in (8, 9, 10)] == [(8, 2), (0, 3), (1, 3)] + assert lvl_texts(numbering, 2)[:2] == ["(%1)", "(%2)"] + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize(("name", "settings"), [ + ("paths", PATH_ITEMS), ("paths", {}), ("ordered paths", ORDERED_PARENT), ("deep", {}), ("deep", PATH_ITEMS), + ("skipped", {"style": "outline"}), ("nested", {"overrides": {"docx.reference_text_switch": "guess"}}), + ("nested", {"overrides": {"docx.reference_text_switch": "off"}})]) +def test_documents_with_detached_levels_deep_lists_and_the_switch_validate(name: str, settings: dict) -> None: + require_schemas() + result = render(STRESS[name], format="docx", **settings) + assert result.binary is not None + assert validation_errors(result.binary) == [] diff --git a/tests/test_docx_wordcheck.py b/tests/test_docx_wordcheck.py new file mode 100644 index 0000000..da9990c --- /dev/null +++ b/tests/test_docx_wordcheck.py @@ -0,0 +1,53 @@ +"""The generator of the Word check document (tests/word_check.py, .github/PUBLISHING.md item 8 and 9): that it makes a +valid package whose rows carry what the planner assumes, from the planner's own code, and that LibreOffice (which does +not read ``\\t``) computes the rows' plain ``REF \\n`` fields as the planner caches them.""" +from __future__ import annotations + +import re + +import pytest +from docx_support import libreoffice_export, parts, require_libreoffice, require_schemas, validation_errors +from word_check import ROWS, build, main + +from legaldown_render.resolve.docx_cover import suppressed +from legaldown_render.resolve.docx_numbering import Count, Lit + + +def test_the_document_is_a_package_that_holds_every_row_and_the_assumption_beside_it() -> None: + contents = parts(build()) + assert {"[Content_Types].xml", "word/document.xml", "word/numbering.xml", "word/styles.xml"} <= set(contents) + text = re.sub(r"<[^>]+>", "", contents["word/document.xml"].decode("utf-8")) + for number, row in enumerate(ROWS, 1): + assert f"Row {number}, {row.title}" in text + # the planner's assumption is what its own code says, per setting of docx.reference_text_switch + assert "Row 6, an abbreviation with a period: No. %1: drawn 'No. 1'" in text + assert "documented: the switch is not used; guess: '1'" in text # No. 1: only the guess drops the period + assert "documented: 'I.1'; guess: 'I.1'" in text # Microsoft's example with a roman counter + assert contents["word/document.xml"].count(b"REF t3 \\n \\t \\h") == 1 + assert suppressed([Lit("Section "), Count("a", "1"), Lit("."), Count("b", "1")]) is not None + + +def test_the_document_is_deterministic_and_the_command_writes_it(tmp_path) -> None: # noqa: ANN001 + assert build() == build() + target = tmp_path / "folder" / "check.docx" + assert main(["word_check.py", str(target)]) == 0 and target.read_bytes() == build() + assert main(["word_check.py"]) == 2 + + +@pytest.mark.ooxml_schema +def test_the_document_validates_against_the_schemas() -> None: + require_schemas() + assert validation_errors(build()) == [] + + +@pytest.mark.libreoffice +def test_libreoffice_computes_the_plain_fields_as_the_planner_caches_them() -> None: + require_libreoffice() + said = libreoffice_export({"check": build()})["check"].text + lines = [line for line in said.splitlines() if "REF \\n gives" in line] + assert len(lines) == len(ROWS) + 6 + 2 # the rows, the nested items of part B, the two headings of part C + for line in lines: + computed = re.search(r"REF \\n gives (.*?)(?:\. REF \\n \\t|$)", line) + assumed = re.search(r"\[planner: (.*?)\]", line) or re.search(r"The planner assumes REF \\n gives '(.*?)';", line) + assert computed and assumed, line + assert computed.group(1).rstrip(".") == assumed.group(1).rstrip("."), line diff --git a/tests/word_check.py b/tests/word_check.py new file mode 100644 index 0000000..6698185 --- /dev/null +++ b/tests/word_check.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""The document for the Word checks of what the DOCX planner assumes (.github/PUBLISHING.md, "Checking the DOCX in +Word", items 8 and 9): what Word's ``REF \\n \\t`` returns for the label forms the plan relies on, whether a level text +of the heading list that names the levels around a never-used one (``%1.%2(%6)`` with levels 3 to 5 unused) draws and +restarts as the plan models it, and what a skipped heading level draws. + +Run it with the project's environment, from the repository root:: + + python tests/word_check.py word-check/numbering-assumptions.docx + +Open the file in Word, update every field (Ctrl+A, F9), and compare each line with its last column: the number Word +draws, ``REF \\n`` and ``REF \\n \\t`` as Word computes them, against what the planner assumes under each setting of +``docx.reference_text_switch``. Every field holds, until it is updated, the planner's own value. Anything that differs +is a finding for the planner (report it, and set ``docx.reference_text_switch`` to what your Word does). + +The planner's side comes from the planner's own code (the simulator of Word's counters, ``suppressed``), never from a +table written here, so a change to the model shows in the file. +""" +from __future__ import annotations + +import sys +import xml.etree.ElementTree as ET +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +from legaldown_render.resolve.docx_cover import suppressed +from legaldown_render.resolve.docx_numbering import Count, Lit, WordNumbering +from legaldown_render.resolve.docx_plan import AbstractNum, LevelDef, Num +from legaldown_render.style import load_style +from legaldown_render.style.docx import docx_settings +from legaldown_render.writers.docx import package as pkg +from legaldown_render.writers.docx.numbering import abstract_num, num +from legaldown_render.writers.docx.ooxml import Bookmarks, complex_field, el, p, serialize, text_run, val +from legaldown_render.writers.docx.properties import build_app, build_core +from legaldown_render.writers.docx.settings import build_numbering, build_settings +from legaldown_render.writers.docx.styles import build_styles + +_STEP = 567 + + +def _level(ilvl: int, num_fmt: str, text: str, *, p_style: str | None = None) -> LevelDef: + return LevelDef(ilvl, num_fmt, text, p_style=p_style, left=_STEP * (ilvl + 1), hanging=_STEP) + + +def _neutral(ilvl: int) -> LevelDef: + return _level(ilvl, "decimal", f"%{ilvl + 1}") + + +@dataclass(frozen=True) +class Row: + """One label form: the levels of its own list, and the level whose paragraph is referred to (the paragraphs of the + levels above it come first).""" + + title: str + levels: tuple[LevelDef, ...] + target: int + + +#: The label forms the cover can meet. Documented: Microsoft's example, ``Section 1.01`` gives ``1.01``. +ROWS = ( + Row("roman counter with a word: Article %1", (_level(0, "upperRoman", "Article %1"),), 0), + Row("section sign: § %1", (_level(0, "decimal", "§ %1"),), 0), + Row("Microsoft's example: Section %1.%2", (_level(0, "decimal", "%1"), _level(1, "decimal", "Section %1.%2")), 1), + Row("word and roman counter before a decimal: Section %1.%2 with a roman %1", + (_level(0, "upperRoman", "%1"), _level(1, "decimal", "Section %1.%2")), 1), + Row("an abbreviation with a period: Čl. %1", (_level(0, "decimal", "Čl. %1"),), 0), + Row("an abbreviation with a period: No. %1", (_level(0, "decimal", "No. %1"),), 0), + Row("parentheses around a letter: (%1)", (_level(0, "lowerLetter", "(%1)"),), 0), + Row("a parenthesis after a letter: %1)", (_level(0, "lowerLetter", "%1)"),), 0), + Row("a colon after a number: %1:", (_level(0, "decimal", "%1:"),), 0), + Row("a path with a parenthesis: %1.%2(%3)", + (_level(0, "decimal", "%1"), _level(1, "decimal", "%1.%2"), _level(2, "lowerLetter", "%1.%2(%3)")), 2), +) + +#: The levels of the heading list as the planner writes them for a style with items on the heading list's second level +#: and a ``{path}`` list nested in them on the reserved level 6: levels 3 to 5 are never used. +HEADING_LIST = ( + _level(0, "decimal", "%1.", p_style="Heading1"), _level(1, "decimal", "%1.%2"), _level(2, "decimal", "%1.%2.%3"), + _level(3, "decimal", "%1.%2.%3.%4"), _level(4, "decimal", "%1.%2.%3.%4.%5"), + _level(5, "lowerLetter", "%1.%2(%6)"), _level(6, "lowerRoman", "%1.%2(%6)(%7)"), _neutral(7), _neutral(8)) +SKIPPING_LIST = ( + _level(0, "decimal", "%1.", p_style="Heading1"), _level(1, "decimal", "%1.%2", p_style="Heading2"), + _level(2, "decimal", "%1.%2.%3", p_style="Heading3"), *(_neutral(k) for k in range(3, 9))) + + +@dataclass +class _Document: + """What the check document is made of: paragraphs, and the numbering definitions they are numbered by.""" + + body: list[ET.Element] + abstracts: list[ET.Element] + nums: list[ET.Element] + bookmarks: Bookmarks + + def list_(self, levels: Sequence[LevelDef], *, overrides: Sequence[tuple[int, int]] = ()) -> tuple[int, WordNumbering]: + """A new abstractNum with a num; returns the numId and the simulator of Word's counters for it.""" + padded = [*levels, *(_neutral(k) for k in range(len(levels), 9))] + abstract = AbstractNum(len(self.abstracts), "enumerated", f"{0x4C440100 + len(self.abstracts):08X}", + "multilevel", tuple(padded)) + instance = Num(len(self.nums) + 1, abstract.id, tuple(overrides)) + self.abstracts.append(abstract_num(abstract)) + self.nums.append(num(instance)) + simulator = WordNumbering(skipped="lo") + simulator.add_abstract(abstract.id, padded) + simulator.add_num(instance) + return instance.id, simulator + + def line(self, *content: object, style: str | None = None, numbering: tuple[int, int] | None = None) -> None: + props = [el("numPr", None, val("ilvl", numbering[1]), val("numId", numbering[0]))] if numbering else [] + self.body.append(p(*content, style=style, props=props)) + + def text(self, text: str, *, style: str | None = None) -> None: + self.line(text_run(text), style=style) + + +def _field(instruction: str, cached: str) -> list[ET.Element]: + return complex_field(instruction, [text_run(cached)]) + + +def _assumed(tokens: Sequence[Lit | Count]) -> str: + """What the planner assumes ``REF \\n \\t`` returns, under each setting of ``docx.reference_text_switch``.""" + said = [] + for mode in ("documented", "guess"): + kept = suppressed(tokens, mode) # type: ignore[arg-type] + said.append(f"{mode}: " + ("the switch is not used" if kept is None else f"'{''.join(t.text for t in kept)}'")) + return "; ".join(said) + + +def _words(tokens: Sequence[Lit | Count]) -> str: + """What ``REF \\n`` returns: the number's text without its trailing periods.""" + return "".join(t.text for t in tokens).rstrip(".") + + +def build() -> bytes: + """The check document as a ``.docx``.""" + doc = _Document([], [], [], Bookmarks()) + doc.text("Word check of the planner's assumptions about numbering", style="Title") + doc.text("Update every field (Ctrl+A, then F9). Each row gives the number as Word draws it, the field REF \\n and " + "the field REF \\n \\t as Word computes them, and the planner's assumption. The fields hold the planner's " + "values until they are updated: report every field that changes, and every row whose number is not the " + "one shown as 'drawn'.") + doc.text("A. The switch \\t on each label form (docx.reference_text_switch)", style="Heading1") + for number, row in enumerate(ROWS, 1): + num_id, simulator = doc.list_(row.levels) + step = None + for ilvl in range(row.target + 1): + step = simulator.step(num_id, ilvl, f"row{number}-{ilvl}") + name = f"t{number}" + runs = text_run(f"target of row {number}") if ilvl == row.target else text_run(f"(parent {ilvl})") + doc.line(*(doc.bookmarks.around(name, runs) if ilvl == row.target else [runs]), + numbering=(num_id, ilvl)) + assert step is not None + tokens = list(step.tokens) + doc.line(text_run(f"Row {number}, {row.title}: drawn '{step.label}'. REF \\n gives "), + *_field(f"REF t{number} \\n \\h", _words(tokens)), text_run(". REF \\n \\t gives "), + *_field(f"REF t{number} \\n \\t \\h", ("".join(t.text for t in (suppressed(tokens, "guess") or tokens)))), + text_run(f". The planner assumes REF \\n gives '{_words(tokens)}'; for REF \\n \\t, {_assumed(tokens)}.")) + doc.text("B. A level text that names the levels around three that are never used", style="Heading1") + doc.text("The heading list below has levels 3 to 5 that nothing numbers, and a nested level 6 whose text is " + "%1.%2(%6). The nested items must read 1.1(a), 1.1(b), then 1.2(a) under the next item.") + num_id, simulator = doc.list_(HEADING_LIST, overrides=((1, 1),)) + for name, ilvl, text in (("r_head", 0, "heading"), ("r_item1", 1, "item 1"), ("r_n1", 5, "nested a"), + ("r_n2", 5, "nested b"), ("r_item2", 1, "item 2"), ("r_n3", 5, "nested under item 2")): + step = simulator.step(num_id, ilvl, name) + doc.line(*doc.bookmarks.around(name, text_run(text)), text_run(f" [planner: {step.label}] REF \\n gives "), + *_field(f"REF {name} \\n \\h", step.word_ref), numbering=(num_id, ilvl)) + doc.text("C. A heading level the document skips", style="Heading1") + doc.text("A level-1 heading followed by a level-3 one. The renderer, and LibreOffice, count the skipped level as 1 " + "(the planner caches that reading, and writes the skipped level's part of a reference as text); Word is " + "expected to draw 0 for it. Record what Word draws.") + heading_id, heading_sim = doc.list_(SKIPPING_LIST) + for name, ilvl, style, text in (("s_one", 0, "Heading1", "First heading"), ("s_deep", 2, "Heading3", "Third level")): + step = heading_sim.step(heading_id, ilvl, name) + doc.line(*doc.bookmarks.around(name, text_run(text)), style=style, numbering=(heading_id, ilvl)) + doc.line(text_run(f"[planner: {step.label}] REF \\n gives "), *_field(f"REF {name} \\n \\h", step.word_ref)) + return _package(doc) + + +def _package(doc: _Document) -> bytes: + style = load_style("default") + settings = docx_settings(style) + package = pkg.Package() + package.relationships.add(pkg.REL_OFFICE_DOCUMENT, "word/document.xml") + package.relationships.add(pkg.REL_CORE_PROPERTIES, "docProps/core.xml") + package.relationships.add(pkg.REL_EXTENDED_PROPERTIES, "docProps/app.xml") + rels = pkg.Relationships() + rels.add(pkg.REL_STYLES, "styles.xml") + rels.add(pkg.REL_NUMBERING, "numbering.xml") + rels.add(pkg.REL_SETTINGS, "settings.xml") + page = settings.page + section = el("sectPr", None, + el("pgSz", {"w": page.width, "h": page.height}), + el("pgMar", {"top": page.top, "right": page.right, "bottom": page.bottom, "left": page.left, + "header": page.header, "footer": page.footer, "gutter": 0})) + package.add("docProps/core.xml", pkg.CT_CORE_PROPERTIES, build_core("Word check of the numbering assumptions", "en")) + package.add("docProps/app.xml", pkg.CT_EXTENDED_PROPERTIES, build_app("legaldown-render word_check")) + package.add("word/document.xml", pkg.CT_DOCUMENT, serialize(el("document", None, el("body", None, doc.body, section))), + rels) + package.add("word/styles.xml", pkg.CT_STYLES, build_styles(style, settings, "en", heading_num_id=None)) + package.add("word/numbering.xml", pkg.CT_NUMBERING, build_numbering(doc.abstracts, doc.nums)) + package.add("word/settings.xml", pkg.CT_SETTINGS, build_settings(update_fields=True)) + return package.to_bytes() + + +def main(argv: Sequence[str]) -> int: + if len(argv) != 2: + print(__doc__) + return 2 + path = Path(argv[1]) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(build()) + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From bcaa4fd7787c6d02eb29cb402ef8c11516defe82 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:44:54 +0000 Subject: [PATCH 109/118] Review follow-ups: skipped-level reasons, bullets past nine levels, page-derived indents, words between counters under guess A reference below a skipped heading level whose number Word draws with another unit's counter now says so (the skipped level's own reason, not a remark on the style's labels), and the label warning no longer claims Word draws something else when it draws the same text from another counter. The warnings about labels that lose counters are one finding for a chain, with how many more places do the same. Bullets go on in bullet+1 to bullet+3 past nine levels like the other families, and a nesting past 36 levels is reported for every family. The indent that nested lists stop stepping in at is 60% of the page's text width (passed to the planner), not a fixed thirteen centimetres, so a deep list on A5 stays readable. docx.reference_text_switch: documented drops only the words before the first counter and after the last; words between counters are dropped only under guess (the documentation does not say what Word does with them). The Word check document has a row for them, no longer links its lists to the heading styles (Word would renumber the document's own titles), and word-check/ is ignored. The ghost carve-out of the property test compares the letters and digits written as text with the text the plan says it wrote. The c2 message says where the delimiters would go. --- .github/PUBLISHING.md | 11 +- .gitignore | 3 + docs/architecture.md | 11 +- docs/decisions/0008-docx-word-native.md | 2 +- docs/style-templates.md | 2 +- src/legaldown_render/api.py | 3 +- src/legaldown_render/resolve/docx_cover.py | 5 +- src/legaldown_render/resolve/docx_plan.py | 174 ++++++++++++------ src/legaldown_render/style/model.py | 8 +- src/legaldown_render/writers/docx/__init__.py | 3 +- tests/test_docx_plan.py | 98 +++++++++- tests/test_docx_roundtrip.py | 2 +- tests/test_docx_wordcheck.py | 1 + tests/word_check.py | 14 +- 14 files changed, 247 insertions(+), 90 deletions(-) diff --git a/.github/PUBLISHING.md b/.github/PUBLISHING.md index fa59f46..cbb2987 100644 --- a/.github/PUBLISHING.md +++ b/.github/PUBLISHING.md @@ -152,8 +152,8 @@ For each file: 8. **The numbering corner cases.** Make the check document with `python tests/word_check.py word-check/numbering-assumptions.docx` (from the repository root, in the project's environment): one row for each label form the planner relies on, each with the planner's assumption beside it, taken from the planner's own code, so that - a change to the model shows in the file. Ctrl+A, F9, then compare each row's `REF \n` and `REF \n \t` with the last - column: + a change to the model shows in the file. Ctrl+A, F9, then compare each row's `REF \n` and `REF \n \t` with what the + row says the planner assumes (the text after "The planner assumes", and the `[planner: ...]` of parts B and C): - `\t` with words before the number (rows 1 to 4: `Article %1`, `§ %1`, Microsoft's own `Section %1.%2` with a decimal and with a roman `%1`): Word shows the bare number (`I`, `1`, `1.1`, `I.1`) where LibreOffice ignores `\t` and shows the words. This is what the default `docx.reference_text_switch: documented` relies on. @@ -161,8 +161,11 @@ For each file: them under `documented`, and takes it to give the bare number under `guess`. If Word gives `1`, a style with such a label may set `docx.reference_text_switch: guess`; if it gives anything else, report it. - delimiters around the counter (rows 7 to 10: `(%1)`, `%1)`, `%1:`, `%1.%2(%3)`): the planner never uses `\t` for - them (what Word does is not documented). Note what Word returns: if it keeps the delimiter, `\t` gives the plain - number and the planner's refusal is only cautious. + them (what Word does is not documented). Note what Word returns. If it leaves the delimiter out (`a` for `(a)`), + `\t` would make a reference to such an item live and the planner's refusal is only cautious (report it); if it + keeps the delimiter (`(a)`), the refusal is right and the reference stays fixed text, reported. + - words between the counters (row 11, `Part %1 of %2`): only `docx.reference_text_switch: guess` expects `\t` to drop + them (the result would be `12`). If Word returns something else, report it. - the reference to such an item reads as the HTML reads it, or the render reported `render-docx-reference` and the reference is fixed text (with the class it falls in and the setting that could change it). - skipped heading levels (a `###` straight after a `#`, an Error for the validator but rendered; part C of the check diff --git a/.gitignore b/.gitignore index fc9e61f..53e0c19 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ dist/ # OOXML schemas fetched for the DOCX tests (tests/schemas/README.md) tests/schemas/ooxml/ + +# The documents made for the manual Word checks (.github/PUBLISHING.md) +word-check/ diff --git a/docs/architecture.md b/docs/architecture.md index 8b115b1..2a95af6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -316,9 +316,9 @@ A level of a list shows only counters of its own list, which settles where the r restarts under a new parent at a fixed value, like any list nested in a list of another kind (`render-docx-static-restart`): a list made new in Word by copying an item of another goes on counting where the renderer would start again. - **Lists nested past Word's nine levels.** Word has nine levels in a list. A list nested deeper goes on in the next - abstractNum of its family (`enumerated+1` to `enumerated+3`, `renumber+1` to `renumber+3`: 36 levels in all), whose + abstractNum of its family (`enumerated+1` to `enumerated+3`, `renumber+1` to `renumber+3`, `bullet+1` to `bullet+3`: 36 levels in all), whose level `j` is the family's level of depth `9k + j` (format and label with its own counter alone, indent stepping a - centimetre per level up to thirteen). Inside such an abstractNum the lists restart by structure; the list that + centimetre per level up to 60% of the page's text width, as for every nested list, `plan_docx(text_width=)`). Inside such an abstractNum the lists restart by structure; the list that begins one restarts at a fixed value (`restart-depth`, reported as information). Only a nesting past 36 levels uses the last level again (`clamped`, `render-docx-list-depth`). - **Alternatives that share a number** (§15.8). A unit that shares its number with the one before it and that Word @@ -369,8 +369,9 @@ LibreOffice ignores it) is used where the number holds words the reference does gives `I`, `§ 3` gives `3`; Microsoft's own example is `Section 1.01`, which gives `1.01`), as far as the style's `docx.reference_text_switch` says Word may be expected to do it: `documented` (the default) drops the words before the first counter and after the last, keeps a delimiter between counters, and does not use the switch for a number with a -delimiter around its counters (`(a)`, `a)`, `1:`, `No. 1`), because what Word does with it is not documented; `off` -never uses it; `guess` also drops a period after the letters that lead a number (`No. 1`, `Čl. 1`). What Word does is +delimiter around its counters (`(a)`, `a)`, `1:`, `No. 1`) or words between counters, because what Word does with them is +not documented; `off` never uses it; `guess` also drops a period after the letters that lead a number (`No. 1`, +`Čl. 1`) and words between counters. What Word does is on the manual Word checklist (`.github/PUBLISHING.md`). Each field's cached result is **Word's** value (the simulator's), and `cached_lo` what LibreOffice shows. A unit that a field points at has a bookmark (its anchor's, or a hidden one); a reference to an item or paragraph the style does not number is the @@ -393,7 +394,7 @@ that part, is literal text (`RefPlan.ghost`, `render-docx-reference` names it); there is LibreOffice's reading of a skipped level (its start value); Word itself is expected to read it as one less (the checklist). A heading's title that refers back to itself under the `none` scheme is the third case (`render-docx-reference-cycle`). Nothing else that Word could compute is written as literal text; the built-in styles -have no fixed reference in a document without a skipped heading level, whatever its nesting. +have no fixed reference in a document without a skipped heading level, with lists nested up to 36 levels. **Bookmarks** (`resolve/docx_bookmarks.py`). The planner names bookmarks and the writer gives them ids. Every anchor of the tree has one, referenced or not: `ld_` for a section, list item or paragraph, `lddef_` for a diff --git a/docs/decisions/0008-docx-word-native.md b/docs/decisions/0008-docx-word-native.md index 8ff68ee..64bc701 100644 --- a/docs/decisions/0008-docx-word-native.md +++ b/docs/decisions/0008-docx-word-native.md @@ -100,7 +100,7 @@ which a field returns whole or not at all; and, in a document that skips a headi Error), **the counter of the skipped level**, held by no unit, which is text between the fields of the units that do hold theirs. A heading's title that refers back to itself under the `none` scheme is the other case (`render-docx-reference-cycle`). The built-in styles have none of these in a document without errors, and a -reference to an item in a list nested as deep as the document likes is made of fields. +reference to an item in a list nested up to 36 levels is made of fields. Under the `none` scheme a reference is a `REF` to the heading text. ### Numbering the hard cases diff --git a/docs/style-templates.md b/docs/style-templates.md index e9fcb30..e312a00 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -267,7 +267,7 @@ style in Word applies to every paragraph that uses it. | Setting | Meaning | |---|---| | `docx.update_fields_on_open` | `true` (the default) asks Word to update every field when it opens the file: the cross-references, the table of contents and its page numbers. Word asks the reader once ("This document contains fields that may refer to other files. Update the fields?"). The fields hold the values the renderer computed either way, so a viewer that never updates fields (Word Online, Quick Look, Outlook's preview) shows the document right, but the table of contents without page numbers. LibreOffice and Word Online ignore the setting. `false` writes no such request | -| `docx.reference_text_switch` | What a cross-reference may expect of Word's `\t` switch (`REF bookmark \n \t`), which leaves the words of a number out. A reference is built from `REF` fields so that it stays right when a clause is inserted; where the reference shows less than the number Word draws (`Section {designation}` for a number `Article I`, `čl. {designation}` for `Čl. 1`), the switch leaves the words out and the `references.format` text goes back in as literal text. LibreOffice ignores the switch (and shows the whole number there, which the render reports once as `render-docx-reference-word-only`). Microsoft documents it with one example (`Section 1.01` gives `1.01`: the words around the number go, a delimiter between its counters stays); what Word does with a delimiter in the text around the counters is not documented. **`documented`** (the default) uses the switch only where that example settles the result: words before the first counter or after the last go, a delimiter between counters stays, and a label with a delimiter around its number (`(a)`, `a)`, `1:`, `No. 1`) is not asked for it. **`off`** never uses it: a reference that needs it is fixed text, reported (`render-docx-reference`). **`guess`** also expects Word to leave out a period after the letters that lead a number (`No. 1`, `Čl. 1`): the checklist in `.github/PUBLISHING.md` says how to check what your Word does, and a reference that is wrong in Word is fixed by setting `documented` | +| `docx.reference_text_switch` | What a cross-reference may expect of Word's `\t` switch (`REF bookmark \n \t`), which leaves the words of a number out. A reference is built from `REF` fields so that it stays right when a clause is inserted; where the reference shows less than the number Word draws (`Section {designation}` for a number `Article I`, `čl. {designation}` for `Čl. 1`), the switch leaves the words out and the `references.format` text goes back in as literal text. LibreOffice ignores the switch (and shows the whole number there, which the render reports once as `render-docx-reference-word-only`). Microsoft documents it with one example (`Section 1.01` gives `1.01`: the words around the number go, a delimiter between its counters stays); what Word does with a delimiter in the text around the counters is not documented. **`documented`** (the default) uses the switch only where that example settles the result: words before the first counter or after the last go, a delimiter between counters stays (words between counters are not documented: only `guess` drops them), and a label with a delimiter around its number (`(a)`, `a)`, `1:`, `No. 1`) is not asked for it. **`off`** never uses it: a reference that needs it is fixed text, reported (`render-docx-reference`). **`guess`** also expects Word to leave out a period after the letters that lead a number (`No. 1`, `Čl. 1`) and words between counters (`Part 1 of 2` gives `12`): the checklist in `.github/PUBLISHING.md` says how to check what your Word does, and a reference that is wrong in Word is fixed by setting `documented` | Settings that were already part of the template gain a DOCX meaning: diff --git a/src/legaldown_render/api.py b/src/legaldown_render/api.py index c4e55ab..f585b73 100644 --- a/src/legaldown_render/api.py +++ b/src/legaldown_render/api.py @@ -175,7 +175,8 @@ def render(source: str, options: RenderOptions | None = None, /, **settings: Any if isinstance(writer, DocxWriter): # What Word will draw differently from the other formats (docs/decisions/0008) is the planner's to report; its # findings are the renderer's, and are ordered with them. - plan, planned = plan_docx(tree, file=document.filename, text_switch=writer.settings.reference_text_switch) + plan, planned = plan_docx(tree, file=document.filename, text_switch=writer.settings.reference_text_switch, + text_width=writer.settings.page.text_width) extra = [*extra, *planned] # The renderer's findings that the validator's or the assembly's already # name (the same rule, level and message), wherever they are, are not repeated; diff --git a/src/legaldown_render/resolve/docx_cover.py b/src/legaldown_render/resolve/docx_cover.py index 251bddf..50a17b6 100644 --- a/src/legaldown_render/resolve/docx_cover.py +++ b/src/legaldown_render/resolve/docx_cover.py @@ -92,7 +92,8 @@ def suppressed(tokens: Sequence[Shown], mode: TextSwitch = "documented") -> tupl ``§ 3`` gives ``3``). With a delimiter in it (``(a)``, ``a)``, ``1:``) Word's result is not documented: None. Mode ``guess`` also drops leading text that has a period after letters in it (``No. 1``, ``Čl. 1`` give ``1``). * Text between two counters that is nothing but delimiters stays (Microsoft's example: ``Section 1.01`` gives - ``1.01``); text between them with no delimiter in it is dropped; a mixture is None. + ``1.01``). Words between them, with no delimiter, are dropped only in mode ``guess``: the documentation does not + say what Word does with them. A mixture is None. """ if mode == "off": return None @@ -114,6 +115,8 @@ def suppressed(tokens: Sequence[Shown], mode: TextSwitch = "documented") -> tupl if not all(character in DELIMITERS for character in token.text): return None kept.append(token) + elif mode != "guess": + return None return tuple(kept) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 0f83e96..75ad47a 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -117,10 +117,11 @@ class AbstractNum: id: int # 0.., in order of first use (headings first) #: ``detached``: the enumerated levels with each level's own counter alone, for a list nested where the enumerated list #: would have to show the counter of a unit that is not on it (§13.2). ``enumerated+1`` to ``enumerated+3`` (and - #: ``renumber+k``): Word has nine levels, so a list nested deeper takes the next abstractNum of its family, whose level - #: *j* is the family's level of depth ``9k + j`` with its own counter alone. + #: ``renumber+k``, ``bullet+k``): Word has nine levels, so a list nested deeper takes the next abstractNum of its + #: family, whose level *j* is the family's level of depth ``9k + j`` with its own counter alone. kind: Literal["headings", "enumerated", "detached", "renumber", "paragraphs", "bullet", - "enumerated+1", "enumerated+2", "enumerated+3", "renumber+1", "renumber+2", "renumber+3"] + "enumerated+1", "enumerated+2", "enumerated+3", "renumber+1", "renumber+2", "renumber+3", + "bullet+1", "bullet+2", "bullet+3"] #: Eight hex digits, deterministic (``0x4C440000 + id``). Equal in every LegalDown document, so Word may #: merge lists on paste between two of them (cosmetic). nsid: str @@ -202,7 +203,7 @@ class UnitPlan: #: ``restart-family`` / ``restart-section`` / ``restart-alternative`` / ``restart-depth`` (a nested list in a list of #: another kind, in an item whose number carries the section, in an alternative, past Word's nine levels: a num of #: its own that restarts by a fixed setting), ``label-fallback`` (the label is written without a number Word cannot - #: show there), ``clamped`` (nested past :data:`MAX_DEPTH`, or a bullet past Word's nine levels), ``skipped-level``. + #: show there), ``clamped`` (nested past :data:`MAX_DEPTH`, in any family), ``skipped-level``. cause: tuple[str, ...] #: Names to wrap around the paragraph's title/first paragraph content: the bookmark of the unit's anchor when it #: has one, else a hidden one when a reference field points at it, else none. @@ -228,10 +229,10 @@ class RefPlan: #: No cover exists, so the reference is the renderer's text with no field: right when the file is opened, but #: fixed (an edit in Word does not change it). The plan reports each (``render-docx-reference``). fixed: bool = False - #: The cover writes the number of a heading level the document skips (a Core error, ``heading-skip``, §4.1) as text, - #: since no unit holds that level's counter and so no field returns it; the rest is fields. Reported like a fixed - #: reference (``render-docx-reference``). - ghost: bool = False + #: The text of the counters of heading levels the document skips (a Core error, ``heading-skip``, §4.1) that the cover + #: writes as literal text, since no unit holds those counters and so no field returns them; the rest is fields. + #: Empty when the cover has none. Reported like a fixed reference (``render-docx-reference``). + ghost: str = "" @dataclass(frozen=True, slots=True) @@ -333,13 +334,22 @@ def docx_style_errors(style: Style) -> list[str]: #: the last level is used again and the lists restart by fixed values. _LEVELS = 9 MAX_DEPTH = 4 * _LEVELS -#: Twips: the indent items stop stepping at (thirteen centimetres), so that a list nested very deeply stays on the page. -_MAX_LEFT = 13 * _STEP +#: The width of the text of an A4 page with 25 mm margins, in twips: what the indents of nested lists are limited by when +#: the planner is not told the page (:func:`plan_docx`). +DEFAULT_TEXT_WIDTH = 9072 +#: The part of the text width an item's indent may reach: a list nested very deeply stops stepping in there and stays +#: readable on any page (A5 included). +_MAX_INDENT = 0.6 + + +def max_indent(text_width: int) -> int: + """The widest indent, in twips, a nested list steps in to on a page whose text is *text_width* twips wide.""" + return max(_STEP, int(text_width * _MAX_INDENT)) def _level(ilvl: int, num_fmt: str, parts: LevelParts, *, p_style: str | None = None, widths: Mapping[int, str] | None = None, unwritable: list[tuple[str, int]] | None = None, - kind: str = "", depth: int | None = None) -> LevelDef: + kind: str = "", depth: int | None = None, max_left: int | None = None) -> LevelDef: """One level. Headings (*widths*: the ``numFmt`` of each level a ``%k`` names) get an indent wide enough for their widest label; items step one centimetre per level of nesting (*depth*, the level's own by default: a level of a list that goes on past Word's nine levels is nested deeper). A label with a word in it (``Article {n}``) is @@ -348,7 +358,7 @@ def _level(ilvl: int, num_fmt: str, parts: LevelParts, *, p_style: str | None = words = any(isinstance(part, str) and any(c.isalpha() for c in part) for part in parts) depth = ilvl if depth is None else depth if widths is None: - left, hanging = min(_STEP * (depth + 1), _MAX_LEFT), _STEP + left, hanging = min(_STEP * (depth + 1), max_left or max_indent(DEFAULT_TEXT_WIDTH)), _STEP else: sample = sum(len(part) if isinstance(part, str) else len(SAMPLE_COUNTER[widths[part]]) for part in parts) left = hanging = max(_STEP, _CHARACTER * (sample + 2)) @@ -364,8 +374,8 @@ def _neutral(ilvl: int) -> LevelDef: return LevelDef(ilvl, "decimal", f"%{ilvl + 1}", left=_STEP * (ilvl + 1), hanging=_STEP) -def _own(formats: NumberingFormats, ilvl: int, depth: int, textual: bool, unwritable: list[tuple[str, int]] | None - ) -> LevelDef: +def _own(formats: NumberingFormats, ilvl: int, depth: int, textual: bool, unwritable: list[tuple[str, int]] | None, + max_left: int | None = None) -> LevelDef: """Level *ilvl* of an enumerated list nested *depth* levels deep, with its own counter alone (a ``{path}`` shows its own ``ref`` and nothing of the units it is nested in): the levels of the detached list and of the lists that go on past Word's nine levels, where no unit of the list holds the levels above.""" @@ -373,11 +383,11 @@ def _own(formats: NumberingFormats, ilvl: int, depth: int, textual: bool, unwrit here = levels[depth % len(levels)] return _level(ilvl, NUM_FMT[here.counter], level_text(here.label, [""] * ilvl + [here.ref], ilvl, section=(), textual=textual), - unwritable=unwritable, kind="enumerated", depth=depth) + unwritable=unwritable, kind="enumerated", depth=depth, max_left=max_left) -def _definitions(formats: NumberingFormats, unwritable: list[tuple[str, int]] | None = None - ) -> dict[str, list[LevelDef]]: +def _definitions(formats: NumberingFormats, unwritable: list[tuple[str, int]] | None = None, + max_left: int | None = None) -> dict[str, list[LevelDef]]: """The nine levels of every abstractNum *formats* needs, by kind. A label that needs a number of the section (``{section}``, or ``{path}`` of an item or paragraph in one) is written without it, as for a preamble item: Word shows only counters of the list itself in a level.""" @@ -396,26 +406,32 @@ def _definitions(formats: NumberingFormats, unwritable: list[tuple[str, int]] | out["enumerated"] = [ _level(d, NUM_FMT[formats.enumerated[d % size].counter], level_text(formats.enumerated[d % size].label, refs, d, section=(), textual=textual), - unwritable=unwritable, kind="enumerated") + unwritable=unwritable, kind="enumerated", max_left=max_left) for d in range(9)] # The same levels with nothing but their own counter (a ``{path}`` shows its own ``ref`` alone): where a list is # nested in a unit that is not on the enumerated list, Word has no counter of the enclosing units to show. - out["detached"] = [_own(formats, d, d, textual, unwritable) for d in range(_LEVELS)] + out["detached"] = [_own(formats, d, d, textual, unwritable, max_left) for d in range(_LEVELS)] for k in range(1, MAX_DEPTH // _LEVELS): - out[f"enumerated+{k}"] = [_own(formats, d, _LEVELS * k + d, textual, unwritable) for d in range(_LEVELS)] + out[f"enumerated+{k}"] = [_own(formats, d, _LEVELS * k + d, textual, unwritable, max_left) + for d in range(_LEVELS)] if formats.renumber is not None: f = formats.renumber for k in range(MAX_DEPTH // _LEVELS): out["renumber" if k == 0 else f"renumber+{k}"] = [ _level(d, NUM_FMT[f.counter], level_text(f.label, [f.ref] * 9, d, section=(), textual=textual), - unwritable=unwritable, kind="renumber", depth=_LEVELS * k + d) for d in range(_LEVELS)] + unwritable=unwritable, kind="renumber", depth=_LEVELS * k + d, max_left=max_left) + for d in range(_LEVELS)] if formats.paragraphs is not None: f = formats.paragraphs out["paragraphs"] = [ _level(0, NUM_FMT[f.counter], level_text(f.label, [f.ref], 0, section=(), textual=textual), unwritable=unwritable, kind="paragraphs"), *(_neutral(k) for k in range(1, 9))] - out["bullet"] = [LevelDef(d, "bullet", _BULLETS[d % 3], left=_STEP * (d + 1), hanging=_STEP) for d in range(9)] + limit = max_left or max_indent(DEFAULT_TEXT_WIDTH) + for k in range(MAX_DEPTH // _LEVELS): + out["bullet" if k == 0 else f"bullet+{k}"] = [ + LevelDef(d, "bullet", _BULLETS[(_LEVELS * k + d) % 3], left=min(_STEP * (_LEVELS * k + d + 1), limit), + hanging=_STEP) for d in range(_LEVELS)] return out @@ -584,7 +600,7 @@ class _Built(NamedTuple): exact: bool lo_text: str | None fixed: bool - ghost: bool = False + ghost: str = "" class _Finding(NamedTuple): @@ -607,8 +623,10 @@ def __init__(self, node: _Unit, kind: str, step: Step, delta: int) -> None: class _Planner: - def __init__(self, tree: RenderTree, file: str | None, text_switch: TextSwitch = "documented") -> None: + def __init__(self, tree: RenderTree, file: str | None, text_switch: TextSwitch = "documented", + text_width: int = DEFAULT_TEXT_WIDTH) -> None: self.tree, self.file, self.text_switch = tree, file or "", text_switch + self.max_left = max_indent(text_width) self.formats = tree.formats self.textual = tree.formats.scheme == "none" self.diagnostics: list[Diagnostic] = [] @@ -638,12 +656,15 @@ def __init__(self, tree: RenderTree, file: str | None, text_switch: TextSwitch = self.reserve5 = False self.flagged: set[str] = set() self.reported: set[tuple[str, object]] = set() + #: The labels Word draws differently for want of counters it cannot show, by what they have in common, as + #: (line, message) of each place; reported once for each group (:meth:`report_section_labels`). + self.label_findings: dict[tuple[object, ...], list[tuple[int | None, str]]] = {} self.drifts: list[_Drift] = [] self.open_drift: dict[tuple[int, int], _Drift] = {} self.heading_num: Num | None = None - self.bullet_num: Num | None = None + self.bullet_nums: dict[str, Num] = {} self.depth_reported = False - self.definitions = _definitions(self.formats) + self.definitions = _definitions(self.formats, max_left=self.max_left) self.by_kind: dict[str, _Abstract] = {} #: Every unit's node, in document order; what Word numbers each as (:class:`Step`); and its designation as #: the resolver wrote it, as tokens (§13.1-§13.3), each counter with the unit that holds it. @@ -745,12 +766,17 @@ def why_uncovered(self, designation: Sequence[Shown], fields: Sequence[Field]) - ``{section}.{n}`` under a ref ``{n}``): Word returns the whole number or none of it.""" wanted = atoms(designation) _, owner = wanted[uncovered(designation, fields) or 0] + mine = {t.owner for t in designation if isinstance(t, Count)} + skips = any(isinstance(owner_, Ghost) for owner_ in mine) for candidate in fields: plain = candidate.number and not candidate.suppress_text if not (plain and any(isinstance(t, Count) and t.owner == owner for t in candidate.tokens)): continue drawn = printable(text_of(candidate.tokens)) shape = "".join("#" if isinstance(t, Count) else t.text for t in candidate.tokens) + if skips and any(isinstance(t, Count) and t.owner not in mine for t in candidate.tokens): + return ("a heading level is skipped before it (§4.1) and Word's counter for that level belongs to another " + "unit, so no field returns the number the renderer shows", "skipped") if any(isinstance(t, Count) and t.owner != owner for t in candidate.tokens): return (f"Word draws its number as '{drawn}', which holds the counters of other units that the reference " f"writes itself before it (the section's, the enclosing items'), and a field returns the whole " @@ -764,7 +790,8 @@ def why_uncovered(self, designation: Sequence[Shown], fields: Sequence[Field]) - else: how = "Word is not documented to leave such punctuation out with its \\t switch" return (f"Word draws its number as '{drawn}', with text or punctuation around the counter that the " - f"reference does not have, and {how}", f"delimiters {shape}") + f"reference does not have, and {how}. The reference is made of fields when the level's `ref` has " + f"them too (put it in `ref`, for example `({{n}})`)", f"delimiters {shape}") return "no combination of Word's fields reads like it", "other" def reference(self, found: _Reference) -> _Built | None: @@ -804,7 +831,7 @@ def reference(self, found: _Reference) -> _Built | None: if any(isinstance(piece, Field) and piece.suppress_text for piece in pieces): lo_text = "".join(piece if isinstance(piece, str) else text_of(self.words(piece.owner) if piece.number else piece.tokens) for piece in pieces) - return _Built(pieces, shown == found.text, lo_text, False, bool(ghost)) + return _Built(pieces, shown == found.text, lo_text, False, ghost) @staticmethod def skipped_part(whole: Sequence[Shown], pieces: Sequence[str | Field]) -> str: @@ -1036,6 +1063,7 @@ def run(self) -> tuple[DocxPlan, list[Diagnostic]]: self.section(section) for block in section.blocks: self.block(block, section) + self.report_section_labels() for drift in self.drifts: self.report_drift(drift) self.report_collisions() @@ -1213,11 +1241,26 @@ def detached_label(self, fmt: LevelFormat, abstract: _Abstract, ilvl: int, kind: why = "the levels of the heading list kept for nested lists are used up or read otherwise" else: why = "the list is nested in a unit that is on another list" - self.report("render-docx-section-label", "warning", - f"The label '{printable(fmt.label)}' of {kind}s shows the number of {self.describe(lost)}, " - f"which Word cannot show in this list ({why}): in Word it reads " - f"'{printable(_reads(levels[ilvl], levels))}'. A reference to such an item is still made of " - f"fields.", first.line) + self.section_label(("detached", fmt.label, self.describe(lost), why), + f"The label '{printable(fmt.label)}' of {kind}s shows the number of {self.describe(lost)}, " + f"which Word cannot show in this list ({why}): in Word it reads " + f"'{printable(_reads(levels[ilvl], levels))}'. A reference to such an item is still made of " + f"fields.", first.line) + + def section_label(self, group: tuple[object, ...], message: str, line: int | None) -> None: + """Note a label Word draws differently for want of counters it cannot show; the ones that have *group* in + common are one finding.""" + self.label_findings.setdefault(group, []).append((line, message)) + + def report_section_labels(self) -> None: + """One ``render-docx-section-label`` for each group: the first place by line, and how many more do the same + (as for the references that fail alike).""" + for members in self.label_findings.values(): + members.sort(key=lambda m: (m[0] is None, m[0] or 0)) + line, message = members[0] + if len(members) > 1: + message += f" {len(members) - 1} more {'places do' if len(members) > 2 else 'place does'} the same." + self.report("render-docx-section-label", "warning", message, line) def shared_counter(self, message: str, line: int | None) -> None: self.report("render-docx-shared-counter", "warning", message, line) @@ -1431,7 +1474,7 @@ def indent(self, abstract: _Abstract, ilvl: int, unit: str, parent: str | None, plan sets nothing.""" level = abstract.levels[ilvl] if unit in self.item_units and parent in self.text_left: - left, hanging = min(self.text_left[parent] + _STEP, _MAX_LEFT), _STEP + left, hanging = min(self.text_left[parent] + _STEP, self.max_left), _STEP elif unit in self.item_units and abstract.kind == "headings": left, hanging = level.left, level.hanging else: @@ -1493,10 +1536,17 @@ def numbered(self, node: _Unit, kind: Literal["section", "item", "paragraph"], n # a level nothing numbers as 0, or the counter of whatever last counted it, and the resolver's label # is that of the enclosing unit. Not left to Word's reading. cause += ("label-fallback",) - self.report("render-docx-section-label", "warning", - f"The label of {word}s '{printable(node.label or '')}' shows the number of an enclosing " - f"item that Word cannot show here (the list is not nested directly in an item of its own " - f"kind): in Word it reads '{printable(step.label)}'.", node.line) + skipped = any(isinstance(t, Count) and isinstance(t.owner, Ghost) for t in self.desig.get(parent or "", ())) + if skipped: + message = (f"The label of {word}s '{printable(node.label or '')}' holds the number of a heading level " + f"that the document skips (a heading level must follow the one above, §4.1): Word draws " + f"that level with the counter of another unit, so it reads '{printable(step.label)}' only " + f"as long as that counter is what the renderer counts, and a unit inserted there changes it.") + else: + message = (f"The label of {word}s '{printable(node.label or '')}' shows the number of an enclosing " + f"item that Word cannot show here (the list is not nested directly in an item of its own " + f"kind): in Word it reads '{printable(step.label)}'.") + self.section_label(("chain", word, skipped), message, node.line) self.add(node, kind, used, ilvl, explicit=explicit, cause=cause, step=step, ind=self.indent(abstract, ilvl, unit, parent, alternative=False)) self.check(node, word, step, counter, group, cause, (abstract.id, ilvl)) @@ -1514,6 +1564,15 @@ def restart_class(self, parent: str, placed: _Placed, abstract: _Abstract, *, de return "restart-section" if placed.abstract.kind == "headings" else "restart-family" return "resync" + def report_depth(self, depth: int, line: int | None) -> None: + """Report, once, a list nested past the levels the plan has (:data:`MAX_DEPTH`), of any family.""" + if not self.depth_reported: + self.depth_reported = True + self.report("render-docx-list-depth", "warning", + f"A list is nested {depth} levels deep: the plan has {MAX_DEPTH} levels of Word lists. Items from " + f"level {MAX_DEPTH + 1} on are numbered in the format of level {MAX_DEPTH} and restart by fixed " + f"values (a bullet has no number to restart).", line) + def report_collisions(self) -> None: """One finding for the units of the heading list that Word would number off the resolver's because they share a counter with others of their section (§2.7): the first, and how many more. What an edit in Word does to @@ -1602,12 +1661,7 @@ def list_(self, lst: List, section: Section | None, quoted: bool = False) -> Non num = self.new_num(abstract, ilvl, first.number or 1) if depth >= MAX_DEPTH: cause = ("clamped",) - if not self.depth_reported: - self.depth_reported = True - self.report("render-docx-list-depth", "warning", - f"A list is nested {depth} levels deep: the plan has {MAX_DEPTH} levels of Word lists. " - f"Items from level {MAX_DEPTH + 1} on are numbered in the format of level {MAX_DEPTH} " - f"and restart by fixed values.", first.line) + self.report_depth(depth, first.line) elif placed is not None and first.number > 1: cause = ("resync",) self.shared_counter( @@ -1647,31 +1701,39 @@ def list_(self, lst: List, section: Section | None, quoted: bool = False) -> Non self.block(child, section, quoted) def bullets(self, lst: List, section: Section | None, quoted: bool) -> None: - """Bullets count nothing, so every bullet list of the document is numbered by one num.""" - abstract = self.abstract("bullet") - if self.bullet_num is None: - self.bullet_num = self.new_num(abstract) - ilvl = min(lst.depth, 8) + """Bullets count nothing, so every bullet list of the document at one stretch of nine levels is numbered by + one num; a list nested deeper goes on in the next abstractNum, as the other families do.""" + reached = min(lst.depth, MAX_DEPTH - 1) + chunk, ilvl = divmod(reached, _LEVELS) + kind = "bullet" if chunk == 0 else f"bullet+{chunk}" + abstract = self.abstract(kind) + if kind not in self.bullet_nums: + self.bullet_nums[kind] = self.new_num(abstract) + bullet_num = self.bullet_nums[kind] parent = lst.items[0].parent_unit + cause: tuple[str, ...] = () + if lst.depth >= MAX_DEPTH: + cause = ("clamped",) + self.report_depth(lst.depth, lst.items[0].line) for item in lst.items: assert item.unit is not None self.parent[item.unit] = item.parent_unit self.nodes[item.unit] = item self.desig[item.unit] = self.desig.get(parent, []) if parent is not None else [] - cause = ("clamped",) if lst.depth > 8 else () - self.units[item.unit] = UnitPlan(item.unit, "item", self.bullet_num.id, ilvl, True, None, None, "", "", + self.units[item.unit] = UnitPlan(item.unit, "item", bullet_num.id, ilvl, True, None, None, "", "", None, cause, (), None) - self.placed[item.unit] = _Placed(self.bullet_num, abstract) + self.placed[item.unit] = _Placed(bullet_num, abstract) for child in item.blocks: self.block(child, section, quoted) -def plan_docx(tree: RenderTree, *, file: str | None = None, text_switch: TextSwitch = "documented" - ) -> tuple[DocxPlan, list[Diagnostic]]: +def plan_docx(tree: RenderTree, *, file: str | None = None, text_switch: TextSwitch = "documented", + text_width: int = DEFAULT_TEXT_WIDTH) -> tuple[DocxPlan, list[Diagnostic]]: """The numbering plan of *tree* for Word, and the diagnostics only planning can produce (all ``render-docx-*``, in the order of their lines, naming *file*). Pure: the same tree and *text_switch* (the style's ``docx.reference_text_switch``: what a reference may expect of Word's ``\\t`` switch, see - :func:`.docx_cover.suppressed`) give an equal plan, and ``tree.formats`` carries the style's numbering. A style + :func:`.docx_cover.suppressed`) and *text_width* (the page's, in twips: nested lists stop stepping in at 60% of it) + give an equal plan, and ``tree.formats`` carries the style's numbering. A style Word's numbering cannot express (:func:`docx_style_errors`) is refused before rendering (ADR 0006), so it never reaches here.""" - return _Planner(tree, file, text_switch).run() + return _Planner(tree, file, text_switch, text_width).run() diff --git a/src/legaldown_render/style/model.py b/src/legaldown_render/style/model.py index f1973c1..d0c4bb1 100644 --- a/src/legaldown_render/style/model.py +++ b/src/legaldown_render/style/model.py @@ -357,10 +357,10 @@ class Docx: update_fields_on_open: bool = True #: What Word's ``\t`` switch (``REF \n \t``, which leaves the words of a number out) is taken to do when a #: cross-reference is built from fields (docs/style-templates.md, "DOCX"). ``off``: never used, so a reference that - #: needs it is fixed text. ``documented``: only where Microsoft's documentation says what Word returns (words around - #: the number are left out, a delimiter between its counters stays). ``guess``: also where Word is expected to do - #: what it is not documented to (a period after the letters that lead a number: ``No. 1``, ``Čl. 1``); to be checked - #: in Word. + #: needs it is fixed text. ``documented``: only where Microsoft's documentation says what Word returns (words before the first counter + #: and after the last are left out, a delimiter between counters stays). ``guess``: also where Word is expected to do + #: what it is not documented to (a period after the letters that lead a number: ``No. 1``, ``Čl. 1``, and words between + #: counters); to be checked in Word. reference_text_switch: Literal["off", "documented", "guess"] = "documented" diff --git a/src/legaldown_render/writers/docx/__init__.py b/src/legaldown_render/writers/docx/__init__.py index fc48e5f..aa175af 100644 --- a/src/legaldown_render/writers/docx/__init__.py +++ b/src/legaldown_render/writers/docx/__init__.py @@ -61,7 +61,8 @@ def write(self, tree: RenderTree, plan: DocxPlan | None = None) -> bytes: one the tree is planned here (:func:`~legaldown_render.resolve.docx_plan.plan_docx`), whose diagnostics are then lost: :func:`~legaldown_render.render` plans first and reports them.""" if plan is None: - plan = plan_docx(tree, text_switch=self.settings.reference_text_switch)[0] + plan = plan_docx(tree, text_switch=self.settings.reference_text_switch, + text_width=self.settings.page.text_width)[0] settings = self.settings language = language_tag(tree.language, self.style.locale) title = document_title(tree) diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index 3f7cb8c..486489c 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -10,12 +10,14 @@ import random import re import time +import xml.etree.ElementTree as ET import zipfile from collections.abc import Iterator, Sequence from pathlib import Path import pytest from conftest import DOCUMENTS, FRONT, spec_root +from docx_support import W, parts from legaldown_render import DocumentError, render from legaldown_render.errors import StyleError @@ -30,6 +32,7 @@ _components, _Planner, docx_style_errors, + max_indent, plan_docx, ) from legaldown_render.resolve.numbering import format_counter, formats_of @@ -424,11 +427,33 @@ def test_the_abstract_nums_past_nine_levels_have_the_families_formats_of_the_dep ("lowerLetter", "(%5)")] assert [(level.num_fmt, level.lvl_text) for level in by_kind["enumerated+2"].levels[:3]] == [ ("lowerRoman", "%1)"), ("upperLetter", "[%2]"), ("decimal", "%3.")] # depth 18, 19, 20 - # the indent goes on stepping a centimetre per level of nesting, up to thirteen centimetres - assert [level.left for level in by_kind["enumerated+1"].levels[:5]] == [567 * k for k in (10, 11, 12, 13, 13)] + # the indent stops stepping in at 60% of the text width of an A4 page (9072 twips), which level 9 has nearly reached + assert [level.left for level in by_kind["enumerated"].levels[6:]] == [567 * 7, 567 * 8, 567 * 9] + assert [level.left for level in by_kind["enumerated+1"].levels[:5]] == [max_indent(9072)] * 5 == [5443] * 5 assert all(len(a.levels) == 9 for a in plan.abstract_nums) +def test_the_indent_of_a_deep_list_is_limited_by_the_width_of_the_page_it_is_on() -> None: + body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(14)) + "\n- bullet\n" + "".join( + " " * (i + 1) + "- bullet\n" for i in range(12)) + result = render(FRONT + body, format="text") + for width in (5557, 14000): # A5 with 25 mm margins; a wide page + plan, _ = plan_docx(result.tree, text_width=width) + lefts = [level.left for a in plan.abstract_nums if a.kind != "headings" for level in a.levels] + assert max(lefts) == max_indent(width) == int(width * 0.6) + # the indents of the items that carry their own (those on the heading list and nested in them) stop there too + clauses = render(FRONT + body, format="text", style="continental") + plan, _ = plan_docx(clauses.tree, text_width=5557) + assert max(unit.ind[0] for unit in plan.units.values() if unit.ind) == max_indent(5557) + # the render plans for the page of the document: on A5 no indent of the file goes past 60% of the text + data = render(FRONT + body, format="docx", style="continental", overrides={"page.size": "A5"}).binary + assert data is not None + document = ET.fromstring(parts(data)["word/document.xml"]) + text_width = 8391 - 2 * 1417 + lefts = [int(ind.get(f"{W}left")) for ind in document.iter(f"{W}ind") if ind.get(f"{W}left")] + assert max(lefts) > 3000 and max(lefts) <= int(text_width * 0.6) + 1 + + def test_ordered_lists_go_on_past_nine_levels_too() -> None: body = "# A\n\n" + "".join(" " * i + f"{1}. level {i}\n" for i in range(11)) tree, plan, diagnostics = planned(body) @@ -440,6 +465,29 @@ def test_ordered_lists_go_on_past_nine_levels_too() -> None: assert rules(diagnostics) == {"render-docx-static-restart"} +def test_bullets_go_on_past_nine_levels_in_further_abstract_nums_too() -> None: + body = "# A\n\n" + "".join(" " * i + f"- bullet {i}\n" for i in range(12)) + tree, plan, diagnostics = planned(body, overrides={"enumeration.enabled": False}) + deep = items(tree) + assert [a.kind for a in plan.abstract_nums] == ["headings", "bullet", "bullet+1"] + assert [plan.units[i.unit].ilvl for i in deep] == [0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2] + assert len({plan.units[i.unit].num_id for i in deep[:9]}) == 1 == len({plan.units[i.unit].num_id for i in deep[9:]}) + assert plan.units[deep[8].unit].num_id != plan.units[deep[9].unit].num_id + assert all(plan.units[i.unit].cause == () for i in deep) and diagnostics == [] + # the glyph goes on cycling, as it would at that depth + bullets = {a.kind: a for a in plan.abstract_nums} + assert [bullets["bullet"].levels[8].lvl_text, bullets["bullet+1"].levels[0].lvl_text] == ["◦", "•"] + + +def test_a_list_of_any_family_nested_past_the_levels_the_plan_has_is_reported_once() -> None: + for marker in ("-", "1."): + width = 3 if marker == "1." else 2 + body = "# A\n\n" + "".join(" " * (width * i) + f"{marker} level {i}\n" for i in range(38)) + _, plan, diagnostics = planned(body, overrides={"enumeration.enabled": marker == "1."}) + assert [d.rule for d in diagnostics if d.rule == "render-docx-list-depth"] == ["render-docx-list-depth"] + assert any("clamped" in unit.cause for unit in plan.units.values()) + + def test_a_list_nested_past_the_levels_the_plan_has_is_clamped_and_reported_once() -> None: body = "# A\n\n" + "".join(" " * i + f"- level {i}\n" for i in range(38)) tree, plan, diagnostics = planned(body) @@ -1991,7 +2039,7 @@ def test_a_skipped_heading_level_is_the_one_part_of_a_reference_that_is_written_ tree, plan, diagnostics = planned(body, style="outline") # no unit holds the number of the level the document skips (an error, §4.1), so it is text between two fields assert references_of(tree, plan) == [("I.A.1", 'REF a + ".A." + REF b', True)] - assert plan.refs["s2"].ghost and not plan.refs["s2"].fixed + assert plan.refs["s2"].ghost == "A" and not plan.refs["s2"].fixed [found] = [d for d in diagnostics if d.rule == "render-docx-reference"] assert found.level == "warning" and found.line == tree.sections[1].blocks[0].line assert found.message.startswith("The cross-reference 'I.A.1' holds the number of a heading level that the document skips") @@ -2321,7 +2369,8 @@ def test_references_that_are_fixed_for_the_same_reason_are_one_finding() -> None # delimiters in the label that the reference leaves out ([{"counter": "lower-alpha", "label": "({n})", "ref": "{n}"}], "documented", ["Word draws its number as '(a)'", "text or punctuation around the counter that the reference does not have", - "Word is not documented to leave such punctuation out with its \\t switch"]), + "Word is not documented to leave such punctuation out with its \\t switch", + "put it in `ref`, for example `({n})`"]), ([{"counter": "decimal", "label": "{n}:", "ref": "{n}"}], "guess", ["Word draws its number as '1:'", "not documented to leave such punctuation out"]), # an abbreviation with a period: the setting that guesses leaves it out @@ -2345,6 +2394,32 @@ def test_the_reason_a_reference_is_fixed_names_what_the_style_could_change( assert all(r.fixed for r in plan.refs.values()) +def test_a_reference_below_a_skipped_level_whose_number_Word_draws_with_another_units_counter_says_so() -> None: + # the paragraph of the section after the skip is numbered 1.1.1.1 by Word too, but through the counter of the + # paragraph before the skip (level 2 of the heading list), not of the level the document skips + body = ("# Heading\n\nA paragraph. {#a1}\n\n### Heading {#h3}\n\nA paragraph. {#a22}\n\nA third. {#a23}\n\n" + "Refs: {{ref: a1}} {{ref: a22}}\n") + tree, plan, diagnostics = planned(body, overrides=PARAGRAPHS) + [reference] = [d for d in diagnostics if d.rule == "render-docx-reference"] + assert "a heading level is skipped before it (§4.1) and Word's counter for that level belongs to another unit" in reference.message + assert "style" not in reference.message and "label" not in reference.message + [label] = [d for d in diagnostics if d.rule == "render-docx-section-label"] + # what Word draws is not stated as if it were different from the renderer's: it is the same text, from another counter + assert "holds the number of a heading level that the document skips" in label.message + assert "another unit" in label.message and "only as long as that counter" in label.message + assert label.message.endswith(" 2 more places do the same.") # the three paragraphs, one finding + + +def test_the_labels_of_the_lists_of_one_chain_that_lose_counters_are_one_finding() -> None: + body = ("# A\n\n- one\n 1. ordered\n - three\n - four\n - three b\n - four c\n" + " 1. ordered two\n - x\n - y\n") + _, plan, diagnostics = planned(body, **ORDERED_PARENT) + found = [d for d in diagnostics if d.rule == "render-docx-section-label"] + assert len(found) == 1 and found[0].message.endswith(" 2 more places do the same.") + assert sum(next(a.kind for a in plan.abstract_nums if a.id == n.abstract_id) == "detached" + for n in plan.nums) == 3 + + def test_every_reference_that_carries_the_counters_of_other_units_is_one_finding_whatever_its_shape() -> None: levels = [{"counter": "decimal", "label": "{section}.{n}", "ref": "{n}"}] body = "# A\n\n- x {#x}\n\n## B\n\n- y {#y}\n\nSee {{ref: x}} {{ref: y}}.\n" @@ -2415,7 +2490,7 @@ def test_the_switch_that_drops_text_is_modelled_only_where_its_result_is_known() assert suppressed([Lit("§ "), Count("a", "3")]) == (Count("a", "3"),) assert suppressed([Lit("Section "), one, Lit("."), two]) == (one, Lit("."), two) assert suppressed([one, Lit("."), two]) == (one, Lit("."), two) # nothing to drop: no field of its own is needed - assert suppressed([Lit("Part "), one, Lit(" of "), two]) == (one, two) # words between counters go too + assert suppressed([Lit("Part "), one, Lit(" of "), two]) is None # words between counters: only `guess` drops them # a delimiter around the counters, or mixed with words between them, is not documented assert suppressed([Lit("("), Count("a", "a"), Lit(")")]) is None assert suppressed([Count("a", "a"), Lit(")")]) is None @@ -2437,6 +2512,8 @@ def test_the_switch_setting_says_what_word_may_be_expected_to_do() -> None: assert suppressed([Lit("("), one], "guess") is None and suppressed([Lit("1. "), one], "guess") is None assert suppressed([one, Lit(" Art.")], "guess") is None and suppressed([Lit("No: "), one], "guess") is None assert suppressed([Lit("Article "), Count("a", "I")], "guess") == (Count("a", "I"),) + assert suppressed([Lit("Part "), one, Lit(" of "), two], "guess") == (one, two) # words between counters go too + assert suppressed([one, Lit(" to."), two], "guess") is None # a mixture with a delimiter never does # -- Bookmarks ---------------------------------------------------------------------------------------------- @@ -2627,9 +2704,8 @@ def assert_references_as_resolved(result, plan, diagnostics) -> None: # noqa: A tree.formats.paragraphs) if f is not None] delimiters_only = not any(c.isalnum() for t in templates for c in re.sub(r"\{(n|path|section)\}", "", t)) # The one counter a cover may write as text is that of a heading level the document skips (a Core error), which no - # unit holds: it reads as the first counter of that level. + # unit holds, and the plan says which it wrote. skipping = any("skipped-level" in u.cause for u in plan.units.values()) - ghost_text = {c for level in tree.formats.headings for c in format_counter(1, level.counter)} needing_a_finding = 0 fixed = ghosts = 0 for target, text in texts.items(): @@ -2646,9 +2722,11 @@ def assert_references_as_resolved(result, plan, diagnostics) -> None: # noqa: A continue assert not reference.ghost or skipping, (target, text) if delimiters_only: - written = {c for c in literal_text_of_a_cover(tree, reference.segments) if c.isalnum()} - assert written <= (ghost_text if reference.ghost else set()), (target, text, show(plan, reference.segments)) - ghosts += reference.ghost + # the letters and digits written as text are exactly the counters the plan says it wrote as text + written = "".join(c for c in literal_text_of_a_cover(tree, reference.segments) if c.isalnum()) + assert written == "".join(c for c in reference.ghost if c.isalnum()), ( + target, text, show(plan, reference.segments)) + ghosts += bool(reference.ghost) assert any(isinstance(s, FieldSeg) for s in reference.segments) or not text, (target, text) if reference.exact: continue diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index 7c1b1a3..4710ee7 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -1187,7 +1187,7 @@ def test_the_live_parts_of_a_reference_below_a_skipped_level_follow_a_heading_in assert [line for line in lines_of(said) if line.startswith("Refs")] == ["Refs: II II.A.1 II.A.1(a) III III.A.1"] result = render(SKIPPED, format="docx", style="outline") plan, diagnostics = plan_docx(result.tree) - assert [r.ghost for r in plan.refs.values()] == [False, True, True, False, True] + assert [r.ghost for r in plan.refs.values()] == ["", "A", "A", "", "A"] assert not any(r.fixed for r in plan.refs.values()) assert any(d.rule == "render-docx-reference" and "heading level that the document skips" in d.message for d in diagnostics) diff --git a/tests/test_docx_wordcheck.py b/tests/test_docx_wordcheck.py index da9990c..ff5c530 100644 --- a/tests/test_docx_wordcheck.py +++ b/tests/test_docx_wordcheck.py @@ -23,6 +23,7 @@ def test_the_document_is_a_package_that_holds_every_row_and_the_assumption_besid assert "Row 6, an abbreviation with a period: No. %1: drawn 'No. 1'" in text assert "documented: the switch is not used; guess: '1'" in text # No. 1: only the guess drops the period assert "documented: 'I.1'; guess: 'I.1'" in text # Microsoft's example with a roman counter + assert "Part %1 of %2" in text and "documented: the switch is not used; guess: '12'" in text # words between counters assert contents["word/document.xml"].count(b"REF t3 \\n \\t \\h") == 1 assert suppressed([Lit("Section "), Count("a", "1"), Lit("."), Count("b", "1")]) is not None diff --git a/tests/word_check.py b/tests/word_check.py index 6698185..6a492da 100644 --- a/tests/word_check.py +++ b/tests/word_check.py @@ -39,8 +39,10 @@ _STEP = 567 -def _level(ilvl: int, num_fmt: str, text: str, *, p_style: str | None = None) -> LevelDef: - return LevelDef(ilvl, num_fmt, text, p_style=p_style, left=_STEP * (ilvl + 1), hanging=_STEP) +def _level(ilvl: int, num_fmt: str, text: str) -> LevelDef: + """A level of a list of the check document. None is linked to a paragraph style: the document's own titles are + ``Heading`` paragraphs that Word must not number, and the numbered lines carry ``w:numPr`` themselves.""" + return LevelDef(ilvl, num_fmt, text, left=_STEP * (ilvl + 1), hanging=_STEP) def _neutral(ilvl: int) -> LevelDef: @@ -71,17 +73,19 @@ class Row: Row("a colon after a number: %1:", (_level(0, "decimal", "%1:"),), 0), Row("a path with a parenthesis: %1.%2(%3)", (_level(0, "decimal", "%1"), _level(1, "decimal", "%1.%2"), _level(2, "lowerLetter", "%1.%2(%3)")), 2), + Row("words between the counters: Part %1 of %2 (only the guess drops them)", + (_level(0, "decimal", "%1"), _level(1, "decimal", "Part %1 of %2")), 1), ) #: The levels of the heading list as the planner writes them for a style with items on the heading list's second level #: and a ``{path}`` list nested in them on the reserved level 6: levels 3 to 5 are never used. HEADING_LIST = ( - _level(0, "decimal", "%1.", p_style="Heading1"), _level(1, "decimal", "%1.%2"), _level(2, "decimal", "%1.%2.%3"), + _level(0, "decimal", "%1."), _level(1, "decimal", "%1.%2"), _level(2, "decimal", "%1.%2.%3"), _level(3, "decimal", "%1.%2.%3.%4"), _level(4, "decimal", "%1.%2.%3.%4.%5"), _level(5, "lowerLetter", "%1.%2(%6)"), _level(6, "lowerRoman", "%1.%2(%6)(%7)"), _neutral(7), _neutral(8)) SKIPPING_LIST = ( - _level(0, "decimal", "%1.", p_style="Heading1"), _level(1, "decimal", "%1.%2", p_style="Heading2"), - _level(2, "decimal", "%1.%2.%3", p_style="Heading3"), *(_neutral(k) for k in range(3, 9))) + _level(0, "decimal", "%1."), _level(1, "decimal", "%1.%2"), + _level(2, "decimal", "%1.%2.%3"), *(_neutral(k) for k in range(3, 9))) @dataclass From c318661458b07534ae9a5208e1ea993136718478 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:57:26 +0000 Subject: [PATCH 110/118] Report where LibreOffice counts a level nothing numbered before a deeper item; one LibreOffice session for the new edit tests An item that is the first of its level in its list after a deeper item of that list (the level above is held by a unit of another list) is 1 in Word and 2 in LibreOffice, which counts the unused level as its start and goes on from it. The simulator now also gives LibreOffice's count (Step.lo_count), and the plan says once per document that such items read higher there (render-docx-libreoffice-count, information); a test shows LibreOffice's reading. The four new edit fixtures run in one libreoffice_edit job, and the label-form test samples ten documents. --- .github/PUBLISHING.md | 2 +- CONFORMANCE.md | 2 +- docs/architecture.md | 4 +- .../resolve/docx_numbering.py | 16 ++++- src/legaldown_render/resolve/docx_plan.py | 29 +++++++++ tests/test_docx_plan.py | 17 ++++- tests/test_docx_roundtrip.py | 65 +++++++++++++------ tests/test_docx_simulator.py | 16 +++++ tests/test_docx_wordcheck.py | 2 +- 9 files changed, 125 insertions(+), 28 deletions(-) diff --git a/.github/PUBLISHING.md b/.github/PUBLISHING.md index cbb2987..bcf49b7 100644 --- a/.github/PUBLISHING.md +++ b/.github/PUBLISHING.md @@ -165,7 +165,7 @@ For each file: `\t` would make a reference to such an item live and the planner's refusal is only cautious (report it); if it keeps the delimiter (`(a)`), the refusal is right and the reference stays fixed text, reported. - words between the counters (row 11, `Part %1 of %2`): only `docx.reference_text_switch: guess` expects `\t` to drop - them (the result would be `12`). If Word returns something else, report it. + them (the result would be `11` for `Part 1 of 1`). If Word returns something else, report it. - the reference to such an item reads as the HTML reads it, or the render reported `render-docx-reference` and the reference is fixed text (with the class it falls in and the setting that could change it). - skipped heading levels (a `###` straight after a `#`, an Error for the validator but rendered; part C of the check diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 2258cc7..486f9a5 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -20,7 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | -| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can; the continental `5.1` items and the numbered paragraphs `1.1` ride the heading list's next level, with the lists nested in them on its reserved levels, so they stay live; alternatives that share a number (§15.8) are `numId 0` paragraphs whose number is a `REF` field of the first, with the sections and lists under them on a num of their own: live for edits elsewhere, but **deleting the first alternative in Word breaks them and every reference to the shared identifier** (reported; render with the condition values for a final document); the built-in continental style uses the heading list's levels 6 and 7 for lists nested in its clauses, a label that shows the path of an item keeps its section's, clause's and own counters on those levels, and a list nested where Word cannot show the counters above it, or in a level the heading list has no room for, is drawn with its own counter alone on a detached list and restarts by a fixed setting, reported; a list nested past Word's nine levels goes on in further lists, up to 36 levels), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference is fixed text, reported, only where Word's fields cannot reproduce the designation: a number with text or punctuation around its counter that the reference does not have (`(a)`, `{n}:`, `No. {n}`) where Word's `\t` switch is not documented to leave it out (`docx.reference_text_switch`: `off`, `documented`, `guess`), and a number that holds the counters of other units the reference writes itself (`{section}.{n}` under a ref `{n}`); the counter of a heading level that the document skips (`heading-skip`, an Error) is text between the fields of the units that hold theirs; the words of `references.format` that a number repeats (`Article I`) are returned by the field itself. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says. **The front and back matter are written too**, in the order the HTML shows them: the title block (header rows and parties), a live table of contents (`TOC \o "1-N" \h \z \u` with cached entries equal to the HTML contents, each a link with a `PAGEREF` page number Word fills in on update; the attachments are listed by outline level, so that no locale-dependent style list is in the field; at `contents.depth: 1` the attachment titles are listed as first-level entries, where the HTML indents them one level more), the attachments (bookmarked so `{{attach:}}` resolves, separated by a page break or a rule per `attachments.separator`) and the signature blocks (§3.6, borderless tables kept together on one page as far as one page allows). A LibreOffice round trip shows that after its indexes are updated the contents lists exactly the HTML entries with page numbers, also after a heading is inserted in the editor, and that the document reads in the plain-text writer's order. Remaining differences are the reported ones: a label that shows the section's number that the heading list cannot show (an outline scheme's `{section}.{n}`, the `none` scheme) is drawn without it. The contents show the shared number for a later alternative. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that shows a number Word cannot show there: the section's, or the enclosing items' on a list that cannot carry them), `render-docx-list-depth` (nested past 36 levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a unit that shares a counter with others in Word, such as a sub-heading and the items of its section that have the same designation in every output format, and that is restarted at a fixed value, which will not follow edits), `render-docx-number-mismatch` (any other difference: a catch-all that does not occur outside documents that skip a heading level), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits, with the class it falls in; one that has the number of a skipped heading level as text; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-alternative` (alternatives share a number as fields of the first, which must not be deleted in Word), `render-docx-static-restart` (a list that restarts by a fixed setting: in a list of another kind, in an item on the heading list that it cannot ride, in an alternative, or the one that begins the next list past nine levels) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). The Word-only checks (the contents' page numbers, what `\t` does, numbering after an insertion) are a manual checklist before a release (`.github/PUBLISHING.md`). HTML is the format this renderer claims §13.6 with | +| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can; the continental `5.1` items and the numbered paragraphs `1.1` ride the heading list's next level, with the lists nested in them on its reserved levels, so they stay live; alternatives that share a number (§15.8) are `numId 0` paragraphs whose number is a `REF` field of the first, with the sections and lists under them on a num of their own: live for edits elsewhere, but **deleting the first alternative in Word breaks them and every reference to the shared identifier** (reported; render with the condition values for a final document); the built-in continental style uses the heading list's levels 6 and 7 for lists nested in its clauses, a label that shows the path of an item keeps its section's, clause's and own counters on those levels, and a list nested where Word cannot show the counters above it, or in a level the heading list has no room for, is drawn with its own counter alone on a detached list and restarts by a fixed setting, reported; a list nested past Word's nine levels goes on in further lists, up to 36 levels), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference is fixed text, reported, only where Word's fields cannot reproduce the designation: a number with text or punctuation around its counter that the reference does not have (`(a)`, `{n}:`, `No. {n}`) where Word's `\t` switch is not documented to leave it out (`docx.reference_text_switch`: `off`, `documented`, `guess`), and a number that holds the counters of other units the reference writes itself (`{section}.{n}` under a ref `{n}`); the counter of a heading level that the document skips (`heading-skip`, an Error) is text between the fields of the units that hold theirs; the words of `references.format` that a number repeats (`Article I`) are returned by the field itself. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says. **The front and back matter are written too**, in the order the HTML shows them: the title block (header rows and parties), a live table of contents (`TOC \o "1-N" \h \z \u` with cached entries equal to the HTML contents, each a link with a `PAGEREF` page number Word fills in on update; the attachments are listed by outline level, so that no locale-dependent style list is in the field; at `contents.depth: 1` the attachment titles are listed as first-level entries, where the HTML indents them one level more), the attachments (bookmarked so `{{attach:}}` resolves, separated by a page break or a rule per `attachments.separator`) and the signature blocks (§3.6, borderless tables kept together on one page as far as one page allows). A LibreOffice round trip shows that after its indexes are updated the contents lists exactly the HTML entries with page numbers, also after a heading is inserted in the editor, and that the document reads in the plain-text writer's order. Remaining differences are the reported ones: a label that shows the section's number that the heading list cannot show (an outline scheme's `{section}.{n}`, the `none` scheme) is drawn without it. The contents show the shared number for a later alternative. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that shows a number Word cannot show there: the section's, or the enclosing items' on a list that cannot carry them), `render-docx-list-depth` (nested past 36 levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a unit that shares a counter with others in Word, such as a sub-heading and the items of its section that have the same designation in every output format, and that is restarted at a fixed value, which will not follow edits), `render-docx-number-mismatch` (any other difference: a catch-all that does not occur outside documents that skip a heading level), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits, with the class it falls in; one that has the number of a skipped heading level as text; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-alternative` (alternatives share a number as fields of the first, which must not be deleted in Word), `render-docx-static-restart` (a list that restarts by a fixed setting: in a list of another kind, in an item on the heading list that it cannot ride, in an alternative, or the one that begins the next list past nine levels) `render-docx-libreoffice-count` (items numbered as in Word but one or more higher in LibreOffice, which counts a level nothing numbered before a deeper item of its list) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). The Word-only checks (the contents' page numbers, what `\t` does, numbering after an insertion) are a manual checklist before a release (`.github/PUBLISHING.md`). HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | diff --git a/docs/architecture.md b/docs/architecture.md index 2a95af6..4af03cb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -440,7 +440,7 @@ already gives; references that fail alike are one finding), `reference-cycle` (a leads back to it: written as text), and, as information, `alternative` (alternatives share a number as fields of the first, which deleting in Word breaks), `static-restart` (a list that restarts at 1 by a fixed setting: nested in a list of another kind, in an item whose number is on the heading list but which the list cannot ride, in an alternative that shares a number, or the list that begins the next abstractNum past Word's nine -levels) and `reference-word-only` (a cover relies on `\t`, which LibreOffice ignores, once per document). A +levels) `libreoffice-count` (items that Word numbers as the renderer does but LibreOffice reads higher, because a level of their list that nothing numbered before a deeper item is counted as 1 there and goes on from it; once per document) and `reference-word-only` (a cover relies on `\t`, which LibreOffice ignores, once per document). A heading level that is skipped is the Core Error `heading-skip`; the plan only notes it on the unit. ### 5. Write (`writers/`) @@ -522,7 +522,7 @@ Diagnostics reuse `legaldown.Diagnostic`. The renderer adds rules only it can ev `ref-not-enumerated`, `path-not-relative` and `path-outside-root` (Errors for a file reference that is not relative, or that leaves the document root, §2.3; reported here because validator 0.4 checks neither), specification rule ids, plus renderer-specific ids prefixed `render-`: -`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and the rules the DOCX planner reports on what Word draws differently from the other formats (`render-docx-section-label`, `-list-depth`, `-letter-overflow`, `-roman-overflow`, `-shared-counter`, `-number-mismatch`, `-reference`, `-reference-cycle`, as Warnings, `-alternative`, `-static-restart` and `-reference-word-only` as Infos; see "The DOCX planner"). The validator's +`render-not-processed`, `render-link-refused` (a Warning for a link whose URL scheme links may not use; its text is shown), `render-locale-fallback` (an Info when no formatting locale is known, a Warning when a duration unit has no CLDR pattern in the locale and English is used), `render-ref-ambiguous`, `render-term-limit` (a Warning when defined terms refer to each other too deeply or too often: past 32 nested terms, 200,000 inline nodes resolved inside repeat computations of a term text (only cycles cause them), or 60 inline levels nested inside an open term, a `{{term:}}` shows its id; one block's own nesting, up to the builder's 100 levels, never counts), `render-designation-cycle` (a Warning for a heading that refers back to itself under the `none` scheme) and `render-designation-limit` (a Warning under the `none` scheme for headings that refer to each other past 32 headings, 60 nested levels, or 1,000 characters, and for a reference to a heading longer than 1,000 characters, which shows its first 1,000), and `render-term-too-long` (a Warning for a defined term whose text, with its references resolved, is longer than 1,000 characters: `{{term:}}` then shows the definition id), and the rules the DOCX planner reports on what Word draws differently from the other formats (`render-docx-section-label`, `-list-depth`, `-letter-overflow`, `-roman-overflow`, `-shared-counter`, `-number-mismatch`, `-reference`, `-reference-cycle`, as Warnings, `-alternative`, `-static-restart`, `-libreoffice-count` and `-reference-word-only` as Infos; see "The DOCX planner"). The validator's diagnostics carry their line (§16.9), and so do the renderer's own, and the CLI prints it as `file:line:`. diff --git a/src/legaldown_render/resolve/docx_numbering.py b/src/legaldown_render/resolve/docx_numbering.py index 4bb20e6..d99d937 100644 --- a/src/legaldown_render/resolve/docx_numbering.py +++ b/src/legaldown_render/resolve/docx_numbering.py @@ -244,6 +244,10 @@ class Step(NamedTuple): word_ref: str #: The label as tokens, each counter with the unit (or ghost) that holds it. tokens: tuple[Token, ...] + #: The counter of the paragraph's own level as LibreOffice counts it (-1: not modelled): it counts a level that + #: nothing numbered before a deeper one as its start value and goes on from it, where Word shows one less and + #: leaves it unused, so the next paragraph of that level is 2 there and 1 in Word. + lo_count: int = -1 class WordNumbering: @@ -262,6 +266,7 @@ def __init__(self, *, skipped: Literal["lo", "word"] = "lo") -> None: self._levels: dict[int, Sequence[LevelDef]] = {} self._nums: dict[int, Num] = {} self._count: dict[int, list[int]] = {} + self._lo: dict[int, list[int]] = {} self._holder: dict[int, list[object | None]] = {} self._used: set[tuple[int, int]] = set() @@ -270,6 +275,7 @@ def add_abstract(self, abstract_id: int, levels: Sequence[LevelDef], *, self._levels[abstract_id] = levels self._skipped[abstract_id] = skipped or self.skipped self._count[abstract_id] = [0] * 9 + self._lo[abstract_id] = [0] * 9 self._holder[abstract_id] = [None] * 9 def add_num(self, num: Num) -> None: @@ -299,14 +305,20 @@ def step(self, num_id: int, ilvl: int, owner: object) -> Step: num = self._nums[num_id] levels = self._levels[num.abstract_id] count, holder = self._count[num.abstract_id], self._holder[num.abstract_id] + lo = self._lo[num.abstract_id] overrides = self._overrides(num_id) if ilvl in overrides and (num_id, ilvl) not in self._used: self._used.add((num_id, ilvl)) - count[ilvl] = overrides[ilvl] - 1 + count[ilvl] = lo[ilvl] = overrides[ilvl] - 1 count[ilvl] += 1 + lo[ilvl] += 1 holder[ilvl] = owner for deeper in range(ilvl + 1, 9): count[deeper], holder[deeper] = 0, None + lo[deeper] = 0 + for above in range(ilvl): + if lo[above] == 0: + lo[above] = levels[above].start skipped = self._skipped[num.abstract_id] for above in range(ilvl): if count[above] == 0: @@ -325,4 +337,4 @@ def step(self, num_id: int, ilvl: int, owner: object) -> Step: count[above], holder[above] = 0, None label = text_of(tokens) return Step(count[ilvl], word_counter(count[ilvl], levels[ilvl].num_fmt), label, - label.rstrip("."), tuple(tokens)) + label.rstrip("."), tuple(tokens), lo[ilvl]) diff --git a/src/legaldown_render/resolve/docx_plan.py b/src/legaldown_render/resolve/docx_plan.py index 75ad47a..6b6ad28 100644 --- a/src/legaldown_render/resolve/docx_plan.py +++ b/src/legaldown_render/resolve/docx_plan.py @@ -659,6 +659,8 @@ def __init__(self, tree: RenderTree, file: str | None, text_switch: TextSwitch = #: The labels Word draws differently for want of counters it cannot show, by what they have in common, as #: (line, message) of each place; reported once for each group (:meth:`report_section_labels`). self.label_findings: dict[tuple[object, ...], list[tuple[int | None, str]]] = {} + #: The units of lists whose number LibreOffice reads higher than Word does (:attr:`.docx_numbering.Step.lo_count`). + self.lo_counts: list[tuple[_Unit, Step]] = [] self.drifts: list[_Drift] = [] self.open_drift: dict[tuple[int, int], _Drift] = {} self.heading_num: Num | None = None @@ -1064,6 +1066,7 @@ def run(self) -> tuple[DocxPlan, list[Diagnostic]]: for block in section.blocks: self.block(block, section) self.report_section_labels() + self.report_libreoffice_counts() for drift in self.drifts: self.report_drift(drift) self.report_collisions() @@ -1522,6 +1525,8 @@ def numbered(self, node: _Unit, kind: Literal["section", "item", "paragraph"], n "are kept as the renderer shows them by restarting at a fixed number, which will not follow " "items added or removed before it.", node.line) step = self.sim.step(used.id, ilvl, unit) + if abstract.kind != "headings" and step.lo_count not in (-1, step.count): + self.lo_counts.append((node, step)) if kind != "section": # The counters of the unit's own chain: the units it stands under, and every part of the number of the # section (a skipped heading level counts, as the ghost of the heading that skipped it). @@ -1564,6 +1569,30 @@ def restart_class(self, parent: str, placed: _Placed, abstract: _Abstract, *, de return "restart-section" if placed.abstract.kind == "headings" else "restart-family" return "resync" + def report_libreoffice_counts(self) -> None: + """One information for the units LibreOffice numbers higher than Word does. A level of a list that nothing + counted before a deeper item of the same list (an item nested in a unit of another list: its own level above is + never used here) is read by Word as unused, so the next unit of that level is 1; LibreOffice counts the + unused level as 1 and goes on from it, so that unit is 2 there. The plan is Word's, the file is right for it.""" + if not self.lo_counts: + return + node, step = self.lo_counts[0] + more = f", and {len(self.lo_counts) - 1} more" if len(self.lo_counts) > 1 else "" + self.report("render-docx-libreoffice-count", "info", + f"{len(self.lo_counts)} list item(s) are numbered as the renderer does in Word but one or more higher " + f"in LibreOffice, which counts a level that nothing in the list numbered before a deeper item as 1: " + f"the item '{printable(node.label or '')}' (the first{more}) reads '{step.own}' in Word and " + f"'{printable(self.in_libreoffice(node, step))}' there. Nothing in the file is wrong for Word.", + node.line) + + def in_libreoffice(self, node: _Unit, step: Step) -> str: + """The counter of *node* as LibreOffice reads it, in the format of its level.""" + assert node.unit is not None + unit = self.units[node.unit] + assert unit.ilvl is not None + abstract = next(a for a in self.abstracts if a.id == self.nums[(unit.num_id or 1) - 1].abstract_id) + return word_counter(step.lo_count, abstract.levels[unit.ilvl].num_fmt) + def report_depth(self, depth: int, line: int | None) -> None: """Report, once, a list nested past the levels the plan has (:data:`MAX_DEPTH`), of any family.""" if not self.depth_reported: diff --git a/tests/test_docx_plan.py b/tests/test_docx_plan.py index 486489c..a2a7cb4 100644 --- a/tests/test_docx_plan.py +++ b/tests/test_docx_plan.py @@ -1783,7 +1783,7 @@ def extra_runs(name: str, switch: str) -> tuple: """(source, plan, diagnostics) of a sample of the random documents without a skipped heading level, under the label forms of :data:`EXTRA_VARIANTS` and a setting of the switch.""" runs = [] - for source in (*sample(name, 4, _random_sources()), *sample(name, 2, _alternative_sources())): + for source in (*sample(name, 8, _random_sources()), *sample(name, 2, _alternative_sources())): prefix = "" if source.startswith("---") else FRONT result = render(prefix + source, format="text", **EXTRA_VARIANTS[name][0]) plan, diagnostics = plan_docx(result.tree, text_switch=switch) # type: ignore[arg-type] @@ -2410,6 +2410,21 @@ def test_a_reference_below_a_skipped_level_whose_number_Word_draws_with_another_ assert label.message.endswith(" 2 more places do the same.") # the three paragraphs, one finding +def test_a_level_nothing_counted_before_a_deeper_item_is_read_higher_by_libreoffice_and_the_plan_says_so() -> None: + # c is an ordered item (another list), d nested in it is on the enumerated list's level 4 with the level 3 above it + # never numbered there; e and f are the first items of that level 3: 1 and 2 in Word, 2 and 3 in LibreOffice + tree, plan, diagnostics = planned("# A\n\n- a\n - b\n 1. c\n - d\n - e\n - f\n") + e, f = [i for i in items(tree) if i.label in ("(A)", "(B)")] + assert [plan.units[i.unit].count for i in (e, f)] == [1, 2] + [found] = [d for d in diagnostics if d.rule == "render-docx-libreoffice-count"] + assert found.level == "info" and found.line == e.line + assert "the item '(A)' (the first, and 1 more) reads 'A' in Word and 'B' there" in found.message + assert "Nothing in the file is wrong for Word" in found.message + # without the deeper item first, nothing differs + _, _, diagnostics = planned("# A\n\n- a\n - b\n 1. c\n - e\n - f\n") + assert "render-docx-libreoffice-count" not in rules(diagnostics) + + def test_the_labels_of_the_lists_of_one_chain_that_lose_counters_are_one_finding() -> None: body = ("# A\n\n- one\n 1. ordered\n - three\n - four\n - three b\n - four c\n" " 1. ordered two\n - x\n - y\n") diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index 4710ee7..a2c3e0f 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -1052,11 +1052,8 @@ def test_the_contents_list_the_headings_as_the_plan_does_and_none_of_the_items_o ] -@functools.cache def path_clauses() -> str: - require_libreoffice() - data = render(PATH_CLAUSES, format="docx", **PATH_ITEMS).binary - return libreoffice_edit({"paths": (data, [(match, new) for match, new, _, _ in PATH_EDITS])})["paths"] # type: ignore[arg-type] + return later_edits()["paths"] def test_a_path_label_on_the_reserved_levels_follows_the_items_a_lawyer_inserts_and_so_do_its_references() -> None: @@ -1086,16 +1083,10 @@ def test_a_path_label_on_the_reserved_levels_follows_the_items_a_lawyer_inserts_ """ -@functools.cache def ordered_paths() -> dict[str, str]: """The ordered-parent document as it is, with a nested item inserted in the detached list, and with a new parent whose nested item is a copy of one of the detached list's (Word's paste keeps the list).""" - require_libreoffice() - data = render(ORDERED_PATHS, format="docx", **ORDERED_PARENT).binary - return libreoffice_edit({ # type: ignore[arg-type] - "original": (data, []), - "nested inserted": (data, [("four b", "NEW four")]), - "new parent": (data, [("three b", "NEW three"), ("three b", "NEW copy", "four b")])}) + return {name: text for name, text in later_edits().items() if name in ("original", "nested inserted", "new parent")} def test_a_detached_list_draws_its_own_counter_and_its_references_are_live() -> None: @@ -1133,11 +1124,8 @@ def test_a_new_parent_whose_nested_item_is_copied_from_a_detached_list_goes_on_c ] -@functools.cache def deep_lists() -> str: - require_libreoffice() - data = render(DEEP, format="docx").binary - return libreoffice_edit({"deep": (data, [(match, new) for match, new, _, _ in DEEP_EDITS])})["deep"] # type: ignore[arg-type] + return later_edits()["deep"] def test_items_nested_past_nine_levels_are_numbered_by_the_list_they_go_on_in_and_follow_edits() -> None: @@ -1168,15 +1156,33 @@ def test_items_nested_past_nine_levels_are_numbered_by_the_list_they_go_on_in_an """ -@functools.cache def skipped() -> tuple[str, str]: """What LibreOffice says about the document that skips a heading level (outline numbering) after a heading is inserted before its first section, and what the text writer prints for the edited source.""" - require_libreoffice() - data = render(SKIPPED, format="docx", style="outline").binary - said = libreoffice_edit({"skipped": (data, [("One", "NEWH")])})["skipped"] # type: ignore[arg-type] edited_source = applied(SKIPPED, [("# One {#one}", "# NEWH\n")]) - return said, render(edited_source, format="text", style="outline").output + return later_edits()["skipped"], render(edited_source, format="text", style="outline").output + + +@functools.cache +def later_edits() -> dict[str, str]: + """What LibreOffice says about the documents of the sections above after their edits, all in one LibreOffice session + (starting it is what costs): path labels in a clause, the ordered-parent list as it is, with a nested item inserted + and with a new parent whose nested item is a copy, the list past nine levels, and the document that skips a level.""" + require_libreoffice() + + def docx(source: str, **settings: object) -> bytes: + data = render(source, format="docx", **settings).binary + assert data is not None + return data + + ordered = docx(ORDERED_PATHS, **ORDERED_PARENT) + return libreoffice_edit({ # type: ignore[arg-type] + "paths": (docx(PATH_CLAUSES, **PATH_ITEMS), [(match, new) for match, new, _, _ in PATH_EDITS]), + "original": (ordered, []), + "nested inserted": (ordered, [("four b", "NEW four")]), + "new parent": (ordered, [("three b", "NEW three"), ("three b", "NEW copy", "four b")]), + "deep": (docx(DEEP), [(match, new) for match, new, _, _ in DEEP_EDITS]), + "skipped": (docx(SKIPPED, style="outline"), [("One", "NEWH")])}) def test_the_live_parts_of_a_reference_below_a_skipped_level_follow_a_heading_inserted_before_it() -> None: @@ -1191,3 +1197,22 @@ def test_the_live_parts_of_a_reference_below_a_skipped_level_follow_a_heading_in assert not any(r.fixed for r in plan.refs.values()) assert any(d.rule == "render-docx-reference" and "heading level that the document skips" in d.message for d in diagnostics) + + +# -- A level that nothing counted before a deeper item: where LibreOffice and Word differ ----------------------------------- + +UNCOUNTED = FRONT + "# A\n\n- a\n - b\n 1. c\n - d\n - e\n - f\n" + + +def test_libreoffice_counts_the_level_nothing_numbered_before_a_deeper_item_and_the_plan_reports_it() -> None: + """The one place LibreOffice's numbers differ from the plan's for a reason of Word's model, not of the file: the items + ``e`` and ``f`` are the first of their level in their list, which Word numbers (A), (B) and LibreOffice (B), (C) + (it counts the level above ``d`` as one). The plan is Word's and reports the difference.""" + require_libreoffice() + result = render(UNCOUNTED, format="docx") + plan, diagnostics = plan_docx(result.tree) + said = lines_of(libreoffice_export({"uncounted": result.binary})["uncounted"].text) # type: ignore[arg-type] + assert [line for line in said if line.endswith((" e", " f"))] == ["(B) e", "(C) f"] + assert [plan.units[u.unit].label for u in units(result.tree) if plan.units[u.unit].count and u.unit in ("u5", "u6")] \ + == ["(A)", "(B)"] + assert any(d.rule == "render-docx-libreoffice-count" and d.level == "info" for d in diagnostics) diff --git a/tests/test_docx_simulator.py b/tests/test_docx_simulator.py index 65b4206..954a17f 100644 --- a/tests/test_docx_simulator.py +++ b/tests/test_docx_simulator.py @@ -232,6 +232,22 @@ def test_each_abstract_num_has_its_own_model_of_a_level_that_nothing_numbered() assert sim.step(2, 1, "u1").label == ".1" # Word's: a letter counter at 0 is nothing +def test_libreoffice_counts_a_level_nothing_numbered_before_a_deeper_one_and_goes_on_from_it() -> None: + """The count the plan gives is Word's (the unused level stays unused, so the next unit of that level is 1); + ``Step.lo_count`` is LibreOffice's (the unused level reads as its start and counts, so the next unit is 2).""" + sim = numbering(Num(1, 1), skipped="word") + first = sim.step(1, 2, "u1") # level 3 first: levels 1 and 2 were never numbered in this list + assert (first.count, first.lo_count) == (1, 1) + second = sim.step(1, 1, "u2") + assert (second.count, second.lo_count) == (1, 2) and second.label == "(i)" + third = sim.step(1, 1, "u3") + assert (third.count, third.lo_count) == (2, 3) + # a level that was numbered is counted the same in both + fresh = numbering(Num(1, 1), skipped="word") + assert [(step.count, step.lo_count) for step in (fresh.step(1, 0, "a"), fresh.step(1, 1, "b"), fresh.step(1, 1, "c"))] \ + == [(1, 1), (1, 1), (2, 2)] + + def test_the_first_paragraph_of_a_list_deeper_than_the_level_it_starts_at_fills_the_levels_above() -> None: sim = numbering(Num(1, 1, ((1, 1),))) assert sim.step(1, 1, "u1").label == "(i)" diff --git a/tests/test_docx_wordcheck.py b/tests/test_docx_wordcheck.py index ff5c530..bdd7caa 100644 --- a/tests/test_docx_wordcheck.py +++ b/tests/test_docx_wordcheck.py @@ -23,7 +23,7 @@ def test_the_document_is_a_package_that_holds_every_row_and_the_assumption_besid assert "Row 6, an abbreviation with a period: No. %1: drawn 'No. 1'" in text assert "documented: the switch is not used; guess: '1'" in text # No. 1: only the guess drops the period assert "documented: 'I.1'; guess: 'I.1'" in text # Microsoft's example with a roman counter - assert "Part %1 of %2" in text and "documented: the switch is not used; guess: '12'" in text # words between counters + assert "Part %1 of %2" in text and "documented: the switch is not used; guess: '11'" in text # words between counters assert contents["word/document.xml"].count(b"REF t3 \\n \\t \\h") == 1 assert suppressed([Lit("Section "), Count("a", "1"), Lit("."), Count("b", "1")]) is not None From 72f8f401fd103f81d16528e5045d27428e443798 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:31:45 +0000 Subject: [PATCH 111/118] Make the attachment-id and page-count tests mean what they say An attachment whose id is a heading's is now one attachment, not a second copy of the duplicate-id case. The page tests compare page numbers and counts with each other and with page breaks instead of with what a particular font gives. Two comments that said the table of contents was not written yet are corrected. --- tests/test_docx_libreoffice.py | 41 ++++++++++++++++++++++++++++++---- tests/test_docx_matter.py | 28 +++++++++++++---------- tests/test_docx_roundtrip.py | 6 ++--- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/tests/test_docx_libreoffice.py b/tests/test_docx_libreoffice.py index 85fd6b2..794e0ec 100644 --- a/tests/test_docx_libreoffice.py +++ b/tests/test_docx_libreoffice.py @@ -53,6 +53,22 @@ LONG = "---\ntitle: Long\n---\n\n" + "".join(f"# Heading {n}\n\nParagraph {n} " + "word " * 120 + "\n\n" for n in range(40)) +#: A few lines and three attachments, each of which starts a page (``attachments.separator: page-break``, the default): +#: how many pages it has is the layout's business, but never fewer than the breaks make, and in the order of the document. +PAGED = """--- +legaldown: '0.2' +title: Paged +attachments: + - {id: alpha, title: Alpha Schedule} + - {id: beta, title: Beta Schedule} + - {id: gamma, title: Gamma Schedule} +--- + +# One + +See {{attach: gamma}}. +""" + HOSTILE = "---\nlegaldown: '0.2'\ntitle: \"a\\x01b\"\n---\n\n# H\x01\x08\x0b\x0c\x1f\x7f\x85x\n\nbody\x01 <&> ]]>\n" @@ -89,6 +105,7 @@ def shown() -> dict[str, Exported]: "features": docx((DOCUMENTS / "features.lgd").read_text(encoding="utf-8")), "template": docx((DOCUMENTS / "template.lgd").read_text(encoding="utf-8"), style="continental"), "long": docx(LONG), + "paged": docx(PAGED, overrides={"contents.enabled": True, "page.footer": {"left": "{page}", "center": "", "right": "{pages}"}}), "hostile": docx(HOSTILE), "no_footer": docx(ROUND_TRIP, overrides={"page.footer": {"center": ""}}), "landscape": docx(ROUND_TRIP, overrides={"page.size": "A4 landscape", "page.footer.right": "{page}"}), @@ -120,20 +137,36 @@ def test_the_header_shows_the_templates_and_the_page_field(shown: dict[str, Expo def test_the_default_footer_counts_the_pages(shown: dict[str, Exported]) -> None: footer = shown["round_trip"].footer - assert re.fullmatch(r"1 / 1", footer), footer - assert shown["round_trip"].pages == 1 + match = re.fullmatch(r"1 / (\d+)", footer) # a few lines: one page in any font + assert match is not None, footer + assert int(match.group(1)) == shown["round_trip"].pages == 1 assert shown["round_trip"].header.count("\t") == 2 def test_the_page_count_field_follows_the_layout(shown: dict[str, Exported]) -> None: - long = shown["long"] - assert long.pages > 2 + """How many pages a text takes depends on the fonts of the machine, so the counts are compared with each other: the + long document has more pages than the short one, and its footer reads the first page and the number of pages the + layout has.""" + long, short = shown["long"], shown["round_trip"] + assert long.pages > short.pages match = re.fullmatch(r"(\d+) / (\d+)", long.footer) assert match is not None, long.footer assert int(match.group(2)) == long.pages assert 1 <= int(match.group(1)) <= long.pages +def test_each_attachment_starts_a_page_and_the_contents_give_their_pages_in_order(shown: dict[str, Exported]) -> None: + """The page breaks hold in any font: three attachments are on three different pages, in the order of the document, the + first not on the first page, and the footer's page count is the layout's.""" + paged = shown["paged"] + pages = [int(m[1]) for title in ("Alpha", "Beta", "Gamma") + for m in [re.search(rf"^\s*{title} Schedule\t(\d+)$", paged.text, re.M)] if m] + assert len(pages) == 3, paged.text + assert 2 <= pages[0] < pages[1] < pages[2] <= paged.pages + left, right = paged.footer.split("\t")[0], paged.footer.split("\t")[-1] + assert 1 <= int(left) <= paged.pages and right == str(paged.pages), paged.footer # {page} is that of the view's cursor + + def test_no_footer_means_none(shown: dict[str, Exported]) -> None: assert shown["no_footer"].footer == "" diff --git a/tests/test_docx_matter.py b/tests/test_docx_matter.py index bf00109..6d3b289 100644 --- a/tests/test_docx_matter.py +++ b/tests/test_docx_matter.py @@ -548,28 +548,31 @@ def test_the_attachments_listed_in_the_contents_carry_its_bookmarks() -> None: assert len(set(bookmark_names(root))) == len(bookmark_names(root)) +#: What the attachments say, and how many there are: two with no id, two with the same id, and one whose id is the +#: identifier of a heading (``# A {#a}`` below; ``attachment-id-collision``). BAD_ATTACHMENTS = { - "no ids": " - {title: NoId, file: a.lgd}\n - {title: NoId2, file: b.lgd}\n", - "a duplicate id": " - {id: x, title: A}\n - {id: x, title: B}\n", - "an id that is a section's": " - {id: a, title: A}\n - {id: a, title: B}\n", + "no ids": (" - {title: NoId, file: a.lgd}\n - {title: NoId2, file: b.lgd}\n", 2), + "a duplicate id": (" - {id: x, title: A}\n - {id: x, title: B}\n", 2), + "an id that is a section's": (" - {id: a, title: A}\n", 1), } @pytest.mark.parametrize("which", BAD_ATTACHMENTS) @pytest.mark.parametrize("overrides", [{}, {"labels.attachments": ""}, {"contents.depth": 1}]) def test_attachments_without_a_usable_id_are_still_written_and_listed(which: str, overrides: dict[str, Any]) -> None: - """Attachments with no id, or an id another has (validator Errors: the document still renders), have no anchor of their - own: the contents lists each by position, and no bookmark is written twice.""" - source = ("---\nlegaldown: '0.2'\ntitle: T\nattachments:\n" + BAD_ATTACHMENTS[which] + """Attachments with no id, or an id another attachment or a heading has (validator Errors: the document still + renders), have no anchor of their own: the contents lists each by position, and no bookmark is written twice.""" + yaml, count = BAD_ATTACHMENTS[which] + source = ("---\nlegaldown: '0.2'\ntitle: T\nattachments:\n" + yaml + "---\n\n# A {#a}\n\nSee {{attach: x}}.\n") result = render(source, format="docx", overrides={"contents.enabled": True, **overrides}) assert result.binary is not None root = document_xml(result.binary) names = bookmark_names(root) assert len(names) == len({n.lower() for n in names}) - assert len(paragraphs(result.binary, "AttachmentTitle")) == 2 + assert len(paragraphs(result.binary, "AttachmentTitle")) == count toc = [text_of(p).rstrip("\t") for p in toc_paragraphs(result.binary)] - assert toc[-2:] == [text_of(p) for p in paragraphs(result.binary, "AttachmentTitle")] + assert toc[-count:] == [text_of(p) for p in paragraphs(result.binary, "AttachmentTitle")] anchors = [p.find(f"{W}hyperlink").get(f"{W}anchor") for p in toc_paragraphs(result.binary)] # type: ignore[union-attr] assert set(anchors) <= set(names) if schema_available(): @@ -815,13 +818,13 @@ def test_page_numbers_appear_after_the_update(shown: dict[str, Exported]) -> Non assert len(pages) == len(toc_lines(shown["master"])) and all(p >= 1 for p in pages) assert pages == sorted(pages) assert max(pages) <= shown["master"].pages - # Where the pages are known: the long document's six headings of about a page each run in order from page 1, and - # the attachments after them end the document. + # What holds in any font: the long document's six headings (a fifth of a page of text each) run in order from page 1, + # and the attachments, which each start a page, come after them and end the document. long = shown["long_break"] by_title = dict(zip(toc_lines(long), toc_pages(long), strict=True)) headings = [by_title[f"{n + 1}. Heading {n}"] for n in range(6)] assert headings[0] == 1 and all(0 <= b - a <= 1 for a, b in zip(headings, headings[1:], strict=False)) - assert headings[-1] > headings[0] and by_title["Attachments"] > headings[-1] + assert by_title["Attachments"] > headings[-1] assert by_title["Third"] == long.pages == by_title["Attachments"] + 2 @@ -866,7 +869,8 @@ def test_a_page_break_separator_puts_each_attachment_on_its_own_page(shown: dict def test_rule_and_none_keep_the_attachments_on_the_pages_they_fall_on(shown: dict[str, Exported]) -> None: for name in ("long_rule", "long_none"): by_title = dict(zip(toc_lines(shown[name]), toc_pages(shown[name]), strict=True)) - assert by_title["First"] == by_title["Second"] == by_title["Third"] + # a few lines each: no break between them, so at most the bottom of one page and the top of the next + assert 0 <= by_title["Third"] - by_title["First"] <= 1 and by_title["First"] <= by_title["Second"] <= by_title["Third"] assert shown["long_break"].pages >= shown["long_rule"].pages + 2 diff --git a/tests/test_docx_roundtrip.py b/tests/test_docx_roundtrip.py index a2c3e0f..5099763 100644 --- a/tests/test_docx_roundtrip.py +++ b/tests/test_docx_roundtrip.py @@ -14,7 +14,8 @@ Every body paragraph's text must also be what the file cached for it, and a second round trip makes the edits a lawyer makes (an item, a heading and an ordered item inserted before the targets, in LibreOffice) and updates the fields: every reference and number must read as the renderer writes them for the edited source. The table of -contents is not written yet, so its entries and the ``_Toc`` bookmarks are not exercised. +contents is not part of this comparison (its entries are in ``test_docx_matter.py`` and ``test_docx_acceptance.py``, +which also compares whole documents with the plain-text writer's). The exported text is then compared with what the renderer wrote (the labels and designations that the text writer prints), **modulo the gaps the plan documents**, each of which the plan reports with a diagnostic: @@ -661,8 +662,7 @@ def shown_lines(shown: Exported) -> list[str]: def check(case: Case) -> None: """The property: every number LibreOffice draws is the renderer's, or the plan's where the plan says Word differs (with a diagnostic), every reference reads as the renderer wrote it, or as the plan says, and every body paragraph - says what the file cached for it. The table of contents is not written yet, so its entries and the ``_Toc`` - bookmarks are not exercised.""" + says what the file cached for it. The table of contents is compared elsewhere (``test_docx_matter.py``).""" assert case.shown is not None lines = marked_lines(case.shown) excused = False From 78f709a9366c3f86d70952c115cb8ccc00663e68 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:31:45 +0000 Subject: [PATCH 112/118] Add the DOCX acceptance corpus LibreOffice's text of every corpus document, with fields and indexes updated, equals the plain-text writer's under the default, outline and continental styles, modulo the differences the formats have and the gaps the plan reports; the documents follow a heading, an item, a nested item, a numbered paragraph inserted and a clause deleted in the editor; every part validates and a second render is the same bytes. The module is left out of a plain pytest run (marker acceptance) and runs in the DOCX job. pytest-xdist joins the dev extra: pytest -n auto runs the suite in a third of the time. --- .github/workflows/ci.yml | 3 + pyproject.toml | 8 +- tests/conftest.py | 12 + tests/test_docx_acceptance.py | 578 ++++++++++++++++++++++++++++++++++ 4 files changed, 598 insertions(+), 3 deletions(-) create mode 100644 tests/test_docx_acceptance.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd2fbec..29a8825 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,10 @@ jobs: - run: pytest -q -m ooxml_schema env: LEGALDOWN_SPEC_DIR: .legaldown-spec + # The acceptance corpus (tests/test_docx_acceptance.py) is part of both runs; it needs the specification's documents - run: pytest -q -m libreoffice + env: + LEGALDOWN_SPEC_DIR: .legaldown-spec validator-main: # Surfaces changes in legaldown-validator's model before they are released diff --git a/pyproject.toml b/pyproject.toml index b1573f1..db62ad2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,14 +6,14 @@ build-backend = "hatchling.build" name = "legaldown-render" # Single-sourced from src/legaldown_render/__init__.py. dynamic = ["version"] -description = "Reference renderer for the LegalDown legal document format: HTML and plain text, DOCX in progress, PDF planned" +description = "Reference renderer for the LegalDown legal document format: HTML, plain text and DOCX (live Word numbering and fields), PDF planned" readme = "README.md" requires-python = ">=3.11" license = "MIT" license-files = ["LICENSE"] authors = [{ name = "ForLegalAI s.r.o." }] maintainers = [{ name = "ForLegalAI s.r.o." }] -keywords = ["legaldown", "legal", "contracts", "legaltech", "markdown", "renderer", "html"] +keywords = ["legaldown", "legal", "contracts", "legaltech", "markdown", "renderer", "html", "docx", "word"] classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", @@ -41,7 +41,8 @@ dependencies = [ [project.optional-dependencies] # lxml validates the DOCX parts against the OOXML schemas in the tests (tests/schemas/README.md). -dev = ["pytest>=9.2", "ruff>=0.17", "build>=1.6", "lxml>=6.1"] +# pytest-xdist is optional: `pytest -n auto` runs the suite on all cores (a third of the time). +dev = ["pytest>=9.2", "ruff>=0.17", "build>=1.6", "lxml>=6.1", "pytest-xdist>=3.8"] [project.scripts] legaldown-render = "legaldown_render.cli:main" @@ -77,6 +78,7 @@ markers = [ "conformance: renders the LegalDown specification's examples and fixtures (needs a checkout of the specification)", "ooxml_schema: validates DOCX parts against the ISO/IEC 29500 schemas (needs lxml and the schemas: python tests/schemas/fetch.py)", "libreoffice: opens DOCX files in LibreOffice and compares what it shows (needs soffice and python3-uno)", + "acceptance: the DOCX acceptance corpus (tests/test_docx_acceptance.py); left out of a run that names no marker (-m), unless LEGALDOWN_ACCEPTANCE is set or the file is named", ] # A deprecated legaldown-validator API fails the tests: the next minor # release removes it, and the dependency range admits only one minor version. diff --git a/tests/conftest.py b/tests/conftest.py index c01c435..117506e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -61,6 +61,18 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="rewrite the golden files from the current output instead of comparing") +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Leave the DOCX acceptance corpus (marker ``acceptance``) out of a plain ``pytest`` run, which stays as quick as it + was: it runs when a marker expression is given (``-m libreoffice``, as the DOCX job does), when + ``LEGALDOWN_ACCEPTANCE`` is set, or when a command-line argument names its file.""" + if config.getoption("markexpr") or os.environ.get("LEGALDOWN_ACCEPTANCE") or any("acceptance" in a for a in config.args): + return + dropped = [item for item in items if item.get_closest_marker("acceptance")] + if dropped: + config.hook.pytest_deselected(items=dropped) + items[:] = [item for item in items if item not in dropped] + + @pytest.fixture def update_golden(request: pytest.FixtureRequest) -> bool: return bool(request.config.getoption("--update-golden")) diff --git a/tests/test_docx_acceptance.py b/tests/test_docx_acceptance.py new file mode 100644 index 0000000..5eb70d7 --- /dev/null +++ b/tests/test_docx_acceptance.py @@ -0,0 +1,578 @@ +"""Acceptance of the DOCX writer: the measurable claims of ADR 0008, over the whole corpus and the three built-in styles. + +The corpus is that of ``test_docx_roundtrip.py`` (``tests/documents``, the specification's examples and valid +fixtures, and the documents written to stress the numbering); that module is the finer instrument (it makes the cached +results wrong on purpose and compares every number and reference unit by unit), and this one the coarse, complete one: + +* **Equivalence.** LibreOffice opens each document with its fields and indexes updated, and what it shows is what the + plain-text writer prints, line by line, **modulo the differences the formats have by nature** (:func:`normalise`: + borders for rules and signing lines, a table's cells, the markup of the plain-text layout) **and the gaps the plan + reports** (:data:`GAPS`: a number Word draws differently from the renderer's, with a ``render-docx-*`` diagnostic). + The numbers, the cross-references, the entries of the contents, the attachments and the signature blocks are in that + text. The page numbers of the contents are checked on their own: they are positive, never decrease and never pass the + number of pages. +* **Live after edits.** In one LibreOffice session, for a handful of documents under each style, a heading, a list + item, a nested item and (under the continental style) a numbered paragraph are inserted, and a clause is deleted, + as a lawyer does (:mod:`tests/lo/uno_edit.py`); the fields and indexes are updated; the result is what the text + writer prints for the source with the same edits made. A document may differ only where the plan reports a number it + fixes (:data:`FIXED`), and every such difference is listed. +* **Valid and deterministic.** Every part of every document validates against the ISO/IEC 29500 schemas, and rendering a + document twice gives the same bytes. + +The LibreOffice tests are skipped when ``soffice`` or ``python3-uno`` is missing and fail instead when +``LEGALDOWN_REQUIRE_DOCX_TOOLS`` is set (``tests/docx_support.py``). The module is part of the DOCX job (``pytest -m +"libreoffice or ooxml_schema"``): ``conftest.py`` leaves it out of a run that names no marker, which keeps the default +``pytest`` as quick as it was. +""" +from __future__ import annotations + +import difflib +import functools +import re +import xml.etree.ElementTree as ET +from collections.abc import Iterator +from dataclasses import dataclass + +import pytest +from conftest import spec_root +from docx_support import ( + Exported, + W, + libreoffice_edit, + libreoffice_export, + parts, + require_libreoffice, + require_schemas, + tool_missing, + validation_errors, +) +from test_docx_roundtrip import EXPLAINING, corpus + +from legaldown_render import DocumentError, render +from legaldown_render.tree import ( + Code, + CrossRef, + DraftingNote, + Emphasis, + List, + ListItem, + Paragraph, + Quote, + RenderTree, + Section, + Strong, + Text, + iter_tree_inlines, +) +from legaldown_render.writers.text import inline_text + +pytestmark = pytest.mark.acceptance + +#: The built-in styles, with the table of contents on (down to the fifth level, so that every heading is in it). +STYLES: dict[str, dict] = { + "default": {"overrides": {"contents.enabled": True, "contents.depth": 5}}, + "outline": {"style": "outline", "overrides": {"contents.enabled": True, "contents.depth": 5}}, + "continental": {"style": "continental", "overrides": {"contents.enabled": True, "contents.depth": 5}}, +} + +#: The diagnostics that say Word (and so LibreOffice, which follows its numbering) draws a number differently from the +#: renderer's: a letter counter past 26, a Roman numeral from 4000, a list nested past Word's levels, a label that shows +#: counters Word's list cannot (``render-docx-section-label``), a number off by a count (``render-docx-number-mismatch``), +#: and the two readings of a reference that only LibreOffice has (``\t`` ignored, a level nothing numbered counted). +GAPS = EXPLAINING | {"render-docx-number-mismatch", "render-docx-reference-word-only", "render-docx-libreoffice-count"} + +#: The diagnostics that say a number is fixed in the file: it is right when the file is opened and does not follow what +#: is inserted or deleted before it (a restart by a fixed value, a counter shared with others, a reference that is text), +#: besides the gaps above. +FIXED = GAPS | {"render-docx-shared-counter", "render-docx-static-restart", "render-docx-reference", + "render-docx-reference-cycle"} + + +# -- The corpus --------------------------------------------------------------------------------------------------- + + +@dataclass +class Shown: + """One document under one style: what the text writer prints, what LibreOffice shows, and what the render said.""" + + name: str + style: str + tree: RenderTree + written: str + rules: frozenset[str] + data: bytes + shown: Exported | None = None + + +@functools.cache +def documents() -> dict[str, str]: + """The corpus. A run that promises its tools (CI) must have the specification's documents in it.""" + found = corpus() + if spec_root() is None: + tool_missing("no LegalDown specification checkout (set LEGALDOWN_SPEC_DIR)") + return found + + +def rendered(source: str, style: str) -> tuple[bytes, str, RenderTree, frozenset[str]] | None: + """The DOCX, the text, the tree and the rules reported for *source* under *style* (None when it is not a document).""" + try: + made = render(source, format="docx", **STYLES[style]) + except DocumentError: + return None + assert made.binary is not None + written = render(source, format="text", **STYLES[style]) + return made.binary, written.output, written.tree, frozenset(d.rule for d in made.diagnostics) + + +@functools.cache +def shown_corpus() -> dict[str, tuple[Shown, ...]]: + """Every document under every style, shown by LibreOffice with its fields and indexes updated: one session.""" + require_libreoffice() + cases: dict[str, list[Shown]] = {style: [] for style in STYLES} + for name, source in documents().items(): + for style in STYLES: + made = rendered(source, style) + if made is not None: + data, written, tree, rules = made + cases[style].append(Shown(name, style, tree, written, rules, data)) + everything = {f"d{index}": case for index, case in enumerate(case for group in cases.values() for case in group)} + exported = libreoffice_export({key: case.data for key, case in everything.items()}) + for key, case in everything.items(): + case.shown = exported[key] + return {style: tuple(group) for style, group in cases.items()} + + +# -- Comparing ----------------------------------------------------------------------------------------------------- + + +def squeeze(line: str) -> str: + return re.sub(r"\s+", " ", line).strip() + + +def blocks_of(tree: RenderTree, *, quoted: bool = True) -> Iterator[Section | Paragraph | ListItem | DraftingNote]: + """Every section, paragraph, list item and drafting note of the tree, in the order of the document; with *quoted* False + not those in a quote or a drafting note (which are neither numbered units nor anything a lawyer edits as a clause).""" + def walk(blocks: tuple) -> Iterator[Paragraph | ListItem | DraftingNote]: + for block in blocks: + match block: + case Paragraph(): + yield block + case DraftingNote(blocks=inner): + yield block + if quoted: + yield from walk(inner) + case List(): + for item in block.items: + yield item + yield from walk(item.blocks) + case Quote(blocks=inner): + if quoted: + yield from walk(inner) + + yield from walk(tree.preamble) + for section in tree.sections: + yield section + yield from walk(section.blocks) + + +def normalise(written: str, shown: str, tree: RenderTree) -> tuple[list[str], list[str], list[int]]: + """The lines of the plain-text writer's output *written* and of the text LibreOffice *shown*, made comparable, and the + page numbers of the contents. The formats differ by nature in these ways, each undone here and in no other: + + * a rule (``---``) and a signing line (``____``) are borders of paragraphs and cells in the DOCX, not text; + * a table is pipes and a separator row in the text and cells in the DOCX, which LibreOffice's text export lays out as a + line a cell or a line a row, as its version does: a row is compared as the words of its cells, however many lines they take; + * a quote's ``>`` is the indent of a style in the DOCX; + * a link's ```` after its text is the target of the hyperlink; + * a condition label (§15.8) and the label of a drafting note are in ``[ ]`` in the text and a styled run in the DOCX, + the label of a heading or an attachment is a line of its own in the text and a run of the title's paragraph in the + DOCX, and the label of a numbered paragraph stands before its number in the text and after it in the DOCX + (the number is drawn by Word before the paragraph); + * ``label: value`` rows of the title block and the parties are two cells in the DOCX (compared like a table row); + * the table of contents of LibreOffice ends its entries with a tab and a page number, which the text has not: they are + taken off and returned (their pages are not the renderer's to promise); + * white space is not compared. + """ + labels = {unit.condition_label for unit in blocks_of(tree) if getattr(unit, "condition_label", None)} + labels |= {unit.label for unit in blocks_of(tree) if isinstance(unit, DraftingNote) and unit.label} + labels |= {entry.condition_label for entry in tree.contents if entry.condition_label} + labels |= {part.condition_label for part in tree.attachments if part.condition_label} + titles = {(squeeze(" ".join(filter(None, (s.label, inline_text(s.title))))), s.condition_label) + for s in tree.sections if s.condition_label} + titles |= {(squeeze(inline_text(a.title)), a.condition_label) for a in tree.attachments if a.condition_label} + swaps = [(unit.condition_label, unit.label) for unit in blocks_of(tree) + if isinstance(unit, Paragraph) and unit.condition_label and unit.label] + rows = [*tree.header, *(row for side in tree.sides for party in side.parties for row in party.details)] + split = {squeeze(f"{label}{tree.colon}{inline_text(value)}"): [squeeze(label), squeeze(inline_text(value))] + for label, value in rows if label} + + #: (the text of a line, whether it is the cells of a table row or the label and value of a title-block row) + items: list[tuple[str, bool]] = [] + for raw in written.splitlines(): + line = raw.strip() + if re.fullmatch(r"-{3,}|_{3,}", line) or re.fullmatch(r"\|[:\-| ]+\|", line): + continue + if line.startswith("|") and line.endswith("|"): + cells = [squeeze(cell) for cell in re.split(r"(?\s]*>", "", re.sub(r"^(>\s?)+", "", line)) + for label in labels: + line = line.replace(f"[{label}]", label) + for condition, number in swaps: + line = line.replace(f"{condition} {number} ", f"{number} {condition} ") + line = squeeze(line) + if not line: + continue + if items and (items[-1][0], line) in titles: + items[-1] = (f"{items[-1][0]} {line}", False) + continue + items.append((" ".join(split[line]) if line in split else line, line in split)) + + raw_lines = shown.lstrip("\ufeff").splitlines() + pages: list[int] = [] + squeezed = [squeeze(raw) for raw in raw_lines] + if tree.contents_label and tree.contents_label in squeezed: + start = squeezed.index(tree.contents_label) + 1 + for position in range(start, min(start + len(tree.contents), len(raw_lines))): + entry = re.fullmatch(r"(.*)\t(\d+)\s*", raw_lines[position]) + if entry: + squeezed[position] = squeeze(entry[1]) + pages.append(int(entry[2])) + shown_lines = [line for line in squeezed if line] + + # A table's cells are one line each in some LibreOffice versions and one line a row in others: a row that the text + # writer prints on one line is read from as many lines of LibreOffice's as it takes to say the same words. + written_lines: list[str] = [] + seen: list[str] = [] + cursor = 0 + for text, cells in items: + written_lines.append(text) + if cursor >= len(shown_lines): + continue + line = shown_lines[cursor] + cursor += 1 + while cells and line != text and text.startswith(line) and cursor < len(shown_lines): + line = squeeze(f"{line} {shown_lines[cursor]}") + cursor += 1 + seen.append(line) + seen += shown_lines[cursor:] + return written_lines, seen, pages + + +def words(lines: list[str]) -> list[str]: + """The words of *lines* that are not numbers or counters: four letters or more, which is what a gap leaves alone.""" + return [word for line in lines for word in re.findall(r"[^\W\d_]{4,}", line)] + + +def difference(written: list[str], shown: list[str]) -> str: + return "\n".join(difflib.unified_diff(written, shown, "text writer", "LibreOffice", lineterm="", n=1)) + + +def check(case: Shown) -> bool: + """Assert the equivalence of *case*; returns True if it holds exactly, False if it holds modulo a reported gap.""" + assert case.shown is not None + written, seen, pages = normalise(case.written, case.shown.text, case.tree) + if case.tree.contents: + assert pages, "the contents lists no page numbers" + assert pages == sorted(pages) and pages[0] >= 1 and pages[-1] <= case.shown.pages, (pages, case.shown.pages) + if written == seen: + return True + assert case.rules & GAPS, f"the numbers differ and the render reports no gap:\n{difference(written, seen)}" + assert words(written) == words(seen), f"a gap changes numbers, not words:\n{difference(written, seen)}" + assert len(written) == len(seen), f"a gap changes numbers, not lines:\n{difference(written, seen)}" + return False + + +@pytest.mark.libreoffice +@pytest.mark.parametrize("style", STYLES) +def test_libreoffice_shows_every_document_as_the_text_writer_prints_it(style: str) -> None: + """Numbers, references, the contents, the attachments, the signatures: everything the plain-text writer prints is what + LibreOffice shows after it has updated the fields and indexes, except in the documents the plan reports a gap for.""" + for case in shown_corpus()[style]: + try: + check(case) + except AssertionError as error: + raise AssertionError(f"{style}/{case.name}: {error}") from error + + +@pytest.mark.libreoffice +def test_the_corpus_is_not_vacuous_and_every_gap_is_one_the_plan_reports() -> None: + """The comparison reaches what it is meant to: the specification's documents, a table of contents and signature + blocks, attachments, template constructs and references; and the documents that differ are few and are the + stress documents that ask for what Word draws differently (letters past 26, labels that need the section).""" + cases = [case for group in shown_corpus().values() for case in group] + assert len(cases) >= 3 * 30 + assert sum(case.name.startswith(("examples/", "fixtures/")) for case in cases) >= 3 * 18 + assert all(case.tree.contents for case in cases if case.tree.sections) + assert any(case.tree.signatures for case in cases) and any(case.tree.attachments for case in cases) + assert any(case.tree.is_template for case in cases) + assert sum(isinstance(i, CrossRef) for case in cases for i in iter_tree_inlines(case.tree)) > 100 + different = [case for case in cases if not check(case)] + assert {case.name for case in different} <= {f"stress/{name}" for name in ( + "letters", "qualified", "paths", "ordered paths", "deep")}, [(c.style, c.name) for c in different] + assert different, "the gaps the plan reports are not reached" + assert all(case.name.startswith("stress/") for case in different) + assert all(case.style != "default" or case.name == "stress/letters" for case in different) + + +@pytest.mark.libreoffice +def test_the_comparison_catches_a_wrong_number_a_missing_line_and_a_wrong_page_order() -> None: + """The check is not vacuous: a number LibreOffice draws differently, a line it leaves out, and a table of contents whose + pages run backwards are each a failure.""" + case = next(c for c in shown_corpus()["default"] if c.name == "documents/features.lgd") + assert case.shown is not None and check(case) + text = case.shown.text + assert "\n2. Services" in text + for broken in (text.replace("\n2. Services", "\n3. Services", 1), + text.replace("Business Day means", "", 1), + re.sub(r"\t(\d+)\n", lambda m: f"\t{9 - int(m[1])}\n", text, count=2)): + assert broken != text + mutated = Shown(case.name, case.style, case.tree, case.written, case.rules, case.data, Exported( + broken, case.shown.header, case.shown.footer, case.shown.pages)) + with pytest.raises(AssertionError): + check(mutated) + + +@pytest.mark.libreoffice +def test_a_table_row_on_one_line_reads_as_the_same_row_on_several() -> None: + """LibreOffice's text export lays a table out as a line a cell (24.2) or a line a row (later versions): both are the + document, so both pass.""" + case = next(c for c in shown_corpus()["default"] if c.name == "documents/features.lgd") + assert case.shown is not None + text = case.shown.text + for cells in (["Service", "Fee", "Notes"], ["Hosting", "$500.00", "per month"], ["Support", "$250.75", "see 3"]): + stacked = "\n".join(cells) + assert stacked in text, cells + text = text.replace(stacked, "\t".join(cells)) + assert text != case.shown.text + assert check(Shown(case.name, case.style, case.tree, case.written, case.rules, case.data, + Exported(text, case.shown.header, case.shown.footer, case.shown.pages))) + + +# -- Edits --------------------------------------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Edit: + """What a lawyer does in Word, and the same thing in the source.""" + + kind: str + #: The text of the paragraph it goes before (UNO inserts before the first paragraph that holds it), or the whole text of + #: the paragraph that is deleted. + match: str + #: The text typed (None: the paragraph is deleted). + new: str | None + #: The line of the source it is made at (1-based), and what takes that line's place. + line: int + source: str + + +#: The documents the edits are made in: the two test documents, specification examples with lists, a template and +#: numbered clauses, and documents written for the numbering. +EDITED = ( + "documents/features.lgd", "documents/template.lgd", "examples/advanced/msa/master-service-agreement.lgd", + "examples/advanced/template/consulting-agreement-template.lgd", "examples/simple/nda/mutual-nda.lgd", + "examples/simple/policy/remote-work-policy.lgd", "fixtures/valid/anchors-and-refs.lgd", "stress/nested", + "stress/lists", "stress/alternatives", +) + +MARKER = re.compile(r"^(\s*(?:[-*+]|\d+[.)])\s+)") + + +def body_texts(data: bytes) -> list[str]: + """The text of every paragraph of the document that is not a line of the table of contents or its label.""" + root = ET.fromstring(parts(data)["word/document.xml"]) + found = [] + for paragraph in root.iter(f"{W}p"): + style = paragraph.find(f"{W}pPr/{W}pStyle") + if style is not None and re.fullmatch(r"TOC\d|ContentsHeading", style.get(f"{W}val", "")): + continue + found.append("".join(t.text or "" for t in paragraph.iter(f"{W}t"))) + return found + + +def plain(inlines: tuple) -> str | None: + """The text of *inlines* if it is words alone, possibly emphasised (a paragraph that holds a field, a value or a break has + text that changes or that Word keeps apart, which words do not find).""" + found = [] + for inline in inlines: + if isinstance(inline, Text | Code): + found.append(inline.text) + elif isinstance(inline, Emphasis | Strong): + inner = plain(inline.children) + if inner is None: + return None + found.append(inner) + else: + return None + return "".join(found) or None + + +def leading(inlines: tuple) -> str | None: + """The words a paragraph starts with, if it starts with some (up to 40 characters of its first text, which a field + later in the paragraph does not change): what UNO needs to find a paragraph to insert before.""" + first = inlines[0] if inlines else None + return first.text[:40].strip() if isinstance(first, Text) and len(first.text.strip()) >= 8 else None + + +def unique(texts: list[str], text: str) -> bool: + """Whether exactly one paragraph holds *text* (so that UNO, which takes the first that does, finds that one).""" + return len(text) >= 4 and sum(text in other for other in texts) == 1 + + +def edits_for(source: str, data: bytes, tree: RenderTree) -> list[Edit]: + """The edits the document allows: a heading, an item and a nested item inserted before the first that qualify, a + numbered paragraph where the style numbers them, and the last clause that is a single line nothing refers to deleted. + A target qualifies when it stands in the source on a line of its own, has no condition, is not a later alternative and + has text alone (so that its words find it).""" + lines = source.splitlines() + texts = body_texts(data) + referred = {i.target_unit for i in iter_tree_inlines(tree) if isinstance(i, CrossRef)} + found: dict[str, Edit] = {} + deletable: Edit | None = None + + def lead(line: int) -> str: + return lines[line - 1] if 0 < line <= len(lines) else "" + + for node in blocks_of(tree, quoted=False): + if getattr(node, "condition", "") or getattr(node, "shares_number_with", None) or node.line is None: + continue + if isinstance(node, Section): + text, row = plain(node.title) or leading(node.title), lead(node.line) + if text and "heading" not in found and re.match(rf"#{{{node.level}}} ", row) and unique(texts, text): + found["heading"] = Edit("heading", text, "ZZ heading", node.line, + f"{'#' * node.level} ZZ heading\n\n{row}") + elif isinstance(node, ListItem) and node.blocks and isinstance(node.blocks[0], Paragraph): + whole, row = plain(node.blocks[0].inlines), lead(node.line) + text = whole or leading(node.blocks[0].inlines) + marker = MARKER.match(row) + if not text or not marker or not unique(texts, text): + continue + nested = node.parent_unit is not None and any( + isinstance(other, ListItem) and other.unit == node.parent_unit for other in blocks_of(tree, quoted=False)) + kind = "nested item" if nested else "item" + if kind not in found: + found[kind] = Edit(kind, text, f"ZZ {kind}", node.line, f"{marker[1]}ZZ {kind}\n{row}") + single = whole is not None and texts.count(whole) == 1 and len(node.blocks) == 1 \ + and row[marker.end():].startswith(whole) + following = lead(node.line + 1) + if (single and node.unit not in referred + and (not following.strip() or MARKER.match(following) or following.startswith("#"))): + deletable = Edit("delete", whole, None, node.line, "") + elif isinstance(node, Paragraph) and node.top_level and node.label: + text, row = plain(node.inlines) or leading(node.inlines), lead(node.line) + if text and "paragraph" not in found and row.startswith(text[:12]) and unique(texts, text): + found["paragraph"] = Edit("paragraph", text, "ZZ paragraph", node.line, f"ZZ paragraph\n\n{row}") + chosen = list(found.values()) + if deletable is not None and all(deletable.line != edit.line for edit in chosen): + chosen.append(deletable) + return chosen + + +def made(source: str, edits: list[Edit]) -> str: + """*source* with the edits made, from the last line up so that the lines of the others stay where they are.""" + lines = source.splitlines() + for edit in sorted(edits, key=lambda e: e.line, reverse=True): + lines[edit.line - 1:edit.line] = edit.source.split("\n") if edit.source else [] + return "\n".join(lines) + "\n" + + +@dataclass +class Edited: + name: str + style: str + rules: frozenset[str] + edits: list[Edit] + after: RenderTree + written: str + #: The tree and the text of the document before the edits. + before: RenderTree + unedited: str + shown: str = "" + + +@functools.cache +def edited_corpus() -> dict[str, tuple[Edited, ...]]: + """The edited documents under each style, edited in LibreOffice and updated: one session for all of them.""" + require_libreoffice() + jobs: dict[str, tuple[bytes, list[tuple[str, ...]]]] = {} + cases: dict[str, list[Edited]] = {style: [] for style in STYLES} + keyed: dict[str, Edited] = {} + for name in EDITED: + source = documents()[name] + for style in STYLES: + made_ = rendered(source, style) + assert made_ is not None, name + data, unedited, tree, rules = made_ + edits = edits_for(source, data, tree) + if not edits: + continue + after = render(made(source, edits), format="text", **STYLES[style]) + case = Edited(name, style, rules, edits, after.tree, after.output, tree, unedited) + cases[style].append(case) + key = f"e{len(keyed)}" + keyed[key] = case + jobs[key] = (data, [(e.match, e.new) for e in edits]) + shown = libreoffice_edit(jobs) + for key, case in keyed.items(): + case.shown = shown[key] + return {style: tuple(group) for style, group in cases.items()} + + +@pytest.mark.libreoffice +@pytest.mark.parametrize("style", STYLES) +def test_the_document_follows_the_edits_a_lawyer_makes(style: str) -> None: + """After a heading, an item, a nested item and a numbered paragraph are inserted and a clause is deleted, and the fields + and indexes are updated, the document reads as the plain-text writer prints the source with the same edits: every number + and reference moved, the contents list the new heading. A document may differ only where its render reported a number + that is fixed in the file, and then the difference is in numbers alone.""" + live = 0 + for case in edited_corpus()[style]: + written, seen, _ = normalise(case.written, case.shown, case.after) + if written == seen: + live += 1 + continue + fixed = case.rules & FIXED + assert fixed, f"{style}/{case.name}: not live after {[e.kind for e in case.edits]}:\n{difference(written, seen)}" + assert words(written) == words(seen), f"{style}/{case.name}:\n{difference(written, seen)}" + assert live >= 7, [(c.name, [e.kind for e in c.edits]) for c in edited_corpus()[style]] + + +@pytest.mark.libreoffice +@pytest.mark.parametrize("style", STYLES) +def test_the_edits_move_the_numbers_after_them(style: str) -> None: + """The edits are not no-ops that a document with static numbers would pass: in every document, what the edits did not + type or delete reads differently afterwards (the headings after an inserted one are numbered one higher, and so on).""" + for case in edited_corpus()[style]: + before, _, _ = normalise(case.unedited, "", case.before) + after, _, _ = normalise(case.written, "", case.after) + typed = [line for line in after if "ZZ " in line] + deleted = {edit.match for edit in case.edits if edit.new is None} + rest = [line for line in after if "ZZ " not in line] + assert typed and rest != [line for line in before if not any(m in line for m in deleted)], (style, case.name) + + +@pytest.mark.libreoffice +def test_every_kind_of_edit_is_made_in_the_corpus() -> None: + """The subset is not vacuous: a heading, an item, a nested item and a deleted clause are edited under every style, and a + numbered paragraph under the continental style (the others do not number paragraphs).""" + for style, group in edited_corpus().items(): + kinds = {edit.kind for case in group for edit in case.edits} + assert {"heading", "item", "nested item", "delete"} <= kinds, (style, kinds) + assert ("paragraph" in kinds) == (style == "continental"), (style, kinds) + + +# -- The schemas and determinism ----------------------------------------------------------------------------------- + + +@pytest.mark.ooxml_schema +@pytest.mark.parametrize("style", STYLES) +def test_every_part_of_every_corpus_document_validates_and_a_second_render_is_the_same_bytes(style: str) -> None: + require_schemas() + for name, source in documents().items(): + first, second = rendered(source, style), rendered(source, style) + assert (first is None) == (second is None), f"{style}/{name}" + if first is not None and second is not None: + assert first[0] == second[0], f"{style}/{name}: two renders differ" + assert validation_errors(first[0]) == [], f"{style}/{name}" From 8877dccf21c3b720955cf723397bf4dccf652734 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:31:45 +0000 Subject: [PATCH 113/118] Word checklist: the level nothing numbered, viewers, links, look; check that its snippet runs The snippet skips the attachment variants for the template (it has no attachments) and makes the document of the render-docx-libreoffice-count case. A test runs the snippet and checks what the checklist names. --- .github/PUBLISHING.md | 65 ++++++++++++++++++++++++++---------- tests/test_docx_wordcheck.py | 54 ++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 17 deletions(-) diff --git a/.github/PUBLISHING.md b/.github/PUBLISHING.md index bcf49b7..c3a083e 100644 --- a/.github/PUBLISHING.md +++ b/.github/PUBLISHING.md @@ -96,18 +96,26 @@ for doc in ("features", "template"): source = Path(f"tests/documents/{doc}.lgd").read_text(encoding="utf-8") for style in ("default", "continental", "outline"): (out / f"{doc}.{style}.docx").write_bytes(render(source, format="docx", style=style, overrides=contents).binary) - # the settings whose effect only Word shows (see item 7) - for name, extra in { - "attachments-not-listed": {"contents.attachments": False}, - "depth-1": {"contents.depth": 1}, - "no-attachments-label": {"labels.attachments": ""}, - "separator-none": {"attachments.separator": "none"}, - "separator-rule": {"attachments.separator": "rule"}, - "scheme-none": {"numbering.scheme": "none"}, - "paragraphs-numbered": {"paragraphs.numbered": True}, # `1.1` paragraphs on the heading list (item 9) + # the settings whose effect only Word shows (see items 7 and 9); `True` marks the ones about attachments, which + # template.lgd has none of + for name, (about_attachments, extra) in { + "attachments-not-listed": (True, {"contents.attachments": False}), + "depth-1": (False, {"contents.depth": 1}), + "no-attachments-label": (True, {"labels.attachments": ""}), + "separator-none": (True, {"attachments.separator": "none"}), + "separator-rule": (True, {"attachments.separator": "rule"}), + "scheme-none": (False, {"numbering.scheme": "none"}), + "paragraphs-numbered": (False, {"paragraphs.numbered": True}), # `1.1` paragraphs on the heading list (item 9) }.items(): + if about_attachments and doc == "template": + continue data = render(source, format="docx", overrides={**contents, **extra}).binary (out / f"{doc}.{name}.docx").write_bytes(data) + +# the document of item 8's "a level nothing numbered" line: items e and f stand on a level of their list that nothing +# in the list numbered before them +uncounted = "---\ntitle: T\n---\n# A\n\n- a\n - b\n 1. c\n - d\n - e\n - f\n" +(out / "uncounted.default.docx").write_bytes(render(uncounted, format="docx").binary) ``` (On the command line, `legaldown-render tests/documents/features.lgd -s continental --set contents.enabled=true @@ -117,19 +125,25 @@ for doc in ("features", "template"): For each file: 1. **Opens clean.** No "unreadable content" or repair prompt, and Word does not open it in Compatibility Mode. - Answer *Yes* to the prompt to update the fields (`docx.update_fields_on_open`), or press Ctrl+A, F9. + Answer *Yes* to the prompt to update the fields (`docx.update_fields_on_open`), or press Ctrl+A, F9. (With + `--set docx.update_fields_on_open=false` there is no prompt; the numbers and references already read right, since + the file caches what Word computes, and only the page numbers of the contents are missing until the fields are + updated.) The document's title is in File, Info, and its language in Review, Language. 2. **Table of contents.** It lists exactly the entries of the HTML contents (numbers and titles; the condition labels of a template after the title), indented by level, with a dotted leader and page numbers that are right. The attachments heading and each attachment's title are listed (at the levels the HTML shows) and nothing else is: not the "Contents" label, no quote or note. A click on an entry (Ctrl+click) goes to the heading. Right-click the - TOC, *Update field*, *Update entire table*: the same entries come back. + TOC, *Update field*, *Update entire table*: the same entries come back. Before the update (answer *No* to the + prompt) the entries are already there and read the same, without page numbers. The headings, and nothing else, are in + the navigation pane. 3. **References.** Every `{{ref:}}` reads as in the HTML before and after the update (Ctrl+A, F9). Insert a clause (a heading, a list item, a numbered paragraph) above a referenced one, update the fields: the numbers after it move and every reference and the contents follow. A reference to an attachment (`{{attach:}}`) shows its title and links - to it. + to it, and a `{{term:}}` is a link (Ctrl+click) to the definition of the term; an external link opens its address. 4. **Numbering after inserting a clause.** Press Enter at the end of a numbered item and at the start of one: the new paragraph is numbered, the rest renumbers, nested lists restart under it. In the continental style a new - sub-item continues `a)`, `b)`. + sub-item continues `a)`, `b)`. A bullet list counts nothing: an ordered list after a bullet list starts at its first + number, and two bullet lists show the same bullet. 5. **Zero-length bookmarks.** A `REF \n` to the zero-length (point) bookmark at the start of a numbered paragraph returns the number. Press Enter at the start of that paragraph and update the fields: the references still point at the paragraph they pointed at and read right. @@ -173,8 +187,17 @@ For each file: planner caches the skipped level as 1, as LibreOffice and the renderer count it, and writes that level's part of a reference as text (`render-docx-reference` says so). Word is expected to draw 0 for the level a heading skips: record what it draws. The rest of such a reference is fields and follows edits. + - a level that nothing numbered before the item that stands on it (`uncounted.default.docx`: items `e` and `f` stand + on the third level of their list, `(A)`, after a list of another kind and a bullet have come between): Word is + expected to read the unused level as 0 and number `e` as `(A)` and `f` as `(B)`, as the cached results say; LibreOffice + counts the unused level as 1 and shows `(B)` and `(C)` (the render reports `render-docx-libreoffice-count`, which + names `e` and what each reads). Record what Word shows. If it shows `(B)` and `(C)`, the plan's model is wrong for + every such item: Word numbers it one higher than the renderer does, and the cached results of references to it are + one too low until the fields are updated; the model in `resolve/docx_numbering.py` needs the other reading. - counters shared across nums: a nested list of the same kind restarts under each new parent; a list of another - kind restarts by a fixed setting (the render reported `render-docx-static-restart`). + kind restarts by a fixed setting (the render reported `render-docx-static-restart`): add a parent item above a nested + list of another kind (an ordered list under an enumerated item, say) and the list under the new item starts at + 1 only if you restart it by hand, which is the documented limit. - Roman numerals of 4000 and over, and letters past 26: Word's text (`render-docx-roman-overflow`, `render-docx-letter-overflow`) is the one the render reported, not a different one. 9. **Numbering that shares a list or a counter** (`template.continental.docx`, the continental `features` file, and @@ -210,9 +233,17 @@ For each file: "Error! Reference source not found" (expected: the render reported `render-docx-alternative`, and the final document is made by rendering with the condition values). - In the `paragraphs-numbered` files the paragraphs read `1.1`, `1.2` on the heading list; a sub-heading after a - list of clauses in the same section is restarted (as above). -10. **Page layout.** The running footer shows `n / N`; a header template with `{title}`, `{version}` and `{date}` shows - them; a landscape page is landscape when printed (print preview). + list of clauses in the same section is restarted (as above). A heading added in Word with a paragraph under it + continues the numbering of the section before it (`2.3`, not `3.1`): the documented limit, which asks the lawyer to + restart the numbering by hand. +10. **Page layout and look.** The running footer shows `n / N`; a header template with `{title}`, `{version}` and + `{date}` shows them; a landscape page is landscape when printed (print preview). The paper is the style's (A4 by + default), the margins are its margins, the text is in the style's font and size, numbers hang to the left of their + text and nested lists step in, a table keeps its header row at the top of each page when it runs over a page (add + rows to the table of `features` until it does), and a drafting note is shaded and kept on one page. +11. **A viewer that never updates fields** (optional: Word on the web, or a preview such as Quick Look or Outlook's). The + numbers, the references and the contents entries read as in item 2 and 3 before any update, and the contents have no + page numbers (the documented difference). Record what differs in the release notes. A difference from LibreOffice is expected where the documentation says Word and LibreOffice differ (`\t`, `lvlRestart`, `SEQ \s`, `w:updateFields`). diff --git a/tests/test_docx_wordcheck.py b/tests/test_docx_wordcheck.py index bdd7caa..445dcf6 100644 --- a/tests/test_docx_wordcheck.py +++ b/tests/test_docx_wordcheck.py @@ -4,13 +4,19 @@ from __future__ import annotations import re +from pathlib import Path import pytest from docx_support import libreoffice_export, parts, require_libreoffice, require_schemas, validation_errors from word_check import ROWS, build, main +from legaldown_render import render from legaldown_render.resolve.docx_cover import suppressed from legaldown_render.resolve.docx_numbering import Count, Lit +from legaldown_render.resolve.docx_plan import plan_docx + +REPOSITORY = Path(__file__).resolve().parents[1] +CHECKLIST = REPOSITORY / ".github" / "PUBLISHING.md" def test_the_document_is_a_package_that_holds_every_row_and_the_assumption_beside_it() -> None: @@ -52,3 +58,51 @@ def test_libreoffice_computes_the_plain_fields_as_the_planner_caches_them() -> N assumed = re.search(r"\[planner: (.*?)\]", line) or re.search(r"The planner assumes REF \\n gives '(.*?)';", line) assert computed and assumed, line assert computed.group(1).rstrip(".") == assumed.group(1).rstrip("."), line + + +# -- The checklist's own snippet ----------------------------------------------------------------------------------- + + +def checklist_snippet() -> str: + """The Python the checklist gives for making the documents to open in Word.""" + if not CHECKLIST.is_file(): + pytest.skip("the checklist is not part of this distribution") + section = CHECKLIST.read_text(encoding="utf-8").split("## Checking the DOCX in Word", 1)[1] + found = re.search(r"```python\n(.*?)```", section, re.S) + assert found is not None + return found.group(1) + + +def test_the_snippet_of_the_checklist_makes_the_documents_it_names(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """What the checklist tells a maintainer to run still runs, and makes every file its items name: the three styles and the + settings variants of both golden documents (the template has no attachments, so none of the variants about them), and + the document of the level nothing numbered.""" + monkeypatch.chdir(REPOSITORY) + exec(compile(checklist_snippet().replace('Path("word-check")', f"Path({str(tmp_path)!r})"), "PUBLISHING.md", "exec"), {}) + made = sorted(path.name for path in tmp_path.iterdir()) + styles = ("default", "continental", "outline") + variants = ("attachments-not-listed", "depth-1", "no-attachments-label", "separator-none", "separator-rule", + "scheme-none", "paragraphs-numbered") + about_attachments = {"attachments-not-listed", "no-attachments-label", "separator-none", "separator-rule"} + expected = [f"features.{name}.docx" for name in (*styles, *variants)] + expected += [f"template.{name}.docx" for name in (*styles, *(v for v in variants if v not in about_attachments))] + expected.append("uncounted.default.docx") + assert made == sorted(expected) + checklist = CHECKLIST.read_text(encoding="utf-8") + assert all(name in checklist for name in (*variants, "uncounted.default.docx")) + assert all(parts((tmp_path / name).read_bytes())["word/document.xml"] for name in made) + + +def test_the_document_of_the_level_nothing_numbered_is_the_one_the_checklist_describes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Items ``e`` and ``f`` are numbered ``(A)`` and ``(B)`` as the checklist says Word is expected to number them, with the + information that LibreOffice (``(B)``, ``(C)``) does not (tests/test_docx_roundtrip.py shows what it does).""" + monkeypatch.chdir(REPOSITORY) + code = checklist_snippet().replace('Path("word-check")', f"Path({str(tmp_path)!r})") + namespace: dict = {} + exec(compile(code, "PUBLISHING.md", "exec"), namespace) + result = render(namespace["uncounted"], format="docx") + plan, diagnostics = plan_docx(result.tree) + items = [unit for unit in plan.units.values() if unit.label in ("(A)", "(B)")] + assert [unit.label for unit in items] == ["(A)", "(B)"] + assert any(d.rule == "render-docx-libreoffice-count" and d.level == "info" for d in diagnostics) From 21c551c2943a70cec4e8025f0d7f9ef0899a3977 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:31:48 +0000 Subject: [PATCH 114/118] Document the DOCX writer as it is: README, conformance rules, architecture, styles, roadmap, ADR 0008 The conformance notes list every render-docx-* rule with its level and meaning; the roadmap entry says what is done and what behaviour changes (-o - is standard output for every format, RenderResult.binary, .docx is written). --- CONFORMANCE.md | 69 +++++++++++++++- README.md | 102 +++++++++++++++--------- docs/architecture.md | 14 +++- docs/decisions/0008-docx-word-native.md | 16 +++- docs/roadmap.md | 71 ++++++++--------- docs/style-templates.md | 12 +++ 6 files changed, 204 insertions(+), 80 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 486f9a5..c894f42 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -20,7 +20,7 @@ It is checked against the specification's own examples and fixtures corpus (`tes | `{{include:}}` below Full (§17.5) | ✅ Renders `[NOT PROCESSED: include path]` with a Warning (`render-not-processed`); the path is checked as a file reference (§2.3) | | Party and side display rules (§3.6) | ✅ | | At least one output format (§13.6) | ✅ HTML (recommended) and plain text (optional) | -| DOCX output (§13.6, recommended) | 🚧 In progress, not claimed. Written, as bytes, by `-o x.docx` / `format="docx"`: a valid OOXML package (every part validates against the ISO/IEC 29500-4 transitional schemas in CI; LibreOffice opens it) with styles, page setup, running header and footer (`PAGE`/`NUMPAGES` fields), document properties and deterministic output. **Numbering and cross-references are live Word features:** headings, list items and numbered paragraphs take their numbers from Word numbering definitions (the heading styles are linked to a multilevel list; lists restart by structure where Word can; the continental `5.1` items and the numbered paragraphs `1.1` ride the heading list's next level, with the lists nested in them on its reserved levels, so they stay live; alternatives that share a number (§15.8) are `numId 0` paragraphs whose number is a `REF` field of the first, with the sections and lists under them on a num of their own: live for edits elsewhere, but **deleting the first alternative in Word breaks them and every reference to the shared identifier** (reported; render with the condition values for a final document); the built-in continental style uses the heading list's levels 6 and 7 for lists nested in its clauses, a label that shows the path of an item keeps its section's, clause's and own counters on those levels, and a list nested where Word cannot show the counters above it, or in a level the heading list has no room for, is drawn with its own counter alone on a detached list and restarts by a fixed setting, reported; a list nested past Word's nine levels goes on in further lists, up to 36 levels), and every `{{ref:}}` is the literal delimiters and `REF` fields on bookmarks with the values the renderer computed as cached results (`1.1.1`, `I.A.1.a(i)`, `1(b)(ii)` read as the plain-text writer reads them, and stay right when a clause is inserted and fields are updated); `{{term:}}` and `{{attach:}}` are internal links. A reference is fixed text, reported, only where Word's fields cannot reproduce the designation: a number with text or punctuation around its counter that the reference does not have (`(a)`, `{n}:`, `No. {n}`) where Word's `\t` switch is not documented to leave it out (`docx.reference_text_switch`: `off`, `documented`, `guess`), and a number that holds the counters of other units the reference writes itself (`{section}.{n}` under a ref `{n}`); the counter of a heading level that the document skips (`heading-skip`, an Error) is text between the fields of the units that hold theirs; the words of `references.format` that a number repeats (`Article I`) are returned by the field itself. A LibreOffice round trip over the test and specification documents (the cached results made wrong on purpose) draws the same numbers and reads the same references as the text writer, except where the plan reports otherwise. The body writer lays out every block and inline (headings, paragraphs, lists, quotes, drafting notes, code, tables, rules, links, defined terms, blanks and choices) as a plan says. **The front and back matter are written too**, in the order the HTML shows them: the title block (header rows and parties), a live table of contents (`TOC \o "1-N" \h \z \u` with cached entries equal to the HTML contents, each a link with a `PAGEREF` page number Word fills in on update; the attachments are listed by outline level, so that no locale-dependent style list is in the field; at `contents.depth: 1` the attachment titles are listed as first-level entries, where the HTML indents them one level more), the attachments (bookmarked so `{{attach:}}` resolves, separated by a page break or a rule per `attachments.separator`) and the signature blocks (§3.6, borderless tables kept together on one page as far as one page allows). A LibreOffice round trip shows that after its indexes are updated the contents lists exactly the HTML entries with page numbers, also after a heading is inserted in the editor, and that the document reads in the plain-text writer's order. Remaining differences are the reported ones: a label that shows the section's number that the heading list cannot show (an outline scheme's `{section}.{n}`, the `none` scheme) is drawn without it. The contents show the shared number for a later alternative. The design is that numbers, references, the table of contents and page numbers are Word numbering definitions and fields, never text ([ADR 0008](docs/decisions/0008-docx-word-native.md)). What Word draws differently from the other formats is a diagnostic with a `render-docx-*` rule id, the renderer's own (not the specification's): Warnings `render-docx-section-label` (a label that shows a number Word cannot show there: the section's, or the enclosing items' on a list that cannot carry them), `render-docx-list-depth` (nested past 36 levels), `render-docx-letter-overflow` and `render-docx-roman-overflow` (Word repeats letters after 26 and writes Roman numerals from 4000 differently), `render-docx-shared-counter` (a unit that shares a counter with others in Word, such as a sub-heading and the items of its section that have the same designation in every output format, and that is restarted at a fixed value, which will not follow edits), `render-docx-number-mismatch` (any other difference: a catch-all that does not occur outside documents that skip a heading level), `render-docx-reference` (a cross-reference that cannot be built from fields is the renderer's text, right at open but fixed: it will not follow edits, with the class it falls in; one that has the number of a skipped heading level as text; or one that reads differently in Word), `render-docx-reference-cycle` (a heading's title refers back to the heading, under the `none` scheme: written as text); Infos `render-docx-alternative` (alternatives share a number as fields of the first, which must not be deleted in Word), `render-docx-static-restart` (a list that restarts by a fixed setting: in a list of another kind, in an item on the heading list that it cannot ride, in an alternative, or the one that begins the next list past nine levels) `render-docx-libreoffice-count` (items numbered as in Word but one or more higher in LibreOffice, which counts a level nothing numbered before a deeper item of its list) and `render-docx-reference-word-only` (a reference relies on Word's `\t` switch, which LibreOffice ignores). The Word-only checks (the contents' page numbers, what `\t` does, numbering after an insertion) are a manual checklist before a release (`.github/PUBLISHING.md`). HTML is the format this renderer claims §13.6 with | +| DOCX output (§13.6, recommended) | ⚠️ Written by `-o x.docx` / `format="docx"` (0.5): numbers, cross-references, the table of contents and page numbers are Word features, not text. Checked against the ISO/IEC 29500 schemas and in LibreOffice in CI; **not yet checked in Word itself**, which is a manual step before the release. See "DOCX output", below. HTML is the format this renderer claims §13.6 with | | A heading in a list item or a quote | ✅ Shown as a bold line: it is not a section (§4.1), so it has no number or anchor | | Comment stripping (§8.6) | ✅ Inline comments, and comment blocks across any number of lines. As in CommonMark, a comment block left unclosed runs to the end of the document | | Templates with answers (§15.8) | ✅ Assembled first with `legaldown-validator`'s Assembly capability (§15.7), then rendered: `answers=` / `--answers answers.yaml`. A template that needs other files is refused (see below) | @@ -48,6 +48,73 @@ It is checked against the specification's own examples and fixtures corpus (`tes | Question prompts next to blanks (§15.8, MAY) | ✅ `placeholders.show_prompt` | | Final check (§15.9, SHOULD) | ✅ `--final` / `final=True`: the validator's `placeholder-unfilled` and `template-construct-present` Errors; with `--strict`, the document is refused | +## DOCX output (§13.6) + +[ADR 0008](docs/decisions/0008-docx-word-native.md) decides the design: in the `.docx` every number, cross-reference, +table of contents entry and page number is a Word numbering definition or field, never literal text, and the values the +renderer computed are only the cached results of the fields. The document therefore reads right before any field is +updated, and stays right when a lawyer inserts a clause and updates fields. + +**What is written**, in the order the HTML shows it: + +- The title block (header rows and parties, borderless tables), a live table of contents (`TOC \o "1-N" \h \z \u` with cached + entries equal to the HTML contents, each a link with a `PAGEREF` page number that Word fills in when it updates the + field), the sections, the attachments (a heading and a titled placeholder, separated by a page break or a rule per + `attachments.separator`), and the signature blocks (§3.6, borderless tables with signing lines, kept together on one page as + far as one page allows). +- Headings, list items and numbered paragraphs take their numbers from Word numbering definitions: the heading styles are + linked to a multilevel list; lists restart by structure where Word can; the continental `5.1` items and the numbered + paragraphs ride the heading list's next level, with the lists nested in them on its reserved levels; a list nested past + Word's nine levels goes on in further lists, up to 36 levels. +- Every `{{ref:}}` is the delimiters of its designation and `REF` fields on bookmarks, with the values the renderer computed as the cached results (`1.1.1`, + `I.A.1.a(i)`, `1(b)(ii)`). `{{term:}}` and `{{attach:}}` are internal links; under the `none` scheme a reference is a + field for the text of the heading. +- The template view (§15.8): condition labels, drafting notes and choices are shown as in the other formats. Alternatives that + share a number are paragraphs with no number of their own whose number is a `REF` field of the first. +- Images are their alt text (no file is read, §2.3, §17.6). A link is a hyperlink for the same schemes as in HTML. +- A style setting that Word cannot express (a `%` length, a colour name Word does not know) is a style error when the + format is `docx`, before anything is rendered. + +**Differences from the other formats**, each reported with a `render-docx-*` diagnostic (the renderer's own rule ids, not the +specification's) at the line of the unit or the reference, once per finding and once for references that fail alike: + +| Rule | Level | Meaning | +|---|---|---| +| `render-docx-reference` | Warning | A `{{ref:}}` that Word's fields cannot compose is written as its text: right when the file is opened, but it does not follow edits. The message names the class: a number with text or punctuation around its counter that the reference does not have, where Word's `\t` switch may not be relied on to leave it out (`docx.reference_text_switch`: `off`, `documented`, `guess`); or a number holding the counters of other units that the reference writes itself (`{section}.{n}` under a ref `{n}`). Also this rule: the counter of a heading level the document skips (the Core Error `heading-skip`) is text between the fields of the units that do hold theirs, and a reference that reads differently in Word for another reason | +| `render-docx-reference-cycle` | Warning | Under the `none` scheme, a heading's title refers back to the heading: the reference is written as text | +| `render-docx-section-label` | Warning | A label shows a number that Word's list cannot show there (the section's number on a list that is not the heading list, or the counters of the enclosing items on a list that cannot carry them): Word draws a shorter label, named in the message. References stay fields | +| `render-docx-letter-overflow` | Warning | Past 26 a letter counter reads `aa`, `bb`, `cc` in Word and `aa`, `ab`, `ac` in the renderer | +| `render-docx-roman-overflow` | Warning | Word writes Roman numerals of 4000 and over differently | +| `render-docx-list-depth` | Warning | A list is nested past 36 levels: the items past it are numbered in the format of the last level and restart by fixed values | +| `render-docx-shared-counter` | Warning | A unit shares a counter with others in Word (a sub-heading and the items of its section, a list that goes on from another) and is restarted at its number: right when opened, but a fixed value that does not follow edits made before it | +| `render-docx-number-mismatch` | Warning | Any other difference between Word's number and the renderer's: a catch-all that does not occur outside documents that skip a heading level | +| `render-docx-alternative` | Info | Alternatives that share a number show it as a field of the first: deleting the first in Word breaks the others and every reference to the shared identifier. Render with the condition values or answers for a final document. Once per document | +| `render-docx-static-restart` | Info | A list restarts by a fixed setting, not by structure: nested in a list of another kind, in an item on the heading list that it cannot ride, in an alternative, or the one that begins the next list past nine levels. A parent item added in Word does not restart it | +| `render-docx-libreoffice-count` | Info | Items are numbered as the renderer does in Word but one or more higher in LibreOffice, which counts a level that nothing in the list numbered before a deeper item as 1. Nothing in the file is wrong for Word. Once per document | +| `render-docx-reference-word-only` | Info | A reference relies on Word's `\t` switch, which LibreOffice ignores: there it shows the whole number. Once per document | + +Two more differences are not reported, since they follow from the format: at `contents.depth: 1` the attachment titles are +first-level entries of the contents, where the HTML indents them one level more; and the contents show a later +alternative's shared number. + +**How it is checked.** In CI, on every part of every document of the corpus (the specification's examples and valid +fixtures, the test documents and documents written to stress the numbering) under the `default`, `outline` and +`continental` styles: + +- every part validates against the ISO/IEC 29500-4 transitional schemas, and rendering twice gives the same bytes; +- LibreOffice opens the document, updates its fields and indexes, and what it shows is what the plain-text writer prints + (numbers, references, contents entries, attachments, signatures), except where a diagnostic above says Word draws a + number differently (`tests/test_docx_acceptance.py`); +- after a heading, a list item, a nested item and a numbered paragraph are inserted and a clause is deleted in + LibreOffice, as a lawyer does, and the fields are updated, the document reads as the plain-text writer prints the source + with the same edits (the same module; the finer cases, with the cached results made wrong on purpose, are in + `tests/test_docx_roundtrip.py`). + +LibreOffice is the oracle available to CI, and it differs from Word in a few places: it ignores `\t`, `lvlRestart`, +`SEQ \s` and `w:updateFields`, and counts a level that nothing numbered before a deeper item as 1. What only Word shows +(the contents' page numbers, what `\t` does, the count of an unused level, numbering after an insertion, and every +other Word-only assumption of the plan) is a manual checklist before each release (`.github/PUBLISHING.md`). + ## Known limitations - **Assembly reads no other files.** A template with include fragments, LegalDown attachment diff --git a/README.md b/README.md index e0b0d24..16db18a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ **Turn a LegalDown document into HTML, plain text or a Word document, with section numbers, cross-references, defined terms, and formatted values all generated at render time.** -Targets specification **v0.2** · Conformance level **Rendering** · DOCX in progress, PDF planned +Targets specification **v0.2** · Conformance level **Rendering** · HTML, text and Word (DOCX) output, PDF planned
@@ -57,7 +57,7 @@ Python 3.11 or newer. The dependencies are all pure Python: `legaldown-validator ```bash legaldown-render contract.lgd -o contract.html # HTML page legaldown-render contract.lgd -o contract.txt # plain text (format from extension) -legaldown-render contract.lgd -o contract.docx # a Word document (in progress, see below) +legaldown-render contract.lgd -o contract.docx # a Word document with live numbers and references (see "DOCX") legaldown-render contract.lgd --style continental --locale cs-CZ -o smlouva.html # Czech dates, money, durations legaldown-render contract.lgd --set numbering.scheme=legal-outline --set enumeration.enabled=false legaldown-render contract.lgd --set contents.enabled=true --set contents.depth=3 # table of contents @@ -113,6 +113,8 @@ with open("contract.html", "w", encoding="utf-8", newline="\n") as output: output.write(result.output) ``` +A Word document is bytes, not text: `render(text, format="docx").binary` holds the file (and `.output` is empty). + `render()` also accepts a `RenderOptions`. Pass `answers={...}` to render a template filled in. Each diagnostic has its `line` and its `file`: `render_file()` and the command name the file (its name, without the directory); for text given to `render()`, pass `filename=` (a directory in it is dropped, `/` or `\`). @@ -128,46 +130,67 @@ cannot be assembled with its answers (`RenderRefused`) do. All of them are `Rend boundaries (`multiprocessing`, `concurrent.futures`) intact. `overrides=` takes dotted settings and may not contain `version` or `extends`, so `overrides=legaldown_render.style.style_to_dict(style)` must drop those two keys. -## DOCX (in progress) +## DOCX -`-o contract.docx` (or `--format docx`, or `render(text, format="docx")`) writes a Word document. The -file is bytes: the command writes it to a file or, with `-o -` (or no `-o` when standard output is not a -terminal), to standard output, and refuses to write it to a terminal; `--fragment` is a usage error for it; `render()` returns it as `result.binary` (`result.output` is empty). The writer needs no -dependency beyond the base package. +`-o contract.docx` (or `--format docx`, or `render(text, format="docx")`) writes a Word document that a lawyer can +edit. It is written with the standard library alone, so it needs nothing beyond the base install. -The design is that **in the Word file every number, cross-reference, the table of contents and the page -numbers are live Word features**, not text: numbering definitions and fields, with the values the renderer -computed as their cached results, so a lawyer who inserts a clause gets the numbers, references and contents -updated by Word +**The point of it:** in the Word file the numbers, the cross-references, the table of contents and the page +numbers are Word's own features, not typed text. Insert a clause in Word, update the fields, and the clauses after it, +every "see 2.1(b)" and the contents follow. The renderer still works every value out; it writes the value only as the +cached result of a field, so the document reads right before any field is updated ([ADR 0008](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/decisions/0008-docx-word-native.md)). -**What is there today:** a valid `.docx` (checked against the ISO/IEC 29500 schemas, and opened in LibreOffice in CI) -with styles generated from the style template (`Normal`, `heading 1` to `heading 5`, `Title`, `Quote`, `toc 1` to -`toc 5`, `Hyperlink`, ...), the page size and margins, a running header and footer from `page.header` and -`page.footer` (the default footer is "3 / 12" as live page fields), the document's title and language, -`w:updateFields` so Word refreshes fields when it opens the file (`docx.update_fields_on_open`), and deterministic -output. **Numbering and cross-references are live:** headings, list items and numbered paragraphs are numbered by Word -numbering definitions (headings by the heading styles, nested lists restarting by structure), and a -`{{ref: clause}}` is `REF` fields on bookmarks whose cached results are the text the plain-text writer prints, so -"1(b)(ii)" or "I.A.1" stays right when a lawyer inserts a clause and updates fields. A defined term or an attachment -reference is an internal link. A reference is written as its text (it will not follow edits, and the render reports it) only where Word's fields cannot reproduce it: -a number with punctuation around its counter or with the counters of other units that the reference writes itself, and the number of a heading level the document skips (an Error); how far Word's `\t` switch is trusted to leave the words of a number out is the style's `docx.reference_text_switch`. Where Word cannot draw what the renderer does (letters past 26, a label that shows the -section's number or the counters of enclosing items where Word's list cannot show them, a number restarted by a fixed value, a list restarted by a fixed setting), the render -reports it as a `render-docx-*` Warning or Info with the line, see -[the conformance notes](https://github.com/ForLegalAI/legaldown-render/blob/main/CONFORMANCE.md). The body writer also -lays out paragraphs, quotes, drafting notes, code, tables with a repeated header row, rules, links, blanks and -choices. **Front and back matter are there too, in the order the HTML shows them:** the title and subtitle, the header -rows and the parties (borderless tables), the **table of contents** (a live `TOC \o "1-N" \h \z \u` field whose -cached entries are the contents the HTML shows, each a link to its heading with a `PAGEREF` page number that Word -fills in when it updates the field; with `contents.attachments` the attachments are listed by outline level, so -the field has no locale-dependent style list), the attachments (the heading and a titled placeholder each, bookmarked so -that `{{attach:}}` is a `REF` field or a link, separated by a page break or a rule as `attachments.separator` says), -and the signature blocks (borderless tables with signing lines, kept together on one page as far as one page allows). Viewers that never update -fields (Word Online, Quick Look, Google Docs import, Outlook's preview) show the contents without page numbers. A -reference whose form Word cannot compose from fields (a custom style's `{path}` item labels, say) is fixed text and -is reported as `render-docx-reference`. The maintainers also check the golden documents in Word before each release ([the checklist](https://github.com/ForLegalAI/legaldown-render/blob/main/.github/PUBLISHING.md)), since CI cannot run Word. Settings Word cannot express (a `%` length, say) are an error for DOCX only. See -[Styles and preferences](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/style-templates.md#docx) -and the [roadmap](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/roadmap.md). +**What is written.** The same document the HTML shows, in the same order: the title block (header rows and parties), +the table of contents, the sections, the attachments (a heading and a titled placeholder each) and the signature blocks. +Paragraphs, lists, quotes, drafting notes and the template view's condition labels, code, tables with a repeated header +row, rules, links, defined terms, blanks and choices have Word styles generated from the style template, so a change to +a style in Word applies everywhere. The page size, margins, fonts and colours are the style's, and the file carries the +document's title and language. + +**What is live:** + +- Heading numbers (the heading styles are linked to a multilevel list), list items and numbered paragraphs are Word + numbering definitions. A nested list restarts under a new parent item, and a clause inserted before another renumbers it. +- A `{{ref: clause}}` is `REF` fields on bookmarks, joined by the delimiters of the designation, so "1(b)(ii)" or + "I.A.1" is recomputed. `{{term:}}` and `{{attach:}}` are links (an attachment's title is a field). +- The table of contents is a `TOC` field (`contents.enabled`, `contents.depth`), each entry a link with a `PAGEREF` + page number. The running header and footer are `page.header` and `page.footer`: the page number and the page count + are fields (the default footer reads "3 / 12"). + +**Opening it.** Word asks once whether to update the fields (`docx.update_fields_on_open`, on by default), and then the +cross-references and the contents with their page numbers are up to date. After an edit, press Ctrl+A, then F9, or +right-click the contents and choose *Update field*. LibreOffice ignores the request (Tools, Update, All Indexes and +Tables). A viewer that never updates fields (Word on the web, Quick Look, a Google Docs import, Outlook's preview) shows +the numbers and references right, as cached, and the contents without page numbers. + +**Settings** (the style template, [Styles and preferences](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/style-templates.md#docx)): + +| Setting | Effect in the DOCX | +|---|---| +| `page.size`, `page.margin` | paper and margins; a length Word cannot express (`%`, `ch`, `ex`) is a style error, reported before anything is rendered | +| `page.header`, `page.footer` | left, centre and right text with `{page}`, `{pages}`, `{title}`, `{version}`, `{date}` | +| `docx.update_fields_on_open` | `true` (the default): Word updates every field when it opens the file; `false` writes no such request | +| `docx.reference_text_switch` | `off`, `documented` (the default) or `guess`: how far a cross-reference may rely on Word's `\t` switch to leave the words of a number out (`Article I` shown as `I`) | +| `contents.*`, `attachments.separator`, `signatures.enabled`, `numbering.*`, `enumeration.*`, `paragraphs.*`, `typography.*`, `headings.*` | as in the other formats; the DOCX meaning of each is in the style documentation | + +**Limits.** Word cannot do everything the renderer does, and the render says where, one `render-docx-*` diagnostic +with the line (the rules are listed in [the conformance notes](https://github.com/ForLegalAI/legaldown-render/blob/main/CONFORMANCE.md#docx-output-136)): + +- A cross-reference that Word's fields cannot compose (a number with punctuation around its counter that the reference + leaves out, say, or the number of a heading level the document skips) is written as its text: right when the file is + opened, but it does not follow edits. +- Where Word draws a number differently (letters past 26, Roman numerals from 4000, a label that shows the section's + number on a list that cannot carry it), or restarts a list by a fixed setting instead of by structure, the file is + right when it is opened and does not follow every edit made in Word. +- Alternatives that share a number (§15.8, in the template view) show it as a field of the first: deleting the first in + Word breaks the others. Render with the answers or condition values to get a final document. +- An image is shown as its alt text: no file is read (§2.3, §17.6). +- CI checks every part against the ISO/IEC 29500 schemas and opens the documents in LibreOffice, which updates the + fields and is compared with the plain-text output. It cannot run Word, so the maintainers check the golden documents + in Word before a release ([the checklist](https://github.com/ForLegalAI/legaldown-render/blob/main/.github/PUBLISHING.md)). +- `--fragment` does not apply: a Word document is always whole. On the command line the file goes to `-o FILE.docx`, or + to standard output with `-o -` (or without `-o` when standard output is not a terminal); it is never written to a terminal. ## Rendering documents you did not write @@ -238,9 +261,10 @@ articles, 2.1-style items, numbered paragraphs), and `outline` (I. / A. / 1. / a ```bash pip install -e ".[dev]" -pytest # unit, golden, and CLI tests +pytest # unit, golden, and CLI tests (pytest -n auto: on all cores, about a third of the time) python tests/schemas/fetch.py && pytest -m ooxml_schema # DOCX parts against the OOXML schemas (needs lxml) pytest -m libreoffice # DOCX opened in LibreOffice (needs soffice and python3-uno; skipped without) +LEGALDOWN_SPEC_DIR=../LegalDown pytest -m "libreoffice or ooxml_schema" # also the DOCX acceptance corpus (tests/test_docx_acceptance.py) pytest --update-golden # regenerate tests/golden/ after an intended output change LEGALDOWN_SPEC_DIR=../LegalDown pytest -m conformance # the specification's examples and fixtures ruff check . diff --git a/docs/architecture.md b/docs/architecture.md index 4af03cb..89cb5a6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -671,7 +671,14 @@ out when the validator reported the same path at the same line, or at none (`dia ## Testing Each module is named for its topic. Shared helpers (the minimal document header, the `text` and -`html` renderers, and running the command as a program) are in `tests/conftest.py`. +`html` renderers, and running the command as a program) are in `tests/conftest.py`; those for the DOCX tests +(reading a package, validating it, running LibreOffice) are in `tests/docx_support.py` and `tests/lo/`. + +Markers: `conformance` (the specification's documents), `ooxml_schema` (DOCX parts against the schemas), `libreoffice` +(DOCX opened in LibreOffice; skipped without `soffice` and `python3-uno`, a failure when `LEGALDOWN_REQUIRE_DOCX_TOOLS` +is set, as in the CI job that installs them) and `acceptance`: the DOCX acceptance corpus, whose run takes about half a +minute of LibreOffice time and is left out of a plain `pytest` (it runs with `-m`, with `LEGALDOWN_ACCEPTANCE=1`, or when +its file is named). `pytest -n auto` (pytest-xdist, in the `dev` extra) runs the whole suite in a third of the serial time. | Suite | What it checks | |---|---| @@ -705,7 +712,8 @@ Each module is named for its topic. Shared helpers (the minimal document header, | `tests/test_docx_special.py` | What the planner's hard cases become in the file: the heading list's levels that hold items, the reserved levels of lists nested in them, the detached and the next-nine-levels abstractNums, and later alternatives as `numId 0`, `w:ind` and `REF` fields; and that documents with them validate against the schemas (marker `ooxml_schema`) | | `tests/test_docx_validation.py` | Every part of every generated document against the ISO/IEC 29500-4 schemas (marker `ooxml_schema`; needs lxml and `python tests/schemas/fetch.py`) | | `tests/test_docx_roundtrip.py` | LibreOffice as the oracle of the numbering and the cross-references, in one LibreOffice session: every document of the corpus (the tests', the specification's, stress documents with cycles and unanchored ancestors) under the default, outline, continental, mixed and `none` schemes, with every cached field result made wrong (`XXX`), every numbered paragraph marked and a line per reference added; the number LibreOffice draws, every reference it computes and every body paragraph's text equal what the file cached and the renderer wrote, modulo the gaps the plan reports (`\t` ignored, letters past 26, labels the heading list cannot show); alternatives are checked against the renderer (their `numId 0` number is the holder's, as fields), and so are the continental items on the heading list and the lists on its reserved levels; the table of contents is built by LibreOffice from a `TOC` field the test adds and must list the headings with the numbers the plan gives and none of the items; and edit-then-update (`tests/lo/uno_edit.py`, which inserts and deletes paragraphs, and can put an inserted paragraph on another item's list as pasting a copy does): path labels on the reserved levels, in a detached list (and the documented continuation of a list copied under a new parent) and past nine levels, a heading inserted before a document that skips a level, items, headings, a clause and a nested item inserted before the targets as a lawyer does, fields updated, every reference and the numbers equal the renderer's for the edited source (the new nested item under the continental style reads `a)`), and, for a sub-heading restarted by a fixed value, the plan's warning and what an edit before it keeps right (marker `libreoffice`) | +| `tests/test_docx_acceptance.py` | The measurable claims of ADR 0008 over the whole corpus (the tests' documents, the specification's examples and valid fixtures, the stress documents) under the default, outline and continental styles, with the contents on. **Equivalence:** LibreOffice's text, with fields and indexes updated, equals the plain-text writer's, line by line, modulo the differences the formats have by nature (one documented function: borders for rules and signing lines, table cells, quote and link markup, where a condition label stands, `label: value` rows, the contents' page numbers, which are checked to be positive, never decreasing and within the page count) and modulo the gaps the plan reports; the documents that differ are exactly the stress documents that ask for what Word draws differently. **Live after edits:** in one LibreOffice session, a heading, an item, a nested item, a numbered paragraph (continental) are inserted and a clause deleted in up to ten documents under each style, and the result equals the plain-text writer's for the source with the same edits (a difference is allowed only where the render reports a fixed number). **Validity and determinism:** every part validates and two renders are the same bytes. Negative controls show the comparison fails on a wrong number, a missing line, pages out of order | | `tests/test_docx_matter.py` | The front and back matter: the XML of the title block, the `TOC` field and its cached entries, the attachments and the signature blocks; documents with every part (hostile text too) against the schemas; LibreOffice listing exactly the contents' entries after an update, resolving `{{attach:}}`, paging the attachments as configured, taking a heading inserted in the editor into the contents, and reading in the text writer's order | -| `tests/test_docx_libreoffice.py` | The documents opened in LibreOffice through UNO (`tests/lo/uno_export.py`), fields updated: the words in order, header, footer, page count (marker `libreoffice`; skipped without `soffice` and `python3-uno`) | -| `tests/test_docx_wordcheck.py`, `tests/word_check.py` | The generator of the document for the manual Word checks of what the planner assumes (`python tests/word_check.py OUT.docx`; `.github/PUBLISHING.md` items 8 and 9): one row for each label form the plan relies on (`\t` on `Article %1`, `§ %1`, `Section %1.%2`, `Čl. %1`, `No. %1`, `(%1)`, `%1)`, `%1:`), the heading list's reserved level `%1.%2(%6)` with levels 3 to 5 unused, and a skipped heading level, each with the planner's assumption beside it taken from the planner's own code; that the package is valid (marker `ooxml_schema`) and LibreOffice computes the plain fields as the planner caches them (marker `libreoffice`) | +| `tests/test_docx_libreoffice.py` | The documents opened in LibreOffice through UNO (`tests/lo/uno_export.py`), fields updated: the words in order, header, footer, and the page count and the page numbers compared with each other and with page breaks, not with the page count of a font (marker `libreoffice`; skipped without `soffice` and `python3-uno`) | +| `tests/test_docx_wordcheck.py`, `tests/word_check.py` | The generator of the document for the manual Word checks of what the planner assumes (`python tests/word_check.py OUT.docx`; `.github/PUBLISHING.md` items 8 and 9): one row for each label form the plan relies on (`\t` on `Article %1`, `§ %1`, `Section %1.%2`, `Čl. %1`, `No. %1`, `(%1)`, `%1)`, `%1:`), the heading list's reserved level `%1.%2(%6)` with levels 3 to 5 unused, and a skipped heading level, each with the planner's assumption beside it taken from the planner's own code; that the package is valid (marker `ooxml_schema`) and LibreOffice computes the plain fields as the planner caches them (marker `libreoffice`); and that the Python snippet in the checklist still runs and makes every file the checklist names, including the document of the level nothing numbered | | `tests/conformance/` | Every example and fixture in the specification repository renders, every internal link has a target, and each invalid fixture renders its rule's failure marker. Needs `$LEGALDOWN_SPEC_DIR` or a sibling `../LegalDown` checkout | diff --git a/docs/decisions/0008-docx-word-native.md b/docs/decisions/0008-docx-word-native.md index 64bc701..2e511cd 100644 --- a/docs/decisions/0008-docx-word-native.md +++ b/docs/decisions/0008-docx-word-native.md @@ -65,6 +65,16 @@ Also decided here, because every later part rests on them: - **Compatibility mode 15**, so Word does not open the file in Compatibility Mode. The settings are in the order the schema requires (`updateFields` follows `characterSpacingControl` and precedes `compat`), and contain no revision ids. +- **Front and back matter.** The table of contents is a `TOC \o "1-N" \h \z \u` field whose cached result is the + planner's entries (each a link to a `_Toc` bookmark with a `PAGEREF` field whose cached page is empty, since only + Word knows the pages). The attachments heading and each attachment's title are paragraphs of styles with an outline + level, so `\u` lists them and the field needs no `\t` list of styles (whose separator depends on the locale). The + contents label has no outline level, so it is not an entry, and a condition label is a run inside its heading's + paragraph so that the contents keep it. A page break between attachments is the paragraph property "page break + before". The title block, the parties and the signature blocks are borderless tables. +- **Output.** A DOCX is bytes: `RenderResult.binary` holds the file and `RenderResult.output` stays `str` (empty), so no + typed caller breaks. The command writes `-o FILE.docx`, or standard output with `-o -`, never to a terminal. Images + are shown as their alt text: below the Full level no file is read (§2.3, §17.6). - **Safety.** One sanitiser is applied to every text, attribute, field instruction, bookmark name, relationship and property value. It drops the characters XML cannot carry. Without it `xml.etree` writes a control character raw and a lone surrogate as a character reference, and Word @@ -174,7 +184,11 @@ Under the `none` scheme a reference is a `REF` to the heading text. - Verification is part of the design: every part is validated against the schemas, the documents are opened in LibreOffice (the oracle available in CI, which updates fields and reports the text), and the same documents are checked by hand in Word before a release. LibreOffice ignores `\t`, - `lvlRestart`, `SEQ \s` and `w:updateFields`, so those differences are normalised in the comparison. + `lvlRestart`, `SEQ \s` and `w:updateFields`, fills a skipped heading level with its start value, and counts + a level that nothing in a list numbered before a deeper item as 1 where Word is expected to read it as 0 + (`render-docx-libreoffice-count`), so those differences are normalised in the comparison and the Word-only ones + are on the checklist. The acceptance corpus compares whole documents with the plain-text writer's output under + three styles, before and after edits made in the editor. - Cost: an OOXML writer of our own, with its schema checks, instead of a library. The numbering planner is the bulk of the work. - Style template additions: `page.header`, `page.footer`, `docx.update_fields_on_open` and diff --git a/docs/roadmap.md b/docs/roadmap.md index b8cef08..f412038 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -90,48 +90,47 @@ Rendering what other people send, and reporting problems where they are. No new self-contained - **Project.** CI also runs on Windows, and publishing to PyPI waits for CI -### v0.5 — DOCX (in progress) +### v0.5 — DOCX (implemented; released after the Word checks) A document lawyers can edit: in the .docx every number, cross-reference, the table of contents and the page numbers are live Word features (numbering definitions and fields), and the renderer's values appear only as the cached results of fields. Decided in [ADR 0008](decisions/0008-docx-word-native.md), which also settles the question 0004 left open. -- **Package core (done).** `docx` writer in the base package, with the standard library only (no - `[docx]` extra): the OOXML parts, styles generated from the style template, settings (compatibility - mode 15, `w:updateFields`), page setup, running headers and footers with `PAGE`/`NUMPAGES` fields, - document properties, deterministic output. `RenderResult.binary`, `-o x.docx`, `-o -`. The body at that point - was an interim one: the title, `heading N` paragraphs and plain-text paragraphs, with no numbers yet -- **Behaviour changes this brings.** `-o -` now writes to standard output for every format (it used to - create a file named `-`); `.docx` is a writable extension, and a DOCX with no `-o` goes to standard output - when that is not a terminal; `--fragment` with DOCX is a usage error; `RenderTree` gains `version` and - `effective_date` (plain text), filled whether or not the title block shows them; `.docx` renders added - the Warning `render-docx-interim` until the body was complete (it is gone since the front and back matter) -- **Style settings (done).** `page.header` and `page.footer` (the HTML writer ignores them for now), - `docx.update_fields_on_open`; settings Word cannot express are a `StyleError` for DOCX -- **Verification (done).** Every part validated against the ISO/IEC 29500 schemas (fetched in CI, not - stored), well-formedness fuzzing, a determinism test, and LibreOffice opening the documents through - UNO as the oracle (a `docx` CI job) -- **Body writer (done).** `numbering.xml`, numbered paragraphs, bookmarks, `REF` fields with cached results, - term hyperlinks, and every block and inline, written from a `DocxPlan` (`resolve/docx_plan.py`) -- **Planner: numbering, cross-references and contents plan (done).** The tree is mapped to Word numbering - (heading list, list families, paragraphs) and checked against the resolver with a model of Word's counters; - every `{{ref:}}` is covered by `REF` fields and delimiters (the designation cover), every anchor has a bookmark, - and the table of contents is planned from Word's numbers; `render` runs the planner and reports its - `render-docx-*` diagnostics. A LibreOffice round trip with wrong cached results checks the numbers and references - against the plain-text oracle -- **Front and back matter (done).** The title block (header rows and parties), the `TOC` field with its cached - entries and `PAGEREF` page numbers, the attachments (bookmarked, separated by a page break or a rule) and the - signature blocks; the `render-docx-interim` Warning is gone. A LibreOffice round trip checks the contents against the - HTML's entries, also after a heading is inserted in the editor; a manual Word checklist for releases is in - `.github/PUBLISHING.md` -- **Planner: qualified labels, alternatives and resynchronised numbers (done).** The continental `5.1` items and the - numbered paragraphs ride the heading list's level after their section's, and lists nested in them its reserved - levels, so they restart by structure; alternatives that share a number (§15.8) are `numId 0` paragraphs whose - number is a `REF` field of the first, with the units under them on a num of their own; a unit Word would count - off the resolver's is restarted at a fixed value and reported. Clauses and nested items inserted through UNO are - numbered as the renderer numbers them -- **Next.** Running the manual Word checklist, which finds what only Word shows +- **The writer.** A `docx` writer in the base package, written with the standard library only (no `[docx]` + extra): the OOXML parts, styles generated from the style template, settings (compatibility mode 15, + `w:updateFields`), page setup, running headers and footers with `PAGE`/`NUMPAGES` fields, document + properties, and deterministic bytes. The body is every block and inline: headings, paragraphs, lists, quotes, + drafting notes, code, tables with a repeated header row, rules, links, defined terms, blanks and choices + (images as their alt text), then the front and back matter: the title block, a live `TOC` field with cached + entries and `PAGEREF` page numbers, the attachments and the signature blocks +- **The planner** (`resolve/docx_plan.py`): the render tree carries the numbering structure of every unit, and a pure + planning step maps it to Word numbering (the heading list, list families, paragraphs, qualified `5.1` labels on the + heading list's levels, alternatives that share a number as `numId 0` paragraphs with a `REF` field, restarts by + structure or by a fixed value), to bookmarks, and to the fields of every `{{ref:}}` (the designation cover), with the + cached results taken from a model of Word's counters. Where Word draws differently from the renderer, the render + reports a `render-docx-*` diagnostic (listed in [`CONFORMANCE.md`](../CONFORMANCE.md)) +- **Style settings.** `page.header` and `page.footer` (the HTML writer ignores them for now), + `docx.update_fields_on_open` and `docx.reference_text_switch`; settings Word cannot express are a `StyleError` + for DOCX only +- **Verification.** Every part validated against the ISO/IEC 29500 schemas (fetched in CI, not stored), well-formedness + fuzzing, determinism, and LibreOffice as the oracle through UNO (a `docx` CI job): the numbers and references + unit by unit with the cached results made wrong on purpose, whole documents against the plain-text writer's + output under three styles, and edit-then-update (a heading, an item, a numbered paragraph inserted, a clause + deleted, as a lawyer does). The manual Word checklist in `.github/PUBLISHING.md` covers what only Word shows +- **Behaviour changes this brings.** + - `-o -` writes to standard output for every format (it used to create a file named `-`). + - `.docx` is a writable extension; a DOCX with no `-o` goes to standard output when that is not a terminal, and is + never written to a terminal. `--fragment` with DOCX is a usage error. + - `render(..., format="docx")` returns the file as `RenderResult.binary` (bytes); `RenderResult.output` stays + `str` and is empty for it. `FORMATS` gains `docx`. + - `RenderTree` gains fields for the numbering structure (`formats`, and on sections, list items and + paragraphs `unit`, `number`, `parent_unit`, `shares_number_with`; `List.family`; `CrossRef.target_unit`, + `section_unit`) and `version` and `effective_date` (plain text, filled whether or not the title block shows + them). HTML and plain-text output are byte-identical to 0.4 (the goldens are unchanged). + - New `render-docx-*` diagnostics for DOCX renders only. +- **Next.** Running the manual Word checklist, which finds what only Word shows; its findings are recorded in the + release notes. Whether the writer then claims §13.6 for DOCX is decided in `CONFORMANCE.md` ### v0.6 — PDF diff --git a/docs/style-templates.md b/docs/style-templates.md index e312a00..6dd95f3 100644 --- a/docs/style-templates.md +++ b/docs/style-templates.md @@ -279,6 +279,18 @@ Settings that were already part of the template gain a DOCX meaning: | `signatures.enabled` | Borderless tables with a signing line, kept together on one page as far as one page allows. They follow the last attachment, as in the HTML. A name, title, date or place row whose label is blanked (`labels.signature_*`) and whose value is empty is left out, as the plain-text writer leaves it out | | `title_block.parties`, `title_block.effective_date`, `title_block.version` | The header rows and the parties, as borderless tables | +The numbering and wording settings are the same as in the other formats; what they become in the file: + +| Setting | In the DOCX | +|---|---| +| `numbering.*` | The multilevel list the heading styles are linked to: `counter` is the level's number format, `label` its level text (`{n}` the level's own counter, `{path}` the counters of the levels above it joined as the label says). A label with a literal `%` before a digit or a counter cannot be written as a level text and is a style error for DOCX. With `scheme: none` the headings are not numbered | +| `enumeration.*`, `paragraphs.*` | The lists of items and the numbered paragraphs, one numbering definition for each kind. A label that shows `{section}` can be live only on the heading list's next level, and is otherwise written without it and reported (`render-docx-section-label`) | +| `references.format`, `numbering.levels[].ref`, `enumeration.levels[].ref` | The literal text around the `REF` fields of a cross-reference and between them; the cached result of the field is what Word draws. See `docx.reference_text_switch` | +| `definitions.style`, `definitions.term_style` | The character styles of a defining occurrence and of a `{{term:}}` reference | +| `typography.justify` | The justification of `Normal` | +| `locale`, `values.*`, `placeholders.*`, `template_view.*`, `labels.*` | As in the other formats: dates, amounts, blanks, choices and labels are written as text (they are not numbers Word can compute) | +| `html.max_width`, `html.extra_css` | Not read | + The styles for this matter (`Title Block Text`, `Party Side`, `Party Name`, `Attachments Heading`, `Attachment Title`, `Attachment File`, `Signature Side`, `Signature Text`, `Signature Line`) are Word styles like the others: a lawyer who changes one in Word changes every paragraph that uses it. From 51a37b31179f763305786155b64401abb94af8e9 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:37:06 +0000 Subject: [PATCH 115/118] Read the LibreOffice tests' text from the document model, not the text export filter The plain-text export lays a table row out as one tab-separated line in some LibreOffice versions (the CI's) and as one line per cell paragraph in others (24.2), so the tests passed locally and failed in CI. The UNO scripts now walk the body (paragraphs, and tables by row and cell, in order) and write one line per paragraph, led by the list label LibreOffice draws, which gives the same lines under every version. The nested items no longer carry the filter's indent. The CI job prints the Ubuntu and LibreOffice versions before the tests. --- .github/workflows/ci.yml | 4 ++ tests/lo/uno_edit.py | 7 +-- tests/lo/uno_export.py | 9 ++-- tests/lo/uno_text.py | 57 +++++++++++++++++++++ tests/test_docx_libreoffice.py | 2 +- tests/test_docx_lo_text.py | 91 ++++++++++++++++++++++++++++++++++ 6 files changed, 162 insertions(+), 8 deletions(-) create mode 100644 tests/lo/uno_text.py create mode 100644 tests/test_docx_lo_text.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd2fbec..54637ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,6 +99,10 @@ jobs: - run: | sudo apt-get update sudo apt-get install -y libreoffice-writer python3-uno + - name: Show the versions of the system and of LibreOffice + run: | + lsb_release -ds || cat /etc/os-release + soffice --version - run: pip install -e ".[dev]" - name: Cache the OOXML schemas id: schemas diff --git a/tests/lo/uno_edit.py b/tests/lo/uno_edit.py index 8be179c..5832d09 100644 --- a/tests/lo/uno_edit.py +++ b/tests/lo/uno_edit.py @@ -3,7 +3,7 @@ the document says: for each job, a paragraph is inserted before the first paragraph that holds some text (so that it takes that paragraph's numbering and style, as pressing Enter at its start does), or that paragraph is deleted, then every field and index is updated (the indexes too, so a table of contents lists what was inserted) and the body -is exported as plain text (``JOB.txt``, UTF-8). +is written as plain text (``JOB.txt``, UTF-8, one line per paragraph, read from the document model: tests/lo/uno_text.py). Run it with the system Python that has the ``uno`` module, **not** with the project's virtual environment: @@ -31,6 +31,7 @@ import uno # type: ignore[import-not-found] # noqa: I001 - provided by python3-uno, not by pip from com.sun.star.beans import PropertyValue # type: ignore[import-not-found] from com.sun.star.text.ControlCharacter import PARAGRAPH_BREAK # type: ignore[import-not-found] +from uno_text import body_text def _value(name: str, value: object) -> PropertyValue: @@ -124,8 +125,8 @@ def edit(out_dir: str, jobs: list[dict]) -> None: indexes.getByIndex(position).update() document.getTextFields().refresh() document.refresh() - document.storeToURL(uno.systemPathToFileUrl(os.path.join(out_dir, f"{job['name']}.txt")), - (_value("FilterName", "Text (encoded)"), _value("FilterOptions", "UTF8"))) + with open(os.path.join(out_dir, f"{job['name']}.txt"), "w", encoding="utf-8", newline="") as handle: + handle.write(body_text(document)) finally: document.close(True) finally: diff --git a/tests/lo/uno_export.py b/tests/lo/uno_export.py index 4167298..9246526 100644 --- a/tests/lo/uno_export.py +++ b/tests/lo/uno_export.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 """Open DOCX files in headless LibreOffice through UNO, update every field and index, and write what the document says: one ``NAME.txt`` per file (the body as plain text, -UTF-8, as LibreOffice's text export writes it) and one ``NAME.json`` with the page -count and the text of the running header and footer. +UTF-8, one line per paragraph, read from the document model: tests/lo/uno_text.py) and one +``NAME.json`` with the page count and the text of the running header and footer. Run it with the system Python that has the ``uno`` module (``/usr/bin/python3`` once ``python3-uno`` is installed), **not** with the project's virtual environment: @@ -31,6 +31,7 @@ import uno # type: ignore[import-not-found] # noqa: I001 - provided by python3-uno, not by pip from com.sun.star.beans import PropertyValue # type: ignore[import-not-found] +from uno_text import body_text def _value(name: str, value: object) -> PropertyValue: @@ -93,8 +94,8 @@ def export(out_dir: str, files: list[str], *, update: bool = True) -> None: if update: _update(document) header, footer = _page_texts(document) - document.storeToURL(uno.systemPathToFileUrl(os.path.join(out_dir, f"{stem}.txt")), - (_value("FilterName", "Text (encoded)"), _value("FilterOptions", "UTF8"))) + with open(os.path.join(out_dir, f"{stem}.txt"), "w", encoding="utf-8", newline="") as handle: + handle.write(body_text(document)) pages = document.getCurrentController().PageCount with open(os.path.join(out_dir, f"{stem}.json"), "w", encoding="utf-8") as handle: json.dump({"header": header, "footer": footer, "pages": pages}, handle) diff --git a/tests/lo/uno_text.py b/tests/lo/uno_text.py new file mode 100644 index 0000000..18e5448 --- /dev/null +++ b/tests/lo/uno_text.py @@ -0,0 +1,57 @@ +"""What a LibreOffice document says, read from its model instead of from a text export filter. + +LibreOffice's plain-text export lays a table row out as one line (cells separated by tabs) in some versions +and as one line per cell paragraph in others, so a test that compared that text would depend on the version of +the LibreOffice that happens to be installed. The text here is made by walking the body the way the document +holds it, so it is the same under every version: one line per paragraph, a table's rows and cells in order (each +cell's paragraphs, and a table inside a cell, in place), a soft line break starting a new line, and, before the +text of a numbered paragraph, the label LibreOffice draws for it (``ListLabelString``, which is what its numbering +computes once the fields are updated) and one space. + +This module needs nothing from LibreOffice itself (it reads what the UNO objects answer), so the tests can +run it over stand-ins (tests/test_docx_lo_text.py). It is imported by tests/lo/uno_export.py and +tests/lo/uno_edit.py, which run under the system Python that has ``uno``. +""" +from __future__ import annotations + +import re + +PARAGRAPH = "com.sun.star.text.Paragraph" +TABLE = "com.sun.star.text.TextTable" + + +def _cell_order(name: str) -> tuple[int, int, str]: + """The reading order of a cell name ("A1", "B1", "A2", ..., "AA1"): by row, then by column.""" + match = re.fullmatch(r"([A-Za-z]+)(\d+)", name) + assert match is not None, name + return int(match[2]), len(match[1]), match[1] + + +def paragraph_lines(paragraph: object) -> list[str]: + """The lines of one paragraph: its text, split at the soft line breaks, the first one led by the list label + (and a space) when the paragraph shows one.""" + label = paragraph.ListLabelString # type: ignore[attr-defined] + text = paragraph.getString() # type: ignore[attr-defined] + lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + if label: + lines[0] = f"{label} {lines[0]}" + return lines + + +def collect(enumeration: object, lines: list[str]) -> None: + """Append the lines of every paragraph and table that *enumeration* (of a text or of a table cell) yields.""" + while enumeration.hasMoreElements(): # type: ignore[attr-defined] + element = enumeration.nextElement() # type: ignore[attr-defined] + if element.supportsService(TABLE): + for name in sorted(element.getCellNames(), key=_cell_order): + collect(element.getCellByName(name).createEnumeration(), lines) + elif element.supportsService(PARAGRAPH): + lines += paragraph_lines(element) + + +def body_text(document: object) -> str: + """The body of *document* as text: every line ends with a line feed, so that an empty last paragraph shows as + an empty last line, as in a plain-text file.""" + lines: list[str] = [] + collect(document.getText().createEnumeration(), lines) # type: ignore[attr-defined] + return "".join(f"{line}\n" for line in lines) diff --git a/tests/test_docx_libreoffice.py b/tests/test_docx_libreoffice.py index 85fd6b2..f02f9ec 100644 --- a/tests/test_docx_libreoffice.py +++ b/tests/test_docx_libreoffice.py @@ -100,7 +100,7 @@ def test_the_words_come_back_in_order(shown: dict[str, Exported]) -> None: # the numbers are drawn by LibreOffice from the numbering definitions (test_docx_roundtrip.py compares them) assert shown["round_trip"].lines == [ "Round Trip", "Sub", "Datum účinnosti", "4. března 2026", "Preamble ž€.", "1. First", "Body with emphasis, strong and code.", "Second line.", - "1.1 Second", " (a) item one", " (b) item two", "A quote", "code line 1", " code line 2", "H1", "H2", + "1.1 Second", "(a) item one", "(b) item two", "A quote", "code line 1", " code line 2", "H1", "H2", "a", "b", ""] diff --git a/tests/test_docx_lo_text.py b/tests/test_docx_lo_text.py new file mode 100644 index 0000000..31f1184 --- /dev/null +++ b/tests/test_docx_lo_text.py @@ -0,0 +1,91 @@ +"""The text tests/lo/uno_text.py reads from a LibreOffice document model: one line per paragraph, whatever the +version of LibreOffice. It is run here over stand-ins for the UNO objects, so no LibreOffice is needed: the +text export filter, which lays a table row out differently in different versions, is not involved at all.""" +from __future__ import annotations + +import importlib.util +from pathlib import Path + +spec = importlib.util.spec_from_file_location("uno_text", Path(__file__).parent / "lo" / "uno_text.py") +assert spec is not None and spec.loader is not None +uno_text = importlib.util.module_from_spec(spec) +spec.loader.exec_module(uno_text) + + +class Enumeration: + def __init__(self, items: list[object]) -> None: + self.items = list(items) + + def hasMoreElements(self) -> bool: # noqa: N802 - the UNO name + return bool(self.items) + + def nextElement(self) -> object: # noqa: N802 + return self.items.pop(0) + + +class Paragraph: + def __init__(self, text: str, label: str = "") -> None: + self.text, self.ListLabelString = text, label + + def supportsService(self, name: str) -> bool: # noqa: N802 + return name == uno_text.PARAGRAPH + + def getString(self) -> str: # noqa: N802 + return self.text + + +class Cell: + def __init__(self, *items: object) -> None: + self.items = items + + def createEnumeration(self) -> Enumeration: # noqa: N802 + return Enumeration(list(self.items)) + + +class Table: + def __init__(self, cells: dict[str, Cell]) -> None: + self.cells = cells + + def supportsService(self, name: str) -> bool: # noqa: N802 + return name == uno_text.TABLE + + def getCellNames(self) -> list[str]: # noqa: N802 + return list(self.cells) + + def getCellByName(self, name: str) -> Cell: # noqa: N802 + return self.cells[name] + + +class Document: + def __init__(self, *items: object) -> None: + self.items = items + + def getText(self) -> Document: # noqa: N802 + return self + + def createEnumeration(self) -> Enumeration: # noqa: N802 + return Enumeration(list(self.items)) + + +def test_a_table_row_is_one_line_per_cell_paragraph_in_reading_order() -> None: + # the cell names in the order a version of LibreOffice may list them: the text is by row, then by column + table = Table({"B2": Cell(Paragraph("d")), "A1": Cell(Paragraph("a")), "B1": Cell(Paragraph("b"), Paragraph("b2")), + "A2": Cell(Paragraph("c"))}) + document = Document(Paragraph("before"), table, Paragraph("")) + assert uno_text.body_text(document) == "before\na\nb\nb2\nc\nd\n\n" + + +def test_a_table_inside_a_cell_is_read_in_place_and_wide_tables_by_column_length() -> None: + inner = Table({"A1": Cell(Paragraph("x")), "B1": Cell(Paragraph("y"))}) + names = ["A1", "B1", "Z1", "AA1", "A2"] + outer = Table({name: Cell(inner if name == "B1" else Paragraph(name)) for name in reversed(names)}) + assert uno_text.body_text(Document(outer)) == "A1\nx\ny\nZ1\nAA1\nA2\n" + + +def test_a_numbered_paragraph_leads_with_its_label_and_a_space_and_a_line_break_starts_a_line() -> None: + document = Document(Paragraph("First", "1."), Paragraph("one\ntwo\tthree", "(a)"), Paragraph("plain\r\nnext")) + assert uno_text.body_text(document) == "1. First\n(a) one\ntwo\tthree\nplain\nnext\n" + + +def test_an_empty_document_has_no_text() -> None: + assert uno_text.body_text(Document()) == "" From ca5343e41250d510843beccacce15f342899368e Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:41:58 +0000 Subject: [PATCH 116/118] Acceptance tests and docs follow the model-based LibreOffice text --- docs/architecture.md | 3 ++- tests/test_docx_acceptance.py | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 89cb5a6..a70a43a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -672,7 +672,7 @@ out when the validator reported the same path at the same line, or at none (`dia Each module is named for its topic. Shared helpers (the minimal document header, the `text` and `html` renderers, and running the command as a program) are in `tests/conftest.py`; those for the DOCX tests -(reading a package, validating it, running LibreOffice) are in `tests/docx_support.py` and `tests/lo/`. +(reading a package, validating it, running LibreOffice) are in `tests/docx_support.py` and `tests/lo/`; the text of a LibreOffice document is read from its model (`tests/lo/uno_text.py`: a line a paragraph, the list label before a numbered one, a table's cells in order), not from its text export, whose table layout differs between versions. Markers: `conformance` (the specification's documents), `ooxml_schema` (DOCX parts against the schemas), `libreoffice` (DOCX opened in LibreOffice; skipped without `soffice` and `python3-uno`, a failure when `LEGALDOWN_REQUIRE_DOCX_TOOLS` @@ -714,6 +714,7 @@ its file is named). `pytest -n auto` (pytest-xdist, in the `dev` extra) runs the | `tests/test_docx_roundtrip.py` | LibreOffice as the oracle of the numbering and the cross-references, in one LibreOffice session: every document of the corpus (the tests', the specification's, stress documents with cycles and unanchored ancestors) under the default, outline, continental, mixed and `none` schemes, with every cached field result made wrong (`XXX`), every numbered paragraph marked and a line per reference added; the number LibreOffice draws, every reference it computes and every body paragraph's text equal what the file cached and the renderer wrote, modulo the gaps the plan reports (`\t` ignored, letters past 26, labels the heading list cannot show); alternatives are checked against the renderer (their `numId 0` number is the holder's, as fields), and so are the continental items on the heading list and the lists on its reserved levels; the table of contents is built by LibreOffice from a `TOC` field the test adds and must list the headings with the numbers the plan gives and none of the items; and edit-then-update (`tests/lo/uno_edit.py`, which inserts and deletes paragraphs, and can put an inserted paragraph on another item's list as pasting a copy does): path labels on the reserved levels, in a detached list (and the documented continuation of a list copied under a new parent) and past nine levels, a heading inserted before a document that skips a level, items, headings, a clause and a nested item inserted before the targets as a lawyer does, fields updated, every reference and the numbers equal the renderer's for the edited source (the new nested item under the continental style reads `a)`), and, for a sub-heading restarted by a fixed value, the plan's warning and what an edit before it keeps right (marker `libreoffice`) | | `tests/test_docx_acceptance.py` | The measurable claims of ADR 0008 over the whole corpus (the tests' documents, the specification's examples and valid fixtures, the stress documents) under the default, outline and continental styles, with the contents on. **Equivalence:** LibreOffice's text, with fields and indexes updated, equals the plain-text writer's, line by line, modulo the differences the formats have by nature (one documented function: borders for rules and signing lines, table cells, quote and link markup, where a condition label stands, `label: value` rows, the contents' page numbers, which are checked to be positive, never decreasing and within the page count) and modulo the gaps the plan reports; the documents that differ are exactly the stress documents that ask for what Word draws differently. **Live after edits:** in one LibreOffice session, a heading, an item, a nested item, a numbered paragraph (continental) are inserted and a clause deleted in up to ten documents under each style, and the result equals the plain-text writer's for the source with the same edits (a difference is allowed only where the render reports a fixed number). **Validity and determinism:** every part validates and two renders are the same bytes. Negative controls show the comparison fails on a wrong number, a missing line, pages out of order | | `tests/test_docx_matter.py` | The front and back matter: the XML of the title block, the `TOC` field and its cached entries, the attachments and the signature blocks; documents with every part (hostile text too) against the schemas; LibreOffice listing exactly the contents' entries after an update, resolving `{{attach:}}`, paging the attachments as configured, taking a heading inserted in the editor into the contents, and reading in the text writer's order | +| `tests/test_docx_lo_text.py` | The reader of a LibreOffice document's text (`tests/lo/uno_text.py`) over stand-ins: one line a paragraph, the list label before a numbered paragraph, a table's cells by row then column, soft breaks as lines | | `tests/test_docx_libreoffice.py` | The documents opened in LibreOffice through UNO (`tests/lo/uno_export.py`), fields updated: the words in order, header, footer, and the page count and the page numbers compared with each other and with page breaks, not with the page count of a font (marker `libreoffice`; skipped without `soffice` and `python3-uno`) | | `tests/test_docx_wordcheck.py`, `tests/word_check.py` | The generator of the document for the manual Word checks of what the planner assumes (`python tests/word_check.py OUT.docx`; `.github/PUBLISHING.md` items 8 and 9): one row for each label form the plan relies on (`\t` on `Article %1`, `§ %1`, `Section %1.%2`, `Čl. %1`, `No. %1`, `(%1)`, `%1)`, `%1:`), the heading list's reserved level `%1.%2(%6)` with levels 3 to 5 unused, and a skipped heading level, each with the planner's assumption beside it taken from the planner's own code; that the package is valid (marker `ooxml_schema`) and LibreOffice computes the plain fields as the planner caches them (marker `libreoffice`); and that the Python snippet in the checklist still runs and makes every file the checklist names, including the document of the level nothing numbered | | `tests/conformance/` | Every example and fixture in the specification repository renders, every internal link has a target, and each invalid fixture renders its rule's failure marker. Needs `$LEGALDOWN_SPEC_DIR` or a sibling `../LegalDown` checkout | diff --git a/tests/test_docx_acceptance.py b/tests/test_docx_acceptance.py index 5eb70d7..b83f2ac 100644 --- a/tests/test_docx_acceptance.py +++ b/tests/test_docx_acceptance.py @@ -180,8 +180,8 @@ def normalise(written: str, shown: str, tree: RenderTree) -> tuple[list[str], li page numbers of the contents. The formats differ by nature in these ways, each undone here and in no other: * a rule (``---``) and a signing line (``____``) are borders of paragraphs and cells in the DOCX, not text; - * a table is pipes and a separator row in the text and cells in the DOCX, which LibreOffice's text export lays out as a - line a cell or a line a row, as its version does: a row is compared as the words of its cells, however many lines they take; + * a table is pipes and a separator row in the text and cells in the DOCX (tests/lo/uno_text.py reads one line a cell): a row is + compared as the words of its cells, however many lines they take, so that a text laid out by row would do as well; * a quote's ``>`` is the indent of a style in the DOCX; * a link's ```` after its text is the target of the hyperlink; * a condition label (§15.8) and the label of a drafting note are in ``[ ]`` in the text and a styled run in the DOCX, @@ -241,8 +241,8 @@ def normalise(written: str, shown: str, tree: RenderTree) -> tuple[list[str], li pages.append(int(entry[2])) shown_lines = [line for line in squeezed if line] - # A table's cells are one line each in some LibreOffice versions and one line a row in others: a row that the text - # writer prints on one line is read from as many lines of LibreOffice's as it takes to say the same words. + # A row that the text writer prints on one line is read from as many lines of LibreOffice's as it takes to say the + # same words (one a cell, as tests/lo/uno_text.py reads the document, but not by that alone). written_lines: list[str] = [] seen: list[str] = [] cursor = 0 @@ -336,8 +336,8 @@ def test_the_comparison_catches_a_wrong_number_a_missing_line_and_a_wrong_page_o @pytest.mark.libreoffice def test_a_table_row_on_one_line_reads_as_the_same_row_on_several() -> None: - """LibreOffice's text export lays a table out as a line a cell (24.2) or a line a row (later versions): both are the - document, so both pass.""" + """A table laid out as a line a cell (as the document is read) or a line a row (as some LibreOffice text filters write + it) is the same table, so both pass.""" case = next(c for c in shown_corpus()["default"] if c.name == "documents/features.lgd") assert case.shown is not None text = case.shown.text From ee70c24b0750465798caff946956b563fe4bc2d5 Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:21:48 +0000 Subject: [PATCH 117/118] Acceptance: edited documents must read exactly as the text writer prints them; pin the reported gap lines The edit-then-update test no longer excuses numeric differences (none occur in the subset), the gap documents are compared line by line against the lines pinned for them, and the determinism test needs no schemas. The test hook matches the acceptance file by name; a CI step runs the acceptance tests that carry no other marker. --- .github/workflows/ci.yml | 3 ++ tests/conftest.py | 6 ++- tests/test_docx_acceptance.py | 76 +++++++++++++++++++++-------------- 3 files changed, 53 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43e3985..b1a860d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,6 +119,9 @@ jobs: - run: pytest -q -m libreoffice env: LEGALDOWN_SPEC_DIR: .legaldown-spec + - run: pytest -q -m "acceptance and not libreoffice and not ooxml_schema" + env: + LEGALDOWN_SPEC_DIR: .legaldown-spec validator-main: # Surfaces changes in legaldown-validator's model before they are released diff --git a/tests/conftest.py b/tests/conftest.py index 117506e..0d6b241 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,8 +64,10 @@ def pytest_addoption(parser: pytest.Parser) -> None: def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: """Leave the DOCX acceptance corpus (marker ``acceptance``) out of a plain ``pytest`` run, which stays as quick as it was: it runs when a marker expression is given (``-m libreoffice``, as the DOCX job does), when - ``LEGALDOWN_ACCEPTANCE`` is set, or when a command-line argument names its file.""" - if config.getoption("markexpr") or os.environ.get("LEGALDOWN_ACCEPTANCE") or any("acceptance" in a for a in config.args): + ``LEGALDOWN_ACCEPTANCE`` is set, or when a command-line argument is its file (``tests/test_docx_acceptance.py``, + with or without a ``::test``).""" + if config.getoption("markexpr") or os.environ.get("LEGALDOWN_ACCEPTANCE") or any( + Path(argument.split("::")[0]).name == "test_docx_acceptance.py" for argument in config.args): return dropped = [item for item in items if item.get_closest_marker("acceptance")] if dropped: diff --git a/tests/test_docx_acceptance.py b/tests/test_docx_acceptance.py index b83f2ac..f738736 100644 --- a/tests/test_docx_acceptance.py +++ b/tests/test_docx_acceptance.py @@ -7,17 +7,18 @@ * **Equivalence.** LibreOffice opens each document with its fields and indexes updated, and what it shows is what the plain-text writer prints, line by line, **modulo the differences the formats have by nature** (:func:`normalise`: borders for rules and signing lines, a table's cells, the markup of the plain-text layout) **and the gaps the plan - reports** (:data:`GAPS`: a number Word draws differently from the renderer's, with a ``render-docx-*`` diagnostic). + reports** (:data:`GAPS`: a number Word draws differently from the renderer's, with a ``render-docx-*`` diagnostic; + :data:`GAP_LINES` pins each line that differs, so that nothing else may). The numbers, the cross-references, the entries of the contents, the attachments and the signature blocks are in that text. The page numbers of the contents are checked on their own: they are positive, never decrease and never pass the number of pages. * **Live after edits.** In one LibreOffice session, for a handful of documents under each style, a heading, a list item, a nested item and (under the continental style) a numbered paragraph are inserted, and a clause is deleted, as a lawyer does (:mod:`tests/lo/uno_edit.py`); the fields and indexes are updated; the result is what the text - writer prints for the source with the same edits made. A document may differ only where the plan reports a number it - fixes (:data:`FIXED`), and every such difference is listed. + writer prints for the source with the same edits made, exactly: in the documents of the subset no number is fixed in a + way these edits reach. * **Valid and deterministic.** Every part of every document validates against the ISO/IEC 29500 schemas, and rendering a - document twice gives the same bytes. + document twice gives the same bytes (that test has the marker ``acceptance`` alone). The LibreOffice tests are skipped when ``soffice`` or ``python3-uno`` is missing and fail instead when ``LEGALDOWN_REQUIRE_DOCX_TOOLS`` is set (``tests/docx_support.py``). The module is part of the DOCX job (``pytest -m @@ -81,12 +82,6 @@ #: and the two readings of a reference that only LibreOffice has (``\t`` ignored, a level nothing numbered counted). GAPS = EXPLAINING | {"render-docx-number-mismatch", "render-docx-reference-word-only", "render-docx-libreoffice-count"} -#: The diagnostics that say a number is fixed in the file: it is right when the file is opened and does not follow what -#: is inserted or deleted before it (a restart by a fixed value, a counter shared with others, a reference that is text), -#: besides the gaps above. -FIXED = GAPS | {"render-docx-shared-counter", "render-docx-static-restart", "render-docx-reference", - "render-docx-reference-cycle"} - # -- The corpus --------------------------------------------------------------------------------------------------- @@ -260,27 +255,46 @@ def normalise(written: str, shown: str, tree: RenderTree) -> tuple[list[str], li return written_lines, seen, pages -def words(lines: list[str]) -> list[str]: - """The words of *lines* that are not numbers or counters: four letters or more, which is what a gap leaves alone.""" - return [word for line in lines for word in re.findall(r"[^\W\d_]{4,}", line)] - - def difference(written: list[str], shown: list[str]) -> str: return "\n".join(difflib.unified_diff(written, shown, "text writer", "LibreOffice", lineterm="", n=1)) +#: The lines that differ, by (style, document), where the plan reports a gap: what the text writer prints, then what LibreOffice +#: (which follows Word's numbering) shows. Nothing else may differ: not a line more, not a word of any length. +GAP_LINES: dict[tuple[str, str], list[tuple[str, str]]] = { + # letters past 26: Word repeats them (bb), the renderer counts on (ab) -- render-docx-letter-overflow + ("default", "stress/letters"): [ + ("(ab) item 28", "(bb) item 28"), ("(ac) item 29", "(cc) item 29"), ("(ad) item 30", "(dd) item 30"), + ("(ae) item 31", "(ee) item 31"), ("Ref 1(a) 1(z) 1(aa) 1(ab) 1(ae)", "Ref 1(a) 1(z) 1(aa) 1(bb) 1(ee)")], + ("outline", "stress/letters"): [ + ("(ab) item 28", "(bb) item 28"), ("(ac) item 29", "(cc) item 29"), ("(ad) item 30", "(dd) item 30"), + ("(ae) item 31", "(ee) item 31"), ("Ref I(a) I(z) I(aa) I(ab) I(ae)", "Ref I(a) I(z) I(aa) I(bb) I(ee)")], + # a list that cannot show the section's number draws its own counter alone -- render-docx-section-label + ("continental", "stress/qualified"): [("1.1 deeper", "1 deeper")], + ("continental", "stress/paths"): [("1.1 four", "1 four"), ("1.1 seven", "1 seven")], + ("continental", "stress/ordered paths"): [ + ("1.1 four", "1 four"), ("1.2 four b", "2 four b"), ("1.1 four c", "1 four c"), ("1.1 y", "1 y"), ("1.1 w", "1 w")], + ("continental", "stress/deep"): [ + ("1.1 level 3", "1 level 3"), ("1.1 level 6", "1 level 6"), ("1.1 level 9", "1 level 9"), + ("1.1 level 9 again", "1 level 9 again")], +} + + def check(case: Shown) -> bool: - """Assert the equivalence of *case*; returns True if it holds exactly, False if it holds modulo a reported gap.""" + """Assert the equivalence of *case*; returns True if it holds exactly, False if it holds modulo the reported gap that + :data:`GAP_LINES` pins line by line (and that the render must report).""" assert case.shown is not None written, seen, pages = normalise(case.written, case.shown.text, case.tree) if case.tree.contents: assert pages, "the contents lists no page numbers" assert pages == sorted(pages) and pages[0] >= 1 and pages[-1] <= case.shown.pages, (pages, case.shown.pages) if written == seen: + assert (case.style, case.name) not in GAP_LINES, "the gap that was pinned is gone: remove it from GAP_LINES" return True assert case.rules & GAPS, f"the numbers differ and the render reports no gap:\n{difference(written, seen)}" - assert words(written) == words(seen), f"a gap changes numbers, not words:\n{difference(written, seen)}" - assert len(written) == len(seen), f"a gap changes numbers, not lines:\n{difference(written, seen)}" + assert len(written) == len(seen), f"a gap changes lines, not their number:\n{difference(written, seen)}" + differing = [(a, b) for a, b in zip(written, seen, strict=True) if a != b] + assert differing == GAP_LINES.get((case.style, case.name)), f"not the pinned gap:\n{difference(written, seen)}" return False @@ -525,18 +539,12 @@ def edited_corpus() -> dict[str, tuple[Edited, ...]]: def test_the_document_follows_the_edits_a_lawyer_makes(style: str) -> None: """After a heading, an item, a nested item and a numbered paragraph are inserted and a clause is deleted, and the fields and indexes are updated, the document reads as the plain-text writer prints the source with the same edits: every number - and reference moved, the contents list the new heading. A document may differ only where its render reported a number - that is fixed in the file, and then the difference is in numbers alone.""" - live = 0 + and reference moved, the contents list the new heading. The text is equal, line for line, in every document: an edit + that reached a number fixed in the file (a restart by a fixed value) would show, and none does in this subset.""" for case in edited_corpus()[style]: written, seen, _ = normalise(case.written, case.shown, case.after) - if written == seen: - live += 1 - continue - fixed = case.rules & FIXED - assert fixed, f"{style}/{case.name}: not live after {[e.kind for e in case.edits]}:\n{difference(written, seen)}" - assert words(written) == words(seen), f"{style}/{case.name}:\n{difference(written, seen)}" - assert live >= 7, [(c.name, [e.kind for e in c.edits]) for c in edited_corpus()[style]] + assert written == seen, (f"{style}/{case.name}: not live after {[e.kind for e in case.edits]}:\n" + f"{difference(written, seen)}") @pytest.mark.libreoffice @@ -568,11 +576,19 @@ def test_every_kind_of_edit_is_made_in_the_corpus() -> None: @pytest.mark.ooxml_schema @pytest.mark.parametrize("style", STYLES) -def test_every_part_of_every_corpus_document_validates_and_a_second_render_is_the_same_bytes(style: str) -> None: +def test_every_part_of_every_corpus_document_validates(style: str) -> None: require_schemas() + for name, source in documents().items(): + made_ = rendered(source, style) + if made_ is not None: + assert validation_errors(made_[0]) == [], f"{style}/{name}" + + +@pytest.mark.parametrize("style", STYLES) +def test_rendering_a_corpus_document_twice_gives_the_same_bytes(style: str) -> None: + """Needs no tool but the corpus (marker ``acceptance`` alone: ``pytest -m acceptance`` runs it).""" for name, source in documents().items(): first, second = rendered(source, style), rendered(source, style) assert (first is None) == (second is None), f"{style}/{name}" if first is not None and second is not None: assert first[0] == second[0], f"{style}/{name}: two renders differ" - assert validation_errors(first[0]) == [], f"{style}/{name}" From 39c989ec6abda2adab53e56286f3d76480e1886e Mon Sep 17 00:00:00 2001 From: george <95045003+dvejsada@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:21:48 +0000 Subject: [PATCH 118/118] Release steps for the Word check; docs: concepts, plan_docx signature, tree fields, DOCX is not yet checked in Word MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PUBLISHING gains the step that follows the Word check (record the Word version and platform, update the places that say the DOCX is unchecked, decide the §13.6 claim) and lists the docx job among those that gate a release. --- .github/PUBLISHING.md | 12 +++++++++--- README.md | 7 ++++--- docs/architecture.md | 4 ++-- docs/concepts.md | 4 ++-- docs/roadmap.md | 2 +- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/PUBLISHING.md b/.github/PUBLISHING.md index c3a083e..17cf7c6 100644 --- a/.github/PUBLISHING.md +++ b/.github/PUBLISHING.md @@ -51,7 +51,13 @@ version number can never be reused once published. `pytest --update-golden tests/test_golden.py`, and check that `git diff tests/golden` shows only that line. Also update the version named in [`CONFORMANCE.md`](../CONFORMANCE.md) and the milestone in [`docs/roadmap.md`](../docs/roadmap.md). -3. Commit, then publish a GitHub Release with the tag `vX.Y.Z`. Its notes are where this +3. For a release that changes the DOCX writer, run the checklist below ("Checking the DOCX in Word") first, and record the + Word version and platform (for example Word for Microsoft 365, Windows 11, build number; or Word for Mac) with + its findings in the release notes. Then update what says the DOCX is not yet checked in Word: the row of + [`CONFORMANCE.md`](../CONFORMANCE.md) ("DOCX output", and its section), the heading of the milestone in + [`docs/roadmap.md`](../docs/roadmap.md), and the README's DOCX section; and decide whether the renderer now claims + DOCX for §13.6 or keeps HTML as the format it claims it with. +4. Commit, then publish a GitHub Release with the tag `vX.Y.Z`. Its notes are where this project records what changed in a release. The workflow first runs the CI workflow on the tagged commit. Only if that passes does it build the @@ -96,7 +102,7 @@ for doc in ("features", "template"): source = Path(f"tests/documents/{doc}.lgd").read_text(encoding="utf-8") for style in ("default", "continental", "outline"): (out / f"{doc}.{style}.docx").write_bytes(render(source, format="docx", style=style, overrides=contents).binary) - # the settings whose effect only Word shows (see items 7 and 9); `True` marks the ones about attachments, which + # the settings whose effect only Word shows (see items 6, 7 and 9); `True` marks the ones about attachments, which # template.lgd has none of for name, (about_attachments, extra) in { "attachments-not-listed": (True, {"contents.attachments": False}), @@ -262,7 +268,7 @@ and LibreOffice differ (`\t`, `lvlRestart`, `SEQ \s`, `w:updateFields`). validator's next minor version removes. - **The upload is gated on the tests.** `publish.yml` first calls [`ci.yml`](workflows/ci.yml) (`on: workflow_call`) as the job `ci`, and `build` needs it, so the lint, test - (Linux and Windows on Python 3.13) and conformance jobs must pass on the commit being published + (Linux and Windows on Python 3.13), conformance and docx (schemas and LibreOffice) jobs must pass on the commit being published before anything is built, and `testpypi` and `pypi` need `build`. The `validator-main` job is informational (`continue-on-error`) and never blocks a release. The caller grants the called workflow `contents: read`; a called workflow cannot hold more permissions than its caller gives. diff --git a/README.md b/README.md index 16db18a..87f2cf6 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,8 @@ may not contain `version` or `extends`, so `overrides=legaldown_render.style.sty edit. It is written with the standard library alone, so it needs nothing beyond the base install. **The point of it:** in the Word file the numbers, the cross-references, the table of contents and the page -numbers are Word's own features, not typed text. Insert a clause in Word, update the fields, and the clauses after it, +numbers are Word's own features, not typed text. (This is how the file is built and how it behaves in LibreOffice, which CI +runs; it has not yet been checked in Word itself, which the maintainers do by hand before a release.) Insert a clause in Word, update the fields, and the clauses after it, every "see 2.1(b)" and the contents follow. The renderer still works every value out; it writes the value only as the cached result of a field, so the document reads right before any field is updated ([ADR 0008](https://github.com/ForLegalAI/legaldown-render/blob/main/docs/decisions/0008-docx-word-native.md)). @@ -148,7 +149,7 @@ row, rules, links, defined terms, blanks and choices have Word styles generated a style in Word applies everywhere. The page size, margins, fonts and colours are the style's, and the file carries the document's title and language. -**What is live:** +**What is live** (by design; confirmed in LibreOffice, to be confirmed in Word): - Heading numbers (the heading styles are linked to a multilevel list), list items and numbered paragraphs are Word numbering definitions. A nested list restarts under a new parent item, and a clause inserted before another renumbers it. @@ -189,7 +190,7 @@ with the line (the rules are listed in [the conformance notes](https://github.co - CI checks every part against the ISO/IEC 29500 schemas and opens the documents in LibreOffice, which updates the fields and is compared with the plain-text output. It cannot run Word, so the maintainers check the golden documents in Word before a release ([the checklist](https://github.com/ForLegalAI/legaldown-render/blob/main/.github/PUBLISHING.md)). -- `--fragment` does not apply: a Word document is always whole. On the command line the file goes to `-o FILE.docx`, or +- `--fragment` (and `standalone=False` in the API, where it is ignored) does not apply: a Word document is always whole, and `--fragment` with it is a usage error. On the command line the file goes to `-o FILE.docx`, or to standard output with `-o -` (or without `-o` when standard output is not a terminal); it is never written to a terminal. ## Rendering documents you did not write diff --git a/docs/architecture.md b/docs/architecture.md index a70a43a..9d65d1c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -245,7 +245,7 @@ references show; the Warning says so. Every other heading reads exactly as its d ### The DOCX planner (`resolve/docx_plan.py`) The DOCX writer contains no numbering rules either (ADR 0003, [ADR 0008](decisions/0008-docx-word-native.md)). -Between the resolved tree and that writer sits a pure function, `plan_docx(tree, *, file=None) -> (DocxPlan, +Between the resolved tree and that writer sits a pure function, `plan_docx(tree, *, file=None, text_switch="documented", text_width=...) -> (DocxPlan, diagnostics)`, which maps the numbering structure above to what Word needs: the abstractNums and nums, for every section, list item and numbered paragraph the `(numId, ilvl)` its paragraph takes, the bookmark of every anchor, the fields of every cross-reference, and the entries of the table of contents. It reads only `tree.formats` and the @@ -676,7 +676,7 @@ Each module is named for its topic. Shared helpers (the minimal document header, Markers: `conformance` (the specification's documents), `ooxml_schema` (DOCX parts against the schemas), `libreoffice` (DOCX opened in LibreOffice; skipped without `soffice` and `python3-uno`, a failure when `LEGALDOWN_REQUIRE_DOCX_TOOLS` -is set, as in the CI job that installs them) and `acceptance`: the DOCX acceptance corpus, whose run takes about half a +is set, as in the CI job that installs them) and `acceptance`: the DOCX acceptance corpus, whose run takes about a minute of LibreOffice time and is left out of a plain `pytest` (it runs with `-m`, with `LEGALDOWN_ACCEPTANCE=1`, or when its file is named). `pytest -n auto` (pytest-xdist, in the `dev` extra) runs the whole suite in a third of the serial time. diff --git a/docs/concepts.md b/docs/concepts.md index 2d66b40..c946c23 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -21,7 +21,7 @@ these. Its inputs are: - **an answers set** (optional, templates only): the values used to assemble a template before rendering (§15.7) -Its outputs are the **rendered file** (HTML or plain text, later DOCX and PDF) and a list of +Its outputs are the **rendered file** (HTML, plain text or DOCX, later PDF) and a list of **diagnostics**. The source never changes during rendering. Given the same source, style template, and options, @@ -99,7 +99,7 @@ style template, and the answers file the CLI is given with `--answers`. | **Render tree** | This package's format-neutral tree of blocks and inlines, built from the source and document model. It is structurally complete: nested lists, inline formatting, and directives are all nodes | | **Resolution** | The stage that computes everything the specification leaves to render time: numbers, designations, link targets, display text, formatted values, markers | | **Resolved document** | The render tree after resolution. It is immutable and holds no unresolved LegalDown semantics. It is the only input to writers | -| **Writer** | A backend that turns the resolved document into a file format (`html`, `text`, later `docx` and `pdf`). A writer contains no LegalDown rules, only layout | +| **Writer** | A backend that turns the resolved document into a file format (`html`, `text`, `docx`, later `pdf`). A writer contains no LegalDown rules, only layout | | **Style template** | The configuration file that controls presentation (§13.7). See [style templates](style-templates.md) | | **Numbering scheme** | How headings are numbered: `decimal`, `legal-outline`, `mixed`, or `none` (§13.1) | | **Enumeration** | How list items are labelled at each nesting level, for example (a) / (i) / (A) (§13.2) | diff --git a/docs/roadmap.md b/docs/roadmap.md index f412038..e6647e7 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -125,7 +125,7 @@ values appear only as the cached results of fields. Decided in - `render(..., format="docx")` returns the file as `RenderResult.binary` (bytes); `RenderResult.output` stays `str` and is empty for it. `FORMATS` gains `docx`. - `RenderTree` gains fields for the numbering structure (`formats`, and on sections, list items and - paragraphs `unit`, `number`, `parent_unit`, `shares_number_with`; `List.family`; `CrossRef.target_unit`, + paragraphs `unit`, `number`, `parent_unit`, `shares_number_with`; `List.family`, `List.depth`, `List.format` and `List.start`; `CrossRef.target_unit`, `section_unit`) and `version` and `effective_date` (plain text, filled whether or not the title block shows them). HTML and plain-text output are byte-identical to 0.4 (the goldens are unchanged). - New `render-docx-*` diagnostics for DOCX renders only.