From 752f3c491af2f1c563a0c5ae31e9452f93a1c0a3 Mon Sep 17 00:00:00 2001 From: andy-zen-dev Date: Wed, 7 Oct 2026 15:03:48 +0300 Subject: [PATCH] feat: analyze local UTF-8 texts as private JSON and offline HTML Refs #269. Reuse the deterministic engine, preserve all metric identities, add 22 positive/negative tests and atomic create-only output. Document working commands without corpus, source-text or parity promotion. --- README.md | 15 + docs/LOCAL_ANALYSIS.md | 78 +++++ .../2026-10-07-scrip-metric-003.md | 13 + scriptorium/analyze.py | 316 ++++++++++++++++++ tests/test_analyze.py | 105 ++++++ tests/test_analyze_safety.py | 172 ++++++++++ 6 files changed, 699 insertions(+) create mode 100644 docs/LOCAL_ANALYSIS.md create mode 100644 project/CHANGELOG.d/2026-10-07-scrip-metric-003.md create mode 100644 scriptorium/analyze.py create mode 100644 tests/test_analyze.py create mode 100644 tests/test_analyze_safety.py diff --git a/README.md b/README.md index 4cec5ff6..a5719737 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,21 @@ legally usable and at least 300,000 characters including spaces; copyrighted tex not committed merely because it is accessible online. Shorter texts and excerpts are still valid analyzer inputs, but their results carry representativeness warnings. +## Analyze your own text + +From a checkout, using Python 3.13 and no third-party dependencies: + +```bash +python -m scriptorium.analyze manuscript.txt --output analysis.json +python -m scriptorium.analyze manuscript.txt --format html --output report.html +``` + +Get 29 deterministic metric rows with explicit units, profile versions and source hashes. +The manuscript stays local; reports contain no source prose. Short texts are supported, +missing values remain explicit, and results are inferred/extension rather than FantLab +parity. Existing output files are never overwritten. See +[local analysis and optional dictionaries](docs/LOCAL_ANALYSIS.md). + ## What works today Scriptorium now has a standard-library-only deterministic analysis core: diff --git a/docs/LOCAL_ANALYSIS.md b/docs/LOCAL_ANALYSIS.md new file mode 100644 index 00000000..c8e16436 --- /dev/null +++ b/docs/LOCAL_ANALYSIS.md @@ -0,0 +1,78 @@ +# Analyze a local text + +From the repository root with Python 3.13, run the existing deterministic engine through +`scriptorium.analyze`. No third-party packages or network access are required. + +```bash +python -m scriptorium.analyze manuscript.txt --output analysis.json +python -m scriptorium.analyze manuscript.txt --format html --title "My manuscript" --output report.html +``` + +JSON is the default; omitting `--output` writes the selected format to standard output. +The HTML file is self-contained and can be opened offline. Its tables include units, +compatibility labels, raw punctuation counts and keyboard-accessible scrolling regions. + +## Reproducibility and scientific limits + +The `scriptorium-local-analysis-v1` wrapper retains all 29 current engine rows without +changing their values or formulas. It records the metric contract, normalization/metric/ +dialogue/vocabulary/punctuation profiles, raw-byte and normalized-text SHA-256 identities, +input size and dictionary identity. `--scriptorium-revision ` records an explicit +caller-supplied revision; without it the revision is null, not guessed. The HTML exposes +profiles under **Analysis profiles**. + +Inputs must be UTF-8. CRLF/CR and Unicode composition use the existing normalization +profile. Empty inputs are supported: counts may be zero, but undefined means or rates +remain null. A null vocabulary metric can also mean a missing dictionary or an incomplete +UASZ window; HTML therefore says **Unavailable**, not a fabricated zero or pass. + +The 300,000-character threshold includes spaces. Shorter inputs are valid local analyses +with limited representativeness; meeting the length floor alone is not corpus admission, +proof of representativeness, an author profile or FantLab parity. FantLab-shaped rows +remain inferred; Scriptorium-only rows remain extensions. No morphology backend is +silently instantiated. + +## Optional external dictionary + +Supply a UTF-8 file with one lexeme per line and a non-empty profile together: + +```bash +python -m scriptorium.analyze manuscript.txt --dictionary dictionary.txt \ + --dictionary-profile my-dictionary-v1 --output with-dictionary.json +``` + +The engine performs its existing NFC/case-folded set normalization. A supplied dictionary +can enable active dictionary/nondictionary counts and complete-window UASZ values; short +inputs still lack complete larger windows. Results retain profile, normalized lexeme-set +SHA-256 and count, not dictionary contents. Reproducibility does not establish equivalence +to FantLab's undisclosed production dictionary. + +## Privacy and file safety + +Manuscripts and dictionary contents are not embedded in the returned aggregate artifacts, +sent to a service, or added to the corpus. Titles, dictionary profile labels and optional +revision metadata are caller-controlled and intentionally appear in reports. Hashes and +aggregate metrics can themselves identify a known work: treat generated reports as +private unless deliberately shared. There is no telemetry or external asset loading. + +Output is create-only, including existing files, hard-link aliases and dangling symlinks. +The writer stages derived output in the destination directory, flushes it, then uses an +atomic hard-link creation without replacement. The filesystem must support hard links; +otherwise the command reports an error rather than falling back to unsafe overwriting. +Handled write failures remove temporary files and leave previous output untouched. A +process/OS crash may leave a hidden `.scriptorium-*.tmp` derived-report file. On POSIX, +new reports have mode 0600. Parent directories are created as needed. + +## Library and checks + +```python +from pathlib import Path +from scriptorium.analyze import build_local_analysis, render_html + +artifact = build_local_analysis(Path("manuscript.txt").read_bytes()) +html = render_html(artifact, title="Local analysis") +``` + +Run focused checks with `python -m unittest discover -s tests -p 'test_analyze*.py' -v`. +Tests exercise the real engine, deterministic output, Unicode normalization, dictionary +limits, privacy, output aliases, failed publication and empty inputs. diff --git a/project/CHANGELOG.d/2026-10-07-scrip-metric-003.md b/project/CHANGELOG.d/2026-10-07-scrip-metric-003.md new file mode 100644 index 00000000..54b6ac49 --- /dev/null +++ b/project/CHANGELOG.d/2026-10-07-scrip-metric-003.md @@ -0,0 +1,13 @@ +# SCRIP-METRIC-003 — local text analysis + +Refs #269. Adopt the pending local-analysis implementation into the real metric engine: +UTF-8 input to deterministic JSON or readable offline HTML, optional explicit dictionary, +29 unchanged engine rows, source hashes/profiles, short-input and unavailable-value labels. + +Add 22 focused tests, including direct engine equality, actual CLI file output, dictionary +privacy, normalization identity, empty input, threshold boundaries, non-finite rejection, +create-only aliases and atomic publication failure. README and LOCAL_ANALYSIS document +working commands and report privacy. No source text, corpus admission, morphology +substitution, analyzer formula, parity gate or Pages activation changes. + +Full exact-head CI and later-run acceptance remain required before merge. diff --git a/scriptorium/analyze.py b/scriptorium/analyze.py new file mode 100644 index 00000000..b99886b5 --- /dev/null +++ b/scriptorium/analyze.py @@ -0,0 +1,316 @@ +"""Privacy-safe local text analysis with JSON and standalone HTML output.""" + +from __future__ import annotations + +import argparse +from hashlib import sha256 +from html import escape +import json +import math +import os +import tempfile +from pathlib import Path +from typing import Final, Iterable + +from .metrics import analyze_deterministic_metrics + + +LOCAL_ANALYSIS_SCHEMA: Final = "scriptorium-local-analysis-v1" +REPRESENTATIVE_CHARACTERS: Final = 300_000 + + +def build_local_analysis( + source_bytes: bytes, + *, + dictionary_words: Iterable[str] | None = None, + dictionary_profile: str | None = None, + scriptorium_revision: str | None = None, +) -> dict[str, object]: + """Analyze UTF-8 source bytes without embedding source text in the result.""" + + text = source_bytes.decode("utf-8") + analysis = analyze_deterministic_metrics( + text, + dictionary_words=dictionary_words, + dictionary_profile=dictionary_profile, + ) + normalized_characters = analysis["metrics"]["fantlab.general.characters"]["value"] + if isinstance(normalized_characters, bool) or not isinstance(normalized_characters, int): + raise ValueError("engine character count must be an integer") + meets_threshold = normalized_characters >= REPRESENTATIVE_CHARACTERS + + return { + "schema_version": LOCAL_ANALYSIS_SCHEMA, + "metric_contract_id": analysis["metric_contract_id"], + "source": { + "raw_sha256": sha256(source_bytes).hexdigest(), + "byte_count": len(source_bytes), + "normalized_sha256": analysis["normalized_sha256"], + "normalized_character_count": normalized_characters, + "source_text_embedded": False, + }, + "analyzer": { + "scriptorium_revision": _none_if_blank(scriptorium_revision), + "profiles": analysis["profiles"], + "dependencies": analysis["dependencies"], + }, + "representativeness": { + "calibration_threshold_characters": REPRESENTATIVE_CHARACTERS, + "meets_calibration_threshold": meets_threshold, + "note": ( + "Length meets the standing calibration/profile corpus floor; legal and provenance " + "admission remain separate gates." + if meets_threshold + else "Short input is valid for local analysis but is less representative for " + "author/profile or parity-corpus use." + ), + }, + "metrics": analysis["metrics"], + } + + +def render_html(artifact: dict[str, object], *, title: str | None = None) -> str: + """Render one local-analysis artifact as deterministic standalone HTML.""" + + if artifact.get("schema_version") != LOCAL_ANALYSIS_SCHEMA: + raise ValueError("unsupported local analysis schema") + source = _mapping(artifact.get("source"), "source") + analyzer = _mapping(artifact.get("analyzer"), "analyzer") + representativeness = _mapping(artifact.get("representativeness"), "representativeness") + metrics = _mapping(artifact.get("metrics"), "metrics") + if source.get("source_text_embedded") is not False: + raise ValueError("local analysis must not embed source text") + + display_title = _nonempty_text(title) if title is not None else "Scriptorium local analysis" + groups = ( + ("General", _rows(metrics, ("fantlab.general.", "scriptorium.general."))), + ("Dialogue", _rows(metrics, ("fantlab.dialogue.",))), + ("Vocabulary", _rows(metrics, ("fantlab.vocabulary.",))), + ("Punctuation", _rows(metrics, ("fantlab.punctuation.",))), + ) + included = {metric_id for _, rows in groups for metric_id, _ in rows} + other = [(metric_id, _mapping(metrics[metric_id], metric_id)) + for metric_id in sorted(metrics) if metric_id not in included] + groups = (*groups, ("Other", other)) + sections = "".join(_render_metric_group(name, rows) for name, rows in groups if rows) + profiles = _mapping(analyzer.get("profiles"), "analyzer.profiles") + profiles_html = "".join( + f"
{escape(str(key))}
{escape(str(value))}
" + for key, value in sorted(profiles.items()) + ) + threshold_state = ( + "Meets the 300,000-character calibration floor" + if representativeness.get("meets_calibration_threshold") is True + else "Short input — valid analysis, limited representativeness" + ) + dependency = _mapping(analyzer.get("dependencies"), "analyzer.dependencies") + dictionary_dependency = dependency.get("vocabulary_dictionary") + if dictionary_dependency is None: + dictionary_html = "

Vocabulary dictionary: not supplied; dictionary-dependent rows are not run.

" + else: + dep = _mapping(dictionary_dependency, "vocabulary_dictionary") + dictionary_html = ( + "

Vocabulary dictionary: " + f"{escape(str(dep.get('profile')))} · {escape(str(dep.get('lexeme_count')))} normalized lexemes · " + f"{escape(str(dep.get('normalized_lexemes_sha256')))}

" + ) + + revision = analyzer.get("scriptorium_revision") + revision_html = "not recorded" if revision is None else f"{escape(str(revision))}" + note = escape(str(representativeness.get("note", ""))) + return f''' + + + + +{escape(display_title)} + + + + +
+

Scriptorium · local-only deterministic analysis

+

{escape(display_title)}

+

Readable derived metrics from a local UTF-8 text. FantLab-namespaced rows are inferred compatibility candidates, not parity claims. Source text is not embedded in this report.

+

{escape(threshold_state)}

+
+

Raw SHA-256: {escape(str(source.get('raw_sha256')))}

+

Normalized SHA-256: {escape(str(source.get('normalized_sha256')))}

+

Bytes / normalized characters: {escape(str(source.get('byte_count')))} / {escape(str(source.get('normalized_character_count')))}

+

Scriptorium revision: {revision_html}

+{dictionary_html} +

Representativeness: {note}

+
Analysis profiles
{profiles_html}
+

Unavailable is not zero. A null metric may need a dictionary, a nonzero +word/sentence denominator, or a complete vocabulary window. No null is a parity pass.

+
+{sections} +
+
Local analysis only. Corpus admission, legal provenance and FantLab source identity are separate gates.
+ + +''' + + +def _render_metric_group(name: str, rows: list[tuple[str, dict[str, object]]]) -> str: + body = [] + for metric_id, row in rows: + status = row.get("compatibility_status") + if status not in ("inferred", "extension"): + raise ValueError("local results must remain inferred or extension") + value = row.get("value") + raw_count = row.get("raw_count") + raw_count_html = "—" if raw_count is None else escape(str(raw_count)) + body.append( + "" + f"{escape(metric_id)}" + f"{escape(_format_value(value))}" + f"{escape(_human_unit(str(row.get('unit', ''))))}" + f"{escape(str(row.get('compatibility_status', 'unknown')))}" + f"{raw_count_html}" + "" + ) + label = escape(name) + return ( + f"

{label}

" + f"" + "" + "" + "".join(body) + "
{label}: local metrics and evidence
MetricValueUnitStatusRaw count
" + ) + + +def _rows(metrics: dict[str, object], prefixes: tuple[str, ...]) -> list[tuple[str, dict[str, object]]]: + result: list[tuple[str, dict[str, object]]] = [] + for metric_id in sorted(metrics): + if not metric_id.startswith(prefixes): + continue + row = _mapping(metrics[metric_id], metric_id) + result.append((metric_id, row)) + return result + + +def _format_value(value: object) -> str: + if value is None: + return "Unavailable" + if isinstance(value, bool): + raise ValueError("metric value must not be boolean") + if isinstance(value, int): + return f"{value:,}" + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError("metric value must be finite") + return format(value, ".6g") + raise ValueError("metric value must be numeric or null") + + +def _human_unit(unit: str) -> str: + return { + "characters/word": "characters / word", + "characters/sentence": "characters / sentence", + "occurrences/1000_words": "occurrences / 1,000 words", + "unique_dictionary_words/window": "unique dictionary words / window", + }.get(unit, unit) + + +def _mapping(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict): + raise ValueError(f"{label} must be an object") + return value + + +def _nonempty_text(value: str) -> str: + stripped = value.strip() + if not stripped: + raise ValueError("title must be non-empty") + return stripped + + +def _none_if_blank(value: str | None) -> str | None: + if value is None: + return None + stripped = value.strip() + return stripped or None + + +def _write_new(path: Path, content: str) -> None: + # Publish complete derived output with no replacement, including symlink targets. + # Keep the temporary file on the same filesystem for atomic hard-link creation. + path.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", newline="\n", + prefix=".scriptorium-", suffix=".tmp", dir=path.parent, delete=False, + ) as handle: + temporary = Path(handle.name) + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.link(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Analyze a local UTF-8 text without publishing source text.") + parser.add_argument("text", type=Path) + parser.add_argument("--format", choices=("json", "html"), default="json") + parser.add_argument("--title") + parser.add_argument("--scriptorium-revision") + parser.add_argument("--dictionary", type=Path) + parser.add_argument("--dictionary-profile") + parser.add_argument("--output", type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + if (args.dictionary is None) != (args.dictionary_profile is None): + parser.error("--dictionary and --dictionary-profile must be supplied together") + + try: + source_bytes = args.text.read_bytes() + dictionary_words = None + if args.dictionary is not None: + dictionary_words = args.dictionary.read_text(encoding="utf-8").splitlines() + artifact = build_local_analysis( + source_bytes, + dictionary_words=dictionary_words, + dictionary_profile=args.dictionary_profile, + scriptorium_revision=args.scriptorium_revision, + ) + if args.format == "json": + rendered = json.dumps(artifact, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False) + "\n" + else: + rendered = render_html(artifact, title=args.title) + if args.output is None: + print(rendered, end="") + else: + _write_new(args.output, rendered) + except (OSError, UnicodeDecodeError, ValueError, TypeError) as exc: + parser.error(str(exc)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_analyze.py b/tests/test_analyze.py new file mode 100644 index 00000000..78ec21b6 --- /dev/null +++ b/tests/test_analyze.py @@ -0,0 +1,105 @@ +import contextlib +from hashlib import sha256 +import io +import json +from pathlib import Path +import tempfile +import unittest + +from scriptorium.analyze import build_local_analysis, main, render_html + + +SOURCE = "Кот, кот.\n— Да? — сказал он. — Нет." + + +class LocalAnalysisTests(unittest.TestCase): + def test_build_binds_source_without_embedding_text(self): + source = SOURCE.encode("utf-8") + artifact = build_local_analysis(source, scriptorium_revision="abc123") + serialized = json.dumps(artifact, ensure_ascii=False) + self.assertEqual(artifact["schema_version"], "scriptorium-local-analysis-v1") + self.assertEqual(artifact["source"]["raw_sha256"], sha256(source).hexdigest()) + self.assertFalse(artifact["source"]["source_text_embedded"]) + self.assertEqual(artifact["analyzer"]["scriptorium_revision"], "abc123") + self.assertNotIn(SOURCE, serialized) + self.assertEqual(len(artifact["metrics"]), 29) + + def test_short_input_is_valid_but_marked_less_representative(self): + artifact = build_local_analysis(b"Short text.") + status = artifact["representativeness"] + self.assertFalse(status["meets_calibration_threshold"]) + self.assertEqual(status["calibration_threshold_characters"], 300000) + self.assertIn("less representative", status["note"]) + + def test_dictionary_dependency_is_explicit_and_not_source_text(self): + artifact = build_local_analysis( + "Кот пёс дракон".encode("utf-8"), + dictionary_words=["кот", "пёс"], + dictionary_profile="demo-v1", + ) + dependency = artifact["analyzer"]["dependencies"]["vocabulary_dictionary"] + self.assertEqual(dependency["profile"], "demo-v1") + self.assertEqual(dependency["lexeme_count"], 2) + self.assertEqual(artifact["metrics"]["fantlab.vocabulary.active_dictionary"]["value"], 2) + self.assertEqual(artifact["metrics"]["fantlab.vocabulary.active_nondictionary"]["value"], 1) + self.assertNotIn("дракон", json.dumps(artifact, ensure_ascii=False)) + + def test_html_is_readable_accessible_and_source_free(self): + artifact = build_local_analysis(SOURCE.encode("utf-8")) + html = render_html(artifact, title="Demo ") + self.assertIn("Demo <report>", html) + self.assertIn('href="#main-content"', html) + self.assertIn('id="main-content" tabindex="-1"', html) + self.assertIn('tabindex="0" role="region"', html) + self.assertIn('scope="col"', html) + self.assertIn("*:focus-visible", html) + self.assertIn("prefers-reduced-motion: reduce", html) + self.assertIn("characters / word", html) + self.assertIn("characters / sentence", html) + self.assertNotIn(SOURCE, html) + + def test_deterministic_json_and_html(self): + a = build_local_analysis(SOURCE.encode("utf-8"), scriptorium_revision="abc") + b = build_local_analysis(SOURCE.encode("utf-8"), scriptorium_revision="abc") + self.assertEqual(a, b) + self.assertEqual(render_html(a), render_html(b)) + + def test_cli_writes_json_and_html_create_only(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "input.txt" + json_out = root / "analysis.json" + html_out = root / "analysis.html" + source.write_text(SOURCE, encoding="utf-8") + self.assertEqual(main([str(source), "--output", str(json_out)]), 0) + self.assertEqual( + main([str(source), "--format", "html", "--title", "Reader report", "--output", str(html_out)]), + 0, + ) + self.assertEqual(json.loads(json_out.read_text(encoding="utf-8"))["schema_version"], "scriptorium-local-analysis-v1") + self.assertIn("Reader report", html_out.read_text(encoding="utf-8")) + before = html_out.read_bytes() + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + main([str(source), "--format", "html", "--output", str(html_out)]) + self.assertEqual(html_out.read_bytes(), before) + + def test_cli_dictionary_pair_is_fail_closed(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "input.txt" + source.write_text(SOURCE, encoding="utf-8") + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + main([str(source), "--dictionary-profile", "demo-v1"]) + + def test_cli_rejects_invalid_utf8(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "input.txt" + source.write_bytes(b"\xff\xfe") + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + main([str(source)]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_analyze_safety.py b/tests/test_analyze_safety.py new file mode 100644 index 00000000..816ae4c9 --- /dev/null +++ b/tests/test_analyze_safety.py @@ -0,0 +1,172 @@ +import contextlib +import copy +from hashlib import sha256 +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from scriptorium import analyze +from scriptorium.metrics import analyze_deterministic_metrics +from scriptorium.text import normalize_text + + +class LocalAnalysisSafetyTests(unittest.TestCase): + def fail_cli(self, argv): + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as error: + analyze.main(argv) + self.assertEqual(error.exception.code, 2) + + def test_actual_engine_values_profiles_and_dependencies_are_preserved(self): + raw = "Е\u0308лка\r\n— Кот? — сказал он.\rКот-пёс!".encode() + kwargs = dict(dictionary_words=["кот", "ёлка"], dictionary_profile="fixture-v1") + direct = analyze_deterministic_metrics(raw.decode(), **kwargs) + result = analyze.build_local_analysis(raw, **kwargs) + self.assertEqual(result["metrics"], direct["metrics"]) + self.assertEqual(result["analyzer"]["profiles"], direct["profiles"]) + self.assertEqual(result["analyzer"]["dependencies"], direct["dependencies"]) + self.assertEqual(result["source"]["normalized_sha256"], sha256(normalize_text(raw.decode()).encode()).hexdigest()) + self.assertNotEqual(result["source"]["raw_sha256"], result["source"]["normalized_sha256"]) + + def test_empty_input_preserves_undefined_mean_and_zero_count(self): + result = analyze.build_local_analysis(b"") + self.assertEqual(result["metrics"]["fantlab.general.words"]["value"], 0) + self.assertIsNone(result["metrics"]["fantlab.general.mean_word_length_chars"]["value"]) + rendered = analyze.render_html(result) + self.assertIn("Unavailable", rendered) + self.assertIn("nonzero", rendered) + + def test_supplied_dictionary_does_not_make_incomplete_window_zero(self): + result = analyze.build_local_analysis("Кот".encode(), dictionary_words=["кот"], dictionary_profile="fixture") + self.assertEqual(result["metrics"]["fantlab.vocabulary.active_dictionary"]["value"], 1) + self.assertIsNone(result["metrics"]["fantlab.vocabulary.uasz_3000"]["value"]) + self.assertIn("complete vocabulary window", analyze.render_html(result)) + + def test_threshold_is_not_corpus_admission(self): + for length, meets in ((299999, False), (300000, True)): + with self.subTest(length=length): + result = analyze.build_local_analysis(b" " * length) + self.assertEqual(result["representativeness"]["meets_calibration_threshold"], meets) + self.assertIn("separate gates", result["representativeness"]["note"]) + + def test_html_shows_all_rows_profiles_and_no_remote_assets(self): + result = analyze.build_local_analysis("Уникальная рукопись".encode()) + rendered = analyze.render_html(result) + self.assertEqual(rendered.count('scope="row"'), 29) + self.assertEqual(rendered.count(""), 4) + for profile in result["analyzer"]["profiles"].values(): + self.assertIn(profile, rendered) + self.assertNotIn("Other", analyze.render_html(result)) + + def test_render_rejects_nonfinite_or_promoted_result(self): + original = analyze.build_local_analysis(b"abc") + for value in (float("inf"), float("nan"), True): + bad = copy.deepcopy(original) + bad["metrics"]["fantlab.general.words"]["value"] = value + with self.assertRaises(ValueError): + analyze.render_html(bad) + original["metrics"]["fantlab.general.words"]["compatibility_status"] = "reproduced" + with self.assertRaises(ValueError): + analyze.render_html(original) + + def test_cli_real_dictionary_and_private_payload(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source, dictionary = root / "source.txt", root / "dict.txt" + source.write_text("Кот пёс дракон", encoding="utf-8") + dictionary.write_text("кот\nпёс\nSECRET_DICTIONARY_TOKEN\n", encoding="utf-8") + for fmt in ("json", "html"): + output = root / f"out.{fmt}" + self.assertEqual(analyze.main([str(source), "--dictionary", str(dictionary), "--dictionary-profile", "test-v1", "--format", fmt, "--output", str(output)]), 0) + rendered = output.read_text(encoding="utf-8") + self.assertNotIn("SECRET_DICTIONARY_TOKEN", rendered) + self.assertNotIn("дракон", rendered) + if fmt == "json": + result = json.loads(rendered) + self.assertEqual(result["metrics"]["fantlab.vocabulary.active_dictionary"]["value"], 2) + + def test_invalid_dictionary_utf8_and_blank_profile_create_no_output(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source, dictionary, output = root / "in.txt", root / "dict.txt", root / "out.json" + source.write_bytes(b"abc") + dictionary.write_bytes(b"\xff") + self.fail_cli([str(source), "--dictionary", str(dictionary), "--dictionary-profile", "v1", "--output", str(output)]) + dictionary.write_bytes(b"abc") + self.fail_cli([str(source), "--dictionary", str(dictionary), "--dictionary-profile", " ", "--output", str(output)]) + self.fail_cli([str(source), "--dictionary", str(dictionary), "--output", str(output)]) + self.assertFalse(output.exists()) + + def test_output_aliases_cannot_destroy_manuscript_or_dictionary(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source, dictionary = root / "in.txt", root / "dict.txt" + source.write_bytes(b"private manuscript") + dictionary.write_bytes(b"dictionary") + for target in (source, dictionary): + before = target.read_bytes() + self.fail_cli([str(source), "--output", str(target)]) + self.assertEqual(target.read_bytes(), before) + linked = root / "linked.txt" + os.link(source, linked) + self.fail_cli([str(source), "--output", str(linked)]) + self.assertEqual(source.read_bytes(), b"private manuscript") + + def test_dangling_symlink_is_preserved(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source, target, missing = root / "in.txt", root / "out.json", root / "missing" + source.write_bytes(b"abc") + try: + target.symlink_to(missing) + except (OSError, NotImplementedError): + self.skipTest("symlinks unavailable on this platform") + self.fail_cli([str(source), "--output", str(target)]) + self.assertTrue(target.is_symlink()) + self.assertFalse(missing.exists()) + + def test_failed_write_does_not_publish_partial_output(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + output = root / "report.json" + with patch.object(analyze.os, "fsync", side_effect=OSError("disk failure")): + with self.assertRaises(OSError): + analyze._write_new(output, "derived report") + self.assertFalse(output.exists()) + self.assertEqual(list(root.iterdir()), []) + + def test_atomic_publish_collision_preserves_existing_report(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + output = root / "report.json" + output.write_text("last valid report") + with self.assertRaises(FileExistsError): + analyze._write_new(output, "new report") + self.assertEqual(output.read_text(), "last valid report") + self.assertEqual(list(root.iterdir()), [output]) + + def test_output_has_private_permissions_on_posix(self): + if os.name != "posix": + self.skipTest("POSIX permission bits") + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "report.json" + analyze._write_new(path, "derived") + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + + +if __name__ == "__main__": + unittest.main()