diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d34565..7cdb773 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,5 +12,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' - - run: python -m pip install pyyaml + - run: python -m pip install pyyaml pytest - run: bash scripts/selftest.sh + - run: python -m pytest -q tests diff --git a/.pan/continues/OKF-11.xbrief.json b/.pan/continues/OKF-11.xbrief.json new file mode 100644 index 0000000..b768809 --- /dev/null +++ b/.pan/continues/OKF-11.xbrief.json @@ -0,0 +1,283 @@ +{ + "version": "1", + "issueId": "OKF-11", + "created": "2026-11-15T19:58:40.672Z", + "updated": "2026-11-15T20:34:10.688Z", + "gitState": {}, + "decisions": [ + { + "id": "D-test-waived", + "summary": "Test gate waived: Python project: tests are pytest files under tests/, which this gate's *.test.ts pattern cannot see. 533c86e adds tests/test_publish_cli.py::test_publish_fixture_pages and test_no_unresolved_links (page per public concept, none for private, no unresolved link); 649906f and 46c6da9 add model and --check tests. 75 pytest tests pass and run in CI after the selftest.", + "recordedAt": "2026-11-15T20:23:13.180Z" + }, + { + "id": "D-test-waived", + "summary": "Test gate waived: Python project: tests are pytest files under tests/, which this gate's *.test.ts pattern cannot see. 533c86e adds tests/test_publish_cli.py::test_publish_fixture_pages and test_no_unresolved_links (page per public concept, none for private, no unresolved link); 649906f and 46c6da9 add model and --check tests. 75 pytest tests pass and run in CI after the selftest.", + "recordedAt": "2026-11-15T20:23:59.950Z" + } + ], + "hazards": [], + "resumePoint": null, + "sessionHistory": [ + { + "reason": "feedback", + "note": "[2026-11-15T20:24Z] verification-gate → FAILED — seq 1", + "timestamp": "2026-11-15T20:24:13.758Z" + }, + { + "reason": "feedback", + "note": "[2026-11-15T20:34Z] review-agent → CHANGES-REQUESTED — seq 2", + "timestamp": "2026-11-15T20:34:10.688Z" + } + ], + "items": { + "pytest-harness": { + "status": "completed", + "claimedBy": "agent-okf-11", + "claimedAt": "2026-11-15T19:58:40.672Z", + "doneAt": "2026-11-15T19:59:23.083Z" + }, + "pytest-harness.ac1": { + "status": "completed", + "doneAt": "2026-11-15T19:59:23.083Z" + }, + "pytest-harness.ac2": { + "status": "completed", + "doneAt": "2026-11-15T19:59:23.083Z" + }, + "pytest-harness.ac3": { + "status": "completed", + "doneAt": "2026-11-15T19:59:23.083Z" + }, + "markdown-renderer": { + "status": "completed", + "claimedBy": "agent-okf-11", + "claimedAt": "2026-11-15T20:01:15.589Z", + "doneAt": "2026-11-15T20:04:15.527Z" + }, + "markdown-renderer.ac1": { + "status": "completed", + "doneAt": "2026-11-15T20:04:15.527Z" + }, + "markdown-renderer.ac2": { + "status": "completed", + "doneAt": "2026-11-15T20:04:15.527Z" + }, + "markdown-renderer.ac3": { + "status": "completed", + "doneAt": "2026-11-15T20:04:15.527Z" + }, + "markdown-renderer.ac4": { + "status": "completed", + "doneAt": "2026-11-15T20:04:15.527Z" + }, + "markdown-renderer.ac5": { + "status": "completed", + "doneAt": "2026-11-15T20:04:15.527Z" + }, + "redaction-rules": { + "status": "completed", + "claimedBy": "agent-okf-11", + "claimedAt": "2026-11-15T20:04:16.200Z", + "doneAt": "2026-11-15T20:05:37.485Z" + }, + "redaction-rules.ac1": { + "status": "completed", + "doneAt": "2026-11-15T20:05:37.485Z" + }, + "redaction-rules.ac2": { + "status": "completed", + "doneAt": "2026-11-15T20:05:37.485Z" + }, + "redaction-rules.ac3": { + "status": "completed", + "doneAt": "2026-11-15T20:05:37.485Z" + }, + "redaction-rules.ac4": { + "status": "completed", + "doneAt": "2026-11-15T20:05:37.485Z" + }, + "redaction-rules.ac5": { + "status": "completed", + "doneAt": "2026-11-15T20:05:37.485Z" + }, + "publish-model": { + "status": "completed", + "claimedBy": "agent-okf-11", + "claimedAt": "2026-11-15T20:05:38.261Z", + "doneAt": "2026-11-15T20:07:41.193Z" + }, + "publish-model.ac1": { + "status": "completed", + "doneAt": "2026-11-15T20:07:41.193Z" + }, + "publish-model.ac2": { + "status": "completed", + "doneAt": "2026-11-15T20:07:41.193Z" + }, + "publish-model.ac3": { + "status": "completed", + "doneAt": "2026-11-15T20:07:41.193Z" + }, + "publish-model.ac4": { + "status": "completed", + "doneAt": "2026-11-15T20:07:41.193Z" + }, + "publish-model.ac5": { + "status": "completed", + "doneAt": "2026-11-15T20:07:41.193Z" + }, + "site-pages": { + "status": "completed", + "claimedBy": "agent-okf-11", + "claimedAt": "2026-11-15T20:07:42.900Z", + "doneAt": "2026-11-15T20:09:58.225Z" + }, + "site-pages.ac1": { + "status": "completed", + "doneAt": "2026-11-15T20:09:58.225Z" + }, + "site-pages.ac2": { + "status": "completed", + "doneAt": "2026-11-15T20:09:58.225Z" + }, + "site-pages.ac3": { + "status": "completed", + "doneAt": "2026-11-15T20:09:58.225Z" + }, + "site-pages.ac4": { + "status": "completed", + "doneAt": "2026-11-15T20:09:58.225Z" + }, + "site-pages.ac5": { + "status": "completed", + "doneAt": "2026-11-15T20:09:58.225Z" + }, + "search-index": { + "status": "completed", + "claimedBy": "agent-okf-11", + "claimedAt": "2026-11-15T20:09:58.980Z", + "doneAt": "2026-11-15T20:12:00.053Z" + }, + "search-index.ac1": { + "status": "completed", + "doneAt": "2026-11-15T20:12:00.053Z" + }, + "search-index.ac2": { + "status": "completed", + "doneAt": "2026-11-15T20:12:00.053Z" + }, + "search-index.ac3": { + "status": "completed", + "doneAt": "2026-11-15T20:12:00.053Z" + }, + "search-index.ac4": { + "status": "completed", + "doneAt": "2026-11-15T20:12:00.053Z" + }, + "search-index.ac5": { + "status": "completed", + "doneAt": "2026-11-15T20:12:00.053Z" + }, + "publish-command": { + "status": "completed", + "claimedBy": "agent-okf-11", + "claimedAt": "2026-11-15T20:12:00.757Z", + "doneAt": "2026-11-15T20:13:37.867Z" + }, + "publish-command.ac1": { + "status": "completed", + "doneAt": "2026-11-15T20:13:37.867Z" + }, + "publish-command.ac2": { + "status": "completed", + "doneAt": "2026-11-15T20:13:37.867Z" + }, + "publish-command.ac3": { + "status": "completed", + "doneAt": "2026-11-15T20:13:37.867Z" + }, + "publish-command.ac4": { + "status": "completed", + "doneAt": "2026-11-15T20:13:37.867Z" + }, + "publish-command.ac5": { + "status": "completed", + "doneAt": "2026-11-15T20:13:37.867Z" + }, + "publish-check-mode": { + "status": "completed", + "claimedBy": "agent-okf-11", + "claimedAt": "2026-11-15T20:13:38.712Z", + "doneAt": "2026-11-15T20:14:39.468Z" + }, + "publish-check-mode.ac1": { + "status": "completed", + "doneAt": "2026-11-15T20:14:39.468Z" + }, + "publish-check-mode.ac2": { + "status": "completed", + "doneAt": "2026-11-15T20:14:39.468Z" + }, + "publish-check-mode.ac3": { + "status": "completed", + "doneAt": "2026-11-15T20:14:39.468Z" + }, + "publish-check-mode.ac4": { + "status": "completed", + "doneAt": "2026-11-15T20:14:39.468Z" + }, + "skill-command-row": { + "status": "completed", + "claimedBy": "agent-okf-11", + "claimedAt": "2026-11-15T20:15:07.647Z", + "doneAt": "2026-11-15T20:15:27.121Z" + }, + "skill-command-row.ac1": { + "status": "completed", + "doneAt": "2026-11-15T20:15:27.121Z" + }, + "skill-command-row.ac2": { + "status": "completed", + "doneAt": "2026-11-15T20:15:27.121Z" + }, + "publishing-docs": { + "status": "completed", + "claimedBy": "agent-okf-11", + "claimedAt": "2026-11-15T20:15:27.786Z", + "doneAt": "2026-11-15T20:17:16.161Z" + }, + "publishing-docs.ac1": { + "status": "completed", + "doneAt": "2026-11-15T20:17:16.161Z" + }, + "publishing-docs.ac2": { + "status": "completed", + "doneAt": "2026-11-15T20:17:16.161Z" + }, + "publishing-docs.ac3": { + "status": "completed", + "doneAt": "2026-11-15T20:17:16.161Z" + }, + "publishing-docs.ac4": { + "status": "completed", + "doneAt": "2026-11-15T20:17:16.161Z" + } + }, + "feedback": [ + { + "seq": 1, + "specialist": "verification-gate", + "outcome": "failed", + "timestamp": "2026-11-15T20:24:13Z", + "markdownBody": "VERIFICATION FAILED for OKF-11 (attempt 1/3):\n\nFailed check: typecheck\n\nVerification FAILED at typecheck (2270ms).\n\nFull gate output: /home/eltmon/Projects/okf/workspaces/feature-okf-11/.overdeck/verification/2026-11-15T20-24-10-922Z-94d64e89.json\n\n## REQUIRED: Fix the failing check, push, and request a new review\n\n1. Read the complete gate output at `/home/eltmon/Projects/okf/workspaces/feature-okf-11/.overdeck/verification/2026-11-15T20-24-10-922Z-94d64e89.json` carefully\n2. Fix the code causing the failure\n3. Run the failing check locally to verify it passes\n4. Commit every change\n5. Invoke the /rebase-and-submit skill for OKF-11 — this is an atomic task. Because verification already ran once (a PR exists), the skill will push your branch and run `pan review request OKF-11 -m \"Fixed typecheck\"` for you. NEVER curl `/api/review/...` or any dashboard endpoint — `pan review request` is the only supported re-entry point.\n\n`pan review request` returns as soon as the request is accepted. When it exits 0 you are done: end your turn and wait. Do not poll `pan show`, files, or terminals — Overdeck will message you when verification passes or fails and when reviewers request changes. If it exits non-zero, reconcile the failure and try again." + }, + { + "seq": 2, + "specialist": "review-agent", + "outcome": "changes-requested", + "timestamp": "2026-11-15T20:34:10Z", + "markdownBody": "# Review CHANGES REQUESTED for OKF-11\n\n# Review — OKF-11\n## Verdict: CHANGES REQUESTED — private concept titles and descriptions leak onto public index pages (FR-3)\n\n## Context\n- Run ID: agent-okf-11-review-2aa74173\n- Manifest: /home/eltmon/Projects/okf/workspaces/feature-okf-11/.pan/review/agent-okf-11-review-2aa74173/context.json\n- Branch / workspace: feature/okf-11 / /home/eltmon/Projects/okf/workspaces/feature-okf-11\n- PR: https://github.com/eltmon/okf/pull/12\n- HEAD reviewed: 2aa74173bb2314309c2d855ca33c3cb21cf5a47d\n- Base / prior cycle SHA: origin/main 92a21febed99895716eabc648f0ba88467fd01ab (PR base); no prior review cycle. The local `main` ref in this worktree points at an unrelated history (fe73d46), so every diff here is against `origin/main`.\n\n## Blocking Findings\n\n### [security/requirements] Index filtering misses listing lines that `reindex.py` generates, so private titles and descriptions get published — `scripts/publish_render.py:59`\n- Severity: ! (MUST)\n- Classification / requirement scope: in_pr_scope (new `kept_index_lines`). It breaks FR-3 (\"A private concept appears nowhere on the site … Its index listing line is dropped\"). It also contradicts the shipped docs: references/publish.md says \"What is never published: Private concepts, and their titles in index listings\", and the README says \"A private concept has no page, no index line\". Triggers 1 and 3 are an **implementation defect**: the filter's link detection is narrower than what reindex produces and the renderer prints. Trigger 2 is a **specification defect**. PRD §4.6 says \"Drop the line when that link is a concept that is not public … **Keep every other line unchanged.**\" The code follows that, so a multi-line description's continuation line survives. Spec correction: inside the `OKF:INDEX` markers, drop a dropped item's continuation lines, and keep only lines that positively resolve to a public concept or a published directory. All three triggers still block, because FR-3 fails either way.\n- Trigger and changed-code connection: `kept_index_lines` decides whether to drop a listing line by running `okf_common.LINK_PATTERN.search(line)`. That regex is `\\[[^\\]]+\\]\\(([^)\\s]+)…\\)`. When it finds no match, the line is kept unchanged. Then `render_markdown` renders the line with its own bracket-depth parser (`_find_closing_bracket` / `_parse_destination`), which does recognise the link. The resolver returns `None` for the private target, so the link text and the rest of the line render as plain text. `scripts/reindex.py` (unchanged, lines 28-35) writes listing lines as `* [{title}]({file}){ - description}` using the raw title and raw description. So the standard `/okf sync` → reindex flow produces lines that hit this gap in two ways:\n 1. **The title contains `]`** (for example `Layoffs [draft]`, `Outage [2026-11-01]`, or an unbalanced `a] b`). LINK_PATTERN cannot match `[Layoffs [draft]](secret-plan.md)`, so the line is kept. The page shows `Layoffs [draft] - `.\n 2. **The description has several lines** (YAML `description: |`). reindex writes the second line of the description as its own line, which does not start with `* `. Only the first line is classified and dropped. The continuation line is kept and renders as a paragraph on the public page.\n 3. **The filename contains a space** (`rules/my file.md`). reindex writes `* [Space file](my file.md) - …`. LINK_PATTERN's target class `[^)\\s]+` does not match it. The renderer's `_parse_destination` does not match it either, so the whole line, filename included, renders as literal text.\n- Expected / observed / impact: expected: the private concept is absent from `rules/index.html`. Observed: its title and description appear in the public HTML. `--check` exits 0, because no redaction rule matches. So nothing warns the author, and the private text ships on a public site. Preventing exactly this is the purpose of the feature.\n- Evidence: runtime reproduction against the fixture copy under the session scratchpad (probe4). I added two private concepts, `rules/secret-plan.md` (`title: Layoffs [draft]`, `description: ZZ-LEAK-DESC …`) and `rules/hr.md` (`description: |` with a second line `ZZ-LEAK-SECOND-LINE about Bob`). Then I ran the real `scripts/reindex.py --bundle b`, `publish.py --bundle b --check` and `publish.py --bundle b --out out`:\n ```\n reindex=0\n ok - publish check passed: 3 public concept(s), no redaction rule matches\n ok - published 3 public concept(s) to out (4 private excluded, 0 redaction(s) applied)\n $ grep -o \"ZZ-LEAK[^<]*\\|Layoffs[^<]*\" out/rules/index.html\n ZZ-LEAK-SECOND-LINE about Bob\n Layoffs [draft] - ZZ-LEAK-DESC fire the night crew\n ```\n Probe 5 (scratchpad probe5) adds `rules/my file.md` (private, `description: ZZ-LEAK-SPACE`) and `rules/unbal.md` (private, `title: \"a] ZZ-LEAK-UNBAL\"`). It also runs reindex, `--check` (exit 0) and `--out`:\n ```\n [Space file](my file.md) - ZZ-LEAK-SPACE\n [a] ZZ-LEAK-UNBAL](unbal.md) - ZZ-LEAK-UNBAL-DESC\n ```\n `validate.py` does not restrict titles, descriptions or filenames, so all these inputs are valid OKF.\n- Fix (must fail closed). Probe 5 shows that switching to the renderer's own parser is **not enough**: `_parse_destination` also fails on the space and unbalanced-bracket lines, so they would still be kept as literal text. Pick one of these:\n - **(a)** Inside the ``…`END` block, keep a line only when it positively resolves to a public concept or a published directory. Drop every other line in the block, continuation lines included.\n - **(b)** Ignore the marker block's contents and regenerate it from the model, reusing `generated_listing`. Keep the hand-written prose outside the markers.\n\n Outside the markers, the current behaviour (link text stays) is documented and acceptable. Add regression tests for a bracketed title, an unbalanced `]` title, a filename with a space and a multi-line description, all on private concepts. Each test asserts that no private title, description or filename reaches any output file.\n\n## Non-blocking Findings\n\n### [security] `search_text` leaves private link targets in `search-index.json` when the link text contains brackets — `scripts/publish_search.py:15`\n- Severity: ~ (SHOULD)\n- Classification: implementation defect, in_pr_scope. It has the same root cause as the blocker: a regex link matcher that disagrees with the renderer.\n- Trigger: a public body contains `[notes [v2]](team-notes.md)`, where `team-notes` is private. `LINK = \\[([^\\]]+)\\]\\([^)]*\\)` does not match, so the raw `[notes [v2]](team-notes.md)` text, target included, lands in the record's `text` field. Probe1 output: `\"text\":\"Proposed in chat. See [notes [v2]](team-notes.md). …\"`. Only the private concept's ID/path leaks, not its content, and only when the author writes nested brackets. FR-3 and §4.7 (\"Link targets never reach the index\") still say it should not happen.\n- Fix: strip links with the same renderer-parser helper suggested above. Add a test with a nested-bracket link to a private concept.\n\n### [security] Redaction runs on raw Markdown, so Markdown syntax can split a match the reader still sees — `scripts/publish.py:88`\n- Severity: ~ (SHOULD; behaviour confirmed, but it needs unusual source text)\n- Classification: specification defect. The PRD's FR-8 design (\"Every rule applies to all published text before rendering\") is followed as written.\n- The redaction rules match the source text before rendering. Probe 5 appended `Esc: /home\\/alice and 10.0.0\\.12 and **Alice** Example` to a public concept. `--check` exited 0 and the redaction count was 0. The page rendered `Esc: /home/alice and 10.0.0.12 and Alice Example`, which a reader sees as the home path, the IP and the private name. Backslash escapes are unlikely by accident, but a bold name is plausible. A later spec revision could also scan (and redact) the rendered plain text.\n\n### [performance] Backlinks and published-directory sets are recomputed per page, which makes publish time quadratic — `scripts/publish_model.py:87`, `scripts/publish_render.py:43`\n- Severity: ~ (SHOULD; a CLI that runs once per publish, not server code)\n- For each page, `backlinks()` re-sorts `public_ids()` and runs `classify_link` over every link of every public concept. `make_resolver`/`kept_index_lines` also rebuild `public_dirs()` each time. Measured end to end on a synthetic bundle with 1,501 public concepts and 10 links each (Python 3.12, this machine): `real 0m29.9s`. Attributing that time to these functions is a static inference; I did not profile it. A bundle of a few hundred concepts takes a few seconds. To make it linear, compute a reverse-link map and `public_dirs` once per model, for example with `functools.cached_property` or a precomputed dict in `build_model`/`render_site`.\n\n### [docs] The private-strings behaviour goes beyond the PRD wording\n- Commit a4de29d changed `private_strings` from `re.escape(s)` to whitespace-run matching (`\\s+` between words). This is stricter, which suits redaction, and README/references document it. The template comment still says \"matched literally and case-insensitively\". That is acceptable, but it could mention the line-break tolerance. No action is required.\n\n## Coverage\n\n### Changed-file ledger (37 files, diff vs origin/main 92a21fe)\n| File | Status |\n| --- | --- |\n| .github/workflows/ci.yml | Reviewed: adds pytest install and a `python -m pytest -q tests` step after the selftest (matches WI-1). |\n| scripts/publish.py | Reviewed: flow, finding ordering, `--out` safety, marker rebuild, redaction copies, check-mode line math. |\n| scripts/publish_model.py | Reviewed: walk, visibility fail-closed, trust tier, classify_link rules 1-10, body_start_line, index frontmatter offsets. |\n| scripts/publish_redact.py | Reviewed: profile validation, rule order, literal replacement, scan never returns matched text. |\n| scripts/publish_markdown.py | Reviewed: escaping (all text runs and attributes escaped, `\\x00` stripped before placeholder use), `safe_href`, fences, lists, tables, footnotes, autolinks. No XSS path found. |\n| scripts/publish_render.py | Reviewed: blocker in `kept_index_lines`; page shell, relative hrefs, generated listings, backlinks are otherwise correct. |\n| scripts/publish_search.py | Reviewed: advisory on `search_text`. |\n| scripts/publish_assets/search.js | Reviewed: uses only `textContent`/`setAttribute`/`createElement`; no `innerHTML`, `eval` or external URL; scoring matches §4.7. |\n| scripts/publish_assets/style.css | Grep-checked only: no `@import`, `url(` or `http`. It has a system font stack, `max-width: 48rem` and a `prefers-color-scheme: dark` block. I did not read individual rules. |\n| scripts/selftest.sh | Reviewed: only the command loop gains `publish` (NFR-3). |\n| templates/okf-publish.yaml | Reviewed: id list matches DEFAULT_RULES; loads cleanly (covered by the test). |\n| README.md, SKILL.md, docs/USAGE.md, references/publish.md | Reviewed: command rows identical in both tables; sections present; the \"never published … titles in index listings\" claim is currently false (blocker). |\n| tests/conftest.py, tests/test_harness.py | Read in full. |\n| tests/test_publish_*.py (6) | Partly read. I checked that every function name the PRD lists is present, plus extras (75 tests pass in CI). I read the bodies of `test_no_unresolved_links`, `test_no_private_or_secret_bytes`, `test_out_safety` and `test_check_index_line_after_frontmatter_and_dropped_lines`; they assert what the PRD requires. For the other tests I rely on the CI pass. No test covers bracketed or unbalanced titles, filenames with spaces, multi-line descriptions or nested-bracket links. |\n| tests/fixtures/publish_bundle/** (11 files) | Read in full (except `rules/index.md`, read through the probe copy). They match the PRD fixture table. |\n| .pan/drafts/OKF-11.md, .pan/specs/…xbrief.json, .pan/continues/OKF-11.xbrief.json | Not applicable for code review: planning artifacts. I read the PRD for requirements. |\n\nNFR-3 check: `git diff origin/main...HEAD --stat` touches no existing script except `scripts/selftest.sh`; okf_common.py, validate.py, reindex.py and the template scripts are unchanged.\n\n### Dimensions\n- Correctness: one blocker (index filtering), one advisory (search text). Visibility resolution, link classification, line numbers, `--out` safety and determinism are correct as traced.\n- Security: the renderer's HTML escaping and href scheme checks hold. The private-content leak is the blocker. The redaction pre-render gap is a hypothesis.\n- Performance: quadratic backlinks (advisory, measured).\n- Requirements/UX: see the matrix below.\n\n### Acceptance criteria\n| AC | Status | Evidence |\n| --- | --- | --- |\n| AC-1 pytest in CI | Met | ci.yml step; CI log `75 passed in 2.58s` (run 43757998023). |\n| AC-2 Markdown renderer | Met | test_publish_markdown passes in CI; escaping traced manually. |\n| AC-3 redaction rules | Met | test_publish_redact passes; `scan` returns ids and lines only. |\n| AC-4 publish model | Met | test_publish_model passes. |\n| AC-5 site pages: \"private listing lines are dropped\" | **Unmet** for reindex-generated lines with bracketed titles or multi-line descriptions (blocker). Met for the fixture's simple lines. |\n| AC-6 search | Met for the unit tests (`search.js` has no `innerHTML`). Browser UAT is the test role's job, not verified here. Advisory on nested-bracket targets. |\n| AC-7 `--out` | Met per the CI tests. `test_no_private_or_secret_bytes` passes only because the fixture has no bracketed titles. |\n| AC-8 `--check` gate | Met (test passes in CI). |\n| AC-9 selftest | Met: CI log `ok - SKILL.md documents /okf publish`. |\n| AC-10 docs | Met: `## Publishing a bundle` at README:102. It contains `x_visibility`, `okf-publish.yaml`, `cloudflare/wrangler-action@v3` and `publish.py --bundle . --check`. references/publish.md lists all seven P_* codes. |\n\n## Verification\n- `gh pr checks 12`: both `selftest` jobs pass on HEAD 2aa7417. The CI job runs `bash scripts/selftest.sh` and `python -m pytest -q tests` (75 passed). I reused this run and did not re-run the full suite.\n- Probe 1 (scratchpad probe1): hand-added bracketed private listing line plus nested-bracket body link. Private title and description appear in `out/rules/index.html`, and the private target appears in `search-index.json`.\n- Probe 4 (scratchpad probe4): the same leak using the real `reindex.py` output for a bracketed title and a multi-line description. `--check` exits 0 and `--out` publishes the private text.\n- Probe 5 (scratchpad probe5): a filename with a space and an unbalanced `]` title, both private, leak through reindex → publish. The escape and emphasis split defeats redaction and `--check`.\n- Probe 2: the `templates/repo` bundle passes `--check` (0 public concepts), so its README.md/CONTRIBUTING.md frontmatter causes no parse errors.\n- Probe 3: synthetic 1,501-concept bundle, publish took 29.9 s.\n- `npm run lint:effect-diagnostics`: not applicable (Python/stdlib repo with no TypeScript or Effect).\n- All probes ran in the session scratchpad. No tracked files were edited.\n\n## Scope Note\n- The manifest's `acceptanceCriteria` list was empty. I used PRD §3 (FR/NFR) and §7 (AC-1 to AC-10) from `.pan/drafts/OKF-11.md` as the acceptance criteria.\n- The local `main` ref in this worktree does not match the PR base (`origin/main` 92a21fe). The PR diff itself is correct.\n\nSource: /home/eltmon/Projects/okf/workspaces/feature-okf-11/.pan/review/agent-okf-11-review-2aa74173/review.md\n\n## Required action\n\nFix every blocking review finding, commit the fixes, then re-request review with:\n\n`pan review request OKF-11 -m \"Fixed review issues\"`" + } + ] +} diff --git a/.pan/drafts/OKF-11.md b/.pan/drafts/OKF-11.md new file mode 100644 index 0000000..26df13d --- /dev/null +++ b/.pan/drafts/OKF-11.md @@ -0,0 +1,642 @@ +# OKF-11: Publish a bundle as a public static site with visibility and redaction + +Issue: https://github.com/eltmon/okf/issues/11 · Planned 2026-11-15 against `main` at `92a21fe`. + +This PRD is the implementation brief. It names every file, function signature, output path, finding code and test. An executor must not re-research or redesign; where a choice exists, this document makes it. + +--- + +## 1. Glossary + +- **Bundle.** A directory tree of UTF-8 Markdown files in Open Knowledge Format (OKF). `references/spec.md` is the vendored spec. +- **Concept.** A `.md` file in the bundle other than the reserved `index.md` and `log.md`. It has YAML frontmatter between `---` lines and a Markdown body. +- **Concept ID.** The concept's bundle-relative path without `.md`, in POSIX form, for example `rules/no-spoilers`. `okf_common.concept_id_for_path` computes it. +- **Index file.** A reserved `index.md` in any bundle directory. `scripts/reindex.py` keeps a generated listing between `` and `` markers; prose outside the markers is hand-written. +- **Visibility.** Whether a concept may appear on the published site: `public` or `private`. +- **Public concept.** A concept whose resolved visibility is `public` (§4.2). Every other concept is private. +- **Published directory.** A bundle directory (root included, written `""`) that contains at least one public concept in itself or in any subdirectory. +- **Schema file.** `schema.md` at the bundle root. OKF v0.2 plans it as a `type: Policy` concept (`.pan/drafts/okf-v0.2-redesign.md` §4.2). For this issue it only carries the bundle visibility default. +- **Redaction profile.** The YAML file `okf-publish.yaml` at the bundle root (or `--profile `). It lists regex rules that rewrite published text. +- **Rule.** One redaction entry: an `id`, a Python `re` pattern and a replacement string. +- **Built-in rules.** The rules in `DEFAULT_RULES` in `scripts/publish_redact.py` (§4.5). They apply even with no profile file. +- **Published text.** Every string the site shows: concept title, description, type, status, tags, source entries, body; and the Markdown of published index pages. +- **Trust tier.** A label derived from OKF v0.2 `verified` frontmatter: `human-reviewed`, `machine-confirmed` or `unverified` (§4.4). +- **Site root.** The `--out` directory. Every site path in this PRD is relative to it. +- **Page-relative href.** An href computed from the linking page's own location, for example `../schema.html` from `rules/no-spoilers.html`. The site never uses `/`-rooted hrefs, so it works under a subpath (GitHub Pages project sites) and on any static host. +- **Output marker.** The file `.okf-publish` that `publish.py` writes at the site root. Its presence proves `publish.py` created the directory and may delete its contents. +- **Finding.** One error line printed by `publish.py`, shaped `ERROR : ` (§4.8). +- **Check mode.** `publish.py --check`: scan without writing; exit 2 if any rule would fire in published text. +- **`PYRUN`.** Shorthand used only in this PRD for `uv run --no-project --python 3.12 --with pyyaml --with pytest`. On the dev machine `python3` is 3.14 without PyYAML, so plain `python3` fails (verified 2026-11-15). Verify commands in the xBRIEF spell the full form. + +## 2. Problem and context + +A bundle can be viewed only locally today (`/okf open`, which delegates to `pan knowledge open`). Some projects want a public, read-only site. The motivating case is a live-stream project whose viewers propose the rules the AI follows; viewers want to see each rule, who suggested it, and what replaced it. + +Publishing the raw bundle is unsafe. Concepts can hold operator-only notes, machine paths, internal incident detail and names of private people. So the site needs per-concept visibility, a deterministic redaction pass, and a gate (`--check`) that fails CI when a rule would fire in a public concept. + +**Current code state (verified 2026-11-15, `main` = `92a21fe`):** + +- Scripts are separate CLIs under `scripts/` sharing `scripts/okf_common.py`. There is no `okf.py` CLI, no `tests/` directory and no pytest. The gate is `scripts/selftest.sh` (bash greps plus inline Python), run by `.github/workflows/ci.yml`. +- OKF-1 (one CLI and pytest), OKF-4 (`schema.md`), OKF-6 (trust fields) and OKF-8 (viewer) are all still open. The issue says publishing "fits after #6". This plan does not wait (decision D1). +- `okf_common.iter_concept_paths` (`scripts/okf_common.py`, line 174, `def iter_concept_paths(bundle_root: Path) -> Iterable[Path]:`) walks every `*.md`, including `.github/` and `.git/`. Publishing needs its own walker (§4.1). +- `okf_common.resolve_concept_link` (line 126) returns `None` for `index.md` and for directory links, so publishing needs its own link classifier (§4.3). +- `okf_common.LINK_PATTERN` (line 17) and `markdown_links` (line 109) skip fenced code. Publishing reuses `markdown_links` for backlinks. +- The house extension-key convention is an `x_` prefix (`x_embed: exclude`, `scripts/embed.py` line 187). The issue names `x_visibility`, so this plan uses `x_visibility` and `x_visibility_default`. + +## 3. Requirements + +### Functional + +- **FR-1.** `python3 scripts/publish.py --bundle --out ` writes one HTML page per public concept at `.html`, an `index.html` for every published directory (root always), `search.html`, `search-index.json`, `assets/style.css`, `assets/search.js` and the output marker `.okf-publish`. It writes nothing else. +- **FR-2.** Visibility resolves per concept from frontmatter `x_visibility`, falling back to the bundle default `x_visibility_default` in root `schema.md` frontmatter. A missing `schema.md` or a missing key means the default is `private`. Any value other than the exact strings `public` or `private` is finding `P_VISIBILITY_INVALID`, and the run exits 2 without writing. +- **FR-3.** A private concept appears nowhere on the site. It has no page. Its index listing line is dropped. It has no search record and appears in no backlink list. A body or source link to it renders as plain text (the link text stays; the issue requires this). +- **FR-4.** Links in published Markdown resolve this way: a link to a public concept becomes a page-relative href to its page, keeping any `#fragment`. A link to a published directory or its `index.md` becomes a page-relative href to that directory's `index.html`. `http:`, `https:` and `mailto:` links stay as written. A `#fragment`-only link stays. Every other target renders as plain text: private concepts, missing concepts, unpublished directories, non-Markdown files and any other scheme. A private target and a missing target render identically. +- **FR-5.** Each concept page shows the concept ID, title, type, status, trust tier, tags, description, rendered body, a "Sources" list and a "Linked from" list (backlinks from public concepts only). +- **FR-6.** The trust tier is `human-reviewed` when any `verified` entry has a `by` string that starts with `human:`. Otherwise it is `machine-confirmed` when there is at least one `verified` entry. Otherwise it is `unverified`. A bare mapping counts as a one-element list. Status is frontmatter `status` or `stable` when absent. +- **FR-7.** Search is client-side. `search-index.json` holds one record per public concept. `search.html` loads `assets/search.js`, which fetches the JSON and lists matches. No server code, no CDN and no third-party script. +- **FR-8.** The redaction profile `okf-publish.yaml` (schema in §4.5) adds regex rules and literal `private_strings`, and can disable built-in rules by id. Built-in rules apply with no profile file. Every rule applies to all published text before rendering, including fenced code. +- **FR-9.** `publish.py --check` scans the published text of public concepts and of published index pages. Each rule that matches gives one `P_REDACTION` finding with the rule id and the location (file and line for bodies and index pages, field name for frontmatter). It never prints the matched text. It exits 2 on any finding and 0 otherwise. It writes no files. +- **FR-10.** A rule that matches a public concept ID or a published directory path gives finding `P_ID_REDACTION` in both modes, and the run exits 2 without writing, because URLs cannot be redacted. +- **FR-11.** `--out` safety. The run refuses (finding `P_OUT_UNSAFE`, exit 2) when `--out` resolves to the bundle root or an ancestor of it, is an existing non-directory, or is a non-empty directory without the output marker. When the marker is present, the run deletes every entry in `--out` and rebuilds, so a page from a concept that has turned private never survives. +- **FR-12.** Bundle walking skips any path with a component that starts with `.` (for example `.git`, `.github`, `.okf`, `.okf-index`) or equals `node_modules`, and skips `--out` when it lies inside the bundle. A concept whose frontmatter does not parse gives finding `P_FRONTMATTER_PARSE`, and the run exits 2 without writing. `log.md` is never published. Non-Markdown files are never copied. +- **FR-13.** Documentation: SKILL.md and README command-table rows and a SKILL.md section for `/okf publish`; a README "Publishing a bundle" section covering visibility, the redaction profile and a sample Cloudflare Pages setup; `references/publish.md`; a `docs/USAGE.md` section. +- **FR-14.** A pytest harness under `tests/` runs in CI next to `scripts/selftest.sh`. + +### Non-functional + +- **NFR-1.** Standard library plus PyYAML only. Python 3.10+ syntax (no `tomllib`, no `typing.Self`, no `ExceptionGroup`). Every new module starts with `#!/usr/bin/env python3`, a one-line docstring and `from __future__ import annotations`. +- **NFR-2.** Deterministic output: two runs on the same bundle produce byte-identical files. No timestamps, sorted iteration everywhere, `json.dumps(..., sort_keys=True, ensure_ascii=False, separators=(",", ":"))`. +- **NFR-3.** No existing script changes except the `scripts/selftest.sh` command loop (§6, WI-9). `okf_common.py`, `validate.py`, `reindex.py` and the template copies under `templates/repo/.okf/scripts/` stay byte-identical. No Overdeck imports (selftest greps for them). +- **NFR-4.** HTML safety. All text from the bundle is HTML-escaped. Raw HTML in Markdown renders as escaped text. No `on*=` attributes, no `style=` attributes, no `javascript:` or `data:` hrefs. The only `` on `search.html`. `search.js` writes results with `textContent` and `setAttribute("href", …)`, never `innerHTML`. +- **NFR-5.** Findings never echo matched secret text, because CI logs of public repos are public. + +## 4. Design (decisions made) + +### 4.1 Module layout + +All new code is flat in `scripts/`, matching the existing `from okf_common import …` style. Files: + +| File | Role | +| --- | --- | +| `scripts/publish_markdown.py` | Stdlib Markdown-subset → escaped HTML renderer. | +| `scripts/publish_redact.py` | Built-in rules, profile loading, `redact`, `scan`. | +| `scripts/publish_model.py` | Bundle walk, visibility, trust tier, sources, link classification, backlinks. | +| `scripts/publish_render.py` | Page shell, concept pages, index pages, index filtering, `assets/style.css`. | +| `scripts/publish_search.py` | `search-index.json`, `search.html`, `assets/search.js`. | +| `scripts/publish.py` | CLI: argparse, profile, `--out` safety, redaction application, `--check`, exit codes. | +| `scripts/publish_assets/style.css` | Site stylesheet, copied verbatim to `assets/style.css`. | +| `scripts/publish_assets/search.js` | Search script, copied verbatim to `assets/search.js`. | +| `templates/okf-publish.yaml` | Starter profile users copy to their bundle root. | + +Modules locate assets with `Path(__file__).resolve().parent / "publish_assets"`. + +### 4.2 Visibility (`publish_model.py`) + +```python +VISIBILITY_KEY = "x_visibility" +DEFAULT_KEY = "x_visibility_default" +SCHEMA_FILE = "schema.md" +VISIBILITY_VALUES = ("public", "private") +``` + +1. Bundle default: if `/schema.md` does not exist → `private`, reason `"no schema.md at bundle root"`. If it parses and lacks `x_visibility_default` → `private`, reason `"schema.md has no x_visibility_default"`. If the value is `public` or `private` → that value, reason `"schema.md x_visibility_default: "`. Any other value → finding `P_VISIBILITY_INVALID schema.md: x_visibility_default must be public or private`, default `private`. +2. Per concept: `x_visibility` absent → bundle default. `public`/`private` → that value. Anything else (including `Public`, `true`, `yes`) → finding `P_VISIBILITY_INVALID : x_visibility must be public or private`, concept treated as private. +3. `schema.md` is an ordinary concept for publishing: its own `x_visibility` or the default decides whether it gets a page. + +Fail closed: no input ever resolves to public by accident. When zero concepts are public, `publish.py` prints the default reason (§4.8). + +### 4.3 Link classification (`publish_model.py`) + +```python +@dataclass(frozen=True) +class LinkTarget: + kind: str # "concept" | "directory" | "external" | "fragment" | "other" + value: str # concept ID, directory path ("" = root), or the raw href + fragment: str # text after "#", without the "#"; "" when none + +def classify_link(source_dir: str, target: str) -> LinkTarget: ... +``` + +`source_dir` is the bundle-relative POSIX directory of the page holding the link (`""` for the root). Rules, in order: + +1. Split `target` on the first `#` into `path` and `fragment`. +2. `path == ""` → `fragment` kind, value `"#" + fragment`. +3. `path` lowercased starts with `http://`, `https://` or `mailto:` → `external`, value = `target` unchanged. +4. `path` contains `:` → `other` (any other scheme, including `javascript:`, `data:`, `tel:`, `repo:`). +5. Resolve the path: a leading `/` makes it bundle-root relative; otherwise join it to `source_dir`. Normalise `.` and `..`. A `..` that would climb above the root → `other`. +6. Normalised path ends with `/`, or is empty after normalising → `directory`, value = path without trailing `/`. +7. Last segment is `index.md` → `directory`, value = its parent (`""` for root). +8. Last segment is `log.md` → `other`. +9. Ends with `.md` → `concept`, value = path without `.md`. +10. Anything else → `other`. + +Whether a `concept` or `directory` target is *published* is decided by the caller (`model.is_public(id)`, `dir in model.public_dirs`). + +### 4.4 Model types (`publish_model.py`) + +```python +@dataclass(frozen=True) +class Source: + title: str # "" when absent + resource: str # required; entries without it are skipped + author: str # "" when absent + +@dataclass(frozen=True) +class PublishConcept: + concept_id: str + path: str # bundle-relative POSIX path, e.g. "rules/no-spoilers.md" + directory: str # "" for root + title: str # frontmatter title, else last ID segment + description: str + type: str + tags: tuple[str, ...] + status: str # default "stable" + trust_tier: str + sources: tuple[Source, ...] + body: str + body_start_line: int # 1-based file line of the first body line + visibility: str # "public" | "private" + links: tuple[str, ...] # raw link targets from markdown_links(body), in order + +@dataclass(frozen=True) +class Finding: + code: str + location: str # "rules/a.md", "rules/a.md:14", "index.md:5" + message: str + def line(self) -> str: + return f"ERROR {self.code} {self.location}: {self.message}" + +@dataclass +class PublishModel: + bundle_root: Path + default_visibility: str + default_reason: str + concepts: dict[str, PublishConcept] # every parsed concept, keyed by ID + index_files: dict[str, str] # directory -> index.md text with any frontmatter removed + def public_ids(self) -> list[str]: ... # sorted + def is_public(self, concept_id: str) -> bool: ... + def public_dirs(self) -> set[str]: ... # "" always included + def backlinks(self, concept_id: str) -> list[str]: ... # sorted public IDs that link to it + +def walk_markdown(bundle_root: Path, exclude: Path | None = None) -> list[Path]: ... +def trust_tier(frontmatter: dict) -> str: ... +def build_model(bundle_root: Path, exclude: Path | None = None) -> tuple[PublishModel, list[Finding]]: ... +``` + +Normalisation rules: + +- `tags`: a list → `str()` of each item; a scalar → one-element tuple; absent → `()`. +- `sources`: a list; each item is either a mapping (use `resource`, `title`, `author`, all `str()`; skip an item without `resource`) or a string (becomes `Source("", s, "")`). A non-list value is ignored. +- `verified` for `trust_tier`: `None`/absent → `[]`; a mapping → `[mapping]`; a list → mapping items only. +- `body_start_line`: index (1-based) of the closing `---` line plus 1; plus 1 more when the following line is empty (because `okf_common.split_frontmatter` drops one leading blank line). +- `backlinks(x)`: IDs `s` with `is_public(s)`, `s != x`, and any `classify_link(s.directory, link)` with kind `concept` and value `x`. Sorted, unique. +- `public_dirs()`: for every public concept, its directory and every ancestor, plus `""`. +- `index_files`: for each `index.md` the walker finds, keyed by its directory. Strip frontmatter when the text starts with `---\n` (the root may carry `okf_version`). +- A concept that fails `okf_common.parse_concept` gives `P_FRONTMATTER_PARSE : ` and is left out of `concepts`. +- `walk_markdown` returns sorted `*.md` paths. It skips any path whose bundle-relative parts include a component starting with `.` or equal to `node_modules`, and skips paths under `exclude` when `exclude` is inside the bundle. It does not skip `log.md`; `build_model` ignores `log.md` itself. + +### 4.5 Redaction (`publish_redact.py`) + +Profile schema (`okf-publish.yaml`): + +```yaml +okf_publish_version: "0.1" # required, must be the string "0.1" +disable_default_rules: [] # optional; ids from DEFAULT_RULES +rules: # optional; applied after built-in rules + - id: stream-box-host # required, unique, matches ^[a-z0-9][a-z0-9-]*$ + pattern: '\bstream-box\.lan\b' # required, Python re syntax; inline flags such as (?i) allowed + replace: '[internal-host]' # optional, default "[redacted]" +private_strings: # optional; literal strings, matched case-insensitively + - Alice Example +``` + +Built-in rules (exact patterns, in this order): + +```python +DEFAULT_RULES: tuple[tuple[str, str, str], ...] = ( + ("home-path-unix", r"/(?:home|Users)/[^/\s'\"`)\]>]+", "~"), + ("home-path-windows", r"[A-Za-z]:\\Users\\[^\\\s'\"`)\]>]+", "~"), + ("ipv4", r"\b(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}\b", "[ip]"), + ("github-token", r"\bgh[pousr]_[A-Za-z0-9]{36,}\b", "[token]"), + ("openai-style-key", r"\bsk-[A-Za-z0-9_-]{20,}", "[token]"), + ("aws-access-key", r"\bAKIA[0-9A-Z]{16}\b", "[token]"), + ("slack-token", r"\bxox[abprs]-[A-Za-z0-9-]{10,}", "[token]"), + ("private-key-block", r"-----BEGIN [A-Z ]*PRIVATE KEY-----", "[private-key]"), +) +``` + +`private_strings` entry *n* (1-based) becomes rule `private-string-` with pattern `re.escape(s)`, flag `re.IGNORECASE`, replacement `[redacted]`. Order of application: enabled built-in rules, then `rules`, then `private_strings`. + +```python +@dataclass(frozen=True) +class Rule: + id: str + pattern: re.Pattern[str] + replace: str + origin: str # "default" | "profile" | "private_strings" + +@dataclass(frozen=True) +class Profile: + rules: tuple[Rule, ...] + path: Path | None # None when built-in rules only + +def default_profile() -> Profile: ... +def load_profile(bundle_root: Path, profile_path: Path | None = None) -> tuple[Profile | None, list[str]]: ... +def redact(text: str, profile: Profile) -> tuple[str, dict[str, int]]: ... +def scan(text: str, profile: Profile) -> list[tuple[str, int]]: ... +``` + +- `load_profile`: with `profile_path` given, that file must exist. Without it, use `/okf-publish.yaml` when it exists, else `default_profile()`. Return `(None, errors)` on any error: unreadable YAML, top level not a mapping, unknown top-level key, `okf_publish_version` not `"0.1"`, unknown id in `disable_default_rules`, rule without `id` or `pattern`, invalid id, duplicate id (including a clash with a built-in id), a pattern that fails `re.compile` (message names the rule id and the `re.error`), an empty private string, or a non-string value. Each error string is one sentence naming the field. +- `redact`: apply each rule with `pattern.sub(replace, text)` in order. Return the new text and `{rule_id: substitutions}` for rules that fired. `replace` is passed as a literal (escape backslashes with `replace.replace("\\", "\\\\")` before `sub`, so `\1` is not a group reference). +- `scan`: for each rule, each match in the *original* text gives `(rule.id, line)` where `line = text.count("\n", 0, match.start()) + 1`. Return sorted unique pairs. +- `templates/okf-publish.yaml` contains `okf_publish_version: "0.1"`, `disable_default_rules: []`, commented examples of one rule and one private string, and a comment listing every built-in rule id. A test asserts the commented id list equals `[r[0] for r in DEFAULT_RULES]`. + +False positives: `ipv4` also matches `127.0.0.1` and four-part version numbers. A bundle that hits this either rewrites the text or adds `ipv4` to `disable_default_rules`. This is documented, and there is no per-match allow list (non-goal). + +### 4.6 Rendering (`publish_markdown.py`, `publish_render.py`) + +```python +LinkResolver = Callable[[str], Optional[str]] +def render_markdown(text: str, resolve_link: LinkResolver) -> str: ... +``` + +`resolve_link(raw_target)` returns an href or `None`; `None` renders only the link text. The renderer re-checks every href it emits: an href whose part before the first `/`, `?` or `#` contains `:` must start with `http:`, `https:` or `mailto:` (case-insensitive), otherwise the link renders as text. + +Supported Markdown (everything else is escaped text inside a paragraph): + +| Construct | Output | +| --- | --- | +| ATX heading `#`–`######` | `…`; slug = lowercase, runs of non-`[a-z0-9]` → `-`, trimmed; repeats get `-2`, `-3` | +| Paragraph (consecutive non-blank lines) | `

…

`, lines joined with `\n` | +| Fenced code ```` ``` ```` or `~~~` with optional info word | `
escaped
`; class only when the word matches `^[A-Za-z0-9_+-]+$` | +| Blockquote `> ` | `
` + recursive render | +| Unordered list `-`, `*`, `+` and ordered `1.` | `
    `/`
      ` with `
    1. `; an item line indented by 2+ more spaces than its parent starts a nested list | +| Thematic break `---`, `***`, `___` alone | `
      ` | +| GFM pipe table (header row, `\|---\|` separator row, body rows) | `……
      ` | +| Footnote definition `[^label]: text` (line start) | collected; emitted at the end as `
      1. …
      ` | +| Inline code `` `x` `` | `escaped` (no further inline parsing inside) | +| Link `[text](target)` and `[text](target "title")` | `text` or plain text per resolver | +| Image `![alt](src)` | escaped alt text only (no ``; assets are never copied) | +| Autolink `` / `` | `` (same scheme check) | +| Footnote reference `[^label]` with a definition | `N`; N = order of first reference; an undefined reference stays as escaped literal text | +| `**x**` / `__x__`, `*x*` / `_x_` | ``, `` | + +Escaping uses `html.escape(s, quote=True)` on every text run and attribute value. Raw HTML such as `` in a body renders as `<script>alert(1)</script>`. + +`publish_render.py`: + +```python +def concept_page_path(concept_id: str) -> str: ... # "rules/no-spoilers.html" +def dir_index_path(directory: str) -> str: ... # "index.html" or "rules/index.html" +def relative_href(from_page: str, to_page: str) -> str: ... # page-relative, URL-quoted segments +def make_resolver(model: PublishModel, source_dir: str, page_path: str) -> LinkResolver: ... +def filter_index_markdown(text: str, directory: str, model: PublishModel) -> str: ... +def page_shell(*, title: str, site_title: str, page_path: str, body_html: str, scripts: tuple[str, ...] = ()) -> str: ... +def site_title(index_texts: dict[str, str]) -> str: ... +def render_site(model: PublishModel, index_texts: dict[str, str], out_dir: Path) -> list[str]: ... +``` + +- `relative_href("rules/no-spoilers.html", "schema.html") == "../schema.html"`; `relative_href("index.html", "rules/a.html") == "rules/a.html"`; same directory → bare file name. Segments are quoted with `urllib.parse.quote(seg, safe="")`. +- `make_resolver` applies FR-4 using `classify_link`: public `concept` → `relative_href(page_path, concept_page_path(id))` plus `#fragment` when present; published `directory` → `relative_href(page_path, dir_index_path(dir))`; `external` and `fragment` → value; all else → `None`. +- `filter_index_markdown`: for each line that starts (after optional indentation) with `* ` or `- ` and contains a Markdown link, classify the **first** link. Drop the line when that link is a `concept` that is not public, or a `directory` not in `public_dirs()`. Keep every other line unchanged. Process every line, including any inside fenced code, because index listings are never fenced. +- `site_title`: the text of the first line starting with `# ` in `index_texts[""]`, stripped; otherwise `"Knowledge"`. +- `page_shell` emits exactly this structure (`rel` = `"../" * page_path.count("/")`): + +```html + + + + + +{title} · {site_title} + + + +
      {site_title} + +
      +
      +{body_html} +
      +
      Published from an Open Knowledge Format bundle.
      +{one line per scripts entry} + + +``` + +- Concept page body (`title`, `site_title` and all text escaped): + +```html +
      +

      {concept_id}

      +

      {title}

      +

      {type} {status} {trust_tier}

      +

      {description}

      +
      • {tag}
      • …
      +
      {render_markdown(body, resolver)}
      +

      Sources

        …
      + +
      +``` + + `status_class` = status lowercased when it matches `^[a-z0-9-]+$`, else `other`. A source `
    2. ` shows the title (or the resource when the title is empty), then ` — {author}` when present. The source resource is linked only when `make_resolver` returns an href for it; a `repo:path@sha` resource therefore renders as text. A backlink `
    3. ` is `{title}`. +- `render_site` writes, for each public concept, `concept_page_path(id)`. For each published directory, it writes `dir_index_path(dir)`: when `dir` is in `index_texts`, it renders that text with `make_resolver(model, dir, page)`; otherwise it renders a generated listing `# {last dir segment}` followed by `* [title](file.md) - description` lines for public concepts directly in `dir` and `* [child/](child/)` lines for published child directories. It copies `publish_assets/style.css` to `assets/style.css`. It returns the sorted list of written paths. Page titles: the concept title; for index pages, the first `# ` heading of the index text, else the directory name, else the site title for the root. + +### 4.7 Search (`publish_search.py`, `publish_assets/search.js`) + +```python +def search_text(body: str) -> str: ... +def search_records(model: PublishModel) -> list[dict[str, str]]: ... +def write_search(model: PublishModel, out_dir: Path, site_title: str) -> None: ... +``` + +- `search_text`: remove fenced-code fence lines (keep their content), replace `![alt](x)` with `alt` and `[text](x)` with `text`, drop footnote markers `[^x]`, collapse whitespace runs to one space, strip. Link targets never reach the index. +- One record per public concept, sorted by `id`: `{"id", "title", "description", "type", "url": concept_page_path(id), "text": search_text(body)}`. +- `write_search` writes `search-index.json` (NFR-2 dump settings), `search.html` (via `page_shell(scripts=("assets/search.js",))`, body `

      Search

        `), and copies `publish_assets/search.js` to `assets/search.js`. +- `search.js` behaviour: read `q` from `location.search`; put it in the header input; if empty, set status "Type a search term."; else `fetch("search-index.json")`, split `q` lowercase on whitespace into terms, keep records where every term occurs in `title + " " + description + " " + text` lowercased, score 3 per term in title, 2 in description, 1 in text, sort by score descending then `id`, show at most 50 as `
      1. title description
      2. ` built with `document.createElement`, `textContent` and `setAttribute`. Status text: `"N result(s)"` or `"No results."`. On fetch failure, status: `"Search needs the site to be served over HTTP."`. No `innerHTML`, no `eval`, no external URLs. + +### 4.8 CLI (`publish.py`) + +``` +python3 scripts/publish.py [--bundle B] (--out DIR | --check) [--profile FILE] +``` + +`--out` and `--check` are a required mutually exclusive argparse group. Flow: + +1. `bundle = Path(args.bundle).resolve()`. Missing directory → `ERROR P_BUNDLE_MISSING : bundle directory not found`, exit 2. +2. `load_profile`; errors → one `ERROR P_PROFILE_INVALID : ` line each, exit 2. +3. `out = Path(args.out).resolve()` when given. `build_model(bundle, exclude=out)`. +4. ID check: for each public ID and each published directory path (non-empty), `scan(value, profile)` hits → `P_ID_REDACTION : rule matches the concept ID; rename the concept or make it private` (directories: `... matches the directory path ...`). +5. Index texts: `filtered = {d: filter_index_markdown(t, d, model) for d, t in model.index_files.items() if d in model.public_dirs()}`. +6. `--check`: for each public concept, scan `title`, `description`, `type`, `status`, each tag, each source's `title`/`resource`/`author` (location ``, message `rule matches published text (field )`), and `body` (location `:`, message `rule matches published text (field body)`). For each filtered index text: location `/index.md:` (root `index.md:`), field `index`. Line numbers in index texts are relative to the stripped text; when the root index had frontmatter, add the stripped line count so numbers match the file. +7. Print every finding sorted by `(location-path, line, code, message)`. Any finding → exit 2. +8. `--check` with no findings: print `ok - publish check passed: public concept(s), no redaction rule matches`, exit 0. +9. `--out`: prepare the directory (FR-11): refuse cases → `P_OUT_UNSAFE : `, exit 2. Otherwise create it, delete all entries if the marker is present, write `.okf-publish` with the text `Generated by OKF publish.py. Every run deletes and rebuilds this directory.\n`. +10. Build redacted copies: every published string field of every public concept via `dataclasses.replace`, and every `filtered` index text, through `redact`; sum the counts. +11. `render_site(redacted_model, redacted_index_texts, out)`, then `write_search(redacted_model, out, site_title(redacted_index_texts))`. +12. Print `ok - published public concept(s) to (

        private excluded, redaction(s) applied)`. When N is 0, also print `note - no public concepts: default visibility is private (); set x_visibility: public on concepts or x_visibility_default: public in schema.md`. Exit 0. + +Finding codes (all `ERROR`, all exit 2): `P_BUNDLE_MISSING`, `P_PROFILE_INVALID`, `P_FRONTMATTER_PARSE`, `P_VISIBILITY_INVALID`, `P_ID_REDACTION`, `P_REDACTION` (check mode only), `P_OUT_UNSAFE`. When any finding exists in `--out` mode, nothing is written and the existing `--out` contents stay untouched. + +Redaction runs over the model's text, including link targets in bodies. If a rule rewrites a link target, that link may become plain text. This is safe (it only removes links) and documented. + +## 5. Repo rules restated for the executor + +- Stdlib + PyYAML only; Python 3.10+ (NFR-1). Do not add dependencies or a `pyproject.toml`. +- Do not import or call Overdeck or `pan` from any script (selftest greps `scripts/`, `.py`, `.js`, `.sh` for it). +- Do not edit `okf_common.py`, `validate.py`, `reindex.py`, `diff_lint.py`, `embed.py`, `build_index.py`, `search.py` or `templates/repo/.okf/scripts/*` (NFR-3; OKF-1 will restructure them). +- Run tests with `uv run --no-project --python 3.12 --with pyyaml --with pytest python -m pytest …`. Plain `python3` on the dev machine is 3.14 without PyYAML. +- Stage files by explicit path. The workspace holds untracked `.claude/` and `.overdeck/` trees, and `.overdeck/config.yaml` there contains live credentials. Never `git add -A`, `git add .` or `git commit -a`. +- The SKILL.md and README command tables must list identical `/okf …` strings in the same order (selftest drift check), so WI-9 changes both tables and the selftest loop in one commit. +- Tests that copy the fixture use pytest's `tmp_path` and `shutil.copytree`; never mutate `tests/fixtures/` in a test. No test sleeps or waits on real time. +- Prose in docs: short, plain sentences; active voice. + +## 6. Work items + +Each item lands as its own commit with an `Item: ` trailer. + +### WI-1 `pytest-harness` — pytest in CI (FR-14) + +Files: `tests/conftest.py`, `tests/test_harness.py`, `.github/workflows/ci.yml`. + +1. `tests/conftest.py`: insert `str(Path(__file__).resolve().parents[1] / "scripts")` at the front of `sys.path`, and define fixtures `scripts_dir` (that path) and `fixture_bundle` (returns `Path(__file__).parent / "fixtures" / "publish_bundle"`). +2. `tests/test_harness.py`: `test_okf_common_importable` imports `okf_common` and asserts `okf_common.concept_id_for_path(Path("a/b.md")) == "a/b"`. +3. `.github/workflows/ci.yml` before: + +```yaml + - run: python -m pip install pyyaml + - run: bash scripts/selftest.sh +``` + +after: + +```yaml + - run: python -m pip install pyyaml pytest + - run: bash scripts/selftest.sh + - run: python -m pytest -q tests +``` + +Test: `uv run --no-project --python 3.12 --with pyyaml --with pytest python -m pytest -q tests/test_harness.py` passes. + +### WI-2 `markdown-renderer` — escaped Markdown subset (FR-4, NFR-4) + +Files: `scripts/publish_markdown.py`, `tests/test_publish_markdown.py`. Implements §4.6 `render_markdown`. Tests (all in `tests/test_publish_markdown.py`): + +- `test_raw_html_is_escaped`: `` and `` render escaped; output contains no ``. +- `test_resolver_none_renders_link_text`: `[Secret](/ops/x.md)` with a resolver returning `None` renders `Secret` and no ``, fenced code with `language-python` and escaped content, a pipe table with ``. +- `test_footnotes`: `Claim.[^s1]` plus `[^s1]: Source text` renders a `fnref` superscript linking `#fn-s1` and a footnotes section; an undefined `[^zz]` stays literal. +- `test_inline`: code span content is not formatted; `**b**`, `*i*`, image renders alt text only. + +### WI-3 `redaction-rules` — profile, built-in rules, redact and scan (FR-8, FR-9, NFR-5) + +Files: `scripts/publish_redact.py`, `templates/okf-publish.yaml`, `tests/test_publish_redact.py`. Implements §4.5. Tests: + +- `test_default_rules_redact_samples`: `/home/alice/x` → `~/x`; `C:\Users\bob\x` → `~\x`; `10.0.0.12` → `[ip]`; a `ghp_` + 36 alphanumerics token → `[token]`; `-----BEGIN RSA PRIVATE KEY-----` → `[private-key]`; counts dict names each rule. +- `test_private_strings_case_insensitive`: profile with `private_strings: [Alice Example]` redacts `alice example` and reports rule id `private-string-1`. +- `test_disable_default_rule`: `disable_default_rules: [ipv4]` leaves `127.0.0.1` unchanged. +- `test_profile_errors`: each invalid profile in §4.5 returns `(None, errors)` with a message naming the field; an invalid regex message names the rule id. +- `test_scan_reports_lines_not_text`: `scan("a\n/home/alice\n", p)` returns `[("home-path-unix", 2)]`. +- `test_replace_is_literal`: a rule with `replace: '\1'` inserts a literal `\1`. +- `test_template_lists_default_ids`: the commented id list in `templates/okf-publish.yaml` equals the `DEFAULT_RULES` ids, and the template loads with no errors via `load_profile(tmp, template_path)`. + +### WI-4 `publish-model` — walk, visibility, trust, links, backlinks, fixture bundle (FR-2, FR-3, FR-6, FR-12) + +Files: `scripts/publish_model.py`, `tests/test_publish_model.py`, `tests/fixtures/publish_bundle/**`. Implements §4.2–§4.4. + +Fixture bundle `tests/fixtures/publish_bundle/` (exact contents up to wording; the named strings are exact): + +| Path | Frontmatter essentials | Body essentials | +| --- | --- | --- | +| `index.md` | none | `# Stream Rules Knowledge`; prose `See [the incident](/ops/incident-42.md).`; marker listing with lines `* [ops/](ops/) - Subdirectory`, `* [rules/](rules/) - Subdirectory`, `* [Schema](schema.md) - Bundle conventions.` | +| `schema.md` | `type: Policy`, `title: Schema`, `description: Bundle conventions.`, `x_visibility: public`, `x_visibility_default: public` | short prose | +| `rules/index.md` | none | `# Rules`; marker listing for `mod-notes.md`, `no-spoilers.md`, `old-spoiler-rule.md` with their titles and descriptions | +| `rules/no-spoilers.md` | `type: Rule`, `title: No spoilers`, `description: Do not reveal plot points.`, `tags: [chat, rules]`, `status: stable`, `sources: [{id: s1, resource: "https://example.com/vod/1", title: Chat proposal, author: viewer-anna}]`, `verified: [{by: "human:host", at: "2026-11-01T00:00:00Z"}]` | `Proposed in chat.[^s1]`; links `[old rule](old-spoiler-rule.md)`, `[incident](/ops/incident-42.md)`, `[future rule](/rules/not-written-yet.md)`, `[VOD](https://example.com/vod/1)`, `[rules home](index.md)`; `[^s1]: Chat proposal from viewer-anna.` | +| `rules/old-spoiler-rule.md` | `type: Rule`, `title: Old spoiler rule`, `description: Replaced rule.`, `status: deprecated`, `verified: {by: "process:ci", at: "2026-10-01T00:00:00Z"}` | `Replaced by [No spoilers](no-spoilers.md).` | +| `rules/mod-notes.md` | `type: Note`, `title: ZZ-PRIVATE-MOD-NOTES`, `description: ZZ-PRIVATE-MOD-DESC`, `x_visibility: private` | mentions `/home/alice/mod.txt` | +| `ops/index.md` | none | marker listing for `incident-42.md` | +| `ops/incident-42.md` | `type: Incident`, `title: ZZ-PRIVATE-INCIDENT`, `description: ZZ-PRIVATE-INCIDENT-DESC`, `x_visibility: private` | `/home/alice/stream/obs.json`, `10.0.0.12`, `Alice Example` | +| `log.md` | none | `# Directory Update Log`, `## 2026-11-01`, `* **Update**: Added [incident](/ops/incident-42.md)` | +| `okf-publish.yaml` | `okf_publish_version: "0.1"`, one rule `stream-box-host` (`\bstream-box\.lan\b` → `[internal-host]`), `private_strings: [Alice Example]` | — | +| `.github/notes.md` | no frontmatter at all | `plain text` (proves dot-dirs are skipped) | + +Public content must contain no rule match, so `--check` passes on the unmodified fixture. + +Tests (`tests/test_publish_model.py`): + +- `test_fixture_public_and_private`: `public_ids() == ["rules/no-spoilers", "rules/old-spoiler-rule", "schema"]`; `ops/incident-42` and `rules/mod-notes` are present and private; findings empty; `.github/notes.md` produced no finding. +- `test_default_private_without_schema`: on a `tmp_path` copy with `schema.md` deleted, `public_ids() == []` and `default_reason == "no schema.md at bundle root"`. +- `test_default_private_without_key`: copy with `x_visibility_default` removed → `public_ids() == ["schema"]`. +- `test_invalid_visibility`: `x_visibility: Public` → one `P_VISIBILITY_INVALID` finding; the concept is private. +- `test_parse_error_finding`: a `notes/bad.md` without frontmatter → one `P_FRONTMATTER_PARSE` finding naming `notes/bad.md`. +- `test_trust_tier`: list with `human:` → `human-reviewed`; bare mapping with `process:ci` → `machine-confirmed`; absent → `unverified`. +- `test_classify_link`: one assertion per rule 1–10 of §4.3, including `../../x.md` from `rules` → `other`, `javascript:x` → `other`, `/rules/` → `directory "rules"`, `index.md` from `rules` → `directory "rules"`. +- `test_backlinks_public_only`: `backlinks("rules/no-spoilers") == ["rules/old-spoiler-rule"]`; a temporary private concept linking to it does not appear. +- `test_public_dirs`: `public_dirs() == {"", "rules"}`. +- `test_body_start_line`: `rules/no-spoilers` `body_start_line` equals the file line of its first body line. + +### WI-5 `site-pages` — concept and index pages with page-relative links (FR-1, FR-3, FR-4, FR-5) + +Files: `scripts/publish_render.py`, `scripts/publish_assets/style.css`, `tests/test_publish_render.py`. Implements §4.6 `publish_render.py` (not search). `style.css`: system font stack, max-width 48rem main column, badge styles per `status-*`/`trust-*`, and a `@media (prefers-color-scheme: dark)` block. No external fonts or `@import`. + +Tests (render the fixture model to `tmp_path` with unredacted texts): + +- `test_relative_href`: the three examples in §4.6 plus a deeper case. +- `test_filter_index_drops_private_lines`: the filtered root index has no `ops/` line; the filtered `rules/index.md` has no `ZZ-PRIVATE-MOD-NOTES`; the hand-written prose line keeps `the incident` as text. +- `test_concept_page_contents`: `rules/no-spoilers.html` contains the concept ID, `human-reviewed`, `stable`, the `Chat proposal — viewer-anna` source, a "Linked from" entry for `Old spoiler rule` with `href="old-spoiler-rule.html"`, `href="../index.html"` in the header, and the words `incident` and `future rule` with no ` ` in an isolated Playwright browser; open `search.html?q=spoilers`; the results list shows `No spoilers` and `Old spoiler rule`, and no `ZZ-PRIVATE` text appears anywhere on the page. + +### WI-7 `publish-command` — `publish.py --out` (FR-1, FR-10, FR-11, NFR-2) + +Files: `scripts/publish.py`, `tests/test_publish_cli.py`. Implements §4.8 steps 1–5 and 9–12. Tests run the CLI with `subprocess.run([sys.executable, str(scripts_dir / "publish.py"), ...])` against `tmp_path` copies: + +- `test_publish_fixture_pages`: exit 0; the set of `*.html` under `--out` equals `{"index.html", "rules/index.html", "rules/no-spoilers.html", "rules/old-spoiler-rule.html", "schema.html", "search.html"}`; `ops/incident-42.html`, `rules/mod-notes.html`, `ops/index.html` and any `log.html` do not exist. +- `test_no_unresolved_links`: parse every `.html` with `html.parser.HTMLParser`; for each `href`/`src` not starting with `http:`, `https:`, `mailto:` or `#`, strip `?…`/`#…`, `urllib.parse.unquote`, resolve against the page's directory, and assert the file exists inside `--out`. +- `test_no_private_or_secret_bytes`: no file under `--out` contains `ZZ-PRIVATE`, `/home/alice`, `10.0.0.12` or `Alice Example` (case-insensitive); running `scan` with the fixture profile over every output file returns no match. +- `test_redaction_applied`: add `Mirror lives at /home/bob/vod` and `stream-box.lan` to the body of `rules/old-spoiler-rule.md` in a copy; publish exits 0, prints a non-zero redaction count, and the page contains `~/vod` and `[internal-host]`. +- `test_deterministic`: two publishes to two directories produce identical file lists and identical bytes. +- `test_out_safety`: `--out` equal to the bundle → exit 2 with `P_OUT_UNSAFE`; a non-empty directory without `.okf-publish` → exit 2 and its files untouched; a second publish into a marked directory after making `rules/old-spoiler-rule.md` private removes `rules/old-spoiler-rule.html`. +- `test_id_redaction_blocks`: a public concept at `hosts/stream-box.lan.md` (ID `hosts/stream-box.lan`, matched by the fixture rule `stream-box-host`) → exit 2 with `P_ID_REDACTION`, and `--out` is not created. +- `test_errors_write_nothing`: an invalid `x_visibility` → exit 2, `P_VISIBILITY_INVALID` printed, `--out` not created. +- `test_zero_public_note`: with `schema.md` deleted → exit 0 and stdout contains `note - no public concepts` and `no schema.md at bundle root`. + +### WI-8 `publish-check-mode` — `publish.py --check` gate (FR-9, NFR-5) + +Files: `scripts/publish.py`, `tests/test_publish_cli.py`. Implements §4.8 steps 6–8. Tests: + +- `test_check_passes_on_fixture`: exit 0, stdout starts with `ok - publish check passed: 3 public concept(s)`; no files written anywhere (the bundle copy's file list is unchanged). +- `test_check_fails_on_home_path_then_passes`: **Given** a copy where `rules/no-spoilers.md` body line `Setup notes: /home/alice/obs.json` is added, **when** `publish.py --check` runs, **then** it exits 2 and prints `ERROR P_REDACTION rules/no-spoilers.md:: rule home-path-unix matches published text (field body)` where `` is that line's file line number, and stdout does not contain `/home/alice`. **Given** the author replaces the line with `Setup notes: ~/obs.json`, **when** `--check` runs again, **then** it exits 0. +- `test_check_ignores_private`: the unmodified fixture's private concepts hold home paths, an IP and a private string, yet check exits 0. +- `test_check_frontmatter_field`: `description: Runs on 10.0.0.12` on a public concept → exit 2 with `rule ipv4 matches published text (field description)`. +- `test_check_index_text`: hand-written prose `Hosted on stream-box.lan` in root `index.md` → exit 2 with location `index.md:`. +- `test_check_and_out_exclusive`: passing both flags exits 2 with an argparse error. + +### WI-9 `skill-command-row` — command tables and selftest loop (FR-13) + +Files: `SKILL.md`, `README.md`, `scripts/selftest.sh`. One commit (drift check). + +1. SKILL.md command table: after the `/okf embed` row (line 38), add + ``| `/okf publish --out

        [--check]` | Build a public static site from public concepts, with redaction. | `references/publish.md` |`` +2. README command table: after the `/okf embed` row (line 85), add + ``| `/okf publish --out [--check]` | Build a public static site from public concepts, with redaction. |`` +3. SKILL.md: append a section after `## \`/okf embed …\``: + +```markdown +## `/okf publish --out [--check]` + +Build a public static site from the bundle. + +- Run `python3 /scripts/publish.py --bundle --check` first; it exits 2 when a redaction rule matches text in a public concept and names the rule and line, never the matched text. +- Then run `python3 /scripts/publish.py --bundle --out `. +- Only concepts with `x_visibility: public`, or covered by `x_visibility_default: public` in root `schema.md`, are published. The default is private. +- Links to private or missing concepts render as plain text. `log.md` and non-Markdown files are never published. +- Redaction rules come from built-in defaults plus `okf-publish.yaml`; see `references/publish.md`. +- Never mark a concept public or edit a redaction profile without the user's request. Deploying the site is outside this skill. +``` + +4. `scripts/selftest.sh` line 27 before: `for command in init author convert sync study retro extract validate lint embed; do` — after: `for command in init author convert sync study retro extract validate lint embed publish; do`. + +Test: `uv run --no-project --python 3.12 --with pyyaml --with pytest bash scripts/selftest.sh` exits 0 and prints `ok - SKILL.md documents /okf publish` and the README drift check line. + +### WI-10 `publishing-docs` — README section, reference page, usage guide (FR-13) + +Files: `README.md`, `references/publish.md`, `docs/USAGE.md`. + +1. README: add `## Publishing a bundle` after `## The Three Loops` (before `## Reading Rules`). Content, in order: + - What it builds (pages, indexes, search, plain static files; any static host works; deploying is outside the skill). + - **Visibility**: `x_visibility: public | private`, `x_visibility_default` in `schema.md`, private by default, private links render as plain text and the link text stays (so the link text should not itself be secret). + - **Redaction profile**: the `okf-publish.yaml` example from §4.5, the built-in rule ids, `disable_default_rules`, `private_strings`, `--check` as the CI gate. + - **Sample Cloudflare Pages setup**: this workflow, verbatim apart from wording in comments: + +```yaml +name: Publish knowledge site +on: + push: + branches: [main] +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + deployments: write + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: eltmon/okf + ref: vX.Y.Z # a release that includes scripts/publish.py + path: .okf-skill # dot-directory: the publisher skips it + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python3 -m pip install PyYAML + - run: python3 .okf-skill/scripts/publish.py --bundle . --check + - run: python3 .okf-skill/scripts/publish.py --bundle . --out _site + - uses: cloudflare/wrangler-action@v3 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + command: pages deploy _site --project-name= +``` + + - One sentence for GitHub Pages: upload `_site` with `actions/upload-pages-artifact` and deploy with `actions/deploy-pages`; page-relative links work under a project subpath. + - One sentence: search fetches `search-index.json`, so preview with `python3 -m http.server --directory _site` rather than opening files directly. +2. `references/publish.md`: command synopsis; visibility rules (§4.2); link rules (FR-4); trust tier rule (FR-6); profile schema and built-in rules table (§4.5) including the `ipv4` false-positive note; output layout (FR-1); finding codes and exit codes (§4.8); the `--out` safety and rebuild rule (FR-11); "What is never published" list (`log.md`, private concepts, non-Markdown files, dot-directories, frontmatter keys other than those in FR-5). +3. `docs/USAGE.md`: add `## \`/okf publish --out [--check]\`` after the `/okf embed` section (before `## Model Ladder`) with two example commands and four bullets matching the SKILL.md section. + +Checks: the grep commands in AC-10. No docs test file is added. + +## 7. Acceptance criteria (map 1:1 to work items) + +- **AC-1 (WI-1).** `python -m pytest -q tests` runs in CI after the selftest step, and `tests/test_harness.py` passes locally with the `uv run` form. +- **AC-2 (WI-2).** `tests/test_publish_markdown.py` passes; raw `` and ``, when render_markdown runs, then the output contains `<script>` and contains no ``.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "markdown-renderer.ac3", + "title": "Given `Claim.[^s1]` and `[^s1]: Source text`, when rendered, then the output contains `href=\"#fn-s1\"` in a `fnref` superscript and a `
        ` with `id=\"fn-s1\"`; an undefined `[^zz]` stays as literal escaped text.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "markdown-renderer.ac4", + "title": "Given headings, nested lists, a blockquote, a fenced python block and a pipe table, when rendered, then the output contains slugged `

        ` ids (duplicate gets `-2`), nested `
          `, `
          `, `` and ``.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "markdown-renderer.ac5", + "title": "Given the repo, when `uv run --no-project --python 3.12 --with pyyaml --with pytest python -m pytest -q tests/test_publish_markdown.py` runs, then all six named tests pass.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + } + ] + }, + { + "id": "redaction-rules", + "title": "Add the redaction profile loader, built-in rules, redact and scan", + "status": "pending", + "priority": "medium", + "created": "2026-11-15T15:18:38Z", + "metadata": { + "difficulty": "medium", + "kind": "backend", + "issueLabel": "okf-11", + "requiresInspection": true, + "inspectionDepth": "deep", + "files_scope": [ + "scripts/publish_redact.py", + "templates/okf-publish.yaml", + "tests/test_publish_redact.py" + ], + "files_scope_confidence": "high", + "verify_commands": [ + "uv run --no-project --python 3.12 --with pyyaml --with pytest python -m pytest -q tests/test_publish_redact.py" + ], + "expected_outputs": [ + "tests/test_publish_redact.py: 7 passed, exit 0" + ], + "readiness": "ready", + "traces": [ + "FR-8", + "FR-9", + "NFR-5" + ], + "foundationFor": [ + "publish-command", + "publish-check-mode" + ] + }, + "narrative": { + "Action": "Follow .pan/drafts/OKF-11.md section 4.5 exactly: the profile schema, the eight DEFAULT_RULES with their exact patterns and order, private_strings as case-insensitive escaped rules named private-string-, load_profile error cases, literal replacement, and scan returning (rule_id, line) pairs without matched text. Write templates/okf-publish.yaml with a comment listing every built-in id. Write the seven named tests from WI-3." + }, + "items": [ + { + "id": "redaction-rules.ac1", + "title": "Given no profile file in the bundle, when load_profile runs, then it returns the built-in profile and redact turns `/home/alice/x` into `~/x`, `10.0.0.12` into `[ip]` and a `ghp_` + 36-character token into `[token]`.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "redaction-rules.ac2", + "title": "Given a profile with `disable_default_rules: [ipv4]`, when redact runs on `127.0.0.1`, then the text is unchanged; given `private_strings: [Alice Example]`, `alice example` is replaced with `[redacted]` and counted under `private-string-1`.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "redaction-rules.ac3", + "title": "Given an invalid profile (unknown top-level key, wrong okf_publish_version, duplicate id, bad regex, unknown disabled id), when load_profile runs, then it returns `(None, errors)` and each error names the offending field or rule id.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "redaction-rules.ac4", + "title": "Given `a\\n/home/alice\\n`, when scan runs, then it returns `[(\"home-path-unix\", 2)]` and no element contains the matched text.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "redaction-rules.ac5", + "title": "Given the repo, when `uv run --no-project --python 3.12 --with pyyaml --with pytest python -m pytest -q tests/test_publish_redact.py` runs, then all seven named tests pass, including the check that templates/okf-publish.yaml lists exactly the DEFAULT_RULES ids.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + } + ] + }, + { + "id": "publish-model", + "title": "Add the publish model: bundle walk, visibility, trust tier, link classes and the fixture bundle", + "status": "pending", + "priority": "high", + "created": "2026-11-15T15:18:38Z", + "metadata": { + "difficulty": "complex", + "kind": "backend", + "issueLabel": "okf-11", + "requiresInspection": true, + "inspectionDepth": "deep", + "files_scope": [ + "scripts/publish_model.py", + "tests/test_publish_model.py", + "tests/fixtures/publish_bundle/*.md", + "tests/fixtures/publish_bundle/okf-publish.yaml", + "tests/fixtures/publish_bundle/rules/*.md", + "tests/fixtures/publish_bundle/ops/*.md", + "tests/fixtures/publish_bundle/.github/notes.md" + ], + "files_scope_confidence": "medium", + "verify_commands": [ + "uv run --no-project --python 3.12 --with pyyaml --with pytest python -m pytest -q tests/test_publish_model.py" + ], + "expected_outputs": [ + "tests/test_publish_model.py: 10 passed, exit 0" + ], + "readiness": "ready", + "traces": [ + "FR-2", + "FR-3", + "FR-6", + "FR-12", + "NFR-1", + "NFR-3" + ], + "foundationFor": [ + "site-pages", + "search-index", + "publish-command", + "publish-check-mode" + ], + "readinessReason": "Creates only new files (publish_model.py, its test, the fixture bundle); no other item writes them, so it can run in its own slot once pytest-harness lands." + }, + "narrative": { + "Action": "Follow .pan/drafts/OKF-11.md sections 4.2-4.4 and WI-4. Implement walk_markdown (skip dot components, node_modules, and the exclude dir), visibility resolution that fails closed (missing schema.md or key -> private, invalid value -> P_VISIBILITY_INVALID and private), trust_tier, Source/PublishConcept/Finding/PublishModel types with the listed fields, classify_link rules 1-10 in order, backlinks among public concepts only, public_dirs, body_start_line, and P_FRONTMATTER_PARSE findings. Create the fixture bundle exactly as the WI-4 table lists, including the ZZ-PRIVATE marker strings, okf-publish.yaml and .github/notes.md without frontmatter. Public fixture text must match no redaction rule. Write the ten named tests." + }, + "items": [ + { + "id": "publish-model.ac1", + "title": "Given the fixture bundle, when build_model runs, then public_ids() returns exactly `[\"rules/no-spoilers\", \"rules/old-spoiler-rule\", \"schema\"]`, findings are empty, and `.github/notes.md` produces no finding.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publish-model.ac2", + "title": "Given a copy without schema.md, when build_model runs, then public_ids() is empty and default_reason is `no schema.md at bundle root`; given `x_visibility: Public` on a concept, then one P_VISIBILITY_INVALID finding is returned and the concept is private.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publish-model.ac3", + "title": "Given classify_link inputs from section 4.3, when classified from source dir `rules`, then `../../x.md` and `javascript:x` return kind `other`, `index.md` and `/rules/` return kind `directory` with value `rules`, and `/ops/incident-42.md#a` returns kind `concept` with fragment `a`.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publish-model.ac4", + "title": "Given the fixture, when backlinks(\"rules/no-spoilers\") runs, then it returns `[\"rules/old-spoiler-rule\"]`, and a private concept linking to it is not included; public_dirs() returns `{\"\", \"rules\"}`.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publish-model.ac5", + "title": "Given the repo, when `uv run --no-project --python 3.12 --with pyyaml --with pytest python -m pytest -q tests/test_publish_model.py` runs, then all ten named tests pass.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + } + ] + }, + { + "id": "site-pages", + "title": "Render concept pages and index pages with page-relative links", + "status": "pending", + "priority": "medium", + "created": "2026-11-15T15:18:38Z", + "metadata": { + "difficulty": "complex", + "kind": "backend", + "issueLabel": "okf-11", + "requiresInspection": false, + "inspectionDepth": "fast", + "files_scope": [ + "scripts/publish_render.py", + "scripts/publish_assets/style.css", + "tests/test_publish_render.py" + ], + "files_scope_confidence": "high", + "verify_commands": [ + "uv run --no-project --python 3.12 --with pyyaml --with pytest python -m pytest -q tests/test_publish_render.py" + ], + "expected_outputs": [ + "tests/test_publish_render.py: 5 passed, exit 0" + ], + "readiness": "ready", + "traces": [ + "FR-1", + "FR-3", + "FR-4", + "FR-5", + "NFR-2", + "NFR-4" + ], + "readinessReason": "Creates only new files (publish_render.py, style.css, its test); it reads publish_markdown.py and publish_model.py, which its blockers produce." + }, + "narrative": { + "Action": "Follow .pan/drafts/OKF-11.md section 4.6 (publish_render.py part) and WI-5: concept_page_path, dir_index_path, relative_href with quoted segments, make_resolver applying FR-4, filter_index_markdown dropping listing lines whose first link is an unpublished concept or directory, site_title, page_shell with the exact HTML structure, concept page body with badges, sources and backlinks, generated listings for published directories without index.md, and copying style.css (no external fonts, no @import, a prefers-color-scheme dark block). Write the five named tests." + }, + "items": [ + { + "id": "site-pages.ac1", + "title": "Given the fixture model, when render_site writes to tmp_path, then it returns exactly `[\"assets/style.css\", \"index.html\", \"rules/index.html\", \"rules/no-spoilers.html\", \"rules/old-spoiler-rule.html\", \"schema.html\"]`.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "site-pages.ac2", + "title": "Given rules/no-spoilers.html, when read, then it contains `human-reviewed`, `stable`, `Chat proposal — viewer-anna`, a Linked from entry `href=\"old-spoiler-rule.html\"`, and header `href=\"../index.html\"`, while `incident` and `future rule` appear with no surrounding ` --out ` runs, then it exits 0 and the set of `.html` files is exactly index.html, rules/index.html, rules/no-spoilers.html, rules/old-spoiler-rule.html, schema.html, search.html, with no ops/ pages and no log.html.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publish-command.ac2", + "title": "Given that output, when every href/src that is not http(s), mailto or `#` is resolved against its page's directory, then each target file exists under --out; and no output file contains `ZZ-PRIVATE`, `/home/alice`, `10.0.0.12` or `Alice Example`.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publish-command.ac3", + "title": "Given --out equal to the bundle, or a non-empty directory without `.okf-publish`, when publish runs, then it exits 2 with `P_OUT_UNSAFE` and leaves existing files untouched; given a marked directory and a concept turned private, the rerun removes that concept's page.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publish-command.ac4", + "title": "Given a public concept at hosts/stream-box.lan.md or an invalid x_visibility value, when publish runs, then it exits 2 with `P_ID_REDACTION` or `P_VISIBILITY_INVALID` and does not create --out.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publish-command.ac5", + "title": "Given two publishes of the same bundle to two directories, when compared, then the file lists and bytes are identical; given `/home/bob/vod` and `stream-box.lan` added to a public body, the page contains `~/vod` and `[internal-host]`.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + } + ] + }, + { + "id": "publish-check-mode", + "title": "Add the publish.py --check redaction gate", + "status": "pending", + "priority": "high", + "created": "2026-11-15T15:18:38Z", + "metadata": { + "difficulty": "medium", + "kind": "backend", + "issueLabel": "okf-11", + "requiresInspection": false, + "inspectionDepth": "fast", + "files_scope": [ + "scripts/publish.py", + "tests/test_publish_cli.py" + ], + "files_scope_confidence": "high", + "verify_commands": [ + "uv run --no-project --python 3.12 --with pyyaml --with pytest python -m pytest -q tests/test_publish_cli.py" + ], + "expected_outputs": [ + "tests/test_publish_cli.py: all tests pass, exit 0" + ], + "readiness": "ready", + "traces": [ + "FR-9", + "NFR-5" + ] + }, + "narrative": { + "Action": "Follow .pan/drafts/OKF-11.md section 4.8 steps 6-8 and WI-8. --check scans each public concept's title, description, type, status, tags, source title/resource/author and body, plus each filtered published index text, with publish_redact.scan. Body locations use file line numbers (body_start_line + line - 1); index locations use the index file's line numbers. Messages name the rule id and field and never the matched text. Exit 2 on any finding, else print `ok - publish check passed: public concept(s), no redaction rule matches` and exit 0. Write nothing. Write the six named tests in tests/test_publish_cli.py." + }, + "items": [ + { + "id": "publish-check-mode.ac1", + "title": "Given the unmodified fixture copy, when `publish.py --bundle --check` runs, then it exits 0, prints `ok - publish check passed: 3 public concept(s)`, and the copy's file list is unchanged even though private concepts hold home paths, an IP and a private string.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publish-check-mode.ac2", + "title": "Given rules/no-spoilers.md with the body line `Setup notes: /home/alice/obs.json`, when --check runs, then it exits 2, prints `ERROR P_REDACTION rules/no-spoilers.md:: rule home-path-unix matches published text (field body)` with `` equal to that file line, and stdout does not contain `/home/alice`.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publish-check-mode.ac3", + "title": "Given that line then replaced with `Setup notes: ~/obs.json`, when --check runs again, then it exits 0.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publish-check-mode.ac4", + "title": "Given `description: Runs on 10.0.0.12` on a public concept or `Hosted on stream-box.lan` in root index.md prose, when --check runs, then it exits 2 with `(field description)` or location `index.md:` respectively; passing both --check and --out exits 2 with an argparse error.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + } + ] + }, + { + "id": "skill-command-row", + "title": "Add the /okf publish command row, SKILL.md section and selftest loop entry", + "status": "pending", + "priority": "medium", + "created": "2026-11-15T15:18:38Z", + "metadata": { + "difficulty": "simple", + "kind": "docs", + "issueLabel": "okf-11", + "requiresInspection": false, + "inspectionDepth": "fast", + "files_scope": [ + "SKILL.md", + "README.md", + "scripts/selftest.sh" + ], + "files_scope_confidence": "high", + "verify_commands": [ + "uv run --no-project --python 3.12 --with pyyaml --with pytest bash scripts/selftest.sh" + ], + "expected_outputs": [ + "selftest prints ok - SKILL.md documents /okf publish and exits 0" + ], + "readiness": "ready", + "traces": [ + "FR-13", + "NFR-3" + ] + }, + "narrative": { + "Action": "Follow .pan/drafts/OKF-11.md WI-9 in one commit: add the identical `/okf publish --out [--check]` row after the `/okf embed` row in the SKILL.md table (3 columns) and the README table (2 columns), append the SKILL.md `## \\`/okf publish --out [--check]\\`` section verbatim from WI-9, and add `publish` to the end of the command loop on scripts/selftest.sh line 27. Change nothing else in these files." + }, + "items": [ + { + "id": "skill-command-row.ac1", + "title": "Given the repo, when `uv run --no-project --python 3.12 --with pyyaml --with pytest bash scripts/selftest.sh` runs, then it exits 0 and prints `ok - SKILL.md documents /okf publish` and `ok - README command names match SKILL.md table`.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "skill-command-row.ac2", + "title": "Given SKILL.md, when read, then the `/okf publish` section states that the default visibility is private, that --check names the rule and line but never the matched text, and that deploying is outside the skill.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + } + ] + }, + { + "id": "publishing-docs", + "title": "Document publishing in README, references/publish.md and docs/USAGE.md", + "status": "pending", + "priority": "medium", + "created": "2026-11-15T15:18:38Z", + "metadata": { + "difficulty": "medium", + "kind": "docs", + "issueLabel": "okf-11", + "requiresInspection": false, + "inspectionDepth": "fast", + "files_scope": [ + "README.md", + "references/publish.md", + "docs/USAGE.md" + ], + "files_scope_confidence": "high", + "verify_commands": [ + "uv run --no-project --python 3.12 --with pyyaml --with pytest bash scripts/selftest.sh", + "grep -c '^## Publishing a bundle' README.md" + ], + "expected_outputs": [ + "selftest exits 0", + "grep prints 1" + ], + "readiness": "ready", + "traces": [ + "FR-13" + ] + }, + "narrative": { + "Action": "Follow .pan/drafts/OKF-11.md WI-10: add `## Publishing a bundle` to README.md between `## The Three Loops` and `## Reading Rules` (visibility, redaction profile, the Cloudflare Pages workflow verbatim, one GitHub Pages sentence, one http.server preview sentence); create references/publish.md with every topic WI-10 lists, including all seven P_* codes and the ipv4 false-positive note; add the `/okf publish` section to docs/USAGE.md before `## Model Ladder`. Do not edit the README command table (the previous item owns it)." + }, + "items": [ + { + "id": "publishing-docs.ac1", + "title": "Given README.md, when `grep -n '^## Publishing a bundle' README.md` runs, then it matches one line, and that section contains `x_visibility`, `x_visibility_default`, `okf-publish.yaml`, `cloudflare/wrangler-action@v3` and `publish.py --bundle . --check`.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publishing-docs.ac2", + "title": "Given references/publish.md, when read, then it lists P_BUNDLE_MISSING, P_PROFILE_INVALID, P_FRONTMATTER_PARSE, P_VISIBILITY_INVALID, P_ID_REDACTION, P_REDACTION and P_OUT_UNSAFE with exit code 2, and states that log.md and non-Markdown files are never published.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publishing-docs.ac3", + "title": "Given docs/USAGE.md, when read, then a `/okf publish --out [--check]` section appears before `## Model Ladder` with a `--check` example and an `--out` example.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + }, + { + "id": "publishing-docs.ac4", + "title": "Given the repo, when `uv run --no-project --python 3.12 --with pyyaml --with pytest bash scripts/selftest.sh` runs after this item, then it still exits 0.", + "status": "pending", + "metadata": { + "kind": "acceptance_criterion" + } + } + ] + } + ], + "edges": [ + { + "from": "pytest-harness", + "to": "markdown-renderer", + "type": "blocks" + }, + { + "from": "pytest-harness", + "to": "redaction-rules", + "type": "blocks" + }, + { + "from": "pytest-harness", + "to": "publish-model", + "type": "blocks" + }, + { + "from": "markdown-renderer", + "to": "site-pages", + "type": "blocks" + }, + { + "from": "publish-model", + "to": "site-pages", + "type": "blocks" + }, + { + "from": "publish-model", + "to": "search-index", + "type": "blocks" + }, + { + "from": "site-pages", + "to": "search-index", + "type": "blocks" + }, + { + "from": "redaction-rules", + "to": "publish-command", + "type": "blocks" + }, + { + "from": "site-pages", + "to": "publish-command", + "type": "blocks" + }, + { + "from": "search-index", + "to": "publish-command", + "type": "blocks" + }, + { + "from": "publish-command", + "to": "publish-check-mode", + "type": "blocks" + }, + { + "from": "skill-command-row", + "to": "publishing-docs", + "type": "blocks" + }, + { + "from": "publish-check-mode", + "to": "publishing-docs", + "type": "informs" + } + ], + "metadata": { + "sizeJustification": "One feature in a small flat repo. The subsystem count is inflated because the lint counts the first two path segments, so each new scripts/publish_*.py and tests/test_publish_*.py file counts as its own subsystem; the real surface is scripts/, tests/, templates/, .github/ and four doc files. The items are not independently user-shippable: the model, renderer, redaction and search modules have no entry point until publish.py lands, and the issue's acceptance needs all of them together. Items are already split per module with their own tests.", + "canonicalFilename": "2026-11-15-OKF-11-publish-a-bundle-as-a-public-static-site-with-visibility-and-redaction.xbrief.json", + "promotionIntent": "automatic" + } + }, + "status": "active" +} \ No newline at end of file diff --git a/README.md b/README.md index d5ce57d..866b8aa 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ The skill remains portable. Its core scripts do not import Overdeck and require | `/okf validate [--strict]` | Run the deterministic conformance gate. | | `/okf lint` | Run advisory semantic patrol. | | `/okf embed [--profile ]` | Update embedding shards and rebuild the local index. | +| `/okf publish --out [--check]` | Build a public static site from public concepts, with redaction. | ## The Three Loops @@ -98,6 +99,76 @@ Run `/okf sync [--topic ""]` when code or docs change. The pass maps diff Run `/okf retro` after implementation. The pass asks what knowledge would have made the work easier, then files those answers as concepts through the knowledge PR flow. +## Publishing a bundle + +`/okf publish` builds a read-only static site from the bundle's public concepts. The site has one page per public concept, an index page per published directory, and a client-side search page. It is plain HTML, CSS, one script and a JSON index, with no server code and no CDN, so any static host works. Deploying the site is outside the skill. + +```bash +python3 /scripts/publish.py --bundle . --check +python3 /scripts/publish.py --bundle . --out _site +``` + +**Visibility.** A concept is published only when its frontmatter says `x_visibility: public`, or when it has no `x_visibility` and the root `schema.md` sets `x_visibility_default: public`. Without `schema.md` or that key, every concept is private. Any value other than `public` or `private` stops the run with an error. + +```yaml +# schema.md frontmatter +x_visibility_default: public +# a concept that must stay off the site +x_visibility: private +``` + +A private concept has no page, no index line, no search record and no backlink. A link to it renders as plain text, and the link text stays, so do not put secrets in link text. `log.md`, non-Markdown files and dot-directories are never published. + +**Redaction profile.** Before rendering, `publish.py` rewrites every published string with regex rules. Built-in rules always apply unless you disable them: `home-path-unix`, `home-path-windows`, `ipv4`, `github-token`, `openai-style-key`, `aws-access-key`, `slack-token` and `private-key-block`. Add your own rules in `okf-publish.yaml` at the bundle root (start from `templates/okf-publish.yaml`): + +```yaml +okf_publish_version: "0.1" +disable_default_rules: [] # for example [ipv4] when version numbers trip it +rules: + - id: stream-box-host + pattern: '\bstream-box\.lan\b' + replace: '[internal-host]' +private_strings: # literal, case-insensitive, matches across line breaks + - Alice Example +``` + +Use `publish.py --bundle . --check` as the CI gate. It exits 2 when any rule matches text in a public concept or a published index page. It names the rule and the file line, never the matched text, because CI logs of public repos are public. Fix the source text, then rerun. A rule that matches a public concept ID or directory name is always an error, because URLs cannot be redacted. + +**Sample Cloudflare Pages setup.** This workflow checks the bundle and deploys `_site` on every push to `main`: + +```yaml +name: Publish knowledge site +on: + push: + branches: [main] +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + deployments: write + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: eltmon/okf + ref: vX.Y.Z # a release that includes scripts/publish.py + path: .okf-skill # dot-directory: the publisher skips it + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python3 -m pip install PyYAML + - run: python3 .okf-skill/scripts/publish.py --bundle . --check + - run: python3 .okf-skill/scripts/publish.py --bundle . --out _site + - uses: cloudflare/wrangler-action@v3 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + command: pages deploy _site --project-name= +``` + +For GitHub Pages, upload `_site` with `actions/upload-pages-artifact` and deploy it with `actions/deploy-pages`; every link is page-relative, so the site works under a project subpath. Search fetches `search-index.json`, so preview the site with `python3 -m http.server --directory _site` rather than opening the files directly. See [references/publish.md](references/publish.md) for the full rules. + ## Reading Rules When using a bundle for answers or prompt context: diff --git a/SKILL.md b/SKILL.md index c00011e..e0255ee 100644 --- a/SKILL.md +++ b/SKILL.md @@ -36,6 +36,7 @@ The bundle is the source of truth. Keep changes small, cited, and PR-gated. Dete | `/okf validate [--strict]` | Run the deterministic conformance gate. | `references/conformance.md` | | `/okf lint` | Run advisory semantic patrol for stale or weak knowledge. | `references/lint-prompt.md`, `references/conformance.md` | | `/okf embed [--profile ]` | Update shareable embedding shards and rebuild the local index. | `references/workflow.md` | +| `/okf publish --out [--check]` | Build a public static site from public concepts, with redaction. | `references/publish.md` | ## Guardrails @@ -184,3 +185,14 @@ Refresh embeddings. - Re-embed only concepts whose normalized content hash changed. - Write sorted JSONL shards under `embeddings/`. - Rebuild the local `.okf-index/` cache, which is derived and gitignored. + +## `/okf publish --out [--check]` + +Build a public static site from the bundle. + +- Run `python3 /scripts/publish.py --bundle --check` first; it exits 2 when a redaction rule matches text in a public concept and names the rule and line, never the matched text. +- Then run `python3 /scripts/publish.py --bundle --out `. +- Only concepts with `x_visibility: public`, or covered by `x_visibility_default: public` in root `schema.md`, are published. The default is private. +- Links to private or missing concepts render as plain text. `log.md` and non-Markdown files are never published. +- Redaction rules come from built-in defaults plus `okf-publish.yaml`; see `references/publish.md`. +- Never mark a concept public or edit a redaction profile without the user's request. Deploying the site is outside this skill. diff --git a/docs/USAGE.md b/docs/USAGE.md index 249e572..4fa4897 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -151,6 +151,20 @@ Refresh embeddings and the local search index. - Writes sorted shard lines to `embeddings/.okfe.jsonl`. - Rebuilds `.okf-index/`, which is derived and gitignored. +## `/okf publish --out [--check]` + +Build a public static site from the bundle's public concepts. + +```bash +python3 /scripts/publish.py --bundle . --check +python3 /scripts/publish.py --bundle . --out _site +``` + +- Run `--check` first. It exits 2 when a redaction rule matches text in a public concept, and names the rule and line but never the matched text. +- Only concepts with `x_visibility: public`, or covered by `x_visibility_default: public` in root `schema.md`, are published. The default is private. +- Links to private or missing concepts render as plain text. `log.md` and non-Markdown files are never published. +- Redaction rules come from built-in defaults plus `okf-publish.yaml`; see `references/publish.md`. Deploying the site is outside the skill. + ## Model Ladder For `author`, `sync`, `study`, and `retro`, `--model ` is a hard request: diff --git a/references/publish.md b/references/publish.md new file mode 100644 index 0000000..0b330f2 --- /dev/null +++ b/references/publish.md @@ -0,0 +1,154 @@ +# OKF Publish + +`scripts/publish.py` builds a read-only static site from the public concepts of a bundle. It uses only the Python standard library and PyYAML. Deploying the site is outside the skill. + +## Synopsis + +```text +python3 scripts/publish.py [--bundle B] (--out DIR | --check) [--profile FILE] +``` + +- `--bundle B`: the bundle root. The default is the current directory. +- `--out DIR`: write the site to `DIR`. +- `--check`: scan published text for redaction matches. Write nothing. +- `--profile FILE`: use this redaction profile instead of `/okf-publish.yaml`. + +`--out` and `--check` are required and mutually exclusive. Run `--check` in CI before `--out`. + +## Visibility + +Each concept resolves to `public` or `private`. Only public concepts are published. + +1. The bundle default comes from `x_visibility_default` in the frontmatter of root `schema.md`. + - No `schema.md`: the default is `private`. + - `schema.md` without `x_visibility_default`: the default is `private`. + - `public` or `private`: that value. + - Any other value: finding `P_VISIBILITY_INVALID`, and the default is `private`. +2. A concept's own `x_visibility` overrides the default. + - Absent: the bundle default applies. + - `public` or `private`: that value. + - Any other value, including `Public`, `true` and `yes`: finding `P_VISIBILITY_INVALID`, and the concept is private. +3. `schema.md` is an ordinary concept. Its own `x_visibility`, or the default, decides whether it gets a page. + +No input resolves to public by accident. When no concept is public, `publish.py` prints why the default is private. + +A private concept appears nowhere on the site. It has no page, its index listing line is dropped, it has no search record and it appears in no "Linked from" list. + +Index listings fail closed. Inside the generated `` … `` block of an `index.md`, a line is kept only when it is a listing line (`* ` or `- `) whose first link resolves to a public concept or a published directory. Every other line in the block is dropped: private entries, continuation lines of a multi-line description, and lines whose link cannot be parsed, such as a file name with a space. Outside the block, a hand-written listing line is dropped when its first link points at a private, missing or unpublished target; all other hand-written lines are kept, with links to unpublished targets rendered as text. + +## Links + +A link in published Markdown resolves this way: + +| Target | Result | +| --- | --- | +| A public concept, with or without `#fragment` | A page-relative link to its page, keeping the fragment. | +| A published directory, or its `index.md` | A page-relative link to that directory's `index.html`. | +| `http:`, `https:` or `mailto:` | Kept as written. | +| `#fragment` only | Kept as written. | +| Anything else | Plain text: the link text stays and the link goes. | + +"Anything else" covers private concepts, missing concepts, unpublished directories, `log.md`, non-Markdown files and every other scheme (`javascript:`, `data:`, `repo:` and so on). A private target and a missing target render identically, so the site does not reveal which private concepts exist. The link text itself is still published, so do not put secrets in link text. + +A published directory is a directory that holds a public concept itself or in a subdirectory. The root is always published. Every href is page-relative (for example `../schema.html`), so the site works under a subpath such as a GitHub Pages project site. + +## Concept pages + +Each concept page shows the concept ID, title, type, status, trust tier, tags, description, rendered body, a "Sources" list and a "Linked from" list. "Linked from" lists public concepts only. Status is frontmatter `status`, or `stable` when absent. + +The trust tier comes from `verified` frontmatter: + +- `human-reviewed`: any `verified` entry has a `by` value that starts with `human:`. +- `machine-confirmed`: there is at least one `verified` entry, but none by `human:`. +- `unverified`: there are no `verified` entries. + +A single mapping counts as a one-element list. + +## Redaction profile + +`okf-publish.yaml` at the bundle root adds rules. Start from `templates/okf-publish.yaml`. + +```yaml +okf_publish_version: "0.1" # required, must be the string "0.1" +disable_default_rules: [] # optional; ids from the built-in rules +rules: # optional; applied after built-in rules + - id: stream-box-host # required, unique, matches ^[a-z0-9][a-z0-9-]*$ + pattern: '\bstream-box\.lan\b' # required, Python re syntax; inline flags such as (?i) allowed + replace: '[internal-host]' # optional, default "[redacted]" +private_strings: # optional; literal strings, matched case-insensitively + - Alice Example +``` + +Rules apply in this order: enabled built-in rules, then `rules`, then `private_strings`. Private string *n* becomes rule `private-string-` with replacement `[redacted]`. It matches case-insensitively, and any run of whitespace, including a line break, matches the spaces between its words, so a name wrapped across two lines is still caught. A replacement is literal text, so `\1` inserts a backslash and a one. Rules apply to every published string before rendering, including fenced code and link targets. A rule that rewrites a link target can turn that link into plain text. + +An invalid profile stops the run with `P_PROFILE_INVALID`. That covers unreadable YAML, a top level that is not a mapping, an unknown key, a wrong `okf_publish_version`, an unknown id in `disable_default_rules`, a rule without `id` or `pattern`, an invalid or duplicate id, a pattern that does not compile, an empty private string and a non-string value. + +### Built-in rules + +Built-in rules apply even with no profile file. + +| Id | Pattern | Replacement | +| --- | --- | --- | +| `home-path-unix` | `` /(?:home\|Users)/[^/\s'"`)\]>]+ `` | `~` | +| `home-path-windows` | `` [A-Za-z]:\\Users\\[^\\\s'"`)\]>]+ `` | `~` | +| `ipv4` | four dot-separated numbers from 0 to 255 | `[ip]` | +| `github-token` | `\bgh[pousr]_[A-Za-z0-9]{36,}\b` | `[token]` | +| `openai-style-key` | `\bsk-[A-Za-z0-9_-]{20,}` | `[token]` | +| `aws-access-key` | `\bAKIA[0-9A-Z]{16}\b` | `[token]` | +| `slack-token` | `\bxox[abprs]-[A-Za-z0-9-]{10,}` | `[token]` | +| `private-key-block` | `-----BEGIN [A-Z ]*PRIVATE KEY-----` | `[private-key]` | + +`scripts/publish_redact.py` holds the exact patterns in `DEFAULT_RULES`. + +`ipv4` also matches `127.0.0.1` and four-part version numbers such as `1.2.3.4`. When that happens, rewrite the text or add `ipv4` to `disable_default_rules`. There is no per-match allow list. + +## Output layout + +`--out DIR` writes exactly these files, and nothing else: + +- `.html` for each public concept, for example `rules/no-spoilers.html`. +- `index.html` for each published directory, the root included. A directory's `index.md` supplies the page, with private listing lines dropped. A directory without `index.md` gets a generated listing. +- `search.html`, `search-index.json`, `assets/style.css` and `assets/search.js`. +- `.okf-publish`, the output marker. + +The site title is the first `# ` heading of the root `index.md`, or `Knowledge`. Two runs on the same bundle produce byte-identical files. Search runs in the browser and fetches `search-index.json`, so serve the site over HTTP (`python3 -m http.server --directory DIR`) rather than opening files directly. + +## `--out` safety and rebuilds + +`publish.py` refuses `--out` with `P_OUT_UNSAFE` when it: + +- resolves to the bundle root or an ancestor of it; +- exists and is not a directory; or +- is a non-empty directory without the `.okf-publish` marker. + +When the marker is present, every run deletes all entries in `--out` and rebuilds. So a page for a concept that has turned private never survives. An `--out` inside the bundle is allowed and is skipped when walking the bundle. + +## Findings and exit codes + +Each finding is one line: `ERROR : `. The location is a file, a file and line (`rules/a.md:14`), or a directory. Findings never echo matched text, because CI logs of public repos are public. + +| Code | Mode | Meaning | +| --- | --- | --- | +| `P_BUNDLE_MISSING` | both | The `--bundle` directory does not exist. | +| `P_PROFILE_INVALID` | both | The redaction profile is invalid. | +| `P_FRONTMATTER_PARSE` | both | A concept's frontmatter does not parse. | +| `P_VISIBILITY_INVALID` | both | `x_visibility` or `x_visibility_default` is not `public` or `private`. | +| `P_ID_REDACTION` | both | A rule matches a public concept ID or a published directory path. URLs cannot be redacted, so rename it or make it private. | +| `P_REDACTION` | `--check` | A rule matches published text. The message names the rule id and the field; body and index findings carry the file line. | +| `P_OUT_UNSAFE` | `--out` | The `--out` directory is unsafe to write (see above). | + +| Exit | Meaning | +| --- | --- | +| 0 | `--out`: the site was written. `--check`: no rule matches published text. | +| 2 | Any finding above, or a usage error. With `--out`, nothing is written and the existing `--out` contents stay untouched. | + +`--check` scans the source text of public concepts and published index pages. It also scans each body and index line as a reader sees it, with link syntax, backslash escapes and emphasis markers removed, so `**Alice** Example` or `/home\/alice` still fails the check. It passes only after the author removes the matching text. It never passes just because `--out` would redact the text. + +## What is never published + +- `log.md` files. +- Private concepts, and their titles in index listings, search and backlinks. +- Non-Markdown files such as images and PDFs. An image renders as its alt text. +- Anything under a path component that starts with `.` (for example `.git`, `.github`, `.okf-index`), or under `node_modules`. +- Frontmatter keys other than title, description, type, status, tags and sources. The `verified` entries show only as the derived trust tier. +- Raw HTML in Markdown. It renders as escaped text. diff --git a/scripts/publish.py b/scripts/publish.py new file mode 100644 index 0000000..f7c8c5a --- /dev/null +++ b/scripts/publish.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Publish an OKF bundle's public concepts as a static site, or check it for redaction matches.""" + +from __future__ import annotations + +import argparse +from dataclasses import replace +from pathlib import Path +import shutil +import sys + +from okf_common import markdown_links +from publish_markdown import visible_text +from publish_model import Finding, PublishConcept, PublishModel, Source, build_model +from publish_redact import PROFILE_FILENAME, Profile, load_profile, redact, scan +from publish_render import filter_index_markdown, kept_index_lines, render_site, site_title +from publish_search import write_search + +MARKER = ".okf-publish" +MARKER_TEXT = "Generated by OKF publish.py. Every run deletes and rebuilds this directory.\n" + + +def _location_key(finding: Finding) -> tuple[str, int, str, str]: + path, _, line = finding.location.rpartition(":") + if path and line.isdigit(): + return (path, int(line), finding.code, finding.message) + return (finding.location, 0, finding.code, finding.message) + + +def _rule_ids(text: str, profile: Profile) -> list[str]: + return sorted({rule_id for rule_id, _line in scan(text, profile)}) + + +def id_findings(model: PublishModel, profile: Profile) -> list[Finding]: + """Rules that match a public concept ID or a published directory path (URLs cannot be redacted).""" + findings: list[Finding] = [] + for concept_id in model.public_ids(): + for rule_id in _rule_ids(concept_id, profile): + findings.append( + Finding( + "P_ID_REDACTION", + model.concepts[concept_id].path, + f"rule {rule_id} matches the concept ID; rename the concept or make it private", + ) + ) + for directory in sorted(model.public_dirs()): + if not directory: + continue + for rule_id in _rule_ids(directory, profile): + findings.append( + Finding( + "P_ID_REDACTION", + f"{directory}/", + f"rule {rule_id} matches the directory path; rename the directory or make its concepts private", + ) + ) + return findings + + +def out_findings(out: Path, bundle: Path, shown: str) -> list[Finding]: + """Refuse an --out that is the bundle, an ancestor of it, a file, or an unmarked non-empty directory.""" + if out == bundle or out in bundle.parents: + return [Finding("P_OUT_UNSAFE", shown, "--out is the bundle root or an ancestor of it")] + if out.exists() and not out.is_dir(): + return [Finding("P_OUT_UNSAFE", shown, "--out exists and is not a directory")] + if out.is_dir() and any(out.iterdir()) and not (out / MARKER).is_file(): + return [ + Finding( + "P_OUT_UNSAFE", + shown, + f"--out is a non-empty directory without {MARKER}; refusing to delete its contents", + ) + ] + return [] + + +def prepare_out(out: Path) -> None: + out.mkdir(parents=True, exist_ok=True) + if (out / MARKER).is_file(): + for entry in sorted(out.iterdir()): + if entry.is_dir() and not entry.is_symlink(): + shutil.rmtree(entry) + else: + entry.unlink() + with (out / MARKER).open("w", encoding="utf-8", newline="\n") as handle: + handle.write(MARKER_TEXT) + + +def _redact_concept(concept: PublishConcept, profile: Profile, counts: list[int]) -> PublishConcept: + def clean(text: str) -> str: + new_text, fired = redact(text, profile) + counts.append(sum(fired.values())) + return new_text + + body = clean(concept.body) + return replace( + concept, + title=clean(concept.title), + description=clean(concept.description), + type=clean(concept.type), + status=clean(concept.status), + tags=tuple(clean(tag) for tag in concept.tags), + sources=tuple(Source(clean(s.title), clean(s.resource), clean(s.author)) for s in concept.sources), + body=body, + links=tuple(markdown_links(body)), + ) + + +def redacted_copies( + model: PublishModel, index_texts: dict[str, str], profile: Profile +) -> tuple[PublishModel, dict[str, str], int]: + counts: list[int] = [] + concepts = { + concept_id: _redact_concept(concept, profile, counts) if concept.visibility == "public" else concept + for concept_id, concept in model.concepts.items() + } + texts: dict[str, str] = {} + for directory, text in index_texts.items(): + texts[directory], fired = redact(text, profile) + counts.append(sum(fired.values())) + return replace(model, concepts=concepts), texts, sum(counts) + + +def _field_findings(location: str, field: str, text: str, profile: Profile) -> list[Finding]: + return [ + Finding("P_REDACTION", location, f"rule {rule_id} matches published text (field {field})") + for rule_id in _rule_ids(text, profile) + ] + + +def _visible_hits(lines: list[str], profile: Profile) -> list[tuple[str, int]]: + """Scan each line as a reader sees it, so Markdown escapes, emphasis or link syntax cannot split a match.""" + hits: list[tuple[str, int]] = [] + for number, line in enumerate(lines, start=1): + hits.extend((rule_id, number) for rule_id, _line in scan(visible_text(line), profile)) + return hits + + +def check_findings(model: PublishModel, profile: Profile) -> list[Finding]: + """Scan the published text of public concepts and published index pages (``--check``).""" + findings: set[Finding] = set() + for concept_id in model.public_ids(): + concept = model.concepts[concept_id] + fields = [ + ("title", concept.title), + ("description", concept.description), + ("type", concept.type), + ("status", concept.status), + ] + fields.extend(("tags", tag) for tag in concept.tags) + for source in concept.sources: + fields.extend( + [("sources.title", source.title), ("sources.resource", source.resource), ("sources.author", source.author)] + ) + for field, text in fields: + findings.update(_field_findings(concept.path, field, text, profile)) + for rule_id, line in scan(concept.body, profile) + _visible_hits(concept.body.split("\n"), profile): + findings.add( + Finding( + "P_REDACTION", + f"{concept.path}:{concept.body_start_line + line - 1}", + f"rule {rule_id} matches published text (field body)", + ) + ) + + public_dirs = model.public_dirs() + for directory, text in sorted(model.index_files.items()): + if directory not in public_dirs: + continue + kept = kept_index_lines(text, directory, model) + filtered = "\n".join(line for _number, line in kept) + index_path = f"{directory}/index.md" if directory else "index.md" + offset = model.index_offsets.get(directory, 0) + hits = scan(filtered, profile) + _visible_hits([text_line for _number, text_line in kept], profile) + for rule_id, line in hits: + file_line = kept[line - 1][0] + offset + findings.add( + Finding("P_REDACTION", f"{index_path}:{file_line}", f"rule {rule_id} matches published text (field index)") + ) + return list(findings) + + +def _print_findings(findings: list[Finding]) -> None: + for finding in sorted(findings, key=_location_key): + print(finding.line()) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Publish an OKF bundle's public concepts as a static site.") + parser.add_argument("--bundle", default=".", help="bundle root directory (default: current directory)") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--out", help="output directory for the static site") + mode.add_argument("--check", action="store_true", help="scan public text for redaction matches; write nothing") + parser.add_argument("--profile", help=f"redaction profile (default: /{PROFILE_FILENAME} when present)") + args = parser.parse_args(argv) + + bundle = Path(args.bundle).resolve() + if not bundle.is_dir(): + print(Finding("P_BUNDLE_MISSING", args.bundle, "bundle directory not found").line()) + return 2 + + profile_path = Path(args.profile).resolve() if args.profile else None + profile, errors = load_profile(bundle, profile_path) + if profile is None: + shown = args.profile if args.profile else PROFILE_FILENAME + for error in errors: + print(Finding("P_PROFILE_INVALID", shown, error).line()) + return 2 + + out = Path(args.out).resolve() if args.out else None + model, findings = build_model(bundle, exclude=out) + findings.extend(id_findings(model, profile)) + public_dirs = model.public_dirs() + filtered = {d: filter_index_markdown(t, d, model) for d, t in model.index_files.items() if d in public_dirs} + + if args.check: + findings.extend(check_findings(model, profile)) + if findings: + _print_findings(findings) + return 2 + print(f"ok - publish check passed: {len(model.public_ids())} public concept(s), no redaction rule matches") + return 0 + + assert out is not None + findings.extend(out_findings(out, bundle, args.out)) + if findings: + _print_findings(findings) + return 2 + + prepare_out(out) + redacted_model, redacted_texts, redactions = redacted_copies(model, filtered, profile) + render_site(redacted_model, redacted_texts, out) + write_search(redacted_model, out, site_title(redacted_texts)) + + public_count = len(model.public_ids()) + private_count = len(model.concepts) - public_count + print( + f"ok - published {public_count} public concept(s) to {args.out} " + f"({private_count} private excluded, {redactions} redaction(s) applied)" + ) + if public_count == 0: + print( + f"note - no public concepts: default visibility is private ({model.default_reason}); " + "set x_visibility: public on concepts or x_visibility_default: public in schema.md" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/publish_assets/search.js b/scripts/publish_assets/search.js new file mode 100644 index 0000000..40730f1 --- /dev/null +++ b/scripts/publish_assets/search.js @@ -0,0 +1,84 @@ +(function () { + "use strict"; + + var MAX_RESULTS = 50; + var status = document.getElementById("okf-status"); + var results = document.getElementById("okf-results"); + var params = new URLSearchParams(window.location.search); + var query = (params.get("q") || "").trim(); + var input = document.querySelector('header.site input[name="q"]'); + if (input) { + input.value = query; + } + if (!status || !results) { + return; + } + if (!query) { + status.textContent = "Type a search term."; + return; + } + + var terms = query.toLowerCase().split(/\s+/).filter(function (term) { + return term.length > 0; + }); + + function count(haystack, term) { + return haystack.indexOf(term) === -1 ? 0 : 1; + } + + function render(records) { + var matches = []; + records.forEach(function (record) { + var title = String(record.title || "").toLowerCase(); + var description = String(record.description || "").toLowerCase(); + var text = String(record.text || "").toLowerCase(); + var all = title + " " + description + " " + text; + var score = 0; + for (var i = 0; i < terms.length; i += 1) { + if (all.indexOf(terms[i]) === -1) { + return; + } + score += 3 * count(title, terms[i]) + 2 * count(description, terms[i]) + count(text, terms[i]); + } + matches.push({ record: record, score: score }); + }); + matches.sort(function (a, b) { + if (b.score !== a.score) { + return b.score - a.score; + } + return a.record.id < b.record.id ? -1 : a.record.id > b.record.id ? 1 : 0; + }); + + if (matches.length === 0) { + status.textContent = "No results."; + return; + } + status.textContent = matches.length + " result(s)"; + matches.slice(0, MAX_RESULTS).forEach(function (match) { + var item = document.createElement("li"); + var link = document.createElement("a"); + link.setAttribute("href", String(match.record.url)); + link.textContent = String(match.record.title); + item.appendChild(link); + if (match.record.description) { + item.appendChild(document.createTextNode(" ")); + var description = document.createElement("span"); + description.textContent = String(match.record.description); + item.appendChild(description); + } + results.appendChild(item); + }); + } + + fetch("search-index.json") + .then(function (response) { + if (!response.ok) { + throw new Error("search index unavailable"); + } + return response.json(); + }) + .then(render) + .catch(function () { + status.textContent = "Search needs the site to be served over HTTP."; + }); +})(); diff --git a/scripts/publish_assets/style.css b/scripts/publish_assets/style.css new file mode 100644 index 0000000..3a3c4e9 --- /dev/null +++ b/scripts/publish_assets/style.css @@ -0,0 +1,124 @@ +:root { + --bg: #ffffff; + --fg: #1d1f23; + --muted: #5c6370; + --line: #d9dce1; + --link: #1f5fbf; + --code-bg: #f3f4f6; + --badge-bg: #eef0f3; + --ok: #1e7a3c; + --warn: #8a5a00; + --bad: #a3262a; + --info: #2f5f9e; +} + +@media (prefers-color-scheme: dark) { + :root { + --bg: #15171a; + --fg: #e4e6ea; + --muted: #a0a6b0; + --line: #30343a; + --link: #7fb0ff; + --code-bg: #20242a; + --badge-bg: #262a31; + --ok: #6fcf8f; + --warn: #e0b45c; + --bad: #f08a8d; + --info: #8fb8f0; + } +} + +* { box-sizing: border-box; } + +body { + margin: 0; + background: var(--bg); + color: var(--fg); + font-family: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; + line-height: 1.6; +} + +header.site, main, footer.site { + max-width: 48rem; + margin: 0 auto; + padding: 0 1rem; +} + +header.site { + display: flex; + flex-wrap: wrap; + gap: 0.75rem; + align-items: center; + justify-content: space-between; + padding-top: 1rem; + padding-bottom: 1rem; + border-bottom: 1px solid var(--line); +} + +header.site .home { font-weight: 600; color: var(--fg); text-decoration: none; } +header.site input[type="search"] { + font: inherit; + padding: 0.3rem 0.5rem; + border: 1px solid var(--line); + border-radius: 4px; + background: var(--bg); + color: var(--fg); +} + +main { padding-top: 1.5rem; padding-bottom: 2rem; } + +footer.site { + padding-top: 1rem; + padding-bottom: 2rem; + border-top: 1px solid var(--line); + color: var(--muted); + font-size: 0.875rem; +} + +a { color: var(--link); } + +code, pre { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 0.9em; } +code { background: var(--code-bg); padding: 0.1em 0.3em; border-radius: 3px; } +pre { background: var(--code-bg); padding: 0.75rem 1rem; overflow-x: auto; border-radius: 4px; } +pre code { background: none; padding: 0; } + +blockquote { margin: 1rem 0; padding: 0 1rem; border-left: 3px solid var(--line); color: var(--muted); } + +table { border-collapse: collapse; width: 100%; margin: 1rem 0; display: block; overflow-x: auto; } +th, td { border: 1px solid var(--line); padding: 0.35rem 0.6rem; text-align: left; vertical-align: top; } + +hr { border: 0; border-top: 1px solid var(--line); margin: 1.5rem 0; } + +.concept-id { color: var(--muted); margin: 0; } +.concept h1 { margin: 0.25rem 0 0.5rem; } +.description { color: var(--muted); font-size: 1.05rem; } + +.badges { display: flex; flex-wrap: wrap; gap: 0.4rem; margin: 0.5rem 0; } +.badge { + display: inline-block; + padding: 0.05rem 0.5rem; + border: 1px solid var(--line); + border-radius: 3px; + background: var(--badge-bg); + font-size: 0.8rem; +} +.status-stable, .status-active { color: var(--ok); } +.status-draft, .status-proposed { color: var(--info); } +.status-deprecated, .status-superseded, .status-retired { color: var(--warn); } +.status-other { color: var(--muted); } +.trust-human-reviewed { color: var(--ok); } +.trust-machine-confirmed { color: var(--info); } +.trust-unverified { color: var(--muted); } + +ul.tags { list-style: none; padding: 0; display: flex; flex-wrap: wrap; gap: 0.4rem; } +ul.tags li { font-size: 0.8rem; color: var(--muted); } +ul.tags li::before { content: "#"; } + +.sources, .backlinks { margin-top: 2rem; border-top: 1px solid var(--line); } +.sources h2, .backlinks h2 { font-size: 1rem; } + +.footnotes { margin-top: 2rem; border-top: 1px solid var(--line); font-size: 0.9rem; } +.fnref a, .fnback { text-decoration: none; } + +#okf-results li { margin-bottom: 0.5rem; } +#okf-results span { color: var(--muted); } diff --git a/scripts/publish_markdown.py b/scripts/publish_markdown.py new file mode 100644 index 0000000..b1f4879 --- /dev/null +++ b/scripts/publish_markdown.py @@ -0,0 +1,497 @@ +#!/usr/bin/env python3 +"""Render a Markdown subset to fully escaped HTML for published OKF pages.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import html +import re +from typing import Callable, Optional + +LinkResolver = Callable[[str], Optional[str]] + +SAFE_SCHEMES = ("http:", "https:", "mailto:") + +FENCE_OPEN = re.compile(r"^ {0,3}(`{3,}|~{3,})\s*([^\s`]*)") +INFO_WORD = re.compile(r"^[A-Za-z0-9_+-]+$") +HEADING = re.compile(r"^ {0,3}(#{1,6})(?:[ \t]+(.*?))?[ \t]*$") +HEADING_CLOSE = re.compile(r"(?:^|[ \t]+)#+$") +THEMATIC_BREAK = re.compile(r"^ {0,3}([-*_])(?:[ \t]*\1){2,}[ \t]*$") +BLOCKQUOTE = re.compile(r"^ {0,3}> ?") +LIST_ITEM = re.compile(r"^( *)([-*+]|\d{1,9}[.)])(?:[ \t]+(.*))?$") +TABLE_SEPARATOR = re.compile(r"^ *\|? *:?-+:? *(?:\| *:?-+:? *)*\|? *$") +FOOTNOTE_DEF = re.compile(r"^\[\^([^\]\s]+)\]:[ \t]?(.*)$") +AUTOLINK = re.compile(r"<((?:https?://|mailto:)[^\s<>]+)>", re.IGNORECASE) +PLACEHOLDER = re.compile("\x00(\\d+)\x00") +ASCII_PUNCTUATION = set("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") + +STRONG_STAR = re.compile(r"\*\*(?=\S)(.+?)(?<=\S)\*\*") +STRONG_UNDERSCORE = re.compile(r"(? str: + return html.escape(text, quote=True) + + +def slugify(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") + + +def safe_href(href: str) -> bool: + """Return True when an href may be emitted: relative, or http/https/mailto.""" + if not href or any(ord(char) <= 0x20 or ord(char) == 0x7F for char in href): + return False + head = re.split(r"[/?#]", href, maxsplit=1)[0] + if ":" not in head: + return True + return href.lower().startswith(SAFE_SCHEMES) + + +@dataclass +class _Context: + resolve_link: LinkResolver + footnote_defs: dict[str, str] = field(default_factory=dict) + footnote_order: list[str] = field(default_factory=list) + slug_counts: dict[str, int] = field(default_factory=dict) + + def heading_id(self, text: str) -> str: + base = slugify(text) or "section" + count = self.slug_counts.get(base, 0) + 1 + self.slug_counts[base] = count + return base if count == 1 else f"{base}-{count}" + + +def render_markdown(text: str, resolve_link: LinkResolver) -> str: + """Render ``text`` to HTML; ``resolve_link`` maps a raw link target to an href or None.""" + context = _Context(resolve_link=resolve_link) + lines = _collect_footnotes(text.replace("\x00", "").replace("\r\n", "\n").replace("\r", "\n").split("\n"), context) + parts = _render_blocks(lines, context) + footnotes = _render_footnotes(context) + if footnotes: + parts.append(footnotes) + return "\n".join(parts) + + +def _collect_footnotes(lines: list[str], context: _Context) -> list[str]: + output: list[str] = [] + fence: Optional[str] = None + for line in lines: + if fence is not None: + if _closes_fence(line, fence): + fence = None + output.append(line) + continue + opening = FENCE_OPEN.match(line) + if opening: + fence = opening.group(1) + output.append(line) + continue + definition = FOOTNOTE_DEF.match(line) + if definition: + context.footnote_defs.setdefault(definition.group(1), definition.group(2)) + continue + output.append(line) + return output + + +def _closes_fence(line: str, fence: str) -> bool: + stripped = line.strip() + return len(stripped) >= len(fence) and set(stripped) == {fence[0]} and len(line) - len(line.lstrip(" ")) <= 3 + + +def _is_table_start(lines: list[str], index: int) -> bool: + return ( + index + 1 < len(lines) + and "|" in lines[index] + and "|" in lines[index + 1] + and TABLE_SEPARATOR.match(lines[index + 1]) is not None + ) + + +def _starts_block(lines: list[str], index: int) -> bool: + line = lines[index] + return bool( + FENCE_OPEN.match(line) + or HEADING.match(line) + or THEMATIC_BREAK.match(line) + or BLOCKQUOTE.match(line) + or (LIST_ITEM.match(line) and LIST_ITEM.match(line).group(3)) + or _is_table_start(lines, index) + ) + + +def _render_blocks(lines: list[str], context: _Context) -> list[str]: + parts: list[str] = [] + index = 0 + while index < len(lines): + line = lines[index] + if not line.strip(): + index += 1 + continue + + opening = FENCE_OPEN.match(line) + if opening: + fence = opening.group(1) + word = opening.group(2) + index += 1 + content: list[str] = [] + while index < len(lines) and not _closes_fence(lines[index], fence): + content.append(lines[index]) + index += 1 + index += 1 + attr = f' class="language-{escape(word)}"' if word and INFO_WORD.match(word) else "" + code = escape("\n".join(content) + ("\n" if content else "")) + parts.append(f"
          {code}
          ") + continue + + heading = HEADING.match(line) + if heading: + level = len(heading.group(1)) + raw = HEADING_CLOSE.sub("", heading.group(2) or "").strip() + parts.append(f'{_render_inline(raw, context)}') + index += 1 + continue + + if THEMATIC_BREAK.match(line): + parts.append("
          ") + index += 1 + continue + + if BLOCKQUOTE.match(line): + quoted: list[str] = [] + while index < len(lines) and BLOCKQUOTE.match(lines[index]): + quoted.append(BLOCKQUOTE.sub("", lines[index], count=1)) + index += 1 + inner = "\n".join(_render_blocks(quoted, context)) + parts.append(f"
          \n{inner}\n
          ") + continue + + if _is_table_start(lines, index): + table_html, index = _render_table(lines, index, context) + parts.append(table_html) + continue + + item = LIST_ITEM.match(line) + if item and item.group(3): + list_html, index = _render_list(lines, index, len(item.group(1)), context) + parts.append(list_html) + continue + + paragraph = [line.strip()] + index += 1 + while index < len(lines) and lines[index].strip() and not _starts_block(lines, index): + paragraph.append(lines[index].strip()) + index += 1 + parts.append(f"

          {_render_inline(chr(10).join(paragraph), context)}

          ") + return parts + + +def _split_cells(row: str) -> list[str]: + stripped = row.strip() + if stripped.startswith("|"): + stripped = stripped[1:] + if stripped.endswith("|") and not stripped.endswith("\\|"): + stripped = stripped[:-1] + cells = re.split(r"(? tuple[str, int]: + header = _split_cells(lines[index]) + index += 2 + rows: list[list[str]] = [] + while index < len(lines) and lines[index].strip() and "|" in lines[index]: + rows.append(_split_cells(lines[index])) + index += 1 + head = "".join(f"{_render_inline(cell, context)}" for cell in header) + out = ["", f"{head}", ""] + for row in rows: + cells = (row + [""] * len(header))[: len(header)] + out.append("" + "".join(f"" for cell in cells) + "") + out.append("") + out.append("
          {_render_inline(cell, context)}
          ") + return "\n".join(out), index + + +def _list_kind(marker: str) -> str: + return "ul" if marker in ("-", "*", "+") else "ol" + + +def _render_list(lines: list[str], index: int, base_indent: int, context: _Context) -> tuple[str, int]: + first = LIST_ITEM.match(lines[index]) + assert first is not None + kind = _list_kind(first.group(2)) + start = "" + if kind == "ol": + number = int(first.group(2)[:-1]) + if number != 1: + start = f' start="{number}"' + items: list[str] = [] + while index < len(lines): + match = LIST_ITEM.match(lines[index]) + if not match or not match.group(3) or len(match.group(1)) != base_indent or _list_kind(match.group(2)) != kind: + break + text_lines = [match.group(3).strip()] + nested: list[str] = [] + index += 1 + while index < len(lines): + line = lines[index] + if not line.strip(): + lookahead = index + 1 + while lookahead < len(lines) and not lines[lookahead].strip(): + lookahead += 1 + following = LIST_ITEM.match(lines[lookahead]) if lookahead < len(lines) else None + if following and following.group(3) and len(following.group(1)) >= base_indent: + index = lookahead + continue + break + child = LIST_ITEM.match(line) + if child and child.group(3): + indent = len(child.group(1)) + if indent >= base_indent + 2: + nested_html, index = _render_list(lines, index, indent, context) + nested.append(nested_html) + continue + break + if not nested and not _starts_block(lines, index): + text_lines.append(line.strip()) + index += 1 + continue + break + body = _render_inline("\n".join(text_lines), context) + items.append("
        • " + body + ("\n" + "\n".join(nested) if nested else "") + "
        • ") + return f"<{kind}{start}>\n" + "\n".join(items) + f"\n", index + + +def _render_footnotes(context: _Context) -> str: + labels = list(context.footnote_order) + rendered: list[str] = [] + position = 0 + while position < len(labels): + label = labels[position] + slug = escape(_footnote_slug(label)) + body = _render_inline(context.footnote_defs[label], context) + labels = list(context.footnote_order) + rendered.append(f'
        • {body} ↩
        • ') + position += 1 + if not rendered: + return "" + return '
          \n
            \n' + "\n".join(rendered) + "\n
          \n
          " + + +def _footnote_slug(label: str) -> str: + return slugify(label) or "note" + + +def _find_closing_bracket(text: str, start: int) -> int: + depth = 0 + index = start + while index < len(text): + char = text[index] + if char == "\\": + index += 2 + continue + if char == "[": + depth += 1 + elif char == "]": + depth -= 1 + if depth == 0: + return index + index += 1 + return -1 + + +def _parse_destination(text: str, start: int) -> Optional[tuple[str, int]]: + """Parse ``(target "title")`` starting at the ``(``; return (target, index after ``)``).""" + index = start + 1 + while index < len(text) and text[index] in " \t": + index += 1 + if index < len(text) and text[index] == "<": + end = text.find(">", index) + if end == -1 or "\n" in text[index:end]: + return None + target = text[index + 1 : end] + index = end + 1 + else: + depth = 0 + begin = index + while index < len(text): + char = text[index] + if char.isspace(): + break + if char == "(": + depth += 1 + elif char == ")": + if depth == 0: + break + depth -= 1 + index += 1 + target = text[begin:index] + while index < len(text) and text[index] in " \t\n": + index += 1 + if index < len(text) and text[index] in "\"'": + quote = text[index] + end = text.find(quote, index + 1) + if end == -1: + return None + index = end + 1 + while index < len(text) and text[index] in " \t": + index += 1 + if index >= len(text) or text[index] != ")": + return None + return target, index + 1 + + +def _render_inline(text: str, context: _Context) -> str: + stash: list[str] = [] + + def hold(fragment: str) -> str: + stash.append(fragment) + return f"\x00{len(stash) - 1}\x00" + + out: list[str] = [] + index = 0 + while index < len(text): + char = text[index] + + if char == "\\" and index + 1 < len(text) and text[index + 1] in ASCII_PUNCTUATION: + out.append(hold(escape(text[index + 1]))) + index += 2 + continue + + if char == "`": + run = len(text[index:]) - len(text[index:].lstrip("`")) + run_end = index + run + marker = "`" * run + closing = re.compile(f"(?= 2 and code.startswith(" ") and code.endswith(" ") and code.strip(): + code = code[1:-1] + out.append(hold(f"{escape(code)}")) + index = close + run + continue + + if char == "<": + autolink = AUTOLINK.match(text, index) + if autolink: + target = autolink.group(1) + if safe_href(target): + out.append(hold(f'{escape(target)}')) + else: + out.append(hold(escape(target))) + index = autolink.end() + continue + + if char == "!" and text.startswith("![", index): + close = _find_closing_bracket(text, index + 1) + if close != -1 and close + 1 < len(text) and text[close + 1] == "(": + parsed = _parse_destination(text, close + 1) + if parsed is not None: + out.append(hold(escape(_unescape_backslashes(text[index + 2 : close])))) + index = parsed[1] + continue + + if char == "[": + if text.startswith("[^", index): + end = text.find("]", index) + label = text[index + 2 : end] if end != -1 else "" + if end != -1 and label and label in context.footnote_defs: + if label not in context.footnote_order: + context.footnote_order.append(label) + anchor_id = f' id="fnref-{escape(_footnote_slug(label))}"' + else: + anchor_id = "" + number = context.footnote_order.index(label) + 1 + slug = escape(_footnote_slug(label)) + out.append(hold(f'{number}')) + index = end + 1 + continue + close = _find_closing_bracket(text, index) + if close != -1 and close + 1 < len(text) and text[close + 1] == "(": + parsed = _parse_destination(text, close + 1) + if parsed is not None: + target, after = parsed + label_html = _render_inline(text[index + 1 : close], context) + href = context.resolve_link(target) + if href is not None and safe_href(href): + out.append(hold(f'{label_html}')) + else: + out.append(hold(label_html)) + index = after + continue + + out.append(char) + index += 1 + + rendered = escape("".join(out)) + rendered = STRONG_STAR.sub(r"\1", rendered) + rendered = STRONG_UNDERSCORE.sub(r"\1", rendered) + rendered = EM_STAR.sub(r"\1", rendered) + rendered = EM_UNDERSCORE.sub(r"\1", rendered) + return PLACEHOLDER.sub(lambda match: stash[int(match.group(1))], rendered) + + +def _unescape_backslashes(text: str) -> str: + return re.sub(r"\\([!-/:-@\[-`{-~])", r"\1", text) + + +@dataclass(frozen=True) +class LinkSpan: + start: int # index of "[" (or "!" for an image) + end: int # index after the closing ")" + is_image: bool + label: str # raw text between the brackets + target: str # raw destination, as the renderer's resolver receives it + + +def find_links(text: str) -> list[LinkSpan]: + """Return inline links and images exactly as ``render_markdown`` parses them, in order.""" + spans: list[LinkSpan] = [] + index = 0 + while index < len(text): + char = text[index] + if char == "\\": + index += 2 + continue + if char != "[": + index += 1 + continue + is_image = index > 0 and text[index - 1] == "!" and (index < 2 or text[index - 2] != "\\") + close = _find_closing_bracket(text, index) + if close != -1 and close + 1 < len(text) and text[close + 1] == "(": + parsed = _parse_destination(text, close + 1) + if parsed is not None: + target, after = parsed + start = index - 1 if is_image else index + spans.append(LinkSpan(start, after, is_image, text[index + 1 : close], target)) + index = after + continue + index += 1 + return spans + + +def strip_links(text: str) -> str: + """Replace each link and image with its label text, dropping every destination.""" + out: list[str] = [] + position = 0 + for span in find_links(text): + out.append(text[position : span.start]) + out.append(span.label if span.is_image else strip_links(span.label)) + position = span.end + out.append(text[position:]) + return "".join(out) + + +def visible_text(line: str) -> str: + """Approximate what a reader sees for one line: links as text, escapes and emphasis markers removed.""" + text = strip_links(line) + text = re.sub(r"\\([!-/:-@\[-`{-~])", r"\1", text) + text = text.replace("`", "").replace("*", "") + return re.sub(r"(? str: + return f"ERROR {self.code} {self.location}: {self.message}" + + +@dataclass +class PublishModel: + bundle_root: Path + default_visibility: str + default_reason: str + concepts: dict[str, PublishConcept] # every parsed concept, keyed by ID + index_files: dict[str, str] # directory -> index.md text with any frontmatter removed + index_offsets: dict[str, int] = field(default_factory=dict) # directory -> frontmatter lines stripped + + _public_ids: list[str] | None = field(default=None, init=False, repr=False, compare=False) + _public_dirs: frozenset[str] | None = field(default=None, init=False, repr=False, compare=False) + _backlinks: dict[str, list[str]] | None = field(default=None, init=False, repr=False, compare=False) + + # The caches assume ``concepts`` is not mutated after the first query; build a new model + # (for example with ``dataclasses.replace``) instead, which starts with empty caches. + + def public_ids(self) -> list[str]: + if self._public_ids is None: + self._public_ids = sorted( + concept_id for concept_id, concept in self.concepts.items() if concept.visibility == "public" + ) + return list(self._public_ids) + + def is_public(self, concept_id: str) -> bool: + concept = self.concepts.get(concept_id) + return concept is not None and concept.visibility == "public" + + def public_dirs(self) -> set[str]: + if self._public_dirs is None: + dirs = {""} + for concept_id in self.public_ids(): + directory = self.concepts[concept_id].directory + parts = directory.split("/") if directory else [] + for depth in range(1, len(parts) + 1): + dirs.add("/".join(parts[:depth])) + self._public_dirs = frozenset(dirs) + return set(self._public_dirs) + + def backlinks(self, concept_id: str) -> list[str]: + if self._backlinks is None: + reverse: dict[str, set[str]] = {} + for source_id in self.public_ids(): + source = self.concepts[source_id] + for link in source.links: + target = classify_link(source.directory, link) + if target.kind == "concept" and target.value != source_id: + reverse.setdefault(target.value, set()).add(source_id) + self._backlinks = {target: sorted(sources) for target, sources in reverse.items()} + return list(self._backlinks.get(concept_id, [])) + + +def classify_link(source_dir: str, target: str) -> LinkTarget: + """Classify a raw Markdown link target found on a page in ``source_dir`` (section 4.3).""" + path, _, fragment = target.partition("#") + if path == "": + return LinkTarget("fragment", "#" + fragment, fragment) + if path.lower().startswith(EXTERNAL_PREFIXES): + return LinkTarget("external", target, fragment) + if ":" in path: + return LinkTarget("other", target, fragment) + + if path.startswith("/"): + raw_parts = path[1:].split("/") + else: + raw_parts = (source_dir.split("/") if source_dir else []) + path.split("/") + + parts: list[str] = [] + for part in raw_parts: + if part in ("", "."): + continue + if part == "..": + if not parts: + return LinkTarget("other", target, fragment) + parts.pop() + continue + parts.append(part) + + last = path.rsplit("/", 1)[-1] + if last in ("", ".", "..") or not parts: + return LinkTarget("directory", "/".join(parts), fragment) + name = parts[-1] + if name == "index.md": + return LinkTarget("directory", "/".join(parts[:-1]), fragment) + if name == "log.md": + return LinkTarget("other", target, fragment) + if name.endswith(".md"): + return LinkTarget("concept", "/".join(parts)[: -len(".md")], fragment) + return LinkTarget("other", target, fragment) + + +def walk_markdown(bundle_root: Path, exclude: Path | None = None) -> list[Path]: + """Return sorted ``*.md`` files, skipping dot-components, ``node_modules`` and ``exclude``.""" + root = bundle_root.resolve() + excluded: Path | None = None + if exclude is not None: + candidate = exclude.resolve() + if candidate != root and root in candidate.parents: + excluded = candidate + paths: list[Path] = [] + for path in root.rglob("*.md"): + if not path.is_file(): + continue + relative = path.relative_to(root) + if any(part.startswith(".") or part == "node_modules" for part in relative.parts): + continue + if excluded is not None and (path == excluded or excluded in path.parents): + continue + paths.append(path) + return sorted(paths, key=lambda item: item.relative_to(root).as_posix()) + + +def trust_tier(frontmatter: dict) -> str: + verified = frontmatter.get("verified") + if verified is None: + entries: list[Any] = [] + elif isinstance(verified, dict): + entries = [verified] + elif isinstance(verified, list): + entries = [entry for entry in verified if isinstance(entry, dict)] + else: + entries = [] + if any(isinstance(entry.get("by"), str) and entry["by"].startswith("human:") for entry in entries): + return "human-reviewed" + if entries: + return "machine-confirmed" + return "unverified" + + +def _text(value: Any) -> str: + return "" if value is None else str(value) + + +def _tags(value: Any) -> tuple[str, ...]: + if value is None: + return () + if isinstance(value, list): + return tuple(str(item) for item in value) + return (str(value),) + + +def _sources(value: Any) -> tuple[Source, ...]: + if not isinstance(value, list): + return () + sources: list[Source] = [] + for item in value: + if isinstance(item, dict): + if item.get("resource") is None: + continue + sources.append(Source(_text(item.get("title")), str(item["resource"]), _text(item.get("author")))) + elif isinstance(item, str): + sources.append(Source("", item, "")) + return tuple(sources) + + +def _body_start_line(lines: list[str]) -> int: + closing = next(index for index, line in enumerate(lines[1:], start=1) if line == "---") + start = closing + 2 + if closing + 1 < len(lines) and lines[closing + 1] == "": + start += 1 + return start + + +def _parse_error(exc: Exception) -> str: + if isinstance(exc, yaml.YAMLError): + problem = getattr(exc, "problem", None) or "invalid YAML" + mark = getattr(exc, "problem_mark", None) + where = f" (frontmatter line {mark.line + 1})" if mark is not None else "" + return f"frontmatter does not parse: {problem}{where}" + return str(exc) + + +def _strip_index_frontmatter(text: str) -> tuple[str, int]: + if not text.startswith("---\n"): + return text, 0 + lines = text.split("\n") + for index, line in enumerate(lines[1:], start=1): + if line == "---": + return "\n".join(lines[index + 1 :]), index + 1 + return text, 0 + + +def _bundle_default(root: Path, findings: list[Finding]) -> tuple[str, str]: + schema = root / SCHEMA_FILE + if not schema.is_file(): + return "private", f"no {SCHEMA_FILE} at bundle root" + try: + frontmatter, _body = split_frontmatter(read_text(schema)) + except (OkfError, yaml.YAMLError, UnicodeDecodeError): + return "private", f"{SCHEMA_FILE} frontmatter does not parse" + if DEFAULT_KEY not in frontmatter: + return "private", f"{SCHEMA_FILE} has no {DEFAULT_KEY}" + value = frontmatter[DEFAULT_KEY] + if isinstance(value, str) and value in VISIBILITY_VALUES: + return value, f"{SCHEMA_FILE} {DEFAULT_KEY}: {value}" + findings.append(Finding("P_VISIBILITY_INVALID", SCHEMA_FILE, f"{DEFAULT_KEY} must be public or private")) + return "private", f"{SCHEMA_FILE} {DEFAULT_KEY} is invalid" + + +def build_model(bundle_root: Path, exclude: Path | None = None) -> tuple[PublishModel, list[Finding]]: + root = bundle_root.resolve() + findings: list[Finding] = [] + default_visibility, default_reason = _bundle_default(root, findings) + concepts: dict[str, PublishConcept] = {} + index_files: dict[str, str] = {} + index_offsets: dict[str, int] = {} + + for path in walk_markdown(root, exclude): + relative = path.relative_to(root) + rel = relative.as_posix() + directory = "" if relative.parent.as_posix() == "." else relative.parent.as_posix() + if relative.name == "log.md": + continue + try: + text = normalize_newlines(read_text(path)) + except UnicodeDecodeError as exc: + findings.append(Finding("P_FRONTMATTER_PARSE", rel, f"file is not UTF-8: {exc.reason}")) + continue + if relative.name == "index.md": + index_files[directory], index_offsets[directory] = _strip_index_frontmatter(text) + continue + try: + frontmatter, body = split_frontmatter(text) + except (OkfError, yaml.YAMLError) as exc: + findings.append(Finding("P_FRONTMATTER_PARSE", rel, _parse_error(exc))) + continue + + concept_id = concept_id_for_path(relative) + if VISIBILITY_KEY not in frontmatter: + visibility = default_visibility + else: + value = frontmatter[VISIBILITY_KEY] + if isinstance(value, str) and value in VISIBILITY_VALUES: + visibility = value + else: + findings.append(Finding("P_VISIBILITY_INVALID", rel, f"{VISIBILITY_KEY} must be public or private")) + visibility = "private" + + concepts[concept_id] = PublishConcept( + concept_id=concept_id, + path=rel, + directory=directory, + title=_text(frontmatter.get("title")) or concept_id.rsplit("/", 1)[-1], + description=_text(frontmatter.get("description")), + type=_text(frontmatter.get("type")), + tags=_tags(frontmatter.get("tags")), + status=_text(frontmatter.get("status")) or "stable", + trust_tier=trust_tier(frontmatter), + sources=_sources(frontmatter.get("sources")), + body=body, + body_start_line=_body_start_line(text.split("\n")), + visibility=visibility, + links=tuple(markdown_links(body)), + ) + + model = PublishModel( + bundle_root=root, + default_visibility=default_visibility, + default_reason=default_reason, + concepts=concepts, + index_files=index_files, + index_offsets=index_offsets, + ) + return model, findings diff --git a/scripts/publish_redact.py b/scripts/publish_redact.py new file mode 100644 index 0000000..ab774a3 --- /dev/null +++ b/scripts/publish_redact.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Load OKF publish redaction profiles and apply or scan their regex rules.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import re +from typing import Any + +import yaml + +PROFILE_FILENAME = "okf-publish.yaml" +PROFILE_VERSION = "0.1" +DEFAULT_REPLACE = "[redacted]" +RULE_ID = re.compile(r"^[a-z0-9][a-z0-9-]*$") +TOP_LEVEL_KEYS = ("okf_publish_version", "disable_default_rules", "rules", "private_strings") +RULE_KEYS = ("id", "pattern", "replace") + +DEFAULT_RULES: tuple[tuple[str, str, str], ...] = ( + ("home-path-unix", r"/(?:home|Users)/[^/\s'\"`)\]>]+", "~"), + ("home-path-windows", r"[A-Za-z]:\\Users\\[^\\\s'\"`)\]>]+", "~"), + ("ipv4", r"\b(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}\b", "[ip]"), + ("github-token", r"\bgh[pousr]_[A-Za-z0-9]{36,}\b", "[token]"), + ("openai-style-key", r"\bsk-[A-Za-z0-9_-]{20,}", "[token]"), + ("aws-access-key", r"\bAKIA[0-9A-Z]{16}\b", "[token]"), + ("slack-token", r"\bxox[abprs]-[A-Za-z0-9-]{10,}", "[token]"), + ("private-key-block", r"-----BEGIN [A-Z ]*PRIVATE KEY-----", "[private-key]"), +) +DEFAULT_RULE_IDS = tuple(rule_id for rule_id, _pattern, _replace in DEFAULT_RULES) + + +@dataclass(frozen=True) +class Rule: + id: str + pattern: re.Pattern[str] + replace: str + origin: str # "default" | "profile" | "private_strings" + + +@dataclass(frozen=True) +class Profile: + rules: tuple[Rule, ...] + path: Path | None # None when built-in rules only + + +def _default_rules(disabled: set[str] | frozenset[str] = frozenset()) -> list[Rule]: + return [ + Rule(rule_id, re.compile(pattern), replace, "default") + for rule_id, pattern, replace in DEFAULT_RULES + if rule_id not in disabled + ] + + +def default_profile() -> Profile: + return Profile(rules=tuple(_default_rules()), path=None) + + +def _string_list(value: Any, field: str, errors: list[str]) -> list[str]: + if value is None: + return [] + if not isinstance(value, list): + errors.append(f"{field} must be a list.") + return [] + items: list[str] = [] + for position, item in enumerate(value, start=1): + if not isinstance(item, str): + errors.append(f"{field} entry {position} must be a string.") + continue + items.append(item) + return items + + +def _parse_rules(value: Any, errors: list[str]) -> list[Rule]: + if value is None: + return [] + if not isinstance(value, list): + errors.append("rules must be a list.") + return [] + rules: list[Rule] = [] + for position, entry in enumerate(value, start=1): + label = f"rules entry {position}" + if not isinstance(entry, dict): + errors.append(f"{label} must be a mapping.") + continue + for key in sorted(str(key) for key in entry if key not in RULE_KEYS): + errors.append(f"{label} has unknown key {key}.") + rule_id = entry.get("id") + pattern = entry.get("pattern") + replace = entry.get("replace", DEFAULT_REPLACE) + if rule_id is None: + errors.append(f"{label} is missing id.") + continue + if not isinstance(rule_id, str): + errors.append(f"{label} id must be a string.") + continue + if not RULE_ID.match(rule_id): + errors.append(f"{label} id {rule_id!r} must match ^[a-z0-9][a-z0-9-]*$.") + continue + if pattern is None: + errors.append(f"rule {rule_id} is missing pattern.") + continue + if not isinstance(pattern, str): + errors.append(f"rule {rule_id} pattern must be a string.") + continue + if not isinstance(replace, str): + errors.append(f"rule {rule_id} replace must be a string.") + continue + try: + compiled = re.compile(pattern) + except re.error as exc: + errors.append(f"rule {rule_id} pattern does not compile: {exc}.") + continue + rules.append(Rule(rule_id, compiled, replace, "profile")) + return rules + + +def _private_pattern(text: str) -> re.Pattern[str]: + """Match ``text`` literally and case-insensitively; any whitespace run, including a line break, matches.""" + return re.compile(r"\s+".join(re.escape(word) for word in text.split()), re.IGNORECASE) + + +def _parse_profile(data: Any, path: Path) -> tuple[Profile | None, list[str]]: + errors: list[str] = [] + if not isinstance(data, dict): + return None, ["profile top level must be a mapping."] + + for key in sorted(str(key) for key in data if key not in TOP_LEVEL_KEYS): + errors.append(f"unknown top-level key {key}.") + + if data.get("okf_publish_version") != PROFILE_VERSION: + errors.append(f'okf_publish_version must be the string "{PROFILE_VERSION}".') + + disabled = _string_list(data.get("disable_default_rules"), "disable_default_rules", errors) + for rule_id in disabled: + if rule_id not in DEFAULT_RULE_IDS: + errors.append(f"disable_default_rules names unknown built-in rule id {rule_id}.") + + profile_rules = _parse_rules(data.get("rules"), errors) + + private_rules: list[Rule] = [] + for position, text in enumerate(_string_list(data.get("private_strings"), "private_strings", errors), start=1): + if not text.strip(): + errors.append(f"private_strings entry {position} must not be empty.") + continue + private_rules.append( + Rule(f"private-string-{position}", _private_pattern(text), DEFAULT_REPLACE, "private_strings") + ) + + seen = set(DEFAULT_RULE_IDS) | {rule.id for rule in private_rules} + for rule in profile_rules: + if rule.id in seen: + errors.append(f"rules id {rule.id} is a duplicate rule id.") + seen.add(rule.id) + + if errors: + return None, errors + rules = _default_rules(set(disabled)) + profile_rules + private_rules + return Profile(rules=tuple(rules), path=path), [] + + +def load_profile(bundle_root: Path, profile_path: Path | None = None) -> tuple[Profile | None, list[str]]: + """Load ``profile_path`` or ``/okf-publish.yaml``; fall back to the built-in rules.""" + if profile_path is None: + candidate = bundle_root / PROFILE_FILENAME + if not candidate.exists(): + return default_profile(), [] + path = candidate + else: + path = profile_path + if not path.is_file(): + return None, ["profile file not found."] + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError) as exc: + return None, [f"profile cannot be read: {exc}."] + except yaml.YAMLError as exc: + problem = getattr(exc, "problem", None) or "parse error" + return None, [f"profile is not valid YAML: {problem}."] + return _parse_profile(data, path) + + +def _literal(replace: str) -> str: + return replace.replace("\\", "\\\\") + + +def redact(text: str, profile: Profile) -> tuple[str, dict[str, int]]: + """Apply every rule in order; return the new text and substitution counts by rule id.""" + counts: dict[str, int] = {} + for rule in profile.rules: + text, count = rule.pattern.subn(_literal(rule.replace), text) + if count: + counts[rule.id] = counts.get(rule.id, 0) + count + return text, counts + + +def scan(text: str, profile: Profile) -> list[tuple[str, int]]: + """Return sorted unique ``(rule_id, line)`` pairs for matches in ``text``; never the matched text.""" + hits: set[tuple[str, int]] = set() + for rule in profile.rules: + for match in rule.pattern.finditer(text): + hits.add((rule.id, text.count("\n", 0, match.start()) + 1)) + return sorted(hits) diff --git a/scripts/publish_render.py b/scripts/publish_render.py new file mode 100644 index 0000000..a31643f --- /dev/null +++ b/scripts/publish_render.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Render published OKF concept pages, index pages and the site stylesheet.""" + +from __future__ import annotations + +from pathlib import Path +import re +from urllib.parse import quote + +from publish_markdown import LinkResolver, escape, find_links, render_markdown, safe_href +from publish_model import PublishConcept, PublishModel, classify_link + +ASSETS_DIR = Path(__file__).resolve().parent / "publish_assets" +DEFAULT_SITE_TITLE = "Knowledge" +INDEX_MARKERS = ("", "") +LISTING_LINE = re.compile(r"^\s*[*-] ") +STATUS_CLASS = re.compile(r"^[a-z0-9-]+$") +MARKDOWN_SPECIALS = re.compile(r"([\\`*_\[\]])") + + +def concept_page_path(concept_id: str) -> str: + return f"{concept_id}.html" + + +def dir_index_path(directory: str) -> str: + return "index.html" if directory == "" else f"{directory}/index.html" + + +def relative_href(from_page: str, to_page: str) -> str: + """Return a page-relative href from ``from_page`` to ``to_page`` (both site-root relative).""" + from_dir = from_page.split("/")[:-1] + to_parts = to_page.split("/") + to_dir, name = to_parts[:-1], to_parts[-1] + common = 0 + while common < len(from_dir) and common < len(to_dir) and from_dir[common] == to_dir[common]: + common += 1 + segments = to_dir[common:] + [name] + return "../" * (len(from_dir) - common) + "/".join(quote(segment, safe="") for segment in segments) + + +def make_resolver(model: PublishModel, source_dir: str, page_path: str) -> LinkResolver: + public_dirs = model.public_dirs() + + def resolve(raw_target: str) -> str | None: + target = classify_link(source_dir, raw_target) + if target.kind == "concept" and model.is_public(target.value): + href = relative_href(page_path, concept_page_path(target.value)) + return f"{href}#{target.fragment}" if target.fragment else href + if target.kind == "directory" and target.value in public_dirs: + return relative_href(page_path, dir_index_path(target.value)) + if target.kind in ("external", "fragment"): + return target.value + return None + + return resolve + + +def _first_link_target(line: str) -> str | None: + links = [span for span in find_links(line) if not span.is_image] + return links[0].target if links else None + + +def _links_to_published(target: str | None, directory: str, model: PublishModel) -> bool: + if target is None: + return False + link = classify_link(directory, target) + if link.kind == "concept": + return model.is_public(link.value) + if link.kind == "directory": + return link.value in model.public_dirs() + return False + + +def _links_to_unpublished(target: str | None, directory: str, model: PublishModel) -> bool: + if target is None: + return False + link = classify_link(directory, target) + if link.kind == "concept": + return not model.is_public(link.value) + if link.kind == "directory": + return link.value not in model.public_dirs() + return False + + +def kept_index_lines(text: str, directory: str, model: PublishModel) -> list[tuple[int, str]]: + """Return ``(1-based line, line)`` pairs that survive index filtering. + + Inside the generated ``OKF:INDEX`` block, a line survives only when it is a listing line whose + first link resolves to a public concept or a published directory; everything else there + (private entries, continuation lines of multi-line descriptions, lines whose link does not + parse) is dropped, so the block fails closed. Outside the block, hand-written listing lines + whose first link points at an unpublished target are dropped and all other lines are kept. + """ + kept: list[tuple[int, str]] = [] + in_block = False + for number, line in enumerate(text.split("\n"), start=1): + stripped = line.strip() + if stripped == INDEX_MARKERS[0]: + in_block = True + kept.append((number, line)) + continue + if stripped == INDEX_MARKERS[1]: + in_block = False + kept.append((number, line)) + continue + is_listing = LISTING_LINE.match(line) is not None + if in_block: + if not stripped: + kept.append((number, line)) + elif is_listing and _links_to_published(_first_link_target(line), directory, model): + kept.append((number, line)) + continue + if is_listing and _links_to_unpublished(_first_link_target(line), directory, model): + continue + kept.append((number, line)) + return kept + + +def filter_index_markdown(text: str, directory: str, model: PublishModel) -> str: + return "\n".join(line for _number, line in kept_index_lines(text, directory, model)) + + +def first_heading(text: str) -> str: + for line in text.split("\n"): + if line.startswith("# "): + return line[2:].strip() + return "" + + +def site_title(index_texts: dict[str, str]) -> str: + return first_heading(index_texts.get("", "")) or DEFAULT_SITE_TITLE + + +def page_shell( + *, title: str, site_title: str, page_path: str, body_html: str, scripts: tuple[str, ...] = () +) -> str: + rel = "../" * page_path.count("/") + lines = [ + "", + '', + "", + '', + '', + f"{escape(title)} · {escape(site_title)}", + f'', + "", + "", + f'
          {escape(site_title)}', + f'', + "
          ", + "
          ", + body_html, + "
          ", + '
          Published from an Open Knowledge Format bundle.
          ', + ] + lines.extend(f'' for path in scripts) + lines.extend(["", ""]) + return "\n".join(lines) + "\n" + + +def _status_class(status: str) -> str: + lowered = status.lower() + return lowered if STATUS_CLASS.match(lowered) else "other" + + +def _link_html(href: str | None, label: str) -> str: + if href is not None and safe_href(href): + return f'{escape(label)}' + return escape(label) + + +def concept_body(concept: PublishConcept, model: PublishModel) -> str: + page = concept_page_path(concept.concept_id) + resolver = make_resolver(model, concept.directory, page) + badges = [] + if concept.type: + badges.append(f'{escape(concept.type)}') + badges.append(f'{escape(concept.status)}') + badges.append(f'{escape(concept.trust_tier)}') + parts = [ + '
          ', + f'

          {escape(concept.concept_id)}

          ', + f"

          {escape(concept.title)}

          ", + '

          ' + " ".join(badges) + "

          ", + ] + if concept.description: + parts.append(f'

          {escape(concept.description)}

          ') + if concept.tags: + parts.append('
            ' + "".join(f"
          • {escape(tag)}
          • " for tag in concept.tags) + "
          ") + parts.append(f'
          {render_markdown(concept.body, resolver)}
          ') + if concept.sources: + items = [] + for source in concept.sources: + label = _link_html(resolver(source.resource), source.title or source.resource) + author = f" — {escape(source.author)}" if source.author else "" + items.append(f"
        • {label}{author}
        • ") + parts.append('

          Sources

            ' + "".join(items) + "
          ") + backlinks = model.backlinks(concept.concept_id) + if backlinks: + items = [ + f'
        • ' + f"{escape(model.concepts[source_id].title)}
        • " + for source_id in backlinks + ] + parts.append('") + parts.append("
          ") + return "\n".join(parts) + + +def _markdown_text(text: str) -> str: + return MARKDOWN_SPECIALS.sub(r"\\\1", text) + + +def _markdown_target(target: str) -> str: + return f"<{target}>" if re.search(r"[\s()<>]", target) else target + + +def generated_listing(directory: str, model: PublishModel, title: str) -> str: + lines = [f"# {_markdown_text(title)}", ""] + for concept_id in model.public_ids(): + concept = model.concepts[concept_id] + if concept.directory != directory: + continue + file_name = concept.path.rsplit("/", 1)[-1] + suffix = f" - {_markdown_text(concept.description)}" if concept.description else "" + lines.append(f"* [{_markdown_text(concept.title)}]({_markdown_target(file_name)}){suffix}") + for child in sorted(model.public_dirs()): + if child == "": + continue + parent, _, name = child.rpartition("/") + if parent != directory: + continue + lines.append(f"* [{_markdown_text(name)}/]({_markdown_target(name + '/')})") + return "\n".join(lines) + "\n" + + +def write_file(out_dir: Path, relative: str, content: str) -> None: + target = out_dir / relative + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(content) + + +def copy_asset(out_dir: Path, name: str) -> str: + relative = f"assets/{name}" + write_file(out_dir, relative, (ASSETS_DIR / name).read_text(encoding="utf-8")) + return relative + + +def render_site(model: PublishModel, index_texts: dict[str, str], out_dir: Path) -> list[str]: + """Write concept pages, directory index pages and the stylesheet; return sorted written paths.""" + title = site_title(index_texts) + written: list[str] = [] + for concept_id in model.public_ids(): + concept = model.concepts[concept_id] + page = concept_page_path(concept_id) + write_file(out_dir, page, page_shell(title=concept.title, site_title=title, page_path=page, body_html=concept_body(concept, model))) + written.append(page) + + for directory in sorted(model.public_dirs()): + page = dir_index_path(directory) + name = directory.rsplit("/", 1)[-1] + if directory in index_texts: + text = "\n".join(line for line in index_texts[directory].split("\n") if line.strip() not in INDEX_MARKERS) + else: + text = generated_listing(directory, model, name or title) + page_title = first_heading(text) or name or title + body = render_markdown(text, make_resolver(model, directory, page)) + write_file(out_dir, page, page_shell(title=page_title, site_title=title, page_path=page, body_html=f'
          {body}
          ')) + written.append(page) + + written.append(copy_asset(out_dir, "style.css")) + return sorted(written) diff --git a/scripts/publish_search.py b/scripts/publish_search.py new file mode 100644 index 0000000..79398bd --- /dev/null +++ b/scripts/publish_search.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Write the client-side search index, search page and search script for a published site.""" + +from __future__ import annotations + +import json +from pathlib import Path +import re + +from publish_markdown import strip_links +from publish_model import PublishModel +from publish_render import concept_page_path, copy_asset, page_shell, relative_href, write_file + +FENCE_LINE = re.compile(r"^ {0,3}(?:`{3,}|~{3,}).*$", re.MULTILINE) +FOOTNOTE_MARK = re.compile(r"\[\^[^\]\s]+\]:?") +WHITESPACE = re.compile(r"\s+") +SEARCH_BODY = '

          Search

            ' + + +def search_text(body: str) -> str: + """Return plain searchable text: no fence lines, link targets or footnote markers.""" + text = FENCE_LINE.sub("", body) + text = strip_links(text) + text = FOOTNOTE_MARK.sub("", text) + return WHITESPACE.sub(" ", text).strip() + + +def search_records(model: PublishModel) -> list[dict[str, str]]: + records = [] + for concept_id in model.public_ids(): + concept = model.concepts[concept_id] + records.append( + { + "id": concept_id, + "title": concept.title, + "description": concept.description, + "type": concept.type, + "url": relative_href("search.html", concept_page_path(concept_id)), + "text": search_text(concept.body), + } + ) + return records + + +def write_search(model: PublishModel, out_dir: Path, site_title: str) -> None: + payload = json.dumps(search_records(model), sort_keys=True, ensure_ascii=False, separators=(",", ":")) + write_file(out_dir, "search-index.json", payload + "\n") + page = page_shell( + title="Search", + site_title=site_title, + page_path="search.html", + body_html=SEARCH_BODY, + scripts=("assets/search.js",), + ) + write_file(out_dir, "search.html", page) + copy_asset(out_dir, "search.js") diff --git a/scripts/selftest.sh b/scripts/selftest.sh index b21af4f..1716ac7 100755 --- a/scripts/selftest.sh +++ b/scripts/selftest.sh @@ -24,7 +24,7 @@ require_grep() { return 1 } -for command in init author convert sync study retro extract validate lint embed; do +for command in init author convert sync study retro extract validate lint embed publish; do require_grep "^## \`/okf ${command}" "$SKILL" "SKILL.md documents /okf ${command}" done diff --git a/templates/okf-publish.yaml b/templates/okf-publish.yaml new file mode 100644 index 0000000..395116e --- /dev/null +++ b/templates/okf-publish.yaml @@ -0,0 +1,27 @@ +# OKF publish redaction profile. Copy this file to your bundle root. +# publish.py applies the built-in rules, then `rules`, then `private_strings` +# to every published string. `publish.py --check` fails when any rule matches +# text in a public concept. +# +# Built-in rule ids (all on unless listed in disable_default_rules): +# - home-path-unix +# - home-path-windows +# - ipv4 +# - github-token +# - openai-style-key +# - aws-access-key +# - slack-token +# - private-key-block +okf_publish_version: "0.1" +disable_default_rules: [] + +# Example rule: replace an internal host name. +# rules: +# - id: stream-box-host +# pattern: '\bstream-box\.lan\b' +# replace: '[internal-host]' + +# Example private string: matched literally and case-insensitively; a line break +# or any run of whitespace between its words still matches. +# private_strings: +# - Alice Example diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9de0b86 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,19 @@ +"""Shared pytest setup: put the repo's scripts/ directory on sys.path.""" + +import sys +from pathlib import Path + +import pytest + +SCRIPTS_DIR = Path(__file__).resolve().parents[1] / "scripts" +sys.path.insert(0, str(SCRIPTS_DIR)) + + +@pytest.fixture +def scripts_dir() -> Path: + return SCRIPTS_DIR + + +@pytest.fixture +def fixture_bundle() -> Path: + return Path(__file__).parent / "fixtures" / "publish_bundle" diff --git a/tests/fixtures/publish_bundle/.github/notes.md b/tests/fixtures/publish_bundle/.github/notes.md new file mode 100644 index 0000000..a52e282 --- /dev/null +++ b/tests/fixtures/publish_bundle/.github/notes.md @@ -0,0 +1 @@ +plain text diff --git a/tests/fixtures/publish_bundle/index.md b/tests/fixtures/publish_bundle/index.md new file mode 100644 index 0000000..a55049a --- /dev/null +++ b/tests/fixtures/publish_bundle/index.md @@ -0,0 +1,9 @@ +# Stream Rules Knowledge + +The rules this stream's AI follows, proposed by viewers. See [the incident](/ops/incident-42.md). + + +* [ops/](ops/) - Subdirectory +* [rules/](rules/) - Subdirectory +* [Schema](schema.md) - Bundle conventions. + diff --git a/tests/fixtures/publish_bundle/log.md b/tests/fixtures/publish_bundle/log.md new file mode 100644 index 0000000..bcaa3f5 --- /dev/null +++ b/tests/fixtures/publish_bundle/log.md @@ -0,0 +1,5 @@ +# Directory Update Log + +## 2026-11-01 + +* **Update**: Added [incident](/ops/incident-42.md) diff --git a/tests/fixtures/publish_bundle/okf-publish.yaml b/tests/fixtures/publish_bundle/okf-publish.yaml new file mode 100644 index 0000000..b322430 --- /dev/null +++ b/tests/fixtures/publish_bundle/okf-publish.yaml @@ -0,0 +1,7 @@ +okf_publish_version: "0.1" +rules: + - id: stream-box-host + pattern: '\bstream-box\.lan\b' + replace: '[internal-host]' +private_strings: + - Alice Example diff --git a/tests/fixtures/publish_bundle/ops/incident-42.md b/tests/fixtures/publish_bundle/ops/incident-42.md new file mode 100644 index 0000000..e82828f --- /dev/null +++ b/tests/fixtures/publish_bundle/ops/incident-42.md @@ -0,0 +1,8 @@ +--- +type: Incident +title: ZZ-PRIVATE-INCIDENT +description: ZZ-PRIVATE-INCIDENT-DESC +x_visibility: private +--- + +The overlay config at /home/alice/stream/obs.json leaked the plot. The box at 10.0.0.12 was restarted by Alice Example. diff --git a/tests/fixtures/publish_bundle/ops/index.md b/tests/fixtures/publish_bundle/ops/index.md new file mode 100644 index 0000000..dce6ecb --- /dev/null +++ b/tests/fixtures/publish_bundle/ops/index.md @@ -0,0 +1,5 @@ +# Ops + + +* [ZZ-PRIVATE-INCIDENT](incident-42.md) - ZZ-PRIVATE-INCIDENT-DESC + diff --git a/tests/fixtures/publish_bundle/rules/index.md b/tests/fixtures/publish_bundle/rules/index.md new file mode 100644 index 0000000..9161f3c --- /dev/null +++ b/tests/fixtures/publish_bundle/rules/index.md @@ -0,0 +1,7 @@ +# Rules + + +* [ZZ-PRIVATE-MOD-NOTES](mod-notes.md) - ZZ-PRIVATE-MOD-DESC +* [No spoilers](no-spoilers.md) - Do not reveal plot points. +* [Old spoiler rule](old-spoiler-rule.md) - Replaced rule. + diff --git a/tests/fixtures/publish_bundle/rules/mod-notes.md b/tests/fixtures/publish_bundle/rules/mod-notes.md new file mode 100644 index 0000000..ed952cd --- /dev/null +++ b/tests/fixtures/publish_bundle/rules/mod-notes.md @@ -0,0 +1,8 @@ +--- +type: Note +title: ZZ-PRIVATE-MOD-NOTES +description: ZZ-PRIVATE-MOD-DESC +x_visibility: private +--- + +Moderator notes live in /home/alice/mod.txt. See [No spoilers](no-spoilers.md). diff --git a/tests/fixtures/publish_bundle/rules/no-spoilers.md b/tests/fixtures/publish_bundle/rules/no-spoilers.md new file mode 100644 index 0000000..fc46f57 --- /dev/null +++ b/tests/fixtures/publish_bundle/rules/no-spoilers.md @@ -0,0 +1,27 @@ +--- +type: Rule +title: No spoilers +description: Do not reveal plot points. +tags: [chat, rules] +status: stable +sources: + - id: s1 + resource: "https://example.com/vod/1" + title: Chat proposal + author: viewer-anna +verified: + - by: "human:host" + at: "2026-11-01T00:00:00Z" +--- + +Proposed in chat.[^s1] + +The AI never reveals plot points of the game on stream. This replaces the [old rule](old-spoiler-rule.md). + +- Background: the [incident](/ops/incident-42.md) that prompted the rule. +- Next: a [future rule](/rules/not-written-yet.md) on hints. +- Evidence: the [VOD](https://example.com/vod/1). + +Back to the [rules home](index.md). + +[^s1]: Chat proposal from viewer-anna. diff --git a/tests/fixtures/publish_bundle/rules/old-spoiler-rule.md b/tests/fixtures/publish_bundle/rules/old-spoiler-rule.md new file mode 100644 index 0000000..72e62b3 --- /dev/null +++ b/tests/fixtures/publish_bundle/rules/old-spoiler-rule.md @@ -0,0 +1,11 @@ +--- +type: Rule +title: Old spoiler rule +description: Replaced rule. +status: deprecated +verified: + by: "process:ci" + at: "2026-10-01T00:00:00Z" +--- + +Replaced by [No spoilers](no-spoilers.md). diff --git a/tests/fixtures/publish_bundle/schema.md b/tests/fixtures/publish_bundle/schema.md new file mode 100644 index 0000000..0c41946 --- /dev/null +++ b/tests/fixtures/publish_bundle/schema.md @@ -0,0 +1,9 @@ +--- +type: Policy +title: Schema +description: Bundle conventions. +x_visibility: public +x_visibility_default: public +--- + +Every rule is a `Rule` concept under `rules/`. Concepts are public unless they set `x_visibility: private`. diff --git a/tests/test_harness.py b/tests/test_harness.py new file mode 100644 index 0000000..f5fcdbb --- /dev/null +++ b/tests/test_harness.py @@ -0,0 +1,7 @@ +from pathlib import Path + +import okf_common + + +def test_okf_common_importable(): + assert okf_common.concept_id_for_path(Path("a/b.md")) == "a/b" diff --git a/tests/test_publish_cli.py b/tests/test_publish_cli.py new file mode 100644 index 0000000..0a02e02 --- /dev/null +++ b/tests/test_publish_cli.py @@ -0,0 +1,376 @@ +from html.parser import HTMLParser +from pathlib import Path +import shutil +import subprocess +import sys +from urllib.parse import unquote + +import pytest + +from publish_redact import load_profile, scan + +EXPECTED_PAGES = { + "index.html", + "rules/index.html", + "rules/no-spoilers.html", + "rules/old-spoiler-rule.html", + "schema.html", + "search.html", +} + + +@pytest.fixture +def bundle_copy(fixture_bundle, tmp_path) -> Path: + target = tmp_path / "bundle" + shutil.copytree(fixture_bundle, target) + return target + + +def run_publish(scripts_dir: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(scripts_dir / "publish.py"), *args], + capture_output=True, + text=True, + check=False, + ) + + +def edit(path: Path, old: str, new: str) -> None: + text = path.read_text(encoding="utf-8") + assert old in text + path.write_text(text.replace(old, new, 1), encoding="utf-8") + + +def files_under(root: Path) -> list[str]: + return sorted(path.relative_to(root).as_posix() for path in root.rglob("*") if path.is_file()) + + +class LinkCollector(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.targets: list[str] = [] + + def handle_starttag(self, tag, attrs): + for name, value in attrs: + if name in ("href", "src", "action") and value is not None: + self.targets.append(value) + + +def test_publish_fixture_pages(scripts_dir, bundle_copy, tmp_path): + out = tmp_path / "site" + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.startswith("ok - published 3 public concept(s) to ") + assert "(2 private excluded, 0 redaction(s) applied)" in result.stdout + assert {path for path in files_under(out) if path.endswith(".html")} == EXPECTED_PAGES + assert set(files_under(out)) == EXPECTED_PAGES | { + ".okf-publish", + "assets/search.js", + "assets/style.css", + "search-index.json", + } + for absent in ("ops/incident-42.html", "rules/mod-notes.html", "ops/index.html", "log.html"): + assert not (out / absent).exists() + + +def test_no_unresolved_links(scripts_dir, bundle_copy, tmp_path): + out = tmp_path / "site" + assert run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)).returncode == 0 + checked = 0 + for page in out.rglob("*.html"): + collector = LinkCollector() + collector.feed(page.read_text(encoding="utf-8")) + for target in collector.targets: + if target.lower().startswith(("http:", "https:", "mailto:", "#")): + continue + assert not target.startswith("/"), (page, target) + path = unquote(target.split("#", 1)[0].split("?", 1)[0]) + resolved = (page.parent / path).resolve() + assert resolved.is_file(), (page.relative_to(out).as_posix(), target) + assert out.resolve() in resolved.parents + checked += 1 + assert checked > 20 + + +def test_no_private_or_secret_bytes(scripts_dir, bundle_copy, tmp_path, fixture_bundle): + out = tmp_path / "site" + assert run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)).returncode == 0 + profile, errors = load_profile(fixture_bundle) + assert errors == [] + for path in out.rglob("*"): + if not path.is_file(): + continue + text = path.read_text(encoding="utf-8") + lowered = text.lower() + for needle in ("zz-private", "/home/alice", "10.0.0.12", "alice example"): + assert needle not in lowered, (path, needle) + assert scan(text, profile) == [], path + + +def test_redaction_applied(scripts_dir, bundle_copy, tmp_path): + edit( + bundle_copy / "rules" / "old-spoiler-rule.md", + "Replaced by", + "Mirror lives at /home/bob/vod on stream-box.lan.\n\nReplaced by", + ) + out = tmp_path / "site" + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)) + assert result.returncode == 0, result.stdout + assert "(2 private excluded, 2 redaction(s) applied)" in result.stdout + page = (out / "rules" / "old-spoiler-rule.html").read_text(encoding="utf-8") + assert "Mirror lives at ~/vod on [internal-host]." in page + assert "/home/bob" not in page and "stream-box.lan" not in page + assert "/home/bob" not in (out / "search-index.json").read_text(encoding="utf-8") + + +def test_deterministic(scripts_dir, bundle_copy, tmp_path): + first, second = tmp_path / "one", tmp_path / "two" + assert run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(first)).returncode == 0 + assert run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(second)).returncode == 0 + assert files_under(first) == files_under(second) + for relative in files_under(first): + assert (first / relative).read_bytes() == (second / relative).read_bytes(), relative + + +def test_out_inside_bundle_is_skipped(scripts_dir, bundle_copy): + out = bundle_copy / "site" + assert run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)).returncode == 0 + again = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)) + assert again.returncode == 0, again.stdout + assert {path for path in files_under(out) if path.endswith(".html")} == EXPECTED_PAGES + + +def test_out_safety(scripts_dir, bundle_copy, tmp_path): + same = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(bundle_copy)) + assert same.returncode == 2 + assert "ERROR P_OUT_UNSAFE" in same.stdout + ancestor = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(tmp_path)) + assert ancestor.returncode == 2 and "P_OUT_UNSAFE" in ancestor.stdout + + a_file = tmp_path / "file.txt" + a_file.write_text("keep\n", encoding="utf-8") + as_file = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(a_file)) + assert as_file.returncode == 2 and "P_OUT_UNSAFE" in as_file.stdout + + unmarked = tmp_path / "unmarked" + unmarked.mkdir() + (unmarked / "keep.txt").write_text("keep\n", encoding="utf-8") + refused = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(unmarked)) + assert refused.returncode == 2 and "P_OUT_UNSAFE" in refused.stdout + assert files_under(unmarked) == ["keep.txt"] + + out = tmp_path / "site" + assert run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)).returncode == 0 + assert (out / "rules" / "old-spoiler-rule.html").is_file() + (out / "stale.txt").write_text("old\n", encoding="utf-8") + edit(bundle_copy / "rules" / "old-spoiler-rule.md", "status: deprecated\n", "status: deprecated\nx_visibility: private\n") + rerun = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)) + assert rerun.returncode == 0, rerun.stdout + assert not (out / "rules" / "old-spoiler-rule.html").exists() + assert not (out / "stale.txt").exists() + assert "Old spoiler rule" not in (out / "search-index.json").read_text(encoding="utf-8") + + +def test_id_redaction_blocks(scripts_dir, bundle_copy, tmp_path): + (bundle_copy / "hosts").mkdir() + (bundle_copy / "hosts" / "stream-box.lan.md").write_text( + "---\ntype: Host\ntitle: Box\n---\n\nThe box.\n", encoding="utf-8" + ) + out = tmp_path / "site" + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)) + assert result.returncode == 2 + assert ( + "ERROR P_ID_REDACTION hosts/stream-box.lan.md: rule stream-box-host matches the concept ID; " + "rename the concept or make it private" + ) in result.stdout + assert not out.exists() + + +def test_errors_write_nothing(scripts_dir, bundle_copy, tmp_path): + edit(bundle_copy / "rules" / "no-spoilers.md", "status: stable\n", "status: stable\nx_visibility: yes\n") + out = tmp_path / "site" + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)) + assert result.returncode == 2 + assert "ERROR P_VISIBILITY_INVALID rules/no-spoilers.md: x_visibility must be public or private" in result.stdout + assert not out.exists() + + +def test_bundle_and_profile_errors(scripts_dir, bundle_copy, tmp_path): + missing = run_publish(scripts_dir, "--bundle", str(tmp_path / "nope"), "--out", str(tmp_path / "site")) + assert missing.returncode == 2 and "ERROR P_BUNDLE_MISSING" in missing.stdout + (bundle_copy / "okf-publish.yaml").write_text('okf_publish_version: "9"\n', encoding="utf-8") + bad = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(tmp_path / "site")) + assert bad.returncode == 2 + assert "ERROR P_PROFILE_INVALID okf-publish.yaml: okf_publish_version" in bad.stdout + assert not (tmp_path / "site").exists() + + +def test_zero_public_note(scripts_dir, bundle_copy, tmp_path): + (bundle_copy / "schema.md").unlink() + out = tmp_path / "site" + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)) + assert result.returncode == 0, result.stdout + assert "ok - published 0 public concept(s)" in result.stdout + assert "note - no public concepts" in result.stdout + assert "no schema.md at bundle root" in result.stdout + assert (out / "index.html").is_file() + + +def file_line(path: Path, text: str) -> int: + return path.read_text(encoding="utf-8").split("\n").index(text) + 1 + + +def test_check_passes_on_fixture(scripts_dir, bundle_copy): + before = files_under(bundle_copy) + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--check") + assert result.returncode == 0, result.stdout + assert result.stdout.startswith("ok - publish check passed: 3 public concept(s)") + assert files_under(bundle_copy) == before + + +def test_check_fails_on_home_path_then_passes(scripts_dir, bundle_copy): + concept = bundle_copy / "rules" / "no-spoilers.md" + edit(concept, "Proposed in chat.[^s1]\n", "Proposed in chat.[^s1]\n\nSetup notes: /home/alice/obs.json\n") + line = file_line(concept, "Setup notes: /home/alice/obs.json") + failed = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--check") + assert failed.returncode == 2 + assert ( + f"ERROR P_REDACTION rules/no-spoilers.md:{line}: rule home-path-unix matches published text (field body)" + in failed.stdout + ) + assert "/home/alice" not in failed.stdout + failed.stderr + + edit(concept, "Setup notes: /home/alice/obs.json", "Setup notes: ~/obs.json") + passed = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--check") + assert passed.returncode == 0, passed.stdout + + +def test_check_ignores_private(scripts_dir, bundle_copy): + private = (bundle_copy / "ops" / "incident-42.md").read_text(encoding="utf-8") + assert "/home/alice" in private and "10.0.0.12" in private and "Alice Example" in private + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--check") + assert result.returncode == 0, result.stdout + + +def test_check_frontmatter_field(scripts_dir, bundle_copy): + edit(bundle_copy / "rules" / "old-spoiler-rule.md", "description: Replaced rule.", "description: Runs on 10.0.0.12") + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--check") + assert result.returncode == 2 + assert ( + "ERROR P_REDACTION rules/old-spoiler-rule.md: rule ipv4 matches published text (field description)" + in result.stdout + ) + assert "10.0.0.12" not in result.stdout + + +def test_check_index_text(scripts_dir, bundle_copy): + index = bundle_copy / "index.md" + edit(index, "(/ops/incident-42.md).\n", "(/ops/incident-42.md).\n\nHosted on stream-box.lan\n") + line = file_line(index, "Hosted on stream-box.lan") + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--check") + assert result.returncode == 2 + assert f"ERROR P_REDACTION index.md:{line}: rule stream-box-host matches published text (field index)" in result.stdout + assert "stream-box.lan" not in result.stdout + + +def test_check_index_line_after_frontmatter_and_dropped_lines(scripts_dir, bundle_copy): + index = bundle_copy / "index.md" + text = index.read_text(encoding="utf-8") + index.write_text('---\nokf_version: "0.1"\n---\n' + text + "\nHosted on stream-box.lan\n", encoding="utf-8") + line = file_line(index, "Hosted on stream-box.lan") + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--check") + assert result.returncode == 2 + assert f"ERROR P_REDACTION index.md:{line}: rule stream-box-host" in result.stdout + + +def test_check_and_out_exclusive(scripts_dir, bundle_copy, tmp_path): + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--check", "--out", str(tmp_path / "site")) + assert result.returncode == 2 + assert "not allowed with argument" in result.stderr + assert not (tmp_path / "site").exists() + + +def all_output_text(out: Path) -> str: + return "\n".join(path.read_text(encoding="utf-8") for path in sorted(out.rglob("*")) if path.is_file()) + + +LEAKY_PRIVATE_CONCEPTS = { + "rules/secret-plan.md": '---\ntitle: "Layoffs [draft]"\ndescription: ZZ-LEAK-BRACKET-DESC\nx_visibility: private\n---\n\nBody.\n', + "rules/unbal.md": '---\ntitle: "a] ZZ-LEAK-UNBAL"\ndescription: ZZ-LEAK-UNBAL-DESC\nx_visibility: private\n---\n\nBody.\n', + "rules/my file.md": "---\ntitle: ZZ-LEAK-SPACE-TITLE\ndescription: ZZ-LEAK-SPACE-DESC\nx_visibility: private\n---\n\nBody.\n", + "rules/hr.md": "---\ntitle: ZZ-LEAK-HR\ndescription: |\n First line ZZ-LEAK-FIRST\n ZZ-LEAK-SECOND-LINE about Bob\nx_visibility: private\n---\n\nBody.\n", +} +LEAK_NEEDLES = ("ZZ-LEAK", "Layoffs", "secret-plan", "unbal.md", "my file", "my%20file", "about Bob") + + +def test_reindexed_private_entries_never_reach_the_site(scripts_dir, bundle_copy, tmp_path): + for relative, text in LEAKY_PRIVATE_CONCEPTS.items(): + (bundle_copy / relative).write_text(text, encoding="utf-8") + (bundle_copy / "rules" / "bracketed.md").write_text( + '---\ntitle: "Public [v2]"\ndescription: Shown rule.\n---\n\nBody.\n', encoding="utf-8" + ) + # reindex.py walks dot-directories and fails on the fixture's frontmatter-less .github/notes.md. + shutil.rmtree(bundle_copy / ".github") + reindex = subprocess.run( + [sys.executable, str(scripts_dir / "reindex.py"), "--bundle", str(bundle_copy)], + capture_output=True, + text=True, + check=False, + ) + assert reindex.returncode == 0, reindex.stdout + reindex.stderr + rules_index = (bundle_copy / "rules" / "index.md").read_text(encoding="utf-8") + assert "* [Layoffs [draft]](secret-plan.md) - ZZ-LEAK-BRACKET-DESC" in rules_index + assert "ZZ-LEAK-SECOND-LINE about Bob" in rules_index + assert "(my file.md)" in rules_index + + check = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--check") + assert check.returncode == 0, check.stdout + out = tmp_path / "site" + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)) + assert result.returncode == 0, result.stdout + + published = all_output_text(out) + for needle in LEAK_NEEDLES: + assert needle not in published, needle + page = (out / "rules" / "index.html").read_text(encoding="utf-8") + assert 'Public [v2] - Shown rule.' in page + assert 'No spoilers' in page + assert "ZZ-PRIVATE" not in published + + +def test_hand_written_lines_outside_markers_keep_prose(scripts_dir, bundle_copy, tmp_path): + index = bundle_copy / "index.md" + edit(index, "", "* [Old notes [v1]](/ops/incident-42.md) - ZZ-LEAK-HAND\n\n") + out = tmp_path / "site" + assert run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)).returncode == 0 + root = (out / "index.html").read_text(encoding="utf-8") + assert "ZZ-LEAK-HAND" not in root + assert "See the incident." in root + + +def test_search_text_drops_nested_bracket_targets(scripts_dir, bundle_copy, tmp_path): + edit(bundle_copy / "rules" / "no-spoilers.md", "Proposed in chat.[^s1]", "Proposed in chat.[^s1] See [notes [v2]](team-notes.md).") + out = tmp_path / "site" + assert run_publish(scripts_dir, "--bundle", str(bundle_copy), "--out", str(out)).returncode == 0 + index = (out / "search-index.json").read_text(encoding="utf-8") + assert "team-notes" not in index + assert "See notes [v2]." in index + + +@pytest.mark.parametrize( + ("line", "rule"), + [ + ("Esc: /home\\/alice/obs.json", "home-path-unix"), + ("Box 10.0.0\\.12 is up", "ipv4"), + ("Thanks **Alice** Example", "private-string-1"), + ("Thanks [Alice](https://example.com/a) Example", "private-string-1"), + ], +) +def test_check_catches_matches_split_by_markup(scripts_dir, bundle_copy, line, rule): + concept = bundle_copy / "rules" / "no-spoilers.md" + edit(concept, "Proposed in chat.[^s1]\n", f"Proposed in chat.[^s1]\n\n{line}\n") + number = file_line(concept, line) + result = run_publish(scripts_dir, "--bundle", str(bundle_copy), "--check") + assert result.returncode == 2, result.stdout + assert f"ERROR P_REDACTION rules/no-spoilers.md:{number}: rule {rule} matches published text (field body)" in result.stdout + assert "alice" not in result.stdout.lower() diff --git a/tests/test_publish_markdown.py b/tests/test_publish_markdown.py new file mode 100644 index 0000000..8be82ec --- /dev/null +++ b/tests/test_publish_markdown.py @@ -0,0 +1,115 @@ +from publish_markdown import render_markdown + + +def no_links(_target): + return None + + +def identity(target): + return target + + +def test_raw_html_is_escaped(): + out = render_markdown("\n\n", identity) + assert "<script>alert(1)</script>" in out + assert "three' in out + assert out.count(" ", identity) + assert 'https://example.com/x' in autolink + assert "See Secret now.

            " + assert " quoted *text*", + "", + "---", + "", + "```python", + "print('')", + "```", + "", + "| A | B |", + "| --- | --- |", + "| 1 | 2 |", + ] + ) + out = render_markdown(text, identity) + assert '

            Title

            ' in out + assert '

            Part

            ' in out + assert '

            Part

            ' in out + assert "

            First line\nsecond line.

            " in out + assert "
              \n
            • one
            • \n
            • two\n
                \n
              • nested
              • \n
            • \n
            • three
            • \n
            " in out + assert "
              \n
            1. first
            2. \n
            3. second
            4. \n
            " in out + assert "
            \n

            quoted text

            \n
            " in out + assert "
            " in out + assert '
            print('<b>')\n
            ' in out + assert "\n\n\n" in out + + +def test_fence_info_word_must_be_safe(): + out = render_markdown('```">code\n" in out + + +def test_footnotes(): + out = render_markdown("Claim.[^s1] Other.[^zz]\n\n[^s1]: Source text", identity) + assert '1' in out + assert '
            ' in out + assert '
          1. Source text' in out + assert "[^zz]" in out + assert "[^s1]:" not in out + + +def test_inline(): + out = render_markdown("Use `**not bold**` and **b** and *i* ![alt ](pic.png)", identity) + assert "**not bold**" in out + assert "b" in out + assert "i" in out + assert "alt <text>" in out + assert "real" in out + assert "strong" in out + assert out.count("") == 1 diff --git a/tests/test_publish_model.py b/tests/test_publish_model.py new file mode 100644 index 0000000..1d36dee --- /dev/null +++ b/tests/test_publish_model.py @@ -0,0 +1,158 @@ +from pathlib import Path +import shutil + +import pytest + +from publish_model import LinkTarget, build_model, classify_link, trust_tier, walk_markdown + + +@pytest.fixture +def bundle_copy(fixture_bundle, tmp_path) -> Path: + target = tmp_path / "bundle" + shutil.copytree(fixture_bundle, target) + return target + + +def edit(path: Path, old: str, new: str) -> None: + text = path.read_text(encoding="utf-8") + assert old in text + path.write_text(text.replace(old, new, 1), encoding="utf-8") + + +def test_fixture_public_and_private(fixture_bundle): + model, findings = build_model(fixture_bundle) + assert findings == [] + assert model.public_ids() == ["rules/no-spoilers", "rules/old-spoiler-rule", "schema"] + assert model.concepts["ops/incident-42"].visibility == "private" + assert model.concepts["rules/mod-notes"].visibility == "private" + assert model.default_visibility == "public" + assert all(not concept_id.startswith(".github") for concept_id in model.concepts) + walked = [path.relative_to(fixture_bundle.resolve()).as_posix() for path in walk_markdown(fixture_bundle)] + assert ".github/notes.md" not in walked + assert "log.md" in walked + assert "log" not in model.concepts + assert sorted(model.index_files) == ["", "ops", "rules"] + + +def test_walk_skips_excluded_out_dir(bundle_copy): + out = bundle_copy / "site" + (out / "nested").mkdir(parents=True) + (out / "nested" / "page.md").write_text("---\ntitle: x\n---\n", encoding="utf-8") + (bundle_copy / "node_modules").mkdir() + (bundle_copy / "node_modules" / "pkg.md").write_text("no frontmatter\n", encoding="utf-8") + model, findings = build_model(bundle_copy, exclude=out) + assert findings == [] + assert "site/nested/page" not in model.concepts + + +def test_default_private_without_schema(bundle_copy): + (bundle_copy / "schema.md").unlink() + model, findings = build_model(bundle_copy) + assert findings == [] + assert model.public_ids() == [] + assert model.default_reason == "no schema.md at bundle root" + + +def test_default_private_without_key(bundle_copy): + edit(bundle_copy / "schema.md", "x_visibility_default: public\n", "") + model, findings = build_model(bundle_copy) + assert findings == [] + assert model.public_ids() == ["schema"] + assert model.default_reason == "schema.md has no x_visibility_default" + + +def test_invalid_default_value(bundle_copy): + edit(bundle_copy / "schema.md", "x_visibility_default: public", "x_visibility_default: yes") + model, findings = build_model(bundle_copy) + assert [(f.code, f.location) for f in findings] == [("P_VISIBILITY_INVALID", "schema.md")] + assert model.default_visibility == "private" + + +def test_invalid_visibility(bundle_copy): + edit(bundle_copy / "rules" / "no-spoilers.md", "status: stable\n", "status: stable\nx_visibility: Public\n") + model, findings = build_model(bundle_copy) + assert len(findings) == 1 + assert findings[0].code == "P_VISIBILITY_INVALID" + assert findings[0].line() == "ERROR P_VISIBILITY_INVALID rules/no-spoilers.md: x_visibility must be public or private" + assert not model.is_public("rules/no-spoilers") + + +def test_parse_error_finding(bundle_copy): + (bundle_copy / "notes").mkdir() + (bundle_copy / "notes" / "bad.md").write_text("no frontmatter here\n", encoding="utf-8") + model, findings = build_model(bundle_copy) + assert [(f.code, f.location) for f in findings] == [("P_FRONTMATTER_PARSE", "notes/bad.md")] + assert "notes/bad" not in model.concepts + + +def test_trust_tier(): + assert trust_tier({"verified": [{"by": "process:ci"}, {"by": "human:host"}]}) == "human-reviewed" + assert trust_tier({"verified": {"by": "process:ci", "at": "x"}}) == "machine-confirmed" + assert trust_tier({}) == "unverified" + assert trust_tier({"verified": None}) == "unverified" + assert trust_tier({"verified": ["human:host"]}) == "unverified" + + +def test_classify_link(): + # Rules 1-2: fragment split and fragment-only links. + assert classify_link("rules", "#part") == LinkTarget("fragment", "#part", "part") + # Rule 3: external schemes stay unchanged. + assert classify_link("rules", "HTTPS://example.com/a#b") == LinkTarget("external", "HTTPS://example.com/a#b", "b") + assert classify_link("", "mailto:a@example.com").kind == "external" + # Rule 4: any other scheme. + assert classify_link("rules", "javascript:x").kind == "other" + assert classify_link("rules", "repo:a.md@abc").kind == "other" + # Rule 5: root-relative, source-relative and climbing above the root. + assert classify_link("rules", "/ops/incident-42.md#a") == LinkTarget("concept", "ops/incident-42", "a") + assert classify_link("rules", "../schema.md") == LinkTarget("concept", "schema", "") + assert classify_link("rules", "../../x.md").kind == "other" + # Rule 6: directory links. + assert classify_link("rules", "/rules/") == LinkTarget("directory", "rules", "") + assert classify_link("", "ops/") == LinkTarget("directory", "ops", "") + assert classify_link("rules", "../") == LinkTarget("directory", "", "") + # Rule 7: index.md. + assert classify_link("rules", "index.md") == LinkTarget("directory", "rules", "") + assert classify_link("rules", "/index.md") == LinkTarget("directory", "", "") + # Rule 8: log.md. + assert classify_link("", "log.md").kind == "other" + # Rule 9: concepts. + assert classify_link("rules", "old-spoiler-rule.md") == LinkTarget("concept", "rules/old-spoiler-rule", "") + # Rule 10: anything else. + assert classify_link("rules", "diagram.png").kind == "other" + + +def test_backlinks_public_only(bundle_copy): + (bundle_copy / "rules" / "secret.md").write_text( + "---\ntitle: Secret\nx_visibility: private\n---\n\nSee [No spoilers](no-spoilers.md).\n", encoding="utf-8" + ) + model, findings = build_model(bundle_copy) + assert findings == [] + assert model.backlinks("rules/no-spoilers") == ["rules/old-spoiler-rule"] + assert model.backlinks("rules/old-spoiler-rule") == ["rules/no-spoilers"] + + +def test_public_dirs(fixture_bundle): + model, _findings = build_model(fixture_bundle) + assert model.public_dirs() == {"", "rules"} + + +def test_body_start_line(fixture_bundle): + model, _findings = build_model(fixture_bundle) + concept = model.concepts["rules/no-spoilers"] + lines = (fixture_bundle / "rules" / "no-spoilers.md").read_text(encoding="utf-8").split("\n") + assert lines[concept.body_start_line - 1] == concept.body.split("\n")[0] + assert lines[concept.body_start_line - 1] == "Proposed in chat.[^s1]" + assert lines[concept.body_start_line - 2] == "" + + +def test_concept_fields(fixture_bundle): + model, _findings = build_model(fixture_bundle) + concept = model.concepts["rules/no-spoilers"] + assert concept.tags == ("chat", "rules") + assert concept.status == "stable" + assert concept.trust_tier == "human-reviewed" + assert [(s.title, s.resource, s.author) for s in concept.sources] == [ + ("Chat proposal", "https://example.com/vod/1", "viewer-anna") + ] + assert model.concepts["rules/old-spoiler-rule"].trust_tier == "machine-confirmed" + assert model.concepts["schema"].status == "stable" diff --git a/tests/test_publish_redact.py b/tests/test_publish_redact.py new file mode 100644 index 0000000..711a2b4 --- /dev/null +++ b/tests/test_publish_redact.py @@ -0,0 +1,112 @@ +from pathlib import Path +import re + +import pytest + +from publish_redact import DEFAULT_RULES, default_profile, load_profile, redact, scan + +TEMPLATE = Path(__file__).resolve().parents[1] / "templates" / "okf-publish.yaml" + + +def write_profile(tmp_path: Path, text: str): + (tmp_path / "okf-publish.yaml").write_text(text, encoding="utf-8") + return load_profile(tmp_path) + + +def test_default_rules_redact_samples(tmp_path): + profile, errors = load_profile(tmp_path) + assert errors == [] + assert profile is not None and profile.path is None + token = "ghp_" + "A1b2C3d4E5" * 4 + text = "\n".join( + [ + "/home/alice/x", + "C:\\Users\\bob\\x", + "10.0.0.12", + token, + "-----BEGIN RSA PRIVATE KEY-----", + ] + ) + redacted, counts = redact(text, profile) + assert redacted.split("\n") == ["~/x", "~\\x", "[ip]", "[token]", "[private-key]"] + assert counts == { + "home-path-unix": 1, + "home-path-windows": 1, + "ipv4": 1, + "github-token": 1, + "private-key-block": 1, + } + + +def test_private_strings_case_insensitive(tmp_path): + profile, errors = write_profile(tmp_path, 'okf_publish_version: "0.1"\nprivate_strings: [Alice Example]\n') + assert errors == [] + redacted, counts = redact("Thanks, alice example!", profile) + assert redacted == "Thanks, [redacted]!" + assert counts == {"private-string-1": 1} + wrapped, wrapped_counts = redact("Thanks, Alice\nExample and ALICE example.", profile) + assert wrapped == "Thanks, [redacted] and [redacted]." + assert wrapped_counts == {"private-string-1": 2} + assert scan("x\nAlice\nExample\n", profile) == [("private-string-1", 2)] + + +def test_disable_default_rule(tmp_path): + profile, errors = write_profile(tmp_path, 'okf_publish_version: "0.1"\ndisable_default_rules: [ipv4]\n') + assert errors == [] + assert redact("serve on 127.0.0.1", profile) == ("serve on 127.0.0.1", {}) + + +@pytest.mark.parametrize( + ("text", "needle"), + [ + ("- just a list\n", "top level"), + ('okf_publish_version: "0.1"\nextra: 1\n', "extra"), + ("okf_publish_version: 0.1\n", "okf_publish_version"), + ("disable_default_rules: []\n", "okf_publish_version"), + ('okf_publish_version: "0.1"\ndisable_default_rules: [nope]\n', "disable_default_rules"), + ('okf_publish_version: "0.1"\nrules:\n - pattern: x\n', "missing id"), + ('okf_publish_version: "0.1"\nrules:\n - id: a\n', "rule a is missing pattern"), + ('okf_publish_version: "0.1"\nrules:\n - id: Bad_Id\n pattern: x\n', "Bad_Id"), + ('okf_publish_version: "0.1"\nrules:\n - id: a\n pattern: x\n - id: a\n pattern: y\n', "rules id a"), + ('okf_publish_version: "0.1"\nrules:\n - id: ipv4\n pattern: x\n', "rules id ipv4"), + ('okf_publish_version: "0.1"\nrules:\n - id: broken\n pattern: "("\n', "rule broken"), + ('okf_publish_version: "0.1"\nprivate_strings: [""]\n', "private_strings"), + ('okf_publish_version: "0.1"\nprivate_strings: [" "]\n', "private_strings"), + ('okf_publish_version: "0.1"\nprivate_strings: [3]\n', "private_strings"), + ('okf_publish_version: "0.1"\nrules:\n - id: a\n pattern: x\n replace: 5\n', "replace"), + ("okf_publish_version: [\n", "YAML"), + ], +) +def test_profile_errors(tmp_path, text, needle): + profile, errors = write_profile(tmp_path, text) + assert profile is None + assert errors + assert any(needle in error for error in errors), errors + + +def test_missing_explicit_profile(tmp_path): + profile, errors = load_profile(tmp_path, tmp_path / "absent.yaml") + assert profile is None + assert errors == ["profile file not found."] + + +def test_scan_reports_lines_not_text(): + hits = scan("a\n/home/alice\n", default_profile()) + assert hits == [("home-path-unix", 2)] + assert all("alice" not in str(part) for hit in hits for part in hit) + + +def test_replace_is_literal(tmp_path): + profile, errors = write_profile( + tmp_path, "okf_publish_version: \"0.1\"\nrules:\n - id: grp\n pattern: '(secret)'\n replace: '\\1'\n" + ) + assert errors == [] + assert redact("a secret b", profile) == ("a \\1 b", {"grp": 1}) + + +def test_template_lists_default_ids(tmp_path): + listed = re.findall(r"^# - ([a-z0-9-]+)$", TEMPLATE.read_text(encoding="utf-8"), flags=re.MULTILINE) + assert listed == [rule[0] for rule in DEFAULT_RULES] + profile, errors = load_profile(tmp_path, TEMPLATE) + assert errors == [] + assert [rule.id for rule in profile.rules] == listed diff --git a/tests/test_publish_render.py b/tests/test_publish_render.py new file mode 100644 index 0000000..3ac9733 --- /dev/null +++ b/tests/test_publish_render.py @@ -0,0 +1,108 @@ +from pathlib import Path +import re +import shutil + +import pytest + +from publish_model import build_model +from publish_render import filter_index_markdown, kept_index_lines, relative_href, render_site, site_title + + +@pytest.fixture +def bundle_copy(fixture_bundle, tmp_path) -> Path: + target = tmp_path / "bundle" + shutil.copytree(fixture_bundle, target) + return target + + +def filtered_texts(model): + return {d: filter_index_markdown(t, d, model) for d, t in model.index_files.items() if d in model.public_dirs()} + + +def render(bundle: Path, out: Path) -> list[str]: + model, findings = build_model(bundle) + assert findings == [] + return render_site(model, filtered_texts(model), out) + + +def test_relative_href(): + assert relative_href("rules/no-spoilers.html", "schema.html") == "../schema.html" + assert relative_href("index.html", "rules/a.html") == "rules/a.html" + assert relative_href("rules/a.html", "rules/b.html") == "b.html" + assert relative_href("a/b/c.html", "a/x/y.html") == "../x/y.html" + assert relative_href("a/b/c.html", "index.html") == "../../index.html" + assert relative_href("index.html", "my notes/a b.html") == "my%20notes/a%20b.html" + + +def test_filter_index_drops_private_lines(fixture_bundle): + model, _findings = build_model(fixture_bundle) + texts = filtered_texts(model) + assert sorted(texts) == ["", "rules"] + assert "[ops/]" not in texts[""] + assert "* [rules/](rules/) - Subdirectory" in texts[""] + assert "See [the incident](/ops/incident-42.md)." in texts[""] + assert "ZZ-PRIVATE-MOD-NOTES" not in texts["rules"] + assert "* [No spoilers](no-spoilers.md) - Do not reveal plot points." in texts["rules"] + kept = [number for number, _line in kept_index_lines(model.index_files[""], "", model)] + assert 6 not in kept and 7 in kept + + +def test_concept_page_contents(fixture_bundle, tmp_path): + render(fixture_bundle, tmp_path) + page = (tmp_path / "rules" / "no-spoilers.html").read_text(encoding="utf-8") + assert "rules/no-spoilers" in page + assert "human-reviewed" in page + assert 'class="badge status status-stable">stable<' in page + assert 'Chat proposal — viewer-anna' in page + assert '
          2. AB
            12