diff --git a/.github/release-notes.md.template b/.github/release-notes.md.template new file mode 100644 index 00000000..4bc753e4 --- /dev/null +++ b/.github/release-notes.md.template @@ -0,0 +1,4 @@ +## Release resources + +- [Release website](https://mboworks.github.io/proto/site/tag/@TAG@/) +- [Coverage report](https://mboworks.github.io/proto/coverage/tag/@VERSION@/) diff --git a/.github/workflows/coverage_pages.yml b/.github/workflows/coverage_pages.yml index 032497df..8de334e3 100644 --- a/.github/workflows/coverage_pages.yml +++ b/.github/workflows/coverage_pages.yml @@ -9,6 +9,7 @@ on: # on the same branch revision would otherwise discard whichever report was pushed first. concurrency: group: coverage-pages + queue: max cancel-in-progress: false # The publishing job below declares its exact elevated scopes. Default to no @@ -175,7 +176,7 @@ jobs: git -C site push fi mkdir -p public - rsync --archive site/coverage/ public/coverage/ + rsync --archive --exclude='.git' site/ public/ - uses: actions/upload-pages-artifact@v5 if: steps.target.outputs.publish == 'true' with: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e1355faf..700d512b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,6 +14,26 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: + release-site-tests: + name: Release site tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - run: python3 -m unittest discover -s tools -p release_site_test.py + - run: python3 -m unittest discover -s tools -p release_notes_test.py + - name: Verify configured documentation and generated links + env: + GH_TOKEN: ${{ github.token }} + run: | + # This is a disposable build on the runner, never a Pages publication. + python3 tools/release_site.py . "${RUNNER_TEMP}/release-site-check" \ + --repository "${GITHUB_REPOSITORY}" --tag 0.0.0-verification --latest "" + trunk: runs-on: ubuntu-latest steps: @@ -296,7 +316,16 @@ jobs: proto_version: ${{ matrix.proto_version }} done: - needs: [trunk, pre-commit, clang-tidy, coverage, test-core, test-special] + needs: + [ + release-site-tests, + trunk, + pre-commit, + clang-tidy, + coverage, + test-core, + test-special, + ] if: always() runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 00000000..3bd9185e --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +name: Publish release site + +on: + # Releases created with GITHUB_TOKEN do not trigger release events. + workflow_run: + workflows: [Release] + types: [completed] + workflow_dispatch: + inputs: + tag: + description: Published release tag (empty selects latest) + type: string + required: false + + config_path: + description: Optional config file on main for backfill (empty uses the tag config) + type: string + required: false + +permissions: {} + +# Share the lock and retained branch with coverage: every deployment includes both. +concurrency: + group: coverage-pages + queue: max + cancel-in-progress: false + +jobs: + publish: + if: >- + github.event_name != 'workflow_run' || + (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push') + runs-on: ubuntu-latest + permissions: + contents: write + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v7 + with: + ref: main + path: source + - name: Resolve published release + id: release + env: + GH_TOKEN: ${{ github.token }} + REQUESTED_TAG: ${{ inputs.tag }} + RUN_TAG: ${{ github.event.workflow_run.head_branch }} + run: | + set -euo pipefail + if gh api "repos/${GITHUB_REPOSITORY}/releases/latest" > latest.json; then + latest="$(jq -r .tag_name latest.json)" + elif jq --exit-status '.status == "404"' latest.json >/dev/null; then + latest="" # A repository may have only prereleases so far. + else + exit 1 + fi + tag="${REQUESTED_TAG:-${RUN_TAG:-${latest}}}" + if [[ ! "${tag}" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "Invalid release tag: ${tag}" >&2 + exit 1 + fi + gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${tag}" > release.json + jq --exit-status '.draft == false' release.json + echo "tag=${tag}" >> "${GITHUB_OUTPUT}" + echo "latest=${latest}" >> "${GITHUB_OUTPUT}" + - uses: actions/checkout@v7 + with: + ref: refs/tags/${{ steps.release.outputs.tag }} + path: release + persist-credentials: false + - name: Restore retained Pages tree + working-directory: source + run: | + set -euo pipefail + git fetch origin + if git show-ref --verify --quiet refs/remotes/origin/coverage-pages; then + git worktree add ../site -B coverage-pages origin/coverage-pages + else + git worktree add --detach ../site + git -C ../site checkout --orphan coverage-pages + git -C ../site rm -rf . + fi + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - name: Test site builder + working-directory: source + run: python3 -m unittest discover -s tools -p release_site_test.py + - name: Convert release documentation and update latest redirect + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ steps.release.outputs.tag }} + LATEST_TAG: ${{ steps.release.outputs.latest }} + CONFIG_PATH: ${{ inputs.config_path }} + run: | + set -euo pipefail + config_args=() + if [[ -n "${CONFIG_PATH}" ]]; then + config_file="$(realpath --canonicalize-existing "source/${CONFIG_PATH}")" + if [[ "${config_file}" != "${GITHUB_WORKSPACE}/source/"* ]]; then + echo "Configuration must be a tracked file inside the main checkout" >&2 + exit 1 + fi + git -C source ls-files --error-unmatch -- "${CONFIG_PATH}" >/dev/null + config_args=(--config "${config_file}") + fi + python3 source/tools/release_site.py release site \ + --repository "${GITHUB_REPOSITORY}" --tag "${RELEASE_TAG}" --latest "${LATEST_TAG}" \ + "${config_args[@]}" + - name: Retain and stage complete Pages tree + env: + RELEASE_TAG: ${{ steps.release.outputs.tag }} + run: | + set -euo pipefail + git -C site config user.name 'github-actions[bot]' + git -C site config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git -C site add --all + if ! git -C site diff --cached --quiet; then + git -C site commit -m "site: publish ${RELEASE_TAG}" + git -C site push origin HEAD:coverage-pages + fi + mkdir -p public + rsync --archive --exclude='.git' site/ public/ + - uses: actions/configure-pages@v6 + - uses: actions/upload-pages-artifact@v5 + with: + path: public + - name: Deploy complete site + id: deployment + uses: actions/deploy-pages@v5 diff --git a/.github/workflows/release_prep.sh b/.github/workflows/release_prep.sh index f9cae848..f5912021 100755 --- a/.github/workflows/release_prep.sh +++ b/.github/workflows/release_prep.sh @@ -124,3 +124,6 @@ Copy [dev.MODULE.bazel](https://github.com/mboworks/${PACKAGE_NAME}/blob/main/ba include("//:dev.MODULE.bazel") \`\`\` EOF + +printf '\n' +bash tools/release_notes.sh "${TAG}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a84b9fbc..8066646e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,15 @@ repos: types: ["yaml"] files: ^\.github/workflows/ entry: actionlint - args: ["-shellcheck", ""] + # GitHub supports queue: max; actionlint does not yet recognize this key. + # https://docs.github.com/en/actions/concepts/workflows-and-actions/concurrency + args: + [ + "-shellcheck", + "", + "-ignore", + 'unexpected key "queue" for "concurrency" section', + ] - repo: https://github.com/codespell-project/codespell rev: v2.4.3 diff --git a/README.md b/README.md index 1777c58a..efabc91c 100644 --- a/README.md +++ b/README.md @@ -360,3 +360,90 @@ the proto Builder or when it was open sourced). This clone was established 2023.07.15. The source has since been moved and modified but remains as close to the original source as possible. + +## Release website + +Release notes use `.github/release-notes.md.template`, rendered by +`tools/release_notes.sh TAG`, to link to that tag's versioned website and related +release resources. The existing changelog and installation notes remain included. + +The [website](https://mboworks.github.io/proto/) forwards to the latest published +stable release at `site/tag//`, preserving the exact Git tag name. +Each release keeps its converted HTML, images, and configured files. Retrying +publication leaves an existing snapshot unchanged; a different commit cannot +replace it. Older versions remain directly accessible. + +[`release-site.json`](release-site.json) defines the layout. Source names are +relative to the repository root; destinations are relative to that release's +site directory. For example: + +```json +{ + "pages": { + "README.md": "index.html", + "docs/guide.md": "guide/index.html" + }, + "files": { + "schema/example.json": "schema/v1.json" + }, + "links": [ + { + "label": "Release", + "href": "https://github.com/{owner}/{repo}/releases/tag/{tag}" + } + ] +} +``` + +Use existing source files in the actual configuration. `pages` converts Markdown; +optional `files` copies other files unchanged. `README.md` must map to `index.html`. +The generated `documents.html`, `release.json`, `release-site.json`, and `assets/` +paths are reserved. Destination paths cannot have hidden components (names starting +with a dot), because the Pages artifact uploader excludes them. Hidden source +paths remain valid; for example, `.github/workflows/README.md` maps to +`workflows/index.html`. +Navigation links support `{owner}`, `{repo}`, `{tag}`, `{version}`, and `{commit}`. +`{version}` omits a leading `v` for compatibility with coverage report paths. +By default, the configuration and content come from the release tag. Every linked +local Markdown page (including directory README links) must have a `pages` mapping. +Publication fails for an omitted mapping, a missing generated file, or a broken +anchor within the snapshot. Links to configured pages follow their destination +mappings; other local source links use the exact release commit. Embedded images are copied, including remote badges. Markdown +conversion uses the [GitHub Markdown API](https://docs.github.com/en/rest/markdown/markdown) +at publication time; browsing the result requires no Markdown renderer or CDN. + +After the Release workflow succeeds, `Publish release site` retains the snapshot +on `coverage-pages` and deploys the complete Pages tree. Coverage and site +publication share a concurrency group to preserve both trees. GitHub's latest +stable release selects the root redirect; backfilling an older release does not +make it latest. The workflow can also be dispatched with a published tag to retry +publication. Enable GitHub Pages with +**GitHub Actions** as its source, and set the repository's About website to +`https://mboworks.github.io/proto/`. + +### Backfill a historical release + +No new release or tag change is needed. Manually dispatch `Publish release site` +with `tag` set to the historical release and `config_path` set to a tracked JSON +file on `main`. Leave `config_path` empty to use a configuration already in the tag. +For example, after selecting a compatible configuration and an existing tag: + +```sh +gh workflow run pages.yml --repo mboworks/proto --ref main \ + -f tag="$RELEASE_TAG" -f config_path=release-site.json +``` + +The override controls only publication layout; all Markdown and copied files come +from the selected tag. Each new snapshot retains the exact configuration as +`release-site.json`, with its SHA-256, origin, and source commit in `release.json`. +A configuration can serve several historical tags when its sources exist in each. +For another layout, commit another configuration and select its path. Missing +sources or links fail publication instead of using newer content. Retrying a +published tag preserves its original HTML and configuration. + +Local regression tests: `python3 -m unittest discover -s tools -p release_site_test.py`. +CI also converts the configured documentation and checks the generated links in +a disposable runner directory. It never commits, retains, or deploys that preview. + +Release coverage links select `https://mboworks.github.io/proto/coverage/tag//`, +matching the coverage publisher rather than the moving main-branch report. diff --git a/release-site.json b/release-site.json new file mode 100644 index 00000000..9ca232ae --- /dev/null +++ b/release-site.json @@ -0,0 +1,21 @@ +{ + "pages": { + "AGENTS.md": "AGENTS.html", + "CHANGELOG.md": "CHANGELOG.html", + "CODE_OF_CONDUCT.md": "CODE_OF_CONDUCT.html", + "CONTRIBUTING.md": "CONTRIBUTING.html", + "GIT_RULES.md": "GIT_RULES.html", + "README.md": "index.html", + "RULES.md": "RULES.html", + "STYLE_CPP.md": "STYLE_CPP.html", + "STYLE_SH.md": "STYLE_SH.html", + "TODO.md": "TODO.html", + "docs/coverage.md": "docs/coverage.html" + }, + "links": [ + { + "label": "Coverage", + "href": "/{repo}/coverage/tag/{version}/" + } + ] +} diff --git a/tools/release_notes.sh b/tools/release_notes.sh new file mode 100755 index 00000000..b29ecbd2 --- /dev/null +++ b/tools/release_notes.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Render release-note links without building, publishing, or changing repository state. +set -euo pipefail + +TAG="${1:?Usage: release_notes.sh TAG}" +if [[ ! "${TAG}" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "Invalid release tag: ${TAG}" >&2 + exit 1 +fi +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +sed -e "s|@TAG@|${TAG}|g" -e "s|@VERSION@|${TAG#v}|g" \ + "${ROOT}/.github/release-notes.md.template" diff --git a/tools/release_notes_test.py b/tools/release_notes_test.py new file mode 100644 index 00000000..21d69e96 --- /dev/null +++ b/tools/release_notes_test.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +"""Exercise the release-note renderer with the actual repository template.""" + +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +REPO = "proto" + + +class ReleaseNotesTest(unittest.TestCase): + def render(self, tag): + with tempfile.TemporaryDirectory() as cwd: + return subprocess.run( + ["bash", str(ROOT / "tools/release_notes.sh"), tag], + cwd=cwd, text=True, capture_output=True, check=False) + + def test_release_resources(self): + for tag in ("1.2.3", "v1.2.3", "v1.2.3-rc.1"): + with self.subTest(tag=tag): + result = self.render(tag) + self.assertEqual(result.returncode, 0, result.stderr) + notes = result.stdout + base = f"https://mboworks.github.io/{REPO}" + self.assertIn(f"{base}/site/tag/{tag}/", notes) + self.assertNotIn("@TAG@", notes) + self.assertNotIn("@VERSION@", notes) + version = tag.removeprefix("v") + if REPO in ("mbo", "xff", "carve", "proto"): + self.assertIn(f"{base}/coverage/tag/{version}/", notes) + else: + self.assertNotIn("/coverage/", notes) + if REPO == "xff": + self.assertIn(f"{base}/releases/{version}/)", notes) + self.assertIn(f"{base}/releases/{version}/XFF.md", notes) + if REPO == "coderef": + self.assertIn(f"/blob/{tag}/CHANGELOG.md", notes) + self.assertIn(f"{base}/site/tag/{tag}/schema/v1.json", notes) + + def test_reject_invalid_tags_without_partial_notes(self): + for tag in ("", "main", "../1.2.3", "v1.2.3|bad", "1.2.3\nmain"): + with self.subTest(tag=tag): + result = self.render(tag) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release_site.py b/tools/release_site.py new file mode 100644 index 00000000..f5634cd4 --- /dev/null +++ b/tools/release_site.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +"""Convert a release checkout to a retained, self-contained documentation site.""" + +import argparse +import hashlib +import html +from html.parser import HTMLParser +import json +from pathlib import Path +import posixpath +import re +import subprocess +import tempfile +from urllib.parse import quote, unquote, urlsplit, urlunsplit +from urllib.request import urlopen + + +STYLE = """ +:root { color-scheme: light dark; font: 17px/1.6 system-ui, sans-serif; } +body { max-width: 76rem; margin: auto; padding: 2rem; } +a { color: light-dark(#075da8, #8cc8ff); } +nav { display: flex; flex-wrap: wrap; gap: 1rem; border-bottom: 1px solid #888; } +pre { padding: 1rem; overflow: auto; background: light-dark(#f3f5f7, #20252b); } +code { font-size: .9em; } img { max-width: 100%; } +table { display: block; overflow: auto; border-collapse: collapse; } +th, td { border: 1px solid #888; padding: .4rem .7rem; } +blockquote { border-left: 4px solid #888; margin-left: 0; padding-left: 1rem; } +""" + + +def git(source, *args): + return subprocess.check_output(["git", "-C", str(source), *args], text=True).strip() + + +def render(markdown, repository): + return subprocess.check_output( + ["gh", "api", "markdown", "--input", "-"], + input=json.dumps({"text": markdown, "mode": "gfm", "context": repository}), + text=True, + ) + + +def version(tag): + if not re.fullmatch(r"v?[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?", tag): + raise ValueError(f"Invalid release tag: {tag!r}") + return tag.removeprefix("v") + + +def configuration(source, override=None): + data = (override if override is not None else source / "release-site.json").read_bytes() + config = json.loads(data) + pages = config["pages"] + if not pages or pages.get("README.md") != "index.html": + raise ValueError("README.md must map to index.html") + destinations = set() + for src, dst in pages.items(): + for path in (src, dst): + if (not isinstance(path, str) or path.startswith("/") + or ".." in Path(path).parts or str(Path(path)) != path + or any(char in path for char in "\\?#")): + raise ValueError(f"Unsafe site path: {path!r}") + if not src.endswith(".md") or not dst.endswith(".html"): + raise ValueError("Page mappings must convert .md sources to .html destinations") + if dst in destinations or dst == "documents.html" or dst.startswith("assets/"): + raise ValueError(f"Duplicate or reserved destination: {dst}") + if any(part.startswith(".") for part in Path(dst).parts): + raise ValueError(f"Pages excludes hidden destinations: {dst}") + destinations.add(dst) + for src, dst in config.get("files", {}).items(): + if src in pages: + raise ValueError(f"Source is mapped as both a page and a file: {src}") + if src.lower().endswith(".md"): + raise ValueError(f"Markdown must be converted through pages: {src}") + for path in (src, dst): + if (not isinstance(path, str) or path.startswith("/") + or ".." in Path(path).parts or str(Path(path)) != path + or any(char in path for char in "\\?#")): + raise ValueError(f"Unsafe asset path: {path!r}") + if dst in destinations or dst in ("documents.html", "release.json", "release-site.json") or dst.startswith("assets/"): + raise ValueError(f"Duplicate or reserved destination: {dst}") + if any(part.startswith(".") for part in Path(dst).parts): + raise ValueError(f"Pages excludes hidden destinations: {dst}") + destinations.add(dst) + return config, data + + +def headings(body): + """The Markdown API omits GitHub's heading anchors; restore their slugs.""" + used = set() + + def heading(match): + level, attrs, text = match.groups() + plain = html.unescape(re.sub(r"<[^>]*>", "", text)).lower() + slug = re.sub(r"[^\w\- ]", "", plain).replace(" ", "-") + anchor = slug + count = 0 + while anchor in used: + count += 1 + anchor = f"{slug}-{count}" + used.add(anchor) + return f'{text}' + + return re.sub(r"]*)>(.*?)", heading, body, flags=re.DOTALL) + + +class Links(HTMLParser): + def __init__(self, source, output, document, documents, repository, sha, tag): + super().__init__(convert_charrefs=False) + self.source = source + self.output = output + self.document = document + self.documents = documents + self.repository = repository + self.sha = sha + self.tag = tag + self.parts = [] + + def local(self, value): + parsed = urlsplit(value) + prefix = f"https://github.com/{self.repository}/" + if value.startswith(prefix): + rest = value[len(prefix):] + for kind in ("blob/", "tree/"): + for ref in ("main", "master", self.tag, self.sha): + start = f"{kind}{ref}/" + if rest.startswith(start): + return posixpath.normpath(unquote(urlsplit(rest[len(start):]).path)) + if parsed.scheme or parsed.netloc or not parsed.path: + return None + path = unquote(parsed.path) + if path.startswith("/"): + return posixpath.normpath(path.lstrip("/")) + return posixpath.normpath(posixpath.join(posixpath.dirname(self.document), path)) + + def rewrite(self, value, image=False): + parsed = urlsplit(value) + owner, repo = self.repository.split("/") + coverage = f"/{repo}/coverage/" + if not image and (value.startswith(coverage) or value.startswith(f"https://{owner}.github.io{coverage}")): + return f"/{repo}/coverage/tag/{version(self.tag)}/" + local = self.local(value) + if local is not None: + if local == ".." or local.startswith("../"): + raise ValueError(f"Link escapes repository: {value}") + readme = f"{local.rstrip('/')}/README.md" + if local not in self.documents and (readme in self.documents or (self.source / readme).is_file()): + local = readme + if local == ".": + local = "README.md" + if local in self.documents and not image: + target = posixpath.relpath(self.documents[local], posixpath.dirname(self.documents[self.document]) or ".") + return urlunsplit(("", "", quote(target), parsed.query, parsed.fragment)) + if not image and local.lower().endswith(".md"): + raise ValueError(f"{self.document}: linked Markdown has no page mapping: {local}") + path = self.source / local + if image: + if not path.is_file() or path.resolve() != path.absolute(): + raise ValueError(f"Missing or symlinked image: {local}") + data = path.read_bytes() + suffix = path.suffix + else: + kind = "tree" if path.is_dir() else "blob" + return f"https://github.com/{self.repository}/{kind}/{self.sha}/{quote(local)}" + ( + f"#{parsed.fragment}" if parsed.fragment else "" + ) + elif image: + if parsed.scheme not in ("https", "http"): + raise ValueError(f"Unsupported image URL: {value}") + with urlopen(value, timeout=60) as response: + data = response.read() + media = response.headers.get_content_type() + suffix = {"image/svg+xml": ".svg", "image/png": ".png", "image/jpeg": ".jpg", + "image/gif": ".gif", "image/webp": ".webp"}.get(media) + if suffix is None: + raise ValueError(f"Unsupported image content type: {media}") + else: + return value + asset = f"assets/{hashlib.sha256(data).hexdigest()}{suffix}" + (self.output / "assets").mkdir(exist_ok=True) + (self.output / asset).write_bytes(data) + return posixpath.relpath(asset, posixpath.dirname(self.documents[self.document]) or ".") + + def handle_starttag(self, tag, attrs): + rewritten = [] + for key, value in attrs: + if key in ("data-canonical-src", "srcset"): + continue + if value is not None and key in ("href", "src"): + value = self.rewrite(value, image=tag == "img" and key == "src") + rewritten.append(key if value is None else f'{key}="{html.escape(value, quote=True)}"') + self.parts.append(f"<{tag}{' ' if rewritten else ''}{' '.join(rewritten)}>") + + def handle_startendtag(self, tag, attrs): + self.handle_starttag(tag, attrs) + + def handle_endtag(self, tag): + self.parts.append(f"") + + def handle_data(self, data): + self.parts.append(data) + + def handle_entityref(self, name): + self.parts.append(f"&{name};") + + def handle_charref(self, name): + self.parts.append(f"&#{name};") + + +class PageReferences(HTMLParser): + """Collect browser-visible anchors and links from final HTML.""" + + def __init__(self, text): + super().__init__() + self.anchors = set() + self.links = [] + self.feed(text) + + def handle_starttag(self, tag, attrs): + for key, value in attrs: + if value is None: + continue + if key == "id" or (tag == "a" and key == "name"): + self.anchors.add(value) + if key in ("href", "src"): + self.links.append(value) + + handle_startendtag = handle_starttag + + +def validate_site(output, repository, tag): + """Reject broken links inside the snapshot before retaining or deploying it.""" + owner, repo = repository.split("/") + base = f"/{repo}/site/tag/{tag}/" + host = f"{owner}.github.io" + pages = {path.relative_to(output).as_posix(): PageReferences(path.read_text()) + for path in output.rglob("*.html")} + for document, page in pages.items(): + for link in page.links: + parsed = urlsplit(link) + if parsed.scheme or parsed.netloc: + if parsed.scheme not in ("http", "https") or parsed.netloc != host: + continue + if not parsed.path.startswith(base): + continue + path = unquote(parsed.path) + if path.startswith("/"): + if not path.startswith(base): + continue # Coverage and other explicitly separate Pages trees. + target = posixpath.normpath(path[len(base):]) + elif path: + target = posixpath.normpath(posixpath.join(posixpath.dirname(document), path)) + else: + target = document + if target == ".." or target.startswith("../"): + raise ValueError(f"{document}: link escapes release snapshot: {link}") + destination = output / target + if destination.is_dir(): + target = posixpath.join(target, "index.html") + destination = output / target + target = posixpath.normpath(target) + if not destination.is_file(): + raise ValueError(f"{document}: missing generated link target: {link}") + fragment = unquote(parsed.fragment) + if fragment and target in pages and fragment not in pages[target].anchors: + raise ValueError(f"{document}: missing generated anchor: {link}") + + +def build(source, retained, repository, tag, renderer=render, config_path=None): + source = source.resolve() + release_version = version(tag) + destination = retained / "site" / "tag" / tag + sha = git(source, "rev-parse", "HEAD") + if destination.exists(): + metadata = json.loads((destination / "release.json").read_text()) + if metadata["commit"] != sha or metadata["tag"] != tag: + raise ValueError("A retained release cannot be replaced by a different commit or tag") + return + config, config_data = configuration(source, config_path) + documents = config["pages"] + files = config.get("files", {}) + tracked = set(git(source, "ls-files", "-z").split("\0")) + for document in [*documents, *files]: + path = source / document + if document not in tracked or not path.is_file() or path.resolve() != path.absolute(): + raise ValueError(f"Missing, untracked, or symlinked document: {document}") + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=destination.parent) as temporary: + output = Path(temporary) + for src, dst in files.items(): + target = output / dst + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes((source / src).read_bytes()) + owner, repo = repository.split("/") + base = f"/{repo}/site/tag/{tag}/" + links = [("Home", base), ("Documentation", base + "documents.html"), + ("Release & downloads", f"https://github.com/{repository}/releases/tag/{tag}"), + ("Source", f"https://github.com/{repository}/tree/{sha}")] + for link in config.get("links", []): + links.append((link["label"], link["href"].format( + repo=repo, owner=owner, tag=tag, version=release_version, commit=sha))) + navigation = "".join(f'{html.escape(label)}' for label, url in links) + + def page(title, body): + return (f'' + f'' + f'{html.escape(title)} - {repo} {tag}' + f'

{repo} {tag}

{body}
' + f'
Release snapshot ยท {sha}
\n') + + for document in sorted(documents): + parser = Links(source, output, document, {**documents, **files}, repository, sha, tag) + parser.feed(headings(renderer((source / document).read_text(), repository))) + target = output / documents[document] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(page(document, "".join(parser.parts))) + index = "

Documentation

" + (output / "documents.html").write_text(page("Documentation", index)) + (output / "release-site.json").write_bytes(config_data) + (output / "release.json").write_text(json.dumps({ + "tag": tag, "commit": sha, + "configuration": {"origin": "override" if config_path is not None else "tag", + "sha256": hashlib.sha256(config_data).hexdigest()}, + }) + "\n") + validate_site(output, repository, tag) + # Rename only after every document and image has been converted successfully. + output.rename(destination) + + +def redirect(retained, tag): + version(tag) # Validate the exact Git tag before using it as a path. + target = f"site/tag/{tag}/" + if not (retained / target / "index.html").is_file(): + return # An older backfill must not redirect to an unpublished latest site. + (retained / "index.html").write_text( + '' + f'' + f'Latest releaseLatest release\n' + ) + (retained / ".nojekyll").touch() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("retained", type=Path) + parser.add_argument("--repository", required=True) + parser.add_argument("--tag", required=True) + parser.add_argument("--latest", required=True) + parser.add_argument("--config", type=Path, help="Explicit configuration override for historical tags") + args = parser.parse_args() + build(args.source, args.retained, args.repository, args.tag, config_path=args.config) + if args.latest: + redirect(args.retained, args.latest) + + +if __name__ == "__main__": + main() diff --git a/tools/release_site_test.py b/tools/release_site_test.py new file mode 100644 index 00000000..07553d74 --- /dev/null +++ b/tools/release_site_test.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +"""Regression tests for release snapshots, configured links, and retained history.""" + +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest import mock + +import release_site as site + + +class ReleaseSiteTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.source = self.root / "source" + self.retained = self.root / "retained" + self.source.mkdir() + subprocess.run(["git", "init", "-q", str(self.source)], check=True) + self.write("README.md", "release readme") + self.write("docs/guide.md", "release guide") + self.write("image.svg", '') + self.config = {"pages": {"README.md": "index.html", "docs/guide.md": "guide/start.html"}, + "links": [{"label": "Coverage", "href": "/{repo}/coverage/tag/{version}/"}]} + self.write("release-site.json", json.dumps(self.config)) + subprocess.run(["git", "-C", str(self.source), "add", "."], check=True) + subprocess.run(["git", "-C", str(self.source), "-c", "user.name=Test", + "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false", + "commit", "-qm", "fixture"], check=True) + + def write(self, name, text): + target = self.source / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + + def render(self, markdown, repository): + self.assertEqual(repository, "mboworks/mbo") + if markdown == "release readme": + return ('

A title!

A title!

' + 'Guide' + 'Guide' + 'Coverage' + 'Source') + self.assertEqual(markdown, "release guide") + return '

Usage

Home' + + def build(self, tag="v1.2.3", renderer=None): + site.build(self.source, self.retained, "mboworks/mbo", tag, renderer or self.render) + return self.retained / "site/tag" / tag + + def test_configured_destinations_resolve_local_and_github_links(self): + output = self.build() + home = (output / "index.html").read_text() + guide = (output / "guide/start.html").read_text() + self.assertIn('href="guide/start.html#usage"', home) + self.assertIn('href="guide/start.html"', home) + self.assertIn('href="../index.html#a-title"', guide) + self.assertIn('id="a-title"', home) + self.assertIn('id="a-title-1"', home) + self.assertIn('/mbo/coverage/tag/1.2.3/', home) + self.assertNotIn('/coverage/main/', home) + sha = site.git(self.source, "rev-parse", "HEAD") + self.assertIn(f'/blob/{sha}/src/code.cc', home) + self.assertEqual(len(list((output / "assets").iterdir())), 1) + self.assertIn('guide/start.html', (output / "documents.html").read_text()) + + def test_rerun_retains_original_bytes_without_rendering(self): + output = self.build() + before = {p.relative_to(output): p.read_bytes() for p in output.rglob("*") if p.is_file()} + self.build(renderer=mock.Mock(side_effect=AssertionError("must not render again"))) + self.assertEqual(before, {p.relative_to(output): p.read_bytes() + for p in output.rglob("*") if p.is_file()}) + + def test_changed_commit_cannot_replace_retained_release(self): + self.build() + with mock.patch.object(site, "git", return_value="different commit"): + with self.assertRaisesRegex(ValueError, "cannot be replaced"): + self.build() + + def test_new_release_and_backfill_preserve_coverage_schema_and_history(self): + for name in ("coverage/tag/1.2.3/index.html", "schema/v1.json", "releases/1.2.3/index.html"): + target = self.retained / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("retained") + self.build("1.2.3") + self.build("2.0.0") + site.redirect(self.retained, "2.0.0") + self.build("1.0.0") + site.redirect(self.retained, "2.0.0") + self.assertIn('url=site/tag/2.0.0/', (self.retained / "index.html").read_text()) + for name in ("coverage/tag/1.2.3/index.html", "schema/v1.json", "releases/1.2.3/index.html"): + self.assertEqual((self.retained / name).read_text(), "retained") + for tag in ("1.0.0", "1.2.3", "2.0.0"): + self.assertTrue((self.retained / f"site/tag/{tag}/index.html").is_file()) + + def test_unpublished_latest_does_not_break_existing_redirect(self): + self.build() + site.redirect(self.retained, "v1.2.3") + before = (self.retained / "index.html").read_bytes() + site.redirect(self.retained, "9.0.0") + self.assertEqual((self.retained / "index.html").read_bytes(), before) + + def test_conversion_failure_leaves_no_partial_release(self): + with self.assertRaisesRegex(RuntimeError, "renderer unavailable"): + self.build(renderer=mock.Mock(side_effect=RuntimeError("renderer unavailable"))) + self.assertFalse((self.retained / "site/tag/v1.2.3").exists()) + + def test_external_images_are_snapshotted(self): + response = mock.MagicMock() + response.__enter__.return_value = response + response.read.return_value = b"image bytes" + response.headers.get_content_type.return_value = "image/png" + with mock.patch.object(site, "urlopen", return_value=response): + output = self.build(renderer=lambda *_: '') + self.assertNotIn('src="https:', (output / "index.html").read_text()) + self.assertEqual(next((output / "assets").iterdir()).read_bytes(), b"image bytes") + + def test_invalid_paths_and_duplicate_destinations_fail(self): + for pages in ({"README.md": "index.html", "../bad.md": "bad.html"}, + {"README.md": "index.html", "docs/guide.md": "../bad.html"}, + {"README.md": "index.html", "docs/guide.md": "index.html"}): + with self.subTest(pages=pages): + self.write("release-site.json", json.dumps({"pages": pages})) + with self.assertRaises(ValueError): + self.build() + + def test_configured_files_are_copied_and_links_use_their_destinations(self): + self.config["files"] = {"image.svg": "static/logo.svg"} + self.write("release-site.json", json.dumps(self.config)) + output = self.build(renderer=lambda *_: 'Download') + self.assertEqual((output / "static/logo.svg").read_bytes(), + (self.source / "image.svg").read_bytes()) + self.assertIn('href="static/logo.svg"', (output / "index.html").read_text()) + self.assertIn('href="../static/logo.svg"', (output / "guide/start.html").read_text()) + + def test_missing_configured_source_fails_before_rendering(self): + self.config["pages"]["missing.md"] = "missing.html" + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "Missing"): + self.build(renderer=mock.Mock(side_effect=AssertionError("must not render"))) + + def test_root_relative_coverage_links_select_release(self): + output = self.build(renderer=lambda *_: 'Report') + home = (output / "index.html").read_text() + self.assertIn('href="/mbo/coverage/tag/1.2.3/"', home) + self.assertNotIn('/coverage/main/', home) + + def test_heading_suffix_collisions_get_unique_anchors(self): + rendered = site.headings("

Title

Title

Title-1

") + self.assertIn('id="title"', rendered) + self.assertIn('id="title-1"', rendered) + self.assertIn('id="title-1-1"', rendered) + + def test_one_source_cannot_have_page_and_file_destinations(self): + self.config["files"] = {"README.md": "readme.txt"} + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "both a page and a file"): + self.build() + + def test_backfill_uses_old_source_and_retains_exact_override(self): + override = self.root / "backfill.json" + data = json.dumps(self.config, indent=4).encode() + b"\n" + override.write_bytes(data) + subprocess.run(["git", "-C", str(self.source), "rm", "-q", "release-site.json"], check=True) + subprocess.run(["git", "-C", str(self.source), "-c", "user.name=Test", + "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false", + "commit", "-qm", "Historical source without a site config"], check=True) + site.build(self.source, self.retained, "mboworks/mbo", "v1.2.3", + self.render, config_path=override) + output = self.retained / "site/tag/v1.2.3" + self.assertEqual((output / "release-site.json").read_bytes(), data) + metadata = json.loads((output / "release.json").read_text()) + self.assertEqual(metadata["commit"], site.git(self.source, "rev-parse", "HEAD")) + self.assertEqual(metadata["configuration"]["origin"], "override") + self.assertEqual(metadata["configuration"]["sha256"], site.hashlib.sha256(data).hexdigest()) + self.assertFalse((self.source / "release-site.json").exists()) + before = (output / "index.html").read_bytes() + override.write_text("invalid replacement config") + site.build(self.source, self.retained, "mboworks/mbo", "v1.2.3", + mock.Mock(side_effect=AssertionError("must not render")), config_path=override) + self.assertEqual((output / "index.html").read_bytes(), before) + self.assertEqual((output / "release-site.json").read_bytes(), data) + + def test_override_never_reads_missing_content_from_publisher_checkout(self): + override = self.root / "backfill.json" + self.config["pages"]["newer.md"] = "newer.html" + override.write_text(json.dumps(self.config)) + (self.root / "newer.md").write_text("content only present alongside publisher config") + with self.assertRaisesRegex(ValueError, "Missing.*newer.md"): + site.build(self.source, self.retained, "mboworks/mbo", "v1.2.3", + mock.Mock(side_effect=AssertionError("must not render")), config_path=override) + + def test_tag_config_remains_default_and_is_retained(self): + output = self.build() + metadata = json.loads((output / "release.json").read_text()) + self.assertEqual(metadata["configuration"]["origin"], "tag") + self.assertEqual((output / "release-site.json").read_bytes(), + (self.source / "release-site.json").read_bytes()) + + def test_unmapped_markdown_links_fail_publication(self): + del self.config["pages"]["docs/guide.md"] + self.write("release-site.json", json.dumps(self.config)) + for link in ("docs/guide.md", "/docs/guide.md", + "https://github.com/mboworks/mbo/blob/main/docs/guide.md"): + with self.subTest(link=link), self.assertRaisesRegex(ValueError, "no page mapping"): + self.build(renderer=lambda *_: f'Guide') + self.assertFalse((self.retained / "site/tag/v1.2.3").exists()) + + def test_directory_readme_requires_a_mapping(self): + self.write("docs/README.md", "directory guide") + with self.assertRaisesRegex(ValueError, "no page mapping"): + self.build(renderer=lambda *_: 'Guide') + + def test_missing_generated_anchor_aborts_without_retaining_snapshot(self): + with self.assertRaisesRegex(ValueError, "missing generated anchor"): + self.build(renderer=lambda *_: 'Guide') + self.assertFalse((self.retained / "site/tag/v1.2.3").exists()) + + def test_generated_audit_checks_files_anchors_and_snapshot_boundaries(self): + output = self.root / "html" + output.mkdir() + (output / "guide.html").write_text('

Guide

') + for link, error in (("missing.html", "missing generated link target"), + ("guide.html#absent", "missing generated anchor"), + ("../outside.html", "escapes release snapshot"), + ("/mbo/site/tag/1.2.3/missing.html", "missing generated link target"), + ("https://mboworks.github.io/mbo/site/tag/1.2.3/guide.html#absent", + "missing generated anchor")): + with self.subTest(link=link): + (output / "index.html").write_text(f'Target') + with self.assertRaisesRegex(ValueError, error): + site.validate_site(output, "mboworks/mbo", "1.2.3") + (output / "index.html").write_text( + 'GuideSelf' + 'Coverage' + 'External') + site.validate_site(output, "mboworks/mbo", "1.2.3") + + def test_reserved_config_destination_cannot_be_overwritten(self): + self.config["files"] = {"image.svg": "release-site.json"} + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "reserved destination"): + self.build() + + def test_pages_packaging_exclusions_are_rejected_before_rendering(self): + for destination in (".github/guide.html", "guide/.hidden.html", ".hidden/guide.html"): + with self.subTest(destination=destination): + self.config["pages"]["docs/guide.md"] = destination + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "excludes hidden"): + self.build(renderer=mock.Mock(side_effect=AssertionError("must not render"))) + self.config["pages"]["docs/guide.md"] = "guide.html" + self.config["files"] = {"image.svg": "assets-custom/.hidden.svg"} + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "excludes hidden"): + self.build() + + def test_invalid_tag_cannot_escape_site_directory(self): + for tag in ("../bad", "v1.2.3/evil", "1.2.3\nextra", "main"): + with self.subTest(tag=tag), self.assertRaises(ValueError): + self.build(tag) + + def test_symlinked_document_is_rejected(self): + target = self.source / "docs/guide.md" + target.unlink() + target.symlink_to(self.source / "README.md") + with self.assertRaisesRegex(ValueError, "symlinked"): + self.build() + + +if __name__ == "__main__": + unittest.main()