From 215ddf872e55b10b15b1ce180cffbd0eee555c00 Mon Sep 17 00:00:00 2001 From: TM Hospitality Strategies Date: Wed, 21 Oct 2026 23:19:31 -0400 Subject: [PATCH 1/8] Type-check list and boolean fields, strip trailing frontmatter comments, one link-scheme rule, build-time duplicate id check - frontmatter: a trailing ' # comment' is dropped outside quotes (#79); a comment or blank line between a key and its block list no longer breaks the list (#84); TEMPLATE.md moves its inline guidance to comment lines - validate: new V24 checkFieldTypes rejects tags/subcategories/formats that parsed as text and quoted booleans, with the fix in the message (#79) - shared: EXTERNAL_SCHEME_RE and ANY_SCHEME_RE used by validate, the link resolver and the page checks; an ftp:/data:/javascript: link is rejected everywhere with the scheme named; a line-start autolink is not raw HTML (#83) - build: a duplicate id fails the build instead of overwriting a page (#81) Closes #79, #81, #83, #84 Co-Authored-By: Claude Fable 5.1 --- catalog/TEMPLATE.md | 9 +- site/build.mjs | 8 + site/checks.mjs | 32 +++ site/checks.test.mjs | 11 + site/lib/frontmatter.mjs | 291 ++++++++++++++------------ site/lib/lib.test.mjs | 38 ++++ site/lib/links.mjs | 5 +- site/lib/markdown.mjs | 429 ++++++++++++++++++++------------------- site/lib/page-checks.mjs | 8 +- site/lib/shared.mjs | 10 + site/validate.mjs | 29 +-- 11 files changed, 500 insertions(+), 370 deletions(-) diff --git a/catalog/TEMPLATE.md b/catalog/TEMPLATE.md index 311624b..3a124ff 100644 --- a/catalog/TEMPLATE.md +++ b/catalog/TEMPLATE.md @@ -11,9 +11,12 @@ commercial: true attribution_required: false formats: [glTF, FBX, PNG] # subcategories and formats spelling: see CONTRIBUTING.md (formats must be in site/format-vocabulary.json) -tags: [low-poly, modular] # describe the content; never the licence or the publisher (V19, V21) -verified: 2026-07-19 # the day you read the licence at the source -status: needs-review # set active once the licence is read and quoted in ## Evidence +# tags describe the content; never the licence or the publisher (V19, V21) +tags: [low-poly, modular] +# verified: the day you read the licence at the source +verified: 2026-07-19 +# status: set active once the licence is read and quoted in ## Evidence +status: needs-review # Optional (omit if N/A): # publisher: Kenney # rights holder when that publisher has more than one entry; not a generic host # attribution_string: "Credit line" # required when attribution_required is true diff --git a/site/build.mjs b/site/build.mjs index cc5fd38..bc5a67c 100644 --- a/site/build.mjs +++ b/site/build.mjs @@ -51,6 +51,9 @@ function loadEntries(vocab) { const entries = []; const errors = []; const bodies = new Map(); + // validate.mjs catches a duplicate id too, but a local `npm run build` must + // not quietly write one entry's page over another's. + const fileOfId = new Map(); for (const file of files) { const text = fs.readFileSync(file, "utf8"); @@ -72,6 +75,11 @@ function loadEntries(vocab) { errors.push(`${rel} missing: ${missing.join(", ")}`); continue; } + if (fileOfId.has(String(meta.id))) { + errors.push(`duplicate id "${meta.id}": ${fileOfId.get(String(meta.id))} and ${rel}`); + continue; + } + fileOfId.set(String(meta.id), rel); bodies.set(String(meta.id), body); entries.push({ diff --git a/site/checks.mjs b/site/checks.mjs index e5dc20f..b0ca821 100644 --- a/site/checks.mjs +++ b/site/checks.mjs @@ -236,6 +236,38 @@ export function checkPublisherConsistency(entries) { return errors; } +/* ----------------------------------------------------------------- V24 */ +/** Fields that must parse as YAML lists. `hardware_tags` is checked in V12. */ +const LIST_FIELDS = ["subcategories", "formats", "tags"]; +/** Fields that must parse as booleans, plus the literal words each one also allows. */ +const BOOLEAN_FIELDS = { commercial: ["varies", "unknown"], attribution_required: ["unknown"] }; + +/** + * The list fields were checked for emptiness but never for being lists, and + * the two boolean fields accepted the quoted strings "true"/"false". A + * trailing comment or a pair of quotes then produced an entry that passed + * validation with no tags, or with "commercial ?" on its card. + */ +export function checkFieldTypes(rel, meta) { + const errors = []; + for (const field of LIST_FIELDS) { + const v = meta[field]; + if (empty(v) || Array.isArray(v)) continue; + errors.push(`${rel} ${field} must be a list like [a, b] or one "- item" per line; got the text "${String(v)}"`); + } + for (const [field, words] of Object.entries(BOOLEAN_FIELDS)) { + const v = meta[field]; + if (empty(v) || typeof v === "boolean" || words.includes(v)) continue; + const s = String(v); + if (s === "true" || s === "false") { + errors.push(`${rel} ${field} "${s}" is quoted text, not a boolean; write ${field}: ${s} without quotes`); + } else { + errors.push(`${rel} ${field} "${s}" is not true|false|${words.join("|")}`); + } + } + return errors; +} + /* ----------------------------------------------------------------- V12 */ /** Documented values for the optional 2D/UI taxonomy. See catalog/TEMPLATE.md. */ export const CAMERA_PERSPECTIVES = new Set([ diff --git a/site/checks.test.mjs b/site/checks.test.mjs index 6a56df0..e88dc53 100644 --- a/site/checks.test.mjs +++ b/site/checks.test.mjs @@ -19,6 +19,7 @@ import { checkEngineFields, checkEntryUrl, checkEvidenceDates, + checkFieldTypes, checkFormatVocabulary, evidenceDates, markdownAnchors, @@ -748,6 +749,16 @@ accepts( formatVocab.formats.filter((f) => Object.prototype.hasOwnProperty.call(aliases.formats, f)) ); +/* V24: field types (#79) ------------------------------------------------ */ +rejects("V24 rejects tags that parsed as text", checkFieldTypes("bad.md", { tags: "[a, b] # note" }), "tags must be a list"); +rejects("V24 rejects a subcategories string", checkFieldTypes("bad.md", { subcategories: "models" }), "subcategories must be a list"); +rejects("V24 rejects a quoted boolean", checkFieldTypes("bad.md", { commercial: "true" }), "write commercial: true without quotes"); +rejects("V24 rejects a quoted attribution_required", checkFieldTypes("bad.md", { attribution_required: "false" }), "write attribution_required: false without quotes"); +rejects("V24 rejects an unknown commercial word", checkFieldTypes("bad.md", { commercial: "maybe" }), 'commercial "maybe" is not true|false|varies|unknown'); +accepts("V24 accepts lists and booleans", checkFieldTypes("ok.md", { tags: ["a"], subcategories: ["b"], formats: ["PNG"], commercial: true, attribution_required: false })); +accepts("V24 accepts the literal words", checkFieldTypes("ok.md", { commercial: "varies", attribution_required: "unknown" })); +accepts("V24 leaves missing fields to the required check", checkFieldTypes("ok.md", {})); + /* ----------------------------------------------------------------------- */ if (failures.length) { console.error(`checks.test failed (${failures.length}):`); diff --git a/site/lib/frontmatter.mjs b/site/lib/frontmatter.mjs index 37af725..00f35fc 100644 --- a/site/lib/frontmatter.mjs +++ b/site/lib/frontmatter.mjs @@ -1,133 +1,158 @@ -/** - * Entry frontmatter and body helpers, shared by build.mjs and validate.mjs so - * the two can never read the same file differently. (They used to carry a - * copy each, and the copies had drifted: one undid quote escapes and the - * other did not.) - * - * The parser reads exactly the YAML subset entries use: `key: value`, quoted - * scalars, flow lists (`[a, "b, c"]`), block lists (`key:` then ` - item`) - * and `#` comment lines. Anything else, and a key given twice, is reported - * with its line number rather than guessed at. - */ -import { renderInline } from "./markdown.mjs"; -import { unquoteScalar } from "./shared.mjs"; - -/** Splits the inside of a flow list on commas that are not inside quotes. */ -function splitFlowList(inner) { - const parts = []; - let cur = ""; - let quote = null; - for (let i = 0; i < inner.length; i += 1) { - const ch = inner[i]; - if (quote) { - cur += ch; - if (quote === '"' && ch === "\\" && i + 1 < inner.length) cur += inner[++i]; - else if (ch === quote) quote = null; - } else if (ch === '"' || ch === "'") { - quote = ch; - cur += ch; - } else if (ch === ",") { - parts.push(cur); - cur = ""; - } else cur += ch; - } - parts.push(cur); - return parts.map((p) => unquoteScalar(p.trim())); -} - -export function parseScalar(raw) { - const v = raw.trim(); - if (v === "true") return true; - if (v === "false") return false; - if (v === "null" || v === "~" || v === "") return null; - if (v.length >= 2 && ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'")))) { - return unquoteScalar(v); - } - if (v.startsWith("[") && v.endsWith("]")) { - const inner = v.slice(1, -1).trim(); - return inner ? splitFlowList(inner) : []; - } - return v; -} - -/** - * `{ meta, body, errors }`, or null when the text has no frontmatter block. - * `errors` are "line N: ..." strings; the caller prefixes the file. - */ -export function parseFrontmatter(text) { - // A byte-order mark is invisible in an editor; ignore it. - const match = String(text) - .replace(/^/, "") - .match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n([\s\S]*))?$/); - if (!match) return null; - const meta = {}; - const errors = []; - const lines = match[1].split(/\r?\n/); - let listKey = null; - lines.forEach((line, i) => { - const lineNo = i + 2; // line 1 is the opening --- - if (!line.trim() || line.trim().startsWith("#")) return; - const item = line.match(/^\s+-\s+(.*)$/); - if (item) { - if (listKey === null) errors.push(`line ${lineNo}: list item without a key above it`); - else meta[listKey].push(parseScalar(item[1])); - return; - } - listKey = null; - const idx = line.indexOf(":"); - if (idx === -1 || /^\s/.test(line)) { - errors.push(`line ${lineNo}: "${line.trim()}" is not a key: value line`); - return; - } - const key = line.slice(0, idx).trim(); - if (Object.prototype.hasOwnProperty.call(meta, key)) { - errors.push(`line ${lineNo}: key "${key}" is given twice`); - return; - } - const value = parseScalar(line.slice(idx + 1)); - const next = lines[i + 1]; - if (value === null && next !== undefined && /^\s+-\s/.test(next)) { - meta[key] = []; - listKey = key; - return; - } - meta[key] = value; - }); - return { meta, body: match[2] || "", errors }; -} - -/** The `## Evidence` section of an entry body, up to the next `## ` heading, or null. */ -export function evidenceSection(body) { - const idx = body.search(/^## Evidence\s*$/m); - if (idx === -1) return null; - const rest = body.slice(idx); - const next = rest.search(/\n## (?!Evidence\s*$)/m); - return next === -1 ? rest : rest.slice(0, next); -} - -const ENTITIES = { "&": "&", "<": "<", ">": ">", """: '"', "'": "'" }; - -/** - * Plain text of a line of entry markdown. It goes through the same inline - * renderer as the entry page and drops the tags, so it strips exactly the - * markup the page renders: underscores and `#` stay literal (Rig_Medium, C#). - */ -export function plainText(markdown) { - const html = renderInline(markdown, (href) => ({ href, external: true })); - return html - .replace(/<[^>]+>/g, "") - .replace(/&(amp|lt|gt|quot|#39);/g, (m) => ENTITIES[m]) - .replace(/\s+/g, " ") - .trim(); -} - -/** The first paragraph of an entry body as plain text, at most 220 characters. */ -export function summaryFromBody(body) { - const first = - String(body) - .split(/\n\s*\n/) - .map((p) => p.trim()) - .find((p) => p && !p.startsWith("#")) || ""; - const clean = plainText(first); - return clean.length > 220 ? `${clean.slice(0, 217)}…` : clean; -} +/** + * Entry frontmatter and body helpers, shared by build.mjs and validate.mjs so + * the two can never read the same file differently. (They used to carry a + * copy each, and the copies had drifted: one undid quote escapes and the + * other did not.) + * + * The parser reads exactly the YAML subset entries use: `key: value`, quoted + * scalars, flow lists (`[a, "b, c"]`), block lists (`key:` then ` - item`) + * and `#` comment lines. Anything else, and a key given twice, is reported + * with its line number rather than guessed at. + */ +import { renderInline } from "./markdown.mjs"; +import { unquoteScalar } from "./shared.mjs"; + +/** Splits the inside of a flow list on commas that are not inside quotes. */ +function splitFlowList(inner) { + const parts = []; + let cur = ""; + let quote = null; + for (let i = 0; i < inner.length; i += 1) { + const ch = inner[i]; + if (quote) { + cur += ch; + if (quote === '"' && ch === "\\" && i + 1 < inner.length) cur += inner[++i]; + else if (ch === quote) quote = null; + } else if (ch === '"' || ch === "'") { + quote = ch; + cur += ch; + } else if (ch === ",") { + parts.push(cur); + cur = ""; + } else cur += ch; + } + parts.push(cur); + return parts.map((p) => unquoteScalar(p.trim())); +} + +/** + * Drops a trailing ` # comment`. As in YAML, a `#` starts a comment only after + * whitespace (or at the start) and outside quotes, so `C#` and `#fff` are + * content. A quote opens only where a scalar can start (the start, or after + * `[`, `,` or whitespace), so an apostrophe in `Kay's` is content too. + */ +function stripComment(raw) { + let quote = null; + for (let i = 0; i < raw.length; i += 1) { + const ch = raw[i]; + if (quote) { + if (quote === '"' && ch === "\\") i += 1; + else if (ch === quote) quote = null; + } else if ((ch === '"' || ch === "'") && (i === 0 || /[\s[,]/.test(raw[i - 1]))) { + quote = ch; + } else if (ch === "#" && (i === 0 || /\s/.test(raw[i - 1]))) { + return raw.slice(0, i); + } + } + return raw; +} + +export function parseScalar(raw) { + const v = stripComment(raw).trim(); + if (v === "true") return true; + if (v === "false") return false; + if (v === "null" || v === "~" || v === "") return null; + if (v.length >= 2 && ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'")))) { + return unquoteScalar(v); + } + if (v.startsWith("[") && v.endsWith("]")) { + const inner = v.slice(1, -1).trim(); + return inner ? splitFlowList(inner) : []; + } + return v; +} + +/** + * `{ meta, body, errors }`, or null when the text has no frontmatter block. + * `errors` are "line N: ..." strings; the caller prefixes the file. + */ +export function parseFrontmatter(text) { + // A byte-order mark is invisible in an editor; ignore it. + const match = String(text) + .replace(/^/, "") + .match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n([\s\S]*))?$/); + if (!match) return null; + const meta = {}; + const errors = []; + const lines = match[1].split(/\r?\n/); + let listKey = null; + lines.forEach((line, i) => { + const lineNo = i + 2; // line 1 is the opening --- + if (!line.trim() || line.trim().startsWith("#")) return; + const item = line.match(/^\s+-\s+(.*)$/); + if (item) { + if (listKey === null) errors.push(`line ${lineNo}: list item without a key above it`); + else meta[listKey].push(parseScalar(item[1])); + return; + } + listKey = null; + const idx = line.indexOf(":"); + if (idx === -1 || /^\s/.test(line)) { + errors.push(`line ${lineNo}: "${line.trim()}" is not a key: value line`); + return; + } + const key = line.slice(0, idx).trim(); + if (Object.prototype.hasOwnProperty.call(meta, key)) { + errors.push(`line ${lineNo}: key "${key}" is given twice`); + return; + } + const value = parseScalar(line.slice(idx + 1)); + // A block list may start after blank or comment lines under its key. + let j = i + 1; + while (j < lines.length && (!lines[j].trim() || lines[j].trim().startsWith("#"))) j += 1; + const next = lines[j]; + if (value === null && next !== undefined && /^\s+-\s/.test(next)) { + meta[key] = []; + listKey = key; + return; + } + meta[key] = value; + }); + return { meta, body: match[2] || "", errors }; +} + +/** The `## Evidence` section of an entry body, up to the next `## ` heading, or null. */ +export function evidenceSection(body) { + const idx = body.search(/^## Evidence\s*$/m); + if (idx === -1) return null; + const rest = body.slice(idx); + const next = rest.search(/\n## (?!Evidence\s*$)/m); + return next === -1 ? rest : rest.slice(0, next); +} + +const ENTITIES = { "&": "&", "<": "<", ">": ">", """: '"', "'": "'" }; + +/** + * Plain text of a line of entry markdown. It goes through the same inline + * renderer as the entry page and drops the tags, so it strips exactly the + * markup the page renders: underscores and `#` stay literal (Rig_Medium, C#). + */ +export function plainText(markdown) { + const html = renderInline(markdown, (href) => ({ href, external: true })); + return html + .replace(/<[^>]+>/g, "") + .replace(/&(amp|lt|gt|quot|#39);/g, (m) => ENTITIES[m]) + .replace(/\s+/g, " ") + .trim(); +} + +/** The first paragraph of an entry body as plain text, at most 220 characters. */ +export function summaryFromBody(body) { + const first = + String(body) + .split(/\n\s*\n/) + .map((p) => p.trim()) + .find((p) => p && !p.startsWith("#")) || ""; + const clean = plainText(first); + return clean.length > 220 ? `${clean.slice(0, 217)}…` : clean; +} diff --git a/site/lib/lib.test.mjs b/site/lib/lib.test.mjs index f6b3cbc..e3d0c18 100644 --- a/site/lib/lib.test.mjs +++ b/site/lib/lib.test.mjs @@ -118,6 +118,11 @@ eq("external untouched", resolve("https://a.test/x").href, "https://a.test/x"); eq("external flagged", resolve("https://a.test/x").external, true); throws("missing target fails", () => resolve("nope.md"), 'link target "nope.md" does not exist'); throws("escaping the repo fails", () => resolve("../../../outside.md"), "does not exist"); +// #83: one scheme rule everywhere, and the unsupported scheme is named. +throws("ftp link names its scheme", () => resolve("ftp://a.test/x"), 'unsupported scheme "ftp:"'); +throws("data link names its scheme", () => resolve("data:text/plain,hi"), 'unsupported scheme "data:"'); +throws("javascript link names its scheme", () => resolve("javascript:alert(1)"), 'unsupported scheme "javascript:"'); +eq("mailto is external", resolve("mailto:a@b.test").external, true); /* entry page --------------------------------------------------------------- */ import { entryPageHtml, jsonLd, scriptJson } from "./entry-page.mjs"; @@ -161,6 +166,9 @@ has("empty title", checkPage(okPage.replace("T", " has("dangling link", checkPage(okPage.replace("../e2/", "../e3/"), opts).join(), "../e3/ does not resolve"); has("link leaving the site", checkPage(okPage.replace("../e2/", "../../../x/"), opts).join(), "leaves the site"); eq("query and hash are ignored", checkPage(okPage.replace("../e2/", "../../?cat=2d#catalog"), opts).length, 0); +has("page check rejects a data: href", checkPage(okPage.replace("../e2/", "data:text/html,x"), opts).join(), 'unsupported scheme "data:"'); +has("page check rejects a javascript: href", checkPage(okPage.replace("../e2/", "javascript:void(0)"), opts).join(), 'unsupported scheme "javascript:"'); +eq("page check accepts mailto", checkPage(okPage.replace("../e2/", "mailto:a@b.test"), opts).length, 0); /* llms ------------------------------------------------------------------- */ import { llmsFullTxt, llmsTxt } from "./llms.mjs"; @@ -492,6 +500,34 @@ eq("frontmatter with CRLF line endings", parseFrontmatter("---\r\nid: a\r\nname: has("frontmatter rejects a duplicate key", fm(["id: a", "id: b"]).errors.join(), 'line 3: key "id" is given twice'); has("frontmatter rejects a line that is not key: value", fm(["id: a", "just words"]).errors.join(), 'line 3: "just words" is not a key: value line'); eq("no frontmatter is null", parseFrontmatter("# Title\n"), null); +// #79: trailing comments. #84: a comment between a key and its block list. +const f2 = fm([ + "id: a", + "tags: [low-poly, modular] # describe the content", + "name: Kay's pack # note", + 'attribution_string: "Art # by X" # keep the hash inside the quotes', + "langs: [c#, f#]", + 'colour: "#fff"', + "ref: a#b", + "verified: 2026-07-19 # the day you read the licence", + "formats:", + " # content words only", + "", + " - PNG # a comment on an item", + " - SVG", + "commercial: true # yes", +]); +eq("frontmatter strips a comment after a flow list", JSON.stringify(f2.meta.tags), JSON.stringify(["low-poly", "modular"])); +eq("frontmatter strips a comment after a bare scalar with an apostrophe", f2.meta.name, "Kay's pack"); +eq("frontmatter keeps a # inside quotes", f2.meta.attribution_string, "Art # by X"); +eq("frontmatter keeps C# and F#", JSON.stringify(f2.meta.langs), JSON.stringify(["c#", "f#"])); +eq("frontmatter keeps a quoted # value", f2.meta.colour, "#fff"); +eq("frontmatter keeps a # with no space before it", f2.meta.ref, "a#b"); +eq("frontmatter strips a comment after a date", f2.meta.verified, "2026-07-19"); +eq("frontmatter block list after a comment and a blank line", JSON.stringify(f2.meta.formats), JSON.stringify(["PNG", "SVG"])); +eq("frontmatter boolean before a comment", f2.meta.commercial, true); +eq("frontmatter has no errors with comments everywhere", f2.errors.join(), ""); +eq("frontmatter keeps a quoted boolean as text", fm(["id: a", 'commercial: "true"']).meta.commercial, "true"); eq("evidence stops at the next heading, whatever its letter", evidenceSection("## Evidence\n\n- e (2026-01-01)\n\n## Extras\n\n- x"), "## Evidence\n\n- e (2026-01-01)\n"); eq("summary keeps underscores", summaryFromBody("Split by Rig_Medium and **KHR_draco**.\n\n## Notes"), "Split by Rig_Medium and KHR_draco."); eq("summary keeps C# and link text", summaryFromBody("# T\n\nA [C#](x.md) library, `code`, *em*."), "A C# library, code, em."); @@ -502,6 +538,8 @@ eq("summary is capped at 220", summaryFromBody("word ".repeat(80)).length, 218); eq("link href keeps balanced parens", renderInline("[Foo](https://en.wikipedia.org/wiki/Foo_(bar)) next", passLink), 'Foo next'); eq("misnested emphasis stays literal", renderInline("**a *b** c*", passLink), "a *b c*"); eq("italic around bold still works", renderInline("*a **b** c*", passLink), "a b c"); +has("a line starting with an autolink is not raw HTML", rb(" is the page."), ' rb("bold"), "raw HTML"); const dupIds = rb("## Notes\n\nx\n\n## Notes\n\ny\n\n## Content\n\nz"); has("first heading keeps its id", dupIds, '

'); has("repeated heading gets a suffix", dupIds, '

'); diff --git a/site/lib/links.mjs b/site/lib/links.mjs index 199ba68..e9b7275 100644 --- a/site/lib/links.mjs +++ b/site/lib/links.mjs @@ -1,13 +1,16 @@ /** Rewrites links in an entry body for the entry's own page. */ import path from "node:path"; +import { ANY_SCHEME_RE, EXTERNAL_SCHEME_RE } from "./shared.mjs"; export class LinkError extends Error {} export function makeLinkResolver({ entryPath, idByPath, repo, kindOf, entryBase = "../" }) { const dir = path.posix.dirname(entryPath); return (raw) => { - if (/^(https?:|mailto:)/i.test(raw)) return { href: raw, external: true }; + if (EXTERNAL_SCHEME_RE.test(raw)) return { href: raw, external: true }; if (raw.startsWith("#")) return { href: raw, external: false }; + const scheme = raw.match(ANY_SCHEME_RE); + if (scheme) throw new LinkError(`${entryPath}: link "${raw}" uses the unsupported scheme "${scheme[1].toLowerCase()}:" (only https, http and mailto links are allowed)`); // Split off ?query and #hash; both are kept on the rewritten link. const m = raw.match(/^([^?#]*)(\?[^#]*)?(#.*)?$/); const target = m[1]; diff --git a/site/lib/markdown.mjs b/site/lib/markdown.mjs index 0a41959..ebd4243 100644 --- a/site/lib/markdown.mjs +++ b/site/lib/markdown.mjs @@ -1,214 +1,215 @@ -/** - * Renders catalog entry bodies to HTML. Deliberately small: it supports the - * markdown the catalog uses (measured 2026-09-24) and throws on anything - * else, so a new construct fails the build instead of rendering wrong. - * Underscores are literal: bodies contain identifiers like Rig_Medium. - */ -import { esc } from "./shared.mjs"; - -export class MarkdownError extends Error {} - -const slot = (i) => `\u0000${i}\u0000`; -const SLOT_RE = /\u0000(\d+)\u0000/g; - -const count = (str, sub) => str.split(sub).length - 1; - -/** - * A markdown link. The href may hold one level of balanced parens - * (Foo_(bar)). Shared with llms.mjs so both read the same href. - */ -export const LINK_RE = /\[([^\]]+)\]\(((?:[^()\s]|\([^()\s]*\))+)\)/g; - -/** - * Bold and italic on already-escaped text. Italic that would cross a bold - * boundary (`**a *b** c*`) stays literal: wrapping it would nest the tags - * wrongly. - */ -function marks(escaped) { - return escaped - .replace(/\*\*(?=\S)(.+?)(?<=\S)\*\*/g, "$1") - .replace(/(^|[^*\w])\*(?=\S)([^*]+?)(?<=\S)\*(?![*\w])/g, (whole, pre, inner) => - count(inner, "") === count(inner, "") ? `${pre}${inner}` : whole - ); -} - -/** Ids the page templates use themselves; a heading must not take one. */ -const RESERVED_IDS = new Set([ - "attribution-string", - "content", - "copy-attribution", - "copy-status", - "credits-all", - "credits-list", - "every-check", - "facts-title", - "owes-title", - "recent", - "sec-gaps", -]); - -export function renderInline(text, resolveLink) { - const slots = []; - const keep = (html) => { - slots.push(html); - return slot(slots.length - 1); - }; - // NUL marks a slot below; one in the input could make a slot hold its own - // token and the restore loop never settle. - let s = String(text).replaceAll("\u0000", ""); - // Code spans first: their contents are literal. - s = s.replace(/`([^`]+)`/g, (_, code) => keep(`${esc(code)}`)); - // Backslash escapes: any ASCII punctuation, as in CommonMark. - s = s.replace(/\\([!-/:-@[-`{-~])/g, (_, ch) => keep(esc(ch))); - // Links. The label keeps bold, italic and code; the href goes through the - // resolver. - s = s.replace(LINK_RE, (_, label, href) => { - const { href: out, external } = resolveLink(href); - const rel = external ? ' rel="noopener noreferrer"' : ""; - return keep(`${marks(esc(label))}`); - }); - // Autolinks in angle brackets. - s = s.replace(/<(https?:\/\/[^\s<>]+)>/g, (_, url) => keep(`${esc(url)}`)); - // Bare URLs, leaving trailing sentence punctuation outside the link. A URL - // stops at `*` (bold around it) and at a slot (a code span right after it), - // and keeps a closing paren that balances one inside it (Foo_(bar)). - s = s.replace(/https?:\/\/[^\s<>"*\u0000]+/g, (url) => { - let [, core, tail] = url.match(/^(.*?)([.,;:!?)\]]*)$/); - while (tail.startsWith(")") && count(core, "(") > count(core, ")")) { - core += ")"; - tail = tail.slice(1); - } - return keep(`${esc(core)}`) + tail; - }); - s = marks(esc(s)); - // Link labels can hold code slots, so restore until stable. - let before; - do { - before = s; - s = s.replace(SLOT_RE, (_, i) => slots[Number(i)]); - } while (s !== before); - return s; -} - -const UNSUPPORTED = [ - [/^\s*\|/, "a table row"], - [/^\s*```/, "a code fence"], - [/^\s*>/, "a blockquote"], - [/^\s*!\[/, "an image"], - [/^\s*<[A-Za-z!/]/, "raw HTML"], - [/^#{3,}\s/, "a heading below level 2"], - [/^\s*\d+[.)]\s/, "an ordered list"], -]; - -const lines = (md) => String(md).replace(/\r\n?/g, "\n").split("\n"); - -export function renderBlocks(md, { file, resolveLink, firstLine = 1 }) { - const html = []; - const stack = []; - const ids = new Set(RESERVED_IDS); - let para = []; - let item = null; - let blank = false; - const inline = (t) => renderInline(t, resolveLink); - const flushPara = () => { - if (para.length) html.push(`

${inline(para.join(" "))}

`); - para = []; - }; - const flushItem = () => { - if (item !== null) html.push(inline(item)); - item = null; - }; - const closeLists = () => { - flushItem(); - while (stack.length) { - html.push(""); - stack.pop(); - } - }; - lines(md).forEach((line, i) => { - for (const [re, what] of UNSUPPORTED) { - if (re.test(line)) { - throw new MarkdownError(`${file}:${firstLine + i}: ${what} is not supported by the site renderer`); - } - } - if (!line.trim()) { - flushPara(); - blank = true; - return; - } - const li = line.match(/^(\s*)[-*]\s+(.*)$/); - if (li) { - flushPara(); - const indent = li[1].replace(/\t/g, " ").length; - if (!stack.length) { - html.push("
  • "); - stack.push(indent); - } else if (indent > stack[stack.length - 1]) { - flushItem(); - html.push("
    • "); - stack.push(indent); - } else { - flushItem(); - while (stack.length > 1 && indent < stack[stack.length - 1]) { - html.push("
    "); - stack.pop(); - } - html.push("
  • "); - } - item = li[2]; - blank = false; - return; - } - if (stack.length && !blank && /^\s/.test(line)) { - item += ` ${line.trim()}`; - return; - } - closeLists(); - blank = false; - const h = line.match(/^(#{1,2})\s+(.*)$/); - if (h) { - flushPara(); - if (h[1] === "#") return; // the page shows the entry name as its h1 - // Unique on the page: a second "Notes" becomes notes-2, and a heading - // never takes an id the page template uses. - const base = h[2].toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "section"; - let id = base; - for (let n = 2; ids.has(id); n += 1) id = `${base}-${n}`; - ids.add(id); - html.push(`

    ${inline(h[2])}

    `); - return; - } - para.push(line.trim()); - }); - flushPara(); - closeLists(); - // Items and their markup were pushed as separate strings; join tight. - return html.join("").replace(/<\/p><(h2|p|ul)/g, "

    \n<$1").replace(/<\/ul><(h2|p)/g, "
\n<$1").replace(/<\/h2><(p|ul)/g, "

\n<$1"); -} - -/** Lead (before the first `## `) and the rest, with the rest's first line number. */ -export function splitEntryBody(body) { - const ls = lines(body); - const i = ls.findIndex((l) => /^##\s/.test(l)); - if (i === -1) return { lead: ls.join("\n"), rest: "", restFirstLine: ls.length + 1 }; - return { lead: ls.slice(0, i).join("\n"), rest: ls.slice(i).join("\n"), restFirstLine: i + 1 }; -} - -/** Raw markdown under `## `, up to the next `## `, or null. */ -export function sectionMarkdown(body, heading) { - const ls = lines(body); - const start = ls.findIndex((l) => l.trim() === `## ${heading}`); - if (start === -1) return null; - let end = ls.findIndex((l, j) => j > start && /^##\s/.test(l)); - if (end === -1) end = ls.length; - return ls.slice(start + 1, end).join("\n").trim(); -} - -/** The text after the `- Deprecated:` marker line, or null. */ -export function deprecationReason(body) { - const m = String(body).match(/^- Deprecated:\s*(.+)$/m); - if (!m) return null; - // The banner reads "Deprecated. ", so the reason starts a sentence. - const reason = m[1].trim(); - return reason.charAt(0).toUpperCase() + reason.slice(1); -} +/** + * Renders catalog entry bodies to HTML. Deliberately small: it supports the + * markdown the catalog uses (measured 2026-09-24) and throws on anything + * else, so a new construct fails the build instead of rendering wrong. + * Underscores are literal: bodies contain identifiers like Rig_Medium. + */ +import { esc } from "./shared.mjs"; + +export class MarkdownError extends Error {} + +const slot = (i) => `\u0000${i}\u0000`; +const SLOT_RE = /\u0000(\d+)\u0000/g; + +const count = (str, sub) => str.split(sub).length - 1; + +/** + * A markdown link. The href may hold one level of balanced parens + * (Foo_(bar)). Shared with llms.mjs so both read the same href. + */ +export const LINK_RE = /\[([^\]]+)\]\(((?:[^()\s]|\([^()\s]*\))+)\)/g; + +/** + * Bold and italic on already-escaped text. Italic that would cross a bold + * boundary (`**a *b** c*`) stays literal: wrapping it would nest the tags + * wrongly. + */ +function marks(escaped) { + return escaped + .replace(/\*\*(?=\S)(.+?)(?<=\S)\*\*/g, "$1") + .replace(/(^|[^*\w])\*(?=\S)([^*]+?)(?<=\S)\*(?![*\w])/g, (whole, pre, inner) => + count(inner, "") === count(inner, "") ? `${pre}${inner}` : whole + ); +} + +/** Ids the page templates use themselves; a heading must not take one. */ +const RESERVED_IDS = new Set([ + "attribution-string", + "content", + "copy-attribution", + "copy-status", + "credits-all", + "credits-list", + "every-check", + "facts-title", + "owes-title", + "recent", + "sec-gaps", +]); + +export function renderInline(text, resolveLink) { + const slots = []; + const keep = (html) => { + slots.push(html); + return slot(slots.length - 1); + }; + // NUL marks a slot below; one in the input could make a slot hold its own + // token and the restore loop never settle. + let s = String(text).replaceAll("\u0000", ""); + // Code spans first: their contents are literal. + s = s.replace(/`([^`]+)`/g, (_, code) => keep(`${esc(code)}`)); + // Backslash escapes: any ASCII punctuation, as in CommonMark. + s = s.replace(/\\([!-/:-@[-`{-~])/g, (_, ch) => keep(esc(ch))); + // Links. The label keeps bold, italic and code; the href goes through the + // resolver. + s = s.replace(LINK_RE, (_, label, href) => { + const { href: out, external } = resolveLink(href); + const rel = external ? ' rel="noopener noreferrer"' : ""; + return keep(`${marks(esc(label))}`); + }); + // Autolinks in angle brackets. + s = s.replace(/<(https?:\/\/[^\s<>]+)>/g, (_, url) => keep(`${esc(url)}`)); + // Bare URLs, leaving trailing sentence punctuation outside the link. A URL + // stops at `*` (bold around it) and at a slot (a code span right after it), + // and keeps a closing paren that balances one inside it (Foo_(bar)). + s = s.replace(/https?:\/\/[^\s<>"*\u0000]+/g, (url) => { + let [, core, tail] = url.match(/^(.*?)([.,;:!?)\]]*)$/); + while (tail.startsWith(")") && count(core, "(") > count(core, ")")) { + core += ")"; + tail = tail.slice(1); + } + return keep(`${esc(core)}`) + tail; + }); + s = marks(esc(s)); + // Link labels can hold code slots, so restore until stable. + let before; + do { + before = s; + s = s.replace(SLOT_RE, (_, i) => slots[Number(i)]); + } while (s !== before); + return s; +} + +const UNSUPPORTED = [ + [/^\s*\|/, "a table row"], + [/^\s*```/, "a code fence"], + [/^\s*>/, "a blockquote"], + [/^\s*!\[/, "an image"], + // An autolink `` at the start of a line is inline markdown, not HTML. + [/^\s*<(?!https?:\/\/)[A-Za-z!/]/, "raw HTML"], + [/^#{3,}\s/, "a heading below level 2"], + [/^\s*\d+[.)]\s/, "an ordered list"], +]; + +const lines = (md) => String(md).replace(/\r\n?/g, "\n").split("\n"); + +export function renderBlocks(md, { file, resolveLink, firstLine = 1 }) { + const html = []; + const stack = []; + const ids = new Set(RESERVED_IDS); + let para = []; + let item = null; + let blank = false; + const inline = (t) => renderInline(t, resolveLink); + const flushPara = () => { + if (para.length) html.push(`

${inline(para.join(" "))}

`); + para = []; + }; + const flushItem = () => { + if (item !== null) html.push(inline(item)); + item = null; + }; + const closeLists = () => { + flushItem(); + while (stack.length) { + html.push(""); + stack.pop(); + } + }; + lines(md).forEach((line, i) => { + for (const [re, what] of UNSUPPORTED) { + if (re.test(line)) { + throw new MarkdownError(`${file}:${firstLine + i}: ${what} is not supported by the site renderer`); + } + } + if (!line.trim()) { + flushPara(); + blank = true; + return; + } + const li = line.match(/^(\s*)[-*]\s+(.*)$/); + if (li) { + flushPara(); + const indent = li[1].replace(/\t/g, " ").length; + if (!stack.length) { + html.push("
  • "); + stack.push(indent); + } else if (indent > stack[stack.length - 1]) { + flushItem(); + html.push("
    • "); + stack.push(indent); + } else { + flushItem(); + while (stack.length > 1 && indent < stack[stack.length - 1]) { + html.push("
    "); + stack.pop(); + } + html.push("
  • "); + } + item = li[2]; + blank = false; + return; + } + if (stack.length && !blank && /^\s/.test(line)) { + item += ` ${line.trim()}`; + return; + } + closeLists(); + blank = false; + const h = line.match(/^(#{1,2})\s+(.*)$/); + if (h) { + flushPara(); + if (h[1] === "#") return; // the page shows the entry name as its h1 + // Unique on the page: a second "Notes" becomes notes-2, and a heading + // never takes an id the page template uses. + const base = h[2].toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "section"; + let id = base; + for (let n = 2; ids.has(id); n += 1) id = `${base}-${n}`; + ids.add(id); + html.push(`

    ${inline(h[2])}

    `); + return; + } + para.push(line.trim()); + }); + flushPara(); + closeLists(); + // Items and their markup were pushed as separate strings; join tight. + return html.join("").replace(/<\/p><(h2|p|ul)/g, "

    \n<$1").replace(/<\/ul><(h2|p)/g, "
\n<$1").replace(/<\/h2><(p|ul)/g, "\n<$1"); +} + +/** Lead (before the first `## `) and the rest, with the rest's first line number. */ +export function splitEntryBody(body) { + const ls = lines(body); + const i = ls.findIndex((l) => /^##\s/.test(l)); + if (i === -1) return { lead: ls.join("\n"), rest: "", restFirstLine: ls.length + 1 }; + return { lead: ls.slice(0, i).join("\n"), rest: ls.slice(i).join("\n"), restFirstLine: i + 1 }; +} + +/** Raw markdown under `## `, up to the next `## `, or null. */ +export function sectionMarkdown(body, heading) { + const ls = lines(body); + const start = ls.findIndex((l) => l.trim() === `## ${heading}`); + if (start === -1) return null; + let end = ls.findIndex((l, j) => j > start && /^##\s/.test(l)); + if (end === -1) end = ls.length; + return ls.slice(start + 1, end).join("\n").trim(); +} + +/** The text after the `- Deprecated:` marker line, or null. */ +export function deprecationReason(body) { + const m = String(body).match(/^- Deprecated:\s*(.+)$/m); + if (!m) return null; + // The banner reads "Deprecated. ", so the reason starts a sentence. + const reason = m[1].trim(); + return reason.charAt(0).toUpperCase() + reason.slice(1); +} diff --git a/site/lib/page-checks.mjs b/site/lib/page-checks.mjs index 6befd0e..b272e22 100644 --- a/site/lib/page-checks.mjs +++ b/site/lib/page-checks.mjs @@ -1,5 +1,6 @@ /** Checks run on every generated page; the build fails on any error. */ import path from "node:path"; +import { ANY_SCHEME_RE, EXTERNAL_SCHEME_RE } from "./shared.mjs"; const LEAKS = [ [/\*\*/, "a double asterisk"], @@ -22,7 +23,12 @@ export function checkPage(html, { file, kindOf }) { const dir = path.posix.dirname(file); for (const m of html.matchAll(/\s(?:href|src)="([^"]+)"/g)) { const href = m[1]; - if (/^(https?:|mailto:|#|data:)/i.test(href)) continue; + if (EXTERNAL_SCHEME_RE.test(href) || href.startsWith("#")) continue; + const scheme = href.match(ANY_SCHEME_RE); + if (scheme) { + errors.push(`${file}: link ${href} uses the unsupported scheme "${scheme[1].toLowerCase()}:"`); + continue; + } const clean = href.split(/[?#]/)[0]; let target = path.posix.normalize(path.posix.join(dir, clean || ".")).replace(/\/$/, ""); if (target === ".") target = ""; diff --git a/site/lib/shared.mjs b/site/lib/shared.mjs index 9314f29..d406fdc 100644 --- a/site/lib/shared.mjs +++ b/site/lib/shared.mjs @@ -78,6 +78,16 @@ export function latestAllowedDate(now = Date.now()) { return new Date(now + 86400000).toISOString().slice(0, 10); } +/** + * The link schemes the catalog treats as external. validate, the build's link + * resolver and the page checks all use this one rule, so a URL is accepted or + * rejected the same way everywhere. + */ +export const EXTERNAL_SCHEME_RE = /^(https?:|mailto:)/i; + +/** Any URL scheme at all (`ftp:`, `data:`, `javascript:`), for naming the unsupported one. */ +export const ANY_SCHEME_RE = /^([a-z][a-z0-9+.-]*):/i; + /** Absolute URL of an entry's page. */ export function entryPageUrl(site, id) { return `${String(site.siteUrl).replace(/\/+$/, "")}/entry/${id}/`; diff --git a/site/validate.mjs b/site/validate.mjs index fe0d597..0f5e247 100644 --- a/site/validate.mjs +++ b/site/validate.mjs @@ -15,6 +15,7 @@ import { checkEngineFields, checkEntryUrl, checkEvidenceDates, + checkFieldTypes, checkFormatVocabulary, checkLicenseVocabulary, checkMaintenance, @@ -29,7 +30,7 @@ import { markdownAnchors, } from "./checks.mjs"; import { evidenceSection, parseFrontmatter } from "./lib/frontmatter.mjs"; -import { isRealDate, latestAllowedDate } from "./lib/shared.mjs"; +import { ANY_SCHEME_RE, EXTERNAL_SCHEME_RE, isRealDate, latestAllowedDate } from "./lib/shared.mjs"; import { listEntryFiles } from "./lib/entry-files.mjs"; import { licenceTerms, listStackFiles } from "./lib/stacks.mjs"; @@ -100,9 +101,7 @@ const SKIP_WALK = new Set(["node_modules", "dist", "RESEARCH", "_scratch"]); const ALLOWED_BINARY_PREFIXES = ["site/dist/", "docs/images/readme/"]; const EMOJI_RE = /\p{Extended_Pictographic}/u; const MD_LINK_RE = /!\[[^\]]*\]\(([^)]+)\)|\[[^\]]*\]\(([^)]+)\)/g; -const COMMERCIAL_VALUES = new Set(["true", "false", "unknown", "varies"]); const STATUS_VALUES = new Set(["active", "needs-review", "deprecated"]); -const ATTRIBUTION_REQUIRED_VALUES = new Set(["true", "false", "unknown"]); function hasEmoji(text) { const stripped = text.replace(/[©®™]/g, ""); @@ -169,7 +168,12 @@ function checkRelativeLinks(file, errors) { if (target.startsWith("<") && target.endsWith(">")) target = target.slice(1, -1); const href = target.split(/\s+/)[0]; if (!href) continue; - if (/^(https?:|mailto:|ftp:)/i.test(href)) continue; + if (EXTERNAL_SCHEME_RE.test(href)) continue; + const scheme = href.match(ANY_SCHEME_RE); + if (scheme) { + errors.push(`${relFromRoot(file)} link "${href}" uses the unsupported scheme "${scheme[1].toLowerCase()}:" (only https, http and mailto links are allowed)`); + continue; + } const hash = href.indexOf("#"); const filePart = hash === -1 ? href : href.slice(0, hash); const fragment = hash === -1 ? "" : href.slice(hash + 1); @@ -283,24 +287,13 @@ function main() { if (meta.category && !categories.has(String(meta.category))) { errors.push(`${rel} category "${meta.category}" is not a catalog/ directory`); } - const commercialKey = enumKey(meta.commercial); - if (!isEmptyField(meta.commercial) && !COMMERCIAL_VALUES.has(commercialKey)) { - errors.push(`${rel} commercial "${commercialKey}" is not true|false|unknown|varies`); - } + // V24 types commercial and attribution_required (a quoted "true" is text). + errors.push(...checkFieldTypes(rel, meta)); const statusKey = enumKey(meta.status); if (!isEmptyField(meta.status) && !STATUS_VALUES.has(statusKey)) { errors.push(`${rel} status "${statusKey}" is not active|needs-review|deprecated`); } - const attrKey = enumKey(meta.attribution_required); - if ( - !isEmptyField(meta.attribution_required) && - !ATTRIBUTION_REQUIRED_VALUES.has(attrKey) - ) { - errors.push( - `${rel} attribution_required "${attrKey}" is not true|false|unknown` - ); - } - if (attrKey === "true" && isEmptyField(meta.attribution_string)) { + if (meta.attribution_required === true && isEmptyField(meta.attribution_string)) { errors.push(`${rel} attribution_required true needs attribution_string`); } errors.push(...checkEntryUrl(rel, meta)); From 571dc4e24868f753da5f1de33926e10cf9aa41a6 Mon Sep 17 00:00:00 2001 From: TM Hospitality Strategies Date: Wed, 21 Oct 2026 23:26:53 -0400 Subject: [PATCH 2/8] Sponsor link instead of iframes, shared head baseline, sitemap lastmod from git, check-links robustness, chip radiogroup, small UX fixes - shared: sponsorButtonHtml is a static link with an inline heart, so no page loads github.com before first paint (#101) - shared: baseHeadHtml gives entry, stack, freshness and home pages the same canonical/sitemap/feed/theme-color/og:image set, with a test (#104) - exports: sitemapXml is pure and every URL has a lastmod: the build date for home and freshness, the later of verified/walked and the file's last commit date for entries and stacks (one git log per build, falls back without git) (#108) - check-links: --stale-days is validated before any fetch; GitHub API failures go to a 'Could not check (N skipped)' section that is not a problem, and a rate limit lists every repository left unchecked (#82) - app: category chips are a radiogroup with aria-checked and arrow keys; the scroll position shows as a 'You are in' line instead of a second chip state; form-control borders use --control-line at 3:1+ in both themes (#107) - app: category and sort changes push history (Back works), keystrokes replace, both in try/catch; '/' ignores Ctrl/Cmd/Alt; one spelling (Licence) in the UI; the perspective select is 'Camera view' with a visible note (#106) Closes #82, #101, #104, #106, #107, #108 Co-Authored-By: Claude Fable 5.1 --- site/build.mjs | 81 +++++----- site/check-links.mjs | 329 +++++++++++++++++++------------------- site/checks.test.mjs | 2 +- site/lib/entry-page.mjs | 19 +-- site/lib/exports.mjs | 253 +++++++++++++++-------------- site/lib/freshness.mjs | 296 +++++++++++++++++----------------- site/lib/lib.test.mjs | 89 ++++++++++- site/lib/link-check.mjs | 159 +++++++++++++------ site/lib/shared.mjs | 57 ++++++- site/lib/stack-page.mjs | 341 +++++++++++++++++++--------------------- site/public/app.js | 66 ++++++-- site/public/index.html | 25 +-- site/public/styles.css | 59 +++++-- 13 files changed, 1003 insertions(+), 773 deletions(-) diff --git a/site/build.mjs b/site/build.mjs index bc5a67c..eaf11be 100644 --- a/site/build.mjs +++ b/site/build.mjs @@ -3,15 +3,16 @@ * Build GitHub Pages site from catalog markdown frontmatter. * Add/edit one catalog//.md → rebuild regenerates everything. */ +import { execFileSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { commercialLabel, ENGINE_LOCK_LABELS, ENGINE_LOCK_NOTES, esc, latestAllowedDate, MAINTENANCE_NOTES, PERSPECTIVE_LABELS, sponsorButtonHtml, verifiedAge } from "./lib/shared.mjs"; +import { baseHeadHtml, commercialLabel, ENGINE_LOCK_LABELS, ENGINE_LOCK_NOTES, esc, latestAllowedDate, MAINTENANCE_NOTES, OG_CARD_NAME, PERSPECTIVE_LABELS, sponsorButtonHtml, verifiedAge } from "./lib/shared.mjs"; import { parseFrontmatter, summaryFromBody } from "./lib/frontmatter.mjs"; import { listEntryFiles } from "./lib/entry-files.mjs"; import { entryPageHtml, scriptJson } from "./lib/entry-page.mjs"; import { LinkError, makeLinkResolver } from "./lib/links.mjs"; -import { atomFeed, catalogCsv, homeJsonLd } from "./lib/exports.mjs"; +import { atomFeed, catalogCsv, homeJsonLd, sitemapXml } from "./lib/exports.mjs"; import { llmsFullTxt, llmsTxt } from "./lib/llms.mjs"; import { deprecationReason, MarkdownError, renderBlocks, renderInline, splitEntryBody } from "./lib/markdown.mjs"; import { checkPage } from "./lib/page-checks.mjs"; @@ -32,7 +33,6 @@ const FORMATS_PATH = path.join(__dirname, "format-vocabulary.json"); // Social preview image: a first-party screenshot of this site, kept with the // other first-party stills (the validator allows binaries there) and copied // into dist at build time. -const OG_CARD_NAME = "og-card.png"; const OG_CARD_SRC = path.join(ROOT, "docs", "images", "readme", OG_CARD_NAME); /** Sort rank for "license permissiveness": least owed first. */ @@ -186,7 +186,7 @@ function entryRowHtml(entry, repo, now) {

${esc(entry.summary || "")}

${esc(entry.category)}${formats}${taxonomy} - ${esc(ageText)} + ${esc(ageText)}
Open source @@ -269,54 +269,43 @@ function guideRowsHtml(guides, repo) { } function headMetaHtml(site, stats, generatedAt, hasCard) { - const url = site.siteUrl; - const title = site.title; - const desc = site.tagline; - const card = `${url.replace(/\/+$/, "")}/${OG_CARD_NAME}`; - const cardMeta = hasCard - ? [ - ``, - ``, - ``, - ``, - ``, - ``, - ] - : [``]; return [ - ``, - // robots.txt is only read at a host's root, so under a project Pages path - // its Sitemap line is never seen; point at the sitemap from the page too. - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ...cardMeta, - ``, - ``, + baseHeadHtml({ site, title: site.title, description: site.tagline, url: site.siteUrl, hasCard }), ``, ``, ``, ].join("\n "); } -function sitemapXml(site, entries, stacks = []) { - const base = site.siteUrl.replace(/\/+$/, ""); - const urls = [ - ` ${esc(base)}/`, - ` ${esc(base)}/freshness/`, - ...stacks.map((s) => ` ${esc(base)}/stack/${esc(s.meta.id)}/${esc(s.meta.walked)}`), - ...entries.map( - (e) => - ` ${esc(base)}/entry/${esc(e.id)}/${e.verified ? `${esc(e.verified)}` : ""}` - ), - ]; - return `\n\n${urls.join("\n")}\n\n`; +/** + * Repo-relative file => date of its last commit, from one `git log` over the + * catalog and stacks. The sitemap's lastmod must never be earlier than the + * last content change, and an entry's `verified` date is not that. Empty when + * git is unavailable (a tarball, a shallow CI checkout with no history). + */ +function lastCommitDates() { + const dates = new Map(); + try { + const out = execFileSync("git", ["log", "--format=%x00%cs", "--name-only", "--", "catalog", "stacks"], { + cwd: ROOT, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + stdio: ["ignore", "pipe", "ignore"], + }); + let date = null; + for (const line of out.split("\n")) { + if (line.startsWith("\0")) { + date = line.slice(1).trim(); + continue; + } + const file = line.trim(); + // Newest commit first, so the first date seen for a file is its latest. + if (file && date && !dates.has(file)) dates.set(file, date); + } + } catch { + // No git: the sitemap falls back to the build date. + } + return dates; } function robotsTxt(site) { @@ -622,7 +611,7 @@ function main() { } fs.writeFileSync(indexPath, html); - fs.writeFileSync(path.join(DIST, "sitemap.xml"), sitemapXml(config.site, visible, stacks)); + fs.writeFileSync(path.join(DIST, "sitemap.xml"), sitemapXml({ site: config.site, entries: visible, stacks, stamp, modified: lastCommitDates() })); fs.writeFileSync(path.join(DIST, "robots.txt"), robotsTxt(config.site)); fs.writeFileSync(path.join(DIST, "404.html"), notFoundHtml(config.site)); fs.mkdirSync(path.join(DIST, "freshness"), { recursive: true }); diff --git a/site/check-links.mjs b/site/check-links.mjs index e4d52e3..dec9967 100644 --- a/site/check-links.mjs +++ b/site/check-links.mjs @@ -1,171 +1,158 @@ -#!/usr/bin/env node -/** - * Fetches every entry's `url` and reports the ones that need a person: - * node site/check-links.mjs [--report ] [--stale-days 365] - * - * - dead: 404/410, 5xx, DNS failure or timeout - * - moved: the final URL is on another domain, or another GitHub repository - * - stale: `verified` is older than --stale-days (365 by default) - * - blocked: 401/403/429, which is usually a site refusing robots rather - * than a dead link; listed for a manual check, never counted as a problem - * - maintenance: a github.com source whose repository is archived or has had - * no push in three years while its `maintenance` field does not say so, or - * the other way round. Uses the GitHub API; set GITHUB_TOKEN to lift the - * 60-requests-an-hour anonymous limit. - * - * Writes a Markdown report (stdout, or --report) and, under GitHub Actions, - * `problems=` to $GITHUB_OUTPUT so the workflow can open or update one - * issue. It never changes an entry: a moved or dead source still needs its - * licence re-read before the entry is edited. - */ -import fs from "node:fs"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { maintenanceFromRepo } from "./checks.mjs"; -import { parseFrontmatter } from "./lib/frontmatter.mjs"; -import { listEntryFiles } from "./lib/entry-files.mjs"; -import { classify, repoOf, reportUrl } from "./lib/link-check.mjs"; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const ROOT = path.resolve(__dirname, ".."); -const CATALOG = path.join(ROOT, "catalog"); -const REPO = JSON.parse(fs.readFileSync(path.join(__dirname, "config.json"), "utf8")).site.repo; -const CONCURRENCY = 8; -const TIMEOUT_MS = 20000; -const UA = - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0 Safari/537.36 " + - "(+https://github.com/TMHSDigital/Free-Game-Dev-Assets link check)"; - -function arg(name, fallback) { - const i = process.argv.indexOf(name); - return i === -1 ? fallback : process.argv[i + 1]; -} - -function loadEntries() { - const out = []; - for (const full of listEntryFiles(CATALOG)) { - const parsed = parseFrontmatter(fs.readFileSync(full, "utf8")); - if (!parsed?.meta.url) continue; - const { id, url, verified, status, maintenance } = parsed.meta; - out.push({ id: String(id), url: String(url), verified: verified ? String(verified) : null, status: String(status), maintenance: maintenance ? String(maintenance) : null, rel: path.relative(ROOT, full).split(path.sep).join("/") }); - } - return out.sort((a, b) => a.id.localeCompare(b.id)); -} - -async function check(entry) { - const opts = { redirect: "follow", headers: { "user-agent": UA, accept: "text/html,*/*;q=0.8" } }; - try { - const res = await fetch(entry.url, { ...opts, signal: AbortSignal.timeout(TIMEOUT_MS) }); - res.body?.cancel().catch(() => {}); - return classify(entry.url, res.status, res.url); - } catch (err) { - const cause = err.cause?.code || err.name || "error"; - return { kind: "dead", detail: cause === "TimeoutError" ? `no answer in ${TIMEOUT_MS / 1000}s` : cause }; - } -} - -async function run(entries) { - const results = new Array(entries.length); - let next = 0; - await Promise.all( - Array.from({ length: CONCURRENCY }, async () => { - while (next < entries.length) { - const i = next++; - let r = await check(entries[i]); - // One retry for anything that did not answer cleanly. - if (r.kind === "dead") r = await check(entries[i]); - results[i] = r; - } - }) - ); - return results; -} - -/** - * GitHub sources whose repository state and `maintenance` field disagree. - * One API call per github.com repository, one at a time: a few dozen calls, - * well inside the limit with a token. An API failure is listed, not guessed. - */ -async function checkMaintenance(entries) { - const headers = { "user-agent": UA, accept: "application/vnd.github+json" }; - if (process.env.GITHUB_TOKEN) headers.authorization = `Bearer ${process.env.GITHUB_TOKEN}`; - const out = []; - for (const e of entries) { - let host; - try { - host = new URL(e.url).hostname; - } catch { - continue; - } - if (host !== "github.com") continue; - const repo = repoOf(e.url); - if (!repo.includes("/")) continue; - try { - const res = await fetch(`https://api.github.com/repos/${repo}`, { headers, signal: AbortSignal.timeout(TIMEOUT_MS) }); - if (!res.ok) { - out.push({ ...e, detail: `GitHub API HTTP ${res.status} for ${repo}` }); - if (res.status === 403 || res.status === 429) break; // rate limited: the rest would fail too - continue; - } - const data = await res.json(); - const want = maintenanceFromRepo(data); - if (want !== e.maintenance) { - const state = data.archived ? "archived" : `last push ${String(data.pushed_at).slice(0, 10)}`; - out.push({ ...e, detail: `${repo} is ${state}; maintenance is ${e.maintenance || "unset"}, expected ${want || "unset"}` }); - } - } catch (err) { - out.push({ ...e, detail: `GitHub API ${err.name || "error"} for ${repo}` }); - } - } - return out; -} - -const staleDays = Number(arg("--stale-days", "365")); -const entries = loadEntries().filter((e) => e.status !== "deprecated"); -const results = await run(entries); -const maintenance = await checkMaintenance(entries); -const cutoff = new Date(Date.now() - staleDays * 86400000).toISOString().slice(0, 10); -const rows = entries.map((e, i) => ({ ...e, ...results[i] })); -const pick = (kind) => rows.filter((r) => r.kind === kind); -const stale = entries.filter((e) => e.verified && e.verified < cutoff); -const line = (r) => `- [ ] [\`${r.id}\`](${REPO}/blob/main/${r.rel}): ${reportUrl(r.url)}${r.detail ? ` (${r.detail})` : ""}`; - -const dead = pick("dead"); -const moved = pick("moved"); -const blocked = pick("blocked"); -const problems = dead.length + moved.length + stale.length + maintenance.length; -const date = new Date().toISOString().slice(0, 10); -const report = [ - `Link check of ${entries.length} active and needs-review entries, run ${date}.`, - "", - "Re-read the licence before changing an entry: a moved or rebuilt source can come back under different terms.", - "", - `### Dead (${dead.length})`, - "", - ...(dead.length ? dead.map(line) : ["None."]), - "", - `### Moved to another domain or repository (${moved.length})`, - "", - ...(moved.length ? moved.map(line) : ["None."]), - "", - `### Verified more than ${staleDays} days ago (${stale.length})`, - "", - ...(stale.length ? stale.map((e) => `- [ ] [\`${e.id}\`](${REPO}/blob/main/${e.rel}): verified ${e.verified}`) : ["None."]), - "", - `### GitHub repository state differs from \`maintenance\` (${maintenance.length})`, - "", - ...(maintenance.length ? maintenance.map(line) : ["None."]), - "", - `
Refused the checker (${blocked.length}): usually bot blocking, check by hand`, - "", - ...(blocked.length ? blocked.map(line) : ["None."]), - "", - "
", - "", -].join("\n"); - -const reportPath = arg("--report", null); -if (reportPath) fs.writeFileSync(reportPath, report); -else process.stdout.write(report); -if (process.env.GITHUB_OUTPUT) fs.appendFileSync(process.env.GITHUB_OUTPUT, `problems=${problems}\n`); -console.error(`${entries.length} checked: ${dead.length} dead, ${moved.length} moved, ${stale.length} stale, ${maintenance.length} maintenance, ${blocked.length} blocked`); +#!/usr/bin/env node +/** + * Fetches every entry's `url` and reports the ones that need a person: + * node site/check-links.mjs [--report ] [--stale-days 365] + * + * - dead: 404/410, 5xx, DNS failure or timeout + * - moved: the final URL is on another domain, or another GitHub repository + * - stale: `verified` is older than --stale-days (365 by default) + * - blocked: 401/403/429, which is usually a site refusing robots rather + * than a dead link; listed for a manual check, never counted as a problem + * - maintenance: a github.com source whose repository is archived or has had + * no push in three years while its `maintenance` field does not say so, or + * the other way round. Uses the GitHub API; set GITHUB_TOKEN to lift the + * 60-requests-an-hour anonymous limit. An API failure or rate limit is + * reported under "Could not check" and never counted as a problem. + * + * Writes a Markdown report (stdout, or --report) and, under GitHub Actions, + * `problems=` to $GITHUB_OUTPUT so the workflow can open or update one + * issue. It never changes an entry: a moved or dead source still needs its + * licence re-read before the entry is edited. + */ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { maintenanceFromRepo } from "./checks.mjs"; +import { parseFrontmatter } from "./lib/frontmatter.mjs"; +import { listEntryFiles } from "./lib/entry-files.mjs"; +import { checkMaintenanceRepos, classify, reportUrl, staleDaysFrom } from "./lib/link-check.mjs"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(__dirname, ".."); +const CATALOG = path.join(ROOT, "catalog"); +const REPO = JSON.parse(fs.readFileSync(path.join(__dirname, "config.json"), "utf8")).site.repo; +const CONCURRENCY = 8; +const TIMEOUT_MS = 20000; +const UA = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0 Safari/537.36 " + + "(+https://github.com/TMHSDigital/Free-Game-Dev-Assets link check)"; + +function arg(name, fallback) { + const i = process.argv.indexOf(name); + return i === -1 ? fallback : process.argv[i + 1]; +} + +function loadEntries() { + const out = []; + for (const full of listEntryFiles(CATALOG)) { + const parsed = parseFrontmatter(fs.readFileSync(full, "utf8")); + if (!parsed?.meta.url) continue; + const { id, url, verified, status, maintenance } = parsed.meta; + out.push({ id: String(id), url: String(url), verified: verified ? String(verified) : null, status: String(status), maintenance: maintenance ? String(maintenance) : null, rel: path.relative(ROOT, full).split(path.sep).join("/") }); + } + return out.sort((a, b) => a.id.localeCompare(b.id)); +} + +async function check(entry) { + const opts = { redirect: "follow", headers: { "user-agent": UA, accept: "text/html,*/*;q=0.8" } }; + try { + const res = await fetch(entry.url, { ...opts, signal: AbortSignal.timeout(TIMEOUT_MS) }); + res.body?.cancel().catch(() => {}); + return classify(entry.url, res.status, res.url); + } catch (err) { + const cause = err.cause?.code || err.name || "error"; + return { kind: "dead", detail: cause === "TimeoutError" ? `no answer in ${TIMEOUT_MS / 1000}s` : cause }; + } +} + +async function run(entries) { + const results = new Array(entries.length); + let next = 0; + await Promise.all( + Array.from({ length: CONCURRENCY }, async () => { + while (next < entries.length) { + const i = next++; + let r = await check(entries[i]); + // One retry for anything that did not answer cleanly. + if (r.kind === "dead") r = await check(entries[i]); + results[i] = r; + } + }) + ); + return results; +} + +/** + * One GitHub API call per github.com repository: a few dozen calls, well + * inside the limit with a token. The classification lives in lib/link-check.mjs. + */ +function checkMaintenance(entries) { + const headers = { "user-agent": UA, accept: "application/vnd.github+json" }; + if (process.env.GITHUB_TOKEN) headers.authorization = `Bearer ${process.env.GITHUB_TOKEN}`; + const fetchRepo = async (repo) => { + const res = await fetch(`https://api.github.com/repos/${repo}`, { headers, signal: AbortSignal.timeout(TIMEOUT_MS) }); + return { ok: res.ok, status: res.status, data: res.ok ? await res.json() : null }; + }; + return checkMaintenanceRepos(entries, fetchRepo, maintenanceFromRepo); +} + +// Flags are checked before the first request, so a typo fails in a second. +const staleArg = staleDaysFrom(process.argv); +if (staleArg.error) { + console.error(staleArg.error); + process.exit(2); +} +const staleDays = staleArg.days; +const entries = loadEntries().filter((e) => e.status !== "deprecated"); +const results = await run(entries); +const { mismatches: maintenance, skipped } = await checkMaintenance(entries); +const cutoff = new Date(Date.now() - staleDays * 86400000).toISOString().slice(0, 10); +const rows = entries.map((e, i) => ({ ...e, ...results[i] })); +const pick = (kind) => rows.filter((r) => r.kind === kind); +const stale = entries.filter((e) => e.verified && e.verified < cutoff); +const line = (r) => `- [ ] [\`${r.id}\`](${REPO}/blob/main/${r.rel}): ${reportUrl(r.url)}${r.detail ? ` (${r.detail})` : ""}`; + +const dead = pick("dead"); +const moved = pick("moved"); +const blocked = pick("blocked"); +const problems = dead.length + moved.length + stale.length + maintenance.length; +const date = new Date().toISOString().slice(0, 10); +const report = [ + `Link check of ${entries.length} active and needs-review entries, run ${date}.`, + "", + "Re-read the licence before changing an entry: a moved or rebuilt source can come back under different terms.", + "", + `### Dead (${dead.length})`, + "", + ...(dead.length ? dead.map(line) : ["None."]), + "", + `### Moved to another domain or repository (${moved.length})`, + "", + ...(moved.length ? moved.map(line) : ["None."]), + "", + `### Verified more than ${staleDays} days ago (${stale.length})`, + "", + ...(stale.length ? stale.map((e) => `- [ ] [\`${e.id}\`](${REPO}/blob/main/${e.rel}): verified ${e.verified}`) : ["None."]), + "", + `### GitHub repository state differs from \`maintenance\` (${maintenance.length})`, + "", + ...(maintenance.length ? maintenance.map(line) : ["None."]), + "", + // API trouble is the checker's problem, not the catalog's: listed, not counted. + `### Could not check (${skipped.length} skipped)`, + "", + ...(skipped.length ? skipped.map(line) : ["None."]), + "", + `
Refused the checker (${blocked.length}): usually bot blocking, check by hand`, + "", + ...(blocked.length ? blocked.map(line) : ["None."]), + "", + "
", + "", +].join("\n"); + +const reportPath = arg("--report", null); +if (reportPath) fs.writeFileSync(reportPath, report); +else process.stdout.write(report); +if (process.env.GITHUB_OUTPUT) fs.appendFileSync(process.env.GITHUB_OUTPUT, `problems=${problems}\n`); +console.error(`${entries.length} checked: ${dead.length} dead, ${moved.length} moved, ${stale.length} stale, ${maintenance.length} maintenance, ${blocked.length} blocked, ${skipped.length} API checks skipped`); diff --git a/site/checks.test.mjs b/site/checks.test.mjs index e88dc53..1f0ae3a 100644 --- a/site/checks.test.mjs +++ b/site/checks.test.mjs @@ -765,4 +765,4 @@ if (failures.length) { for (const f of failures) console.error(` - ${f}`); process.exit(1); } -console.log(`checks.test ok: ${passed} assertions across 22 checks`); +console.log(`checks.test ok: ${passed} assertions across 23 checks`); diff --git a/site/lib/entry-page.mjs b/site/lib/entry-page.mjs index 3b025e0..7237db0 100644 --- a/site/lib/entry-page.mjs +++ b/site/lib/entry-page.mjs @@ -1,5 +1,6 @@ /** One entry's page. Pure: everything it needs is passed in. */ import { + baseHeadHtml, commercialLabel, entryPageUrl, esc, @@ -90,12 +91,6 @@ export function entryPageHtml({ entry, leadHtml, restHtml, deprecatedReasonHtml, ? `

Deprecated. ${deprecatedReasonHtml || "This entry is kept for history only."}

` : ""; const issue = `${repo}/issues/new?template=correction.yml&title=${encodeURIComponent(`[correction] ${entry.id}`)}&entry=${encodeURIComponent(entry.id)}`; - const card = `${String(site.siteUrl).replace(/\/+$/, "")}/og-card.png`; - const cardMeta = hasCard - ? ` - - ` - : ``; const pager = prev || next ? `