diff --git a/benchmarks/fixtures.ts b/benchmarks/fixtures.ts index 3e4d5da4a..29b09343a 100644 --- a/benchmarks/fixtures.ts +++ b/benchmarks/fixtures.ts @@ -253,6 +253,28 @@ ${Array.from( That's it — let me know if you want a full example! ` +/** + * Raw HTML with many attributes, as found in docs written for GitHub/VitePress. + * Every tag goes through `parseHtmlAttributes`, half of the values carry + * entities so both the plain and the decoding path are measured. + */ +export const htmlAttributesMarkdown = `# Raw HTML + +${Array.from( + { length: 30 }, + (_, i) => ` + + +Inline Ctrl + R&R and docs ${i}. +` +).join('\n')} +` + /** Splits `markdown` into `count` growing prefixes, ending with the full text. */ export function toStreamChunks(markdown: string, count: number): string[] { const chunks: string[] = [] diff --git a/benchmarks/html.bench.ts b/benchmarks/html.bench.ts new file mode 100644 index 000000000..0025b3e7a --- /dev/null +++ b/benchmarks/html.bench.ts @@ -0,0 +1,21 @@ +import { bench, describe } from 'vitest' +import { createMarkdownParser } from 'comark' +import { htmlAttributesMarkdown } from './fixtures.ts' + +// Raw HTML attributes are decoded by `parseHtmlAttributes`; the generic parse +// fixtures barely contain HTML, so they cannot catch changes on that path. +const parse = createMarkdownParser() +const parseLiteralHtml = createMarkdownParser({ autoClose: false }) + +await parse(htmlAttributesMarkdown) +await parseLiteralHtml(htmlAttributesMarkdown) + +describe('parse html', () => { + bench('html attributes document', async () => { + await parse(htmlAttributesMarkdown) + }) + + bench('html attributes document without auto-close', async () => { + await parseLiteralHtml(htmlAttributesMarkdown) + }) +}) diff --git a/docs/content/1.getting-started/2.document-model.md b/docs/content/1.getting-started/2.document-model.md index ec64434c2..dfa1b8b5c 100644 --- a/docs/content/1.getting-started/2.document-model.md +++ b/docs/content/1.getting-started/2.document-model.md @@ -250,6 +250,15 @@ const heading = document.nodes[0] console.log(textContent(heading)) // Hello world ``` +Text in parsed documents is already entity-decoded. For trees whose text may still contain character references, such as hand-built nodes, pass a `decodeHTML` function. It runs on each text node: + +```typescript [text-content-decode.ts] +import { decodeHTML } from 'entities' +import { textContent } from 'comark/utils' + +textContent(['p', {}, 'Tom & Jerry'], { decodeHTML }) // Tom & Jerry +``` + ### Visit nodes `visit()` walks every node that matches a checker. A visitor can inspect, replace, or remove matching nodes: diff --git a/packages/comark/src/internal/parse/utils.ts b/packages/comark/src/internal/parse/utils.ts index 8ab0e0884..098016d54 100644 --- a/packages/comark/src/internal/parse/utils.ts +++ b/packages/comark/src/internal/parse/utils.ts @@ -1,5 +1,6 @@ import type { Node, ElementNode, ElementNodeAttributes, MarkdownDocument } from 'comark' -import { decodeHTML, isHtmlVoidElement, TREE_WALK_MAX_DEPTH } from 'comark/utils' +import { decodeHTMLAttribute } from 'entities' +import { isHtmlVoidElement, TREE_WALK_MAX_DEPTH } from 'comark/utils' /** Space, tab, line feed, form feed or carriage return. Defined per file so it stays inlinable on hot paths. */ function isSpaceCode(c: number): boolean { @@ -185,6 +186,15 @@ export function parseHtmlInline(content: string): ParsedHtmlTag { return { kind: 'other', content: trimmed } } +/** + * Decode character references in a raw attribute value the way browsers do: + * legacy semicolon-less references (`&`) are decoded, but not when followed by + * `=` or an alphanumeric (`?a=1©=2` stays literal). Backslashes are literal. + */ +function decodeAttributeValue(value: string): string { + return value.includes('&') ? decodeHTMLAttribute(value) : value +} + /** * Parse HTML attribute string into a key/value map. * Supports `key="value"`, `key='value'`, bare `key=value`, and boolean attributes. @@ -199,7 +209,7 @@ export function parseHtmlAttributes(attrsStr: string): Record { while ((match = re.exec(attrsStr)) !== null) { const key = match[1].toLowerCase() const value = match[2] ?? match[3] ?? match[4] - attrs[key] = value === undefined ? true : decodeHTML(value) + attrs[key] = value === undefined ? true : decodeAttributeValue(value) } return attrs diff --git a/packages/comark/src/utils/index.ts b/packages/comark/src/utils/index.ts index a196a0f9e..e9132e6be 100644 --- a/packages/comark/src/utils/index.ts +++ b/packages/comark/src/utils/index.ts @@ -1,10 +1,7 @@ // #region Tree Utils -import { decodeHTML } from 'entities' import type { Node, MarkdownDocument } from 'comark' -export { decodeHTML } from 'entities' - type VisitResult = Node | false | undefined | void /** @@ -14,25 +11,35 @@ export function isMarkdownDocument(value: unknown): value is MarkdownDocument { return !!value && typeof value === 'object' && Array.isArray((value as MarkdownDocument).nodes) } +export interface TextContentOptions { + /** + * Called on every text node before it is appended, e.g. `decodeHTML` from `entities`. + * Parsed documents already have decoded text, so only pass this for trees whose text + * may still contain character references (hand-built or from another source). + */ + decodeHTML?: (text: string) => string +} + /** * Get the text content of a Markdown AST node * * @param node - The node * @param options - The options - * @param options.decodeUnicodeEntities - Whether to decode Unicode entities + * @param options.decodeHTML - Function applied to each text node * @returns The text content */ -export function textContent(node: Node, options: { decodeUnicodeEntities?: boolean } = {}): string { +export function textContent(node: Node, options?: TextContentOptions): string { + return collectText(node, options?.decodeHTML) +} + +function collectText(node: Node, decode: ((text: string) => string) | undefined): string { if (typeof node === 'string') { - if (options.decodeUnicodeEntities) { - return decodeHTML(node) - } - return node as string + return decode ? decode(node) : node } let out = '' const len = node.length for (let i = 2; i < len; i++) { - out += textContent(node[i] as Node, options) + out += collectText(node[i] as Node, decode) } return out } diff --git a/packages/comark/test/html-escape.test.ts b/packages/comark/test/html-escape.test.ts index 81120661a..788e5ed96 100644 --- a/packages/comark/test/html-escape.test.ts +++ b/packages/comark/test/html-escape.test.ts @@ -39,6 +39,29 @@ describe('HTML attribute escaping', () => { expect(html).not.toContain('title="A "quote""') }) + it('keeps backslashes literal while decoding HTML attribute entities', async () => { + const tree = await parseMarkdown('hi') + const p = tree.nodes[0] as [string, Record, ...Node[]] + const span = (Array.isArray(p[2]) ? p[2] : p) as [string, Record, ...Node[]] + expect(span[1].title).toBe('C:\\*dir\\ & ©') + }) + + it('decodes HTML attribute entities with browser attribute semantics', async () => { + const attrsOf = async (markdown: string) => { + const tree = await parseMarkdown(markdown) + const p = tree.nodes[0] as [string, Record, ...Node[]] + return ((Array.isArray(p[2]) ? p[2] : p) as [string, Record, ...Node[]])[1] + } + + // Legacy semicolon-less reference is decoded... + expect((await attrsOf('hi')).title).toBe('A & B') + // ...but not when followed by `=` or an alphanumeric, so query strings survive. + expect((await attrsOf('hi')).href).toBe('?a=1©=2') + expect((await attrsOf('hi')).title).toBe('¬it;') + // Invalid code points become U+FFFD; references are decoded once. + expect((await attrsOf('hi')).title).toBe('\uFFFD|<') + }) + it('escapes quotes in component attribute props', async () => { const html = await renderHtml(`:span[hi]{title='a" onmouseover=alert(1) x="b'}`) expect(html).toContain('title="a" onmouseover=alert(1) x="b"') diff --git a/packages/comark/test/text-content.test.ts b/packages/comark/test/text-content.test.ts new file mode 100644 index 000000000..c1a95cf77 --- /dev/null +++ b/packages/comark/test/text-content.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest' +import { decodeHTML } from 'entities' +import type { Node } from 'comark' +import { textContent } from '../src/utils' + +describe('textContent', () => { + const tree: Node = ['p', {}, 'Tom & ', ['strong', {}, 'Jerry'], ' ©'] + + it('concatenates text nodes', () => { + expect(textContent('plain')).toBe('plain') + expect(textContent(tree)).toBe('Tom & Jerry ©') + }) + + it('applies decodeHTML to every text node', () => { + expect(textContent(tree, { decodeHTML })).toBe('Tom & Jerry ©') + expect(textContent('<', { decodeHTML })).toBe('<') + }) + + it('decodes each text node separately', () => { + const calls: string[] = [] + const split: Node = ['p', {}, '&am', 'p;'] + const result = textContent(split, { + decodeHTML: (text) => { + calls.push(text) + return decodeHTML(text) + }, + }) + expect(calls).toEqual(['&am', 'p;']) + expect(result).toBe('&') + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5e0fd3891..3cbfb09d6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -321,8 +321,8 @@ catalogs: specifier: ^0.7.6 version: 0.7.6 entities: - specifier: ^8.1.0 - version: 8.1.0 + specifier: ^7.0.0 + version: 7.0.1 github-slugger: specifier: ^2.0.0 version: 2.0.0 @@ -1427,7 +1427,7 @@ importers: version: 1.1.3 entities: specifier: 'catalog:' - version: 8.1.0 + version: 7.0.1 js-yaml: specifier: 'catalog:' version: 5.5.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index bb2a2c8d7..e6f19942c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -90,7 +90,8 @@ catalog: comark: ^0.8.1 # Core comark dependencies - entities: ^8.1.0 + # Keep in the same major as markdown-exit's `entities` so installs dedupe to one copy. + entities: ^7.0.0 github-slugger: ^2.0.0 hast-util-to-string: ^3.0.1 js-yaml: ^5.5.0