From 2f028c43350c2bd6b6be37c0f69173550dcd95bb Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 2 Oct 2026 08:23:17 +0400 Subject: [PATCH 1/6] HTML API: add a parsing benchmark tool under tests/performance/html-api. A PHP CLI that times WP_HTML_Tag_Processor and WP_HTML_Processor parsing (next_token() to the end of the document) over a corpus of real public pages and seeded synthetic documents, and compares two checkouts with interleaved samples, fresh worker processes per round and confidence intervals on the ratio of medians. - bench.php orchestrates one worker per checkout and prints a table, markdown or JSON report; compare.php compares two saved runs. - worker.php loads one checkout's HTML API with function stubs and runs timed jobs over a JSON line protocol. - lib/class-benchmark-synthetic.php generates 13 document shapes (dense tags, long text, entities, UTF-8, comments, attributes, deep nesting, script and style, whitespace, foreign content, formatting, tables, a WordPress post), byte-identical for a given seed across PHP versions. - corpus/manifest.json names twenty public pages with sizes and hashes; fetch-corpus.php restores them into the gitignored corpus/real/. An A/A run with the defaults (4 rounds of 10 samples) reports no difference on all 64 rows. The README documents the method and why the round count defaults to an even number. --- tests/performance/html-api/README.md | 154 ++ tests/performance/html-api/SPEC.md | 176 +++ tests/performance/html-api/bench.php | 965 ++++++++++++ tests/performance/html-api/compare.php | 238 +++ .../performance/html-api/corpus/manifest.json | 245 +++ tests/performance/html-api/fetch-corpus.php | 581 ++++++++ .../lib/class-benchmark-bootstrap.php | 218 +++ .../html-api/lib/class-benchmark-report.php | 293 ++++ .../html-api/lib/class-benchmark-stats.php | 482 ++++++ .../lib/class-benchmark-synthetic.php | 1308 +++++++++++++++++ tests/performance/html-api/worker.php | 297 ++++ 11 files changed, 4957 insertions(+) create mode 100644 tests/performance/html-api/README.md create mode 100644 tests/performance/html-api/SPEC.md create mode 100644 tests/performance/html-api/bench.php create mode 100644 tests/performance/html-api/compare.php create mode 100644 tests/performance/html-api/corpus/manifest.json create mode 100644 tests/performance/html-api/fetch-corpus.php create mode 100644 tests/performance/html-api/lib/class-benchmark-bootstrap.php create mode 100644 tests/performance/html-api/lib/class-benchmark-report.php create mode 100644 tests/performance/html-api/lib/class-benchmark-stats.php create mode 100644 tests/performance/html-api/lib/class-benchmark-synthetic.php create mode 100644 tests/performance/html-api/worker.php diff --git a/tests/performance/html-api/README.md b/tests/performance/html-api/README.md new file mode 100644 index 0000000000000..2a2ae9fbf57f4 --- /dev/null +++ b/tests/performance/html-api/README.md @@ -0,0 +1,154 @@ +# HTML API parsing benchmark + +Measures how fast `WP_HTML_Tag_Processor` and `WP_HTML_Processor` parse documents, and whether a code change made parsing faster or slower, with a confidence interval. It is a development tool under `tests/performance/html-api/`; nothing here ships with WordPress. + +## What is measured + +The timed loop constructs a processor and calls `next_token()` until it returns `false`: + +```php +$p = new WP_HTML_Tag_Processor( $html ); // or WP_HTML_Processor::create_full_parser( $html ) +while ( $p->next_token() ) { ++$tokens; } +``` + +Construction is inside the timed region because real use constructs a processor per document. Nothing else is: no `get_tag()`, no attribute reads, no modifications, no output. A document the HTML Processor cannot finish (`get_last_error()` is not null) is still timed, but it is listed separately and left out of the aggregate. + +Memory is a secondary metric: the single-tree table shows the allocator peak above the pre-run level for one parse (`peak_alloc_bytes` in the JSON); the JSON also carries `peak_bytes`, the delta of `memory_get_peak_usage( true )`, which moves in 2 MB steps. + +The workers load the HTML API files straight from a checkout's `src/wp-includes/` with a few function stubs (`__()`, `esc_url()`, `wp_kses_uri_attributes()` and similar; see `lib/class-benchmark-bootstrap.php`). WordPress itself is not loaded, so hooks, KSES and the rest of the request lifecycle are not measured. + +## Running + +Single tree, from a checkout: + +``` +php tests/performance/html-api/bench.php +``` + +A/B between two checkouts: + +``` +php tests/performance/html-api/bench.php --base ~/a8c/wordpress-develop/trunk --head . +``` + +`--head` defaults to the checkout containing `bench.php`. Both checkouts use the same `worker.php` and bootstrap from the tool directory of the `bench.php` you ran, so `--base` may point at a checkout that predates this tool. + +Options (`--name value` or `--name=value`): + +| Option | Default | Meaning | +| --- | --- | --- | +| `--base ` | none | Baseline checkout; enables A/B mode. | +| `--head ` | the checkout containing bench.php | Checkout under test. | +| `--php ` | `php` | PHP binary for the workers. | +| `--php-args ""` | none | Extra arguments for both workers. | +| `--opcache` / `--no-opcache` | on when the extension is available | Adds `-d opcache.enable_cli=1`. | +| `--jit` | off | Adds `-d opcache.jit_buffer_size=64M -d opcache.jit=tracing`. | +| `--parser tag\|html\|both` | `both` | Which parser to time. | +| `--corpus ` | `corpus/real` when it has files | Directory of `*.html` or `*.htm`; repeatable. The id is the file name. | +| `--no-corpus` | | Skip `corpus/real`; only directories named with `--corpus` are read. | +| `--include-optional` | | Include corpus documents whose manifest entry is `optional` (the 15 MB single-page HTML standard). | +| `--synthetic` / `--no-synthetic` | on | Include the generated shapes. | +| `--synthetic-size ` | 200000 | Target size of each generated document. | +| `--seed ` | 1 | Seed for generation and for the bootstrap. | +| `--filter ` | none | Keep ids containing the text, or matching it when written as `/regex/`. | +| `--samples ` | 10 | Timed samples per document, parser and tree, per round. | +| `--min-sample-ms ` | 25 | Each sample runs enough parses to last at least this long on the slower tree. | +| `--warmup ` | 2 | Untimed parses before sampling. | +| `--rounds ` | 4 | Repeat the whole run with a fresh pair of worker processes each time; see below. | +| `--format table\|markdown\|json` | `table` | Report format. | +| `--save ` | none | Write the full JSON (every sample, config, environment) whatever the format. | +| `--quiet` | | No progress on stderr. | +| `--list` | | Print document ids and sizes and exit. | + +Progress goes to stderr, one line per document and parser; the report goes to stdout. + +## How a run proceeds + +1. `bench.php` starts one worker per tree (`php [ini flags] worker.php --checkout `). Both workers stay alive for the whole run. +2. For each document and parser: the document is loaded into both workers; each runs `--warmup` untimed parses; each runs one calibration parse, and the slower of the two sets the number of parses per sample so that a sample lasts at least `--min-sample-ms`. +3. The samples are taken alternating between trees, and the tree that goes first alternates each sample (A B, B A, A B, ...). Each sample calls `gc_collect_cycles()` first, then times its parses with `hrtime( true )`. +4. The per-sample measure is nanoseconds per parse. Per document, parser and tree: median, min, mean, standard deviation, coefficient of variation (CV), MB/s and tokens/s from the median. + +## Reading the output + +A/B table, one row per document: size, median per tree, change in percent (head over base; negative is faster), a 95% confidence interval on that change, the verdict, and the CV of each tree's samples. + +- `faster`: the whole interval is below 0%. +- `slower`: the whole interval is above 0%. +- `no difference`: the interval contains 0%. This is "not shown to differ at this precision", not "identical". + +The overall line per parser is the geometric mean of the per-document ratios with its own interval, and the throughput of each tree as total bytes over the sum of the median times. Documents that bailed on either tree are listed after it and are not in the aggregate. + +The CV columns say how noisy the samples were. On an otherwise idle machine expect 0.2% to 1%. Several percent means something else was running, or the machine is throttling; rerun. A row whose CV is above 10% on either tree is marked `(noisy)`: its absolute times are not usable, although the interleaving still protects the ratio when both trees were slowed alike. On Apple Silicon a worker that lands on an efficiency core while other work saturates the performance cores runs several times slower for as long as it stays there, and both workers move together; this showed up as 6x slower samples on both trees at once while other jobs ran on this machine. + +### Where the interval comes from + +With one round (`--rounds 1`), the interval is a bootstrap: 2000 times, draw head samples and base samples with replacement, take the ratio of their medians; the interval is the 2.5th to 97.5th percentile of those ratios. The overall interval resamples documents and then samples within each. The bootstrap is seeded from `--seed`, so a saved run recomputes to the same numbers. + +The bootstrap only sees the spread between samples of one process. Two PHP processes running identical code do not run at identical speed: memory layout and scheduling give each process its own offset, which on this machine is 0.3% to 1% and which is constant for the life of the process, so interleaving cannot cancel it. When the samples are tight (CV around 0.3%), the one-round interval is narrower than that offset, and an A/A run returns `faster` or `slower` on some rows at the 0.5% level. + +`--rounds N` with N of 2 or more (the default is 4) starts a fresh pair of workers for each round and repeats the whole schedule. Each round then has its own ratio of medians per document, and the interval is a t-interval over the per-round log ratios (N minus 1 degrees of freedom); the overall interval is a t-interval over the per-round geometric means. The point estimate becomes the geometric mean of the per-round ratios, and the per-tree columns show the mean of the per-round medians. With 3 rounds the interval covers changes of about 2%; with 5 rounds about 1%. Use rounds when the claim is below a few percent. + +### A/A validation + +Before trusting a result on a given machine, run with the same checkout on both sides: + +``` +php tests/performance/html-api/bench.php --base . --head . +``` + +Expect the overall line per parser to say `no difference` and the per-row verdicts to be `no difference` with at most the occasional row crossing 0% by a few tenths of a percent. With 64 rows at 95% confidence, a few false verdicts per run are within expectation even on a perfect machine. If many rows disagree, or the overall interval excludes 0%, the machine is too noisy: close other work, raise `--samples`, or use `--rounds 5`. With `--rounds 1` an A/A run on this machine reported `slower` at +1.8% on every HTML Processor row with CVs of 0.2%: that is the per-process offset, and it is why the default is several rounds. The second-started worker of a pair is the slower one, so the round count defaults to an even number: with 3 rounds, base starts first in two of them and an A/A run reported `slower` on 8 of 64 rows, all in the same direction; with 4 rounds the start order is balanced. + +### Why interleave and bootstrap, rather than run each tree once and compare means + +Machine speed drifts over seconds and minutes (thermal state, background work, frequency scaling). Two back-to-back runs put all of that drift into the difference. Alternating A and B within each sample, and alternating which goes first, puts both trees into the same drift so it cancels in the ratio. The mean of timing samples is pulled by outliers (a GC pass, a scheduler interruption); the median is not. A single difference of means has no error bar; the bootstrap gives the ratio an interval without assuming a distribution for the samples, so a reader can tell a 1% change with a 0.3% interval from a 1% change with a 4% interval. + +`compare.php` exists for saved files from two separate runs. It computes the same statistics but says in its header that the runs were not interleaved, so drift between the runs is in the result. + +## Synthetic documents + +`lib/class-benchmark-synthetic.php` generates one document per shape, each a full document (doctype, html, head, body) sized to `--synthetic-size` within 2%, deterministic for a given shape, size and seed on every PHP version. Shape ids and what each exercises are in that file; `bench.php --list` prints them. + +To add a shape: add its id to `Benchmark_Synthetic::shapes()` and a branch to `generate()` that builds the body with `mt_rand()` only (never `random_int()` or `rand()`), pads or trims to the target size, and wraps it in the shared document skeleton. Check that `WP_HTML_Processor::create_full_parser()` parses it to the end on trunk (`bench.php --filter --parser html --samples 2` shows `bailed` if not), and say in the file header if it cannot. + +## Real corpus + +`corpus/manifest.json` lists about twenty public pages by URL with the size and sha256 of the bytes fetched on the manifest date: the WHATWG HTML standard's parsing section, the W3C HTML5 Recommendation's syntax chapter, RFC 9110, Wikipedia articles in several scripts pinned by revision id, pages rendered by WordPress (wordpress.org, developer.wordpress.org, make.wordpress.org, sirre.al), an MDN reference page, a php.net manual page and the Hacker News front page. Each entry says what the page is, whether its bytes are pinned, and whether the HTML Processor bails on it on trunk (one does: a caching-plugin comment after ``). `corpus/real/` is gitignored. To fetch: + +``` +php tests/performance/html-api/fetch-corpus.php +``` + +It fetches with curl, verifies the hash, and skips documents already present. A hash mismatch on a pinned entry is an error and the file is removed; on an unpinned entry it is a warning, the file is kept, and the page is reported as drifted since the manifest date. `--include-optional` also fetches the single-page HTML standard (15 MB), which `bench.php` likewise skips unless given `--include-optional`. `--update-manifest` refetches everything, rewrites sizes, hashes and dates, and re-checks which documents bail. `bench.php` picks up `corpus/real` automatically when it has files. + +## Your own documents + +Point `--corpus` at any directory of `.html` files, repeatable, with `--no-synthetic` to measure only those: + +``` +php tests/performance/html-api/bench.php --corpus ~/pages --no-synthetic --base ~/a8c/wordpress-develop/trunk +``` + +## Comparing saved runs + +``` +php tests/performance/html-api/bench.php --save before.json +# change code +php tests/performance/html-api/bench.php --save after.json +php tests/performance/html-api/compare.php before.json after.json +``` + +The head tree of each file is used, matched by document id and parser. Prefer a live A/B when both checkouts are available. + +## Files + +- `bench.php`: the orchestrator. +- `worker.php`: a subprocess that loads one checkout's HTML API and runs timed jobs read as JSON lines on stdin, answering on stdout. +- `compare.php`: compares two `--save` files. +- `fetch-corpus.php`: restores `corpus/real/` from `corpus/manifest.json`. +- `lib/class-benchmark-bootstrap.php`: loads the HTML API from a checkout root with stubs for the WordPress functions it calls. +- `lib/class-benchmark-synthetic.php`: the shape generator. +- `lib/class-benchmark-stats.php`: median, bootstrap, t-interval, geometric mean, formatting. +- `lib/class-benchmark-report.php`: table, markdown and JSON output. + +`bench.php` and `compare.php` need PHP 8.1 or newer; the workers run on whatever `--php` names, as long as the checkout's HTML API loads there. diff --git a/tests/performance/html-api/SPEC.md b/tests/performance/html-api/SPEC.md new file mode 100644 index 0000000000000..f79d20cb37245 --- /dev/null +++ b/tests/performance/html-api/SPEC.md @@ -0,0 +1,176 @@ +# HTML API parsing benchmark: design spec (2026-10-01) + +Worktree: ~/a8c/wordpress-develop/html-api-benchmark, branch `html-api/benchmark`, off origin/trunk 4277884c40. +Tool root: `tests/performance/html-api/` in that worktree. Everything below is relative to the tool root unless stated. +PHP on this machine: `php` is 8.5.11; `/opt/homebrew/opt/php@8.4/bin/php` is 8.4. No 7.4. Code must still be PHP 7.2-compatible in syntax where it loads core files, but the tool itself may require PHP 8.1+ (it is a dev tool, not shipped code). Use WordPress coding style (tabs, snake_case, Yoda conditions, spaces inside parens) so it passes the repo's phpcs if ever run. + +## Purpose + +Measure parsing throughput of `WP_HTML_Tag_Processor` and `WP_HTML_Processor` with enough rigor to say whether a code change made parsing faster or slower, and by how much, with a confidence interval. Parsing only: the timed loop calls `next_token()` until it returns false. No `get_tag()`, no attribute reads, no modifications in the timed loop. + +## Layout + +``` +tests/performance/html-api/ + README.md usage, design, how to read the output, how to add a shape + bench.php CLI entry: orchestrates workers, collects samples, computes stats, prints report + worker.php subprocess: loads one checkout's HTML API, runs timed jobs read from stdin as JSON lines, writes JSON lines to stdout + compare.php compares two saved JSON result files (no interleaving; weaker than live A/B, says so in output) + fetch-corpus.php restores corpus/real/*.html from corpus/manifest.json via HTTP range requests to data.commoncrawl.org + lib/ + class-benchmark-bootstrap.php loads the HTML API files from a given checkout root plus WordPress function stubs + class-benchmark-synthetic.php seeded synthetic document generator + class-benchmark-stats.php median, bootstrap CI, geometric mean, formatting + class-benchmark-report.php table / markdown / json output + corpus/ + manifest.json Common Crawl record pointers + sha256 of expected body (committed) + real/ fetched documents (gitignored) + .gitignore `real/` +``` + +## Bootstrap (lib/class-benchmark-bootstrap.php) + +`Benchmark_Bootstrap::load( string $checkout_root ): void` requires, in this order, from `$checkout_root`: + +``` +src/wp-includes/compat.php +src/wp-includes/compat-utf8.php (only if the file exists; older trunks lack it) +src/wp-includes/utf8.php (only if exists) +src/wp-includes/class-wp-token-map.php +src/wp-includes/html-api/html5-named-character-references.php +src/wp-includes/html-api/class-wp-html-attribute-token.php +src/wp-includes/html-api/class-wp-html-span.php +src/wp-includes/html-api/class-wp-html-doctype-info.php +src/wp-includes/html-api/class-wp-html-text-replacement.php +src/wp-includes/html-api/class-wp-html-decoder.php +src/wp-includes/html-api/class-wp-html-tag-processor.php +src/wp-includes/html-api/class-wp-html-unsupported-exception.php +src/wp-includes/html-api/class-wp-html-active-formatting-elements.php +src/wp-includes/html-api/class-wp-html-open-elements.php +src/wp-includes/html-api/class-wp-html-token.php +src/wp-includes/html-api/class-wp-html-stack-event.php +src/wp-includes/html-api/class-wp-html-processor-state.php +src/wp-includes/html-api/class-wp-html-processor.php +``` + +Before those, define stubs guarded by `function_exists`: `__`, `_doing_it_wrong` (no-op), `_deprecated_argument`, `wp_trigger_error`, `esc_url` (identity), `wp_kses_uri_attributes` (the usual list), `wp_scrub_utf8` (identity if missing; check whether utf8.php defines it). Reference: the fuzz harness stubs at ~/a8c/wordpress-develop/html-api-commoncrawl/tools/html-api-fuzz/lib/wp-stubs.php. If a required file is missing in the checkout, fail with a clear message naming the file. + +## Worker protocol (worker.php) + +Started as `php [ini flags] worker.php --checkout `. Loads bootstrap once. Then reads one JSON object per line on stdin and answers one JSON object per line on stdout. The worker never writes anything else to stdout; diagnostics go to stderr. + +Request kinds: + +- `{"op":"hello"}` → `{"ok":true,"php":"8.5.11","opcache":bool,"jit":"","checkout":"","head":""}` +- `{"op":"load","id":"","path":""}` or `{"op":"load","id":"...","html":""}` → stores the document in memory keyed by id; replies `{"ok":true,"id":..., "bytes":N}`. +- `{"op":"calibrate","id":"...","parser":"tag"|"html"}` → runs one parse, replies `{"ok":true,"ns":,"tokens":N,"bailed":null|""}`. +- `{"op":"run","id":"...","parser":"tag"|"html","iterations":N}` → runs N full parses back to back inside one hrtime window, replies `{"ok":true,"ns":,"tokens":,"bailed":null|"","peak_bytes":}`. Before timing, call `gc_collect_cycles()`. Create a fresh processor per iteration (construction is part of parsing a document in real use; keep it in). +- `{"op":"quit"}` → exits 0. + +Tag Processor parse: `$p = new WP_HTML_Tag_Processor( $html ); while ( $p->next_token() ) { ++$tokens; }`. +HTML Processor parse: `$p = WP_HTML_Processor::create_full_parser( $html ); while ( $p->next_token() ) { ++$tokens; }` then if `$p->get_last_error() !== null` the parse bailed: report `get_last_error()` plus `get_unsupported_exception()?->getMessage()` as the reason. For older checkouts without `create_full_parser`, fall back to `create_fragment`. A bailed document is still timed (time to bail), but bench.php reports it separately (see Report). + +Memory: `memory_get_peak_usage(true)` before and after; report peak. Secondary metric only. + +## Orchestrator (bench.php) + +Options (long form, `--name=value` or `--name value`): + +- `--head ` defaults to the checkout containing bench.php (resolve by walking up to the dir containing `src/wp-includes/html-api`). +- `--base ` optional. When given, A/B mode. When omitted, single-tree mode. +- `--php ` default `php`. `--php-args ""` passed to both workers. +- `--opcache` (default on if the opcache extension is available) adds `-d opcache.enable_cli=1`; `--no-opcache`. `--jit` adds `-d opcache.jit_buffer_size=64M -d opcache.jit=tracing`; default off. The report header prints what was in effect. +- `--parser tag|html|both` default both. +- `--corpus ` repeatable; default `corpus/real` if it exists and is non-empty. Reads `*.html` and `*.htm`, document id is the filename. +- `--synthetic` / `--no-synthetic` default on. `--synthetic-size ` default 200000. `--seed ` default 1. +- `--filter ` restrict document ids. +- `--samples N` default 20: number of timed samples per (document, parser, tree). +- `--min-sample-ms N` default 25: iterations per sample are calibrated so a sample lasts at least this long on the slower tree (one calibration parse per tree, take the max, iterations = max(1, ceil(min_sample_ms / ms_per_parse))). +- `--warmup N` default 2 untimed parses per (document, parser, tree) before sampling. +- `--format table|markdown|json` default table. `--save ` writes the full JSON (every sample, config, environment) regardless of format. `--quiet`. +- `--list` prints document ids and sizes and exits. + +Scheduling: both workers start at once and stay alive for the whole run. For each (document, parser): load into both, warm up both, calibrate, then for s in 1..samples: run on A, run on B, alternating which goes first each sample (ABBA ordering: sample 1 A then B, sample 2 B then A, ...). This interleaving is the point of the design: machine noise drifts over time and alternating cancels drift. Print progress to stderr with a one-line-per-document status unless `--quiet`. + +In single-tree mode, run the same loop with one worker. + +## Statistics (lib/class-benchmark-stats.php) + +Per sample, the measure is ns per parse = ns / iterations. Per (document, parser, tree): median, min, mean, stddev, CV (stddev/mean), and throughput MB/s = bytes / median, tokens/s. + +A/B per document: ratio = median_head / median_base (<1 is faster). 95% CI from a bootstrap of the ratio of medians: 2000 resamples, each drawing with replacement from head samples and base samples, percentile interval. Verdict: `faster` if the whole CI is below 1, `slower` if whole CI above 1, else `no difference`. Also report the point change as a percentage. + +Overall per parser: geometric mean of per-document ratios, with a bootstrap CI (resample documents and samples). Also overall throughput: sum(bytes) / sum(median ns) per tree. Report documents that bailed on either tree in their own list, excluded from the aggregate. + +Use `hrtime(true)` only. The bootstrap must be deterministic given `--seed` (mt_srand). + +## Synthetic documents (lib/class-benchmark-synthetic.php) + +`Benchmark_Synthetic::shapes(): array` of ids. `Benchmark_Synthetic::generate( string $shape, int $target_bytes, int $seed ): string`. Deterministic: same (shape, bytes, seed) gives identical bytes across runs and PHP versions (use `mt_srand( $seed, MT_RAND_MT19937 )` and `mt_rand`; never `random_int`). Output size within 2% of target. Every shape is a full document (doctype, html, head, body) so the HTML Processor runs it in full-parser mode without bailing; verify each shape parses to the end with `WP_HTML_Processor::create_full_parser` on trunk (no `get_last_error()`), and note any that cannot in the README. + +Shapes (ids): +- `tags-dense`: short elements with no attributes, little text: `
` style, mixed nesting depth up to 8. +- `text-long`: a few `

` elements holding long runs of plain ASCII prose (no entities, no `<` or `&`). +- `text-entities`: prose with frequent named and numeric character references (`&`, ` `, `’`, `…`, some invalid like `¬anentity;`). +- `text-utf8`: prose of multi-byte UTF-8 (mixed scripts), no entities. +- `comments-many`: thousands of short comments between elements, including some with `--` inside and some bogus comments `` and ``. +- `attributes-heavy`: elements with 10 to 30 attributes each, quoted and unquoted values, duplicates, boolean, long `class` and `style` values, `data-*` names. +- `nesting-deep`: nesting to depth 200 then unwinding, repeated (the HTML Processor stack of open elements under load). +- `script-style`: large `\n" ); + while ( self::append( $body, self::pick( $js_lines ) . "\n", $inner_max ) ) { + if ( strlen( $body ) > mt_rand( 400, 1500 ) ) { + break; + } + } + if ( '' === $body ) { + break; + } + $piece = "\n"; + } elseif ( $roll <= 80 ) { + $body = ''; + $inner_max = $limit - strlen( "\n" ); + while ( self::append( $body, self::pick( $css_lines ) . "\n", $inner_max ) ) { + if ( strlen( $body ) > mt_rand( 300, 1200 ) ) { + break; + } + } + if ( '' === $body ) { + break; + } + $piece = "\n"; + } elseif ( $roll <= 92 ) { + $body = ''; + $inner_max = $limit - strlen( "\n" ); + while ( self::append( $body, self::pick( $rcdata ) . "\n", $inner_max ) ) { + if ( strlen( $body ) > mt_rand( 100, 500 ) ) { + break; + } + } + if ( '' === $body ) { + break; + } + $piece = "\n"; + } else { + $piece = '' . self::pick( $rcdata ) . "\n"; + } + + if ( ! self::append( $out, $piece, $max ) ) { + break; + } + } + + return $out; + } + + /** + * `whitespace-runs`: long whitespace runs between tags and inside text. + * + * @param int $max Byte limit. + * @return string Chunk. + */ + private static function chunk_whitespace_runs( int $max ): string { + static $tags = array( 'div', 'p', 'span', 'section', 'pre', 'li', 'em' ); + + $out = ''; + $misses = 0; + while ( true ) { + $tag = self::pick( $tags ); + $piece = self::whitespace_run() . "<{$tag}>" . self::whitespace_run(); + $words = mt_rand( 0, 2 ); + for ( $i = 0; $i < $words; $i++ ) { + $piece .= self::pick( self::ASCII_WORDS ) . self::whitespace_run(); + } + $piece .= ""; + + if ( ! self::append( $out, $piece, $max ) ) { + // Runs are up to 400 bytes each; try a few more before giving up. + if ( ++$misses < 8 ) { + continue; + } + break; + } + } + + return $out; + } + + /** + * Builds one run of whitespace, 1 to 400 characters. + * + * @return string Whitespace. + */ + private static function whitespace_run(): string { + static $chars = array( ' ', ' ', ' ', "\t", "\n", "\n", "\r\n", "\f" ); + + $length = self::chance( 25 ) ? mt_rand( 1, 20 ) : mt_rand( 20, 400 ); + $out = ''; + $char = self::pick( $chars ); + for ( $i = 0; $i < $length; $i++ ) { + if ( self::chance( 10 ) ) { + $char = self::pick( $chars ); + } + $out .= $char; + } + + return $out; + } + + /** + * `foreign-content`: SVG and MathML subtrees. + * + * @param int $max Byte limit. + * @return string Chunk. + */ + private static function chunk_foreign_content( int $max ): string { + $out = ''; + while ( true ) { + $limit = $max - strlen( $out ); + $piece = self::chance( 65 ) ? self::svg_block( $limit ) : self::math_block( $limit ); + if ( '' === $piece ) { + break; + } + $out .= $piece; + } + + return $out; + } + + /** + * Builds one inline SVG within `$max` bytes. + * + * @param int $max Byte limit. + * @return string SVG markup, or empty when it does not fit. + */ + private static function svg_block( int $max ): string { + $open = '\n"; + if ( strlen( $open ) + strlen( $end ) > $max ) { + return ''; + } + + $inner = ''; + $limit = $max - strlen( $open ) - strlen( $end ); + $count = mt_rand( 3, 12 ); + for ( $i = 0; $i < $count; $i++ ) { + $roll = mt_rand( 1, 100 ); + if ( $roll <= 35 ) { + $d = 'M' . mt_rand( 0, 24 ) . ' ' . mt_rand( 0, 24 ); + $n = mt_rand( 2, 10 ); + for ( $j = 0; $j < $n; $j++ ) { + $d .= self::pick( array( 'L', 'C', 'Q', 'A', 'H', 'V', 'Z' ) ) . mt_rand( 0, 24 ) . '.' . mt_rand( 0, 9 ) . ' ' . mt_rand( 0, 24 ) . '.' . mt_rand( 0, 9 ); + } + $piece = ""; + } elseif ( $roll <= 50 ) { + $piece = ''; + } elseif ( $roll <= 60 ) { + $piece = ''; + } elseif ( $roll <= 70 ) { + $piece = ''; + } elseif ( $roll <= 78 ) { + $piece = ''; + } elseif ( $roll <= 86 ) { + $piece = '' . self::pick( self::ASCII_WORDS ) . ''; + } elseif ( $roll <= 94 ) { + $piece = '

' . self::pick( self::ASCII_WORDS ) . ' ' . self::pick( self::ASCII_WORDS ) . '

  • ' . self::pick( self::ASCII_WORDS ) . '
'; + } else { + $piece = '' . self::pick( self::ASCII_WORDS ) . ' icon' . self::pick( self::ASCII_WORDS ) . ' ' . self::pick( self::ASCII_WORDS ) . ''; + } + + if ( ! self::append( $inner, $piece, $limit ) ) { + break; + } + } + + return $open . $inner . $end; + } + + /** + * Builds one inline MathML block within `$max` bytes. + * + * @param int $max Byte limit. + * @return string MathML markup, or empty when it does not fit. + */ + private static function math_block( int $max ): string { + $open = ''; + $end = "\n"; + if ( strlen( $open ) + strlen( $end ) > $max ) { + return ''; + } + + $inner = ''; + $limit = $max - strlen( $open ) - strlen( $end ); + $count = mt_rand( 2, 8 ); + for ( $i = 0; $i < $count; $i++ ) { + $roll = mt_rand( 1, 100 ); + $a = chr( mt_rand( 97, 122 ) ); + $b = chr( mt_rand( 97, 122 ) ); + if ( $roll <= 30 ) { + $piece = "{$a}=" . mt_rand( 1, 99 ) . ''; + } elseif ( $roll <= 55 ) { + $piece = "{$a}+{$b}" . mt_rand( 2, 9 ) . ''; + } elseif ( $roll <= 70 ) { + $piece = "{$a}" . mt_rand( 2, 5 ) . "−{$b}"; + } elseif ( $roll <= 82 ) { + $piece = "{$a}({$b})"; + } elseif ( $roll <= 92 ) { + $piece = '' . $a . '

' . self::pick( self::ASCII_WORDS ) . ' ' . self::pick( self::ASCII_WORDS ) . '

'; + } else { + $piece = '' . self::pick( self::ASCII_WORDS ) . ' ' . self::pick( self::ASCII_WORDS ) . ''; + } + + if ( ! self::append( $inner, $piece, $limit ) ) { + break; + } + } + + return $open . $inner . $end; + } + + /** + * `formatting-adoption`: misnested formatting elements. + * + * @param int $max Byte limit. + * @return string Chunk. + */ + private static function chunk_formatting_adoption( int $max ): string { + $out = ''; + while ( true ) { + $w = array(); + for ( $i = 0; $i < 8; $i++ ) { + $w[] = self::pick( self::ASCII_WORDS ); + } + $roll = mt_rand( 1, 100 ); + if ( $roll <= 18 ) { + // B and I crossing: the I closer follows the B closer directly. + $piece = "

{$w[0]} {$w[1]} {$w[2]} {$w[3]} {$w[4]} {$w[5]}

\n"; + } elseif ( $roll <= 32 ) { + // A inside A: the second A start tag runs the adoption agency on the first. + $piece = "

{$w[1]} {$w[3]} {$w[4]} {$w[5]}

\n"; + } elseif ( $roll <= 46 ) { + // P closed by DIV, then a stray P closer that inserts an empty P. + $piece = "

{$w[0]} {$w[1]}

{$w[2]} {$w[3]}
{$w[4]}

\n"; + } elseif ( $roll <= 56 ) { + // A non-formatting element popped by the B closer, then its stray closer is ignored. + $piece = "

{$w[0]} {$w[1]} {$w[2]} {$w[3]}

\n"; + } elseif ( $roll <= 66 ) { + $piece = "

{$w[0]} {$w[1]} {$w[2]} {$w[3]}

\n"; + } elseif ( $roll <= 74 ) { + // Three nested B elements; the P closer pops the outermost, whose own closer follows. + $piece = "

{$w[0]} {$w[1]} {$w[2]} {$w[3]}

{$w[4]}\n"; + } elseif ( $roll <= 84 ) { + $piece = "

{$w[0]} {$w[1]} {$w[2]} {$w[3]} {$w[4]}

\n"; + } elseif ( $roll <= 90 ) { + // A second NOBR runs the adoption agency on the first; FONT then pops the rest. + $piece = "
{$w[0]} {$w[1]} {$w[2]} {$w[3]}
\n"; + } elseif ( $roll <= 95 ) { + // P closer pops the open B; the B closer then removes it from the formatting list. + $piece = "

{$w[0]} {$w[1]}

{$w[2]}\n"; + } else { + // P inside P closes the first. + $piece = "

{$w[0]} {$w[1]}

{$w[2]} {$w[3]} {$w[4]}

\n"; + } + + if ( ! self::append( $out, $piece, $max ) ) { + break; + } + } + + return $out; + } + + /** + * `tables`: tables with and without TBODY, implied rows and cells, nested tables. + * + * Stray non-whitespace text and stray tags directly inside a table would + * be foster-parented, which the HTML Processor does not support; this shape + * stays within what it parses so the document never bails. + * + * @param int $max Byte limit. + * @return string Chunk. + */ + private static function chunk_tables( int $max ): string { + $open = self::chance( 50 ) ? "\n" : '
' . "\n"; + $end = "
\n"; + if ( strlen( $open ) + strlen( $end ) > $max ) { + return ''; + } + + $inner = ''; + $limit = $max - strlen( $open ) - strlen( $end ); + + if ( self::chance( 30 ) ) { + self::append( $inner, '' . self::pick( self::ASCII_WORDS ) . ' ' . self::pick( self::ASCII_WORDS ) . "\n", $limit ); + } + if ( self::chance( 30 ) ) { + self::append( $inner, '' . "\n", $limit ); + } + if ( self::chance( 40 ) ) { + self::append( $inner, '' . self::pick( self::ASCII_WORDS ) . '' . self::pick( self::ASCII_WORDS ) . "\n", $limit ); + } + + $style = mt_rand( 1, 4 ); + if ( 1 === $style ) { + self::append( $inner, "\n", $limit ); + } + + $rows = mt_rand( 2, 20 ); + for ( $r = 0; $r < $rows; $r++ ) { + $cells = mt_rand( 1, 6 ); + if ( 4 === $style ) { + // Cells directly in the table imply both TBODY and TR. + $row = ''; + } elseif ( self::chance( 20 ) ) { + $row = ''; + } else { + $row = ''; + } + for ( $c = 0; $c < $cells; $c++ ) { + $tag = self::chance( 15 ) ? 'th' : 'td'; + $cell = "<{$tag}>" . self::pick( self::ASCII_WORDS ); + $sub = mt_rand( 1, 100 ); + if ( $sub <= 10 ) { + $cell .= ' ' . self::pick( self::ASCII_WORDS ) . ' ' . self::pick( self::ASCII_WORDS ) . ''; + } elseif ( $sub <= 16 ) { + $cell .= '
' . self::pick( self::ASCII_WORDS ) . '' . self::pick( self::ASCII_WORDS ) . '
'; + } elseif ( $sub <= 22 ) { + $cell .= '

' . self::pick( self::ASCII_WORDS ) . ' ' . self::pick( self::ASCII_WORDS ) . '

'; + } + // Closers are optional for TD and TH; omit some. + if ( 4 !== $style && self::chance( 50 ) ) { + $cell .= ""; + } + $row .= $cell; + } + if ( 4 !== $style && self::chance( 60 ) ) { + $row .= ''; + } + $row .= "\n"; + + // Tokens the "in table" mode handles between rows: whitespace, comments, hidden inputs. + $between = mt_rand( 1, 100 ); + if ( $between <= 15 ) { + $row .= "\n"; + } elseif ( $between <= 22 ) { + $row .= '\n"; + } elseif ( $between <= 30 ) { + $row .= "\t\t\n"; + } + + if ( ! self::append( $inner, $row, $limit ) ) { + break; + } + } + + if ( 1 === $style ) { + self::append( $inner, "\n", $limit ); + } + if ( self::chance( 20 ) ) { + self::append( $inner, '' . self::pick( self::ASCII_WORDS ) . "\n", $limit ); + } + + if ( '' === $inner ) { + return ''; + } + + return $open . $inner . $end; + } + + /** + * `wordpress-post`: a block-themed page. + * + * @param int $max Byte limit. + * @return string Chunk. + */ + private static function chunk_wordpress_post( int $max ): string { + $out = ''; + $roll = mt_rand( 1, 100 ); + + if ( $roll <= 30 ) { + $frame = strlen( "\n

\n\n" ); + $text = self::prose( min( $max - $frame, mt_rand( 200, 900 ) ), self::ASCII_WORDS, 6, array( '’s', ' ', '&', '“', '”', '…' ) ); + if ( '' === $text ) { + return ''; + } + if ( self::chance( 30 ) ) { + $text = preg_replace( '/^(\S+ \S+)/', '$1', $text, 1 ); + } + if ( self::chance( 30 ) ) { + $text = preg_replace( '/(\S+)\.$/', '$1.', $text, 1 ); + } + $out = "\n

{$text}

\n\n"; + } elseif ( $roll <= 40 ) { + $level = mt_rand( 2, 4 ); + $words = array(); + $n = mt_rand( 2, 8 ); + for ( $i = 0; $i < $n; $i++ ) { + $words[] = self::pick( self::ASCII_WORDS ); + } + $text = ucfirst( implode( ' ', $words ) ); + $id = strtolower( str_replace( ' ', '-', $text ) ); + $out = ( 2 === $level ? '' : "" ) . "\n{$text}\n\n"; + } elseif ( $roll <= 50 ) { + $id = mt_rand( 100, 99999 ); + $slug = self::pick( self::ASCII_WORDS ) . '-' . self::pick( self::ASCII_WORDS ); + $w = self::pick( array( 1024, 1536, 2048 ) ); + $h = intdiv( $w * mt_rand( 50, 80 ), 100 ); + $base = 'https://example.com/wp-content/uploads/2026/' . str_pad( (string) mt_rand( 1, 12 ), 2, '0', STR_PAD_LEFT ) . "/{$slug}"; + $out = "\n" . + "
\""
" . ucfirst( self::pick( self::ASCII_WORDS ) ) . ' ' . self::pick( self::ASCII_WORDS ) . ".
\n" . + "\n"; + } elseif ( $roll <= 60 ) { + $ordered = self::chance( 30 ); + $tag = $ordered ? 'ol' : 'ul'; + $out = ( $ordered ? "\n
    " : "\n
      " ) . "\n"; + $items = mt_rand( 2, 8 ); + for ( $i = 0; $i < $items; $i++ ) { + $out .= "\n
    • " . ucfirst( self::pick( self::ASCII_WORDS ) ) . ' ' . self::pick( self::ASCII_WORDS ) . ' ' . self::pick( self::ASCII_WORDS ) . "
      • \n\n"; + } + $out .= "\n\n"; + } elseif ( $roll <= 78 ) { + // Nested group DIVs with class and style, holding paragraphs. + $depth = mt_rand( 1, 4 ); + $open = ''; + $close = ''; + for ( $i = 0; $i < $depth; $i++ ) { + $layout = self::pick( array( 'constrained', 'flex', 'grid', 'default' ) ); + $open .= "\n" . + "
      \n"; + $close = "
      \n\n" . $close; + } + $text = self::prose( mt_rand( 80, 300 ), self::ASCII_WORDS ); + $out = $open . "\n

      {$text}

      \n\n" . $close; + } elseif ( $roll <= 88 ) { + $services = array( 'wordpress', 'mastodon', 'github', 'bluesky', 'rss' ); + $out = "\n
        \n"; + $n = mt_rand( 2, 5 ); + for ( $i = 0; $i < $n; $i++ ) { + $s = self::pick( $services ); + $out .= "\n" . + "
      • ' . ucfirst( $s ) . "
        • \n"; + } + $out .= "
      \n\n"; + } elseif ( $roll <= 94 ) { + $id = mt_rand( 1, 999 ); + $keys = array(); + $n = mt_rand( 3, 12 ); + for ( $i = 0; $i < $n; $i++ ) { + $keys[] = '"' . self::pick( self::ASCII_WORDS ) . mt_rand( 1, 99 ) . '":' . self::pick( array( 'true', 'false', 'null', (string) mt_rand( 0, 9999 ), '"' . self::pick( self::ASCII_WORDS ) . '"', '{"nested":"<\/div>"}', '["a","b"]' ) ); + } + $out = "\n"; + } else { + $out = "\n
      \n\n" . + '\n
      \n\n"; + } + + if ( strlen( $out ) > $max ) { + // Fall back to the smallest block that fits, else stop. + $small = "\n

      " . self::pick( self::ASCII_WORDS ) . "

      \n\n"; + return strlen( $small ) <= $max ? $small : ''; + } + + return $out; + } +} diff --git a/tests/performance/html-api/worker.php b/tests/performance/html-api/worker.php new file mode 100644 index 0000000000000..ef4b01a430614 --- /dev/null +++ b/tests/performance/html-api/worker.php @@ -0,0 +1,297 @@ + + * + * @package WordPress + * @subpackage HTML-API + */ + +ini_set( 'display_errors', 'stderr' ); +error_reporting( E_ALL ); + +require_once __DIR__ . '/lib/class-benchmark-bootstrap.php'; + +/** + * Writes one JSON line to stdout. + * + * @param array $reply Reply payload. + */ +function benchmark_worker_reply( array $reply ): void { + fwrite( STDOUT, json_encode( $reply, JSON_UNESCAPED_SLASHES ) . "\n" ); + fflush( STDOUT ); +} + +/** + * Writes a diagnostic line to stderr. + * + * @param string $message Message. + */ +function benchmark_worker_log( string $message ): void { + fwrite( STDERR, "worker: {$message}\n" ); +} + +/** + * Parses one document with the Tag Processor. + * + * @param string $html Document. + * @return array{tokens:int, bailed:null} + */ +function benchmark_worker_parse_tag( string $html ): array { + $tokens = 0; + $processor = new WP_HTML_Tag_Processor( $html ); + while ( $processor->next_token() ) { + ++$tokens; + } + return array( + 'tokens' => $tokens, + 'bailed' => null, + ); +} + +/** + * Parses one document with the HTML Processor. + * + * @param string $html Document. + * @return array{tokens:int, bailed:string|null} + */ +function benchmark_worker_parse_html( string $html ): array { + $tokens = 0; + if ( method_exists( 'WP_HTML_Processor', 'create_full_parser' ) ) { + $processor = WP_HTML_Processor::create_full_parser( $html ); + } else { + $processor = WP_HTML_Processor::create_fragment( $html ); + } + if ( null === $processor ) { + return array( + 'tokens' => 0, + 'bailed' => 'processor could not be created', + ); + } + while ( $processor->next_token() ) { + ++$tokens; + } + + $bailed = null; + $error = $processor->get_last_error(); + if ( null !== $error ) { + $bailed = $error; + if ( method_exists( $processor, 'get_unsupported_exception' ) ) { + $exception = $processor->get_unsupported_exception(); + if ( null !== $exception ) { + $bailed .= ': ' . $exception->getMessage(); + } + } + } + + return array( + 'tokens' => $tokens, + 'bailed' => $bailed, + ); +} + +/** + * Returns the parser function for a parser id. + * + * @throws InvalidArgumentException On an unknown parser id. + * + * @param string $parser 'tag' or 'html'. + * @return callable + */ +function benchmark_worker_parser( string $parser ): callable { + switch ( $parser ) { + case 'tag': + return 'benchmark_worker_parse_tag'; + case 'html': + return 'benchmark_worker_parse_html'; + } + throw new InvalidArgumentException( "Unknown parser: {$parser}" ); +} + +/** + * Reads the --checkout argument. + * + * @param string[] $argv Arguments. + * @return string|null Checkout root. + */ +function benchmark_worker_checkout_arg( array $argv ): ?string { + $count = count( $argv ); + for ( $i = 1; $i < $count; $i++ ) { + if ( '--checkout' === $argv[ $i ] && isset( $argv[ $i + 1 ] ) ) { + return $argv[ $i + 1 ]; + } + if ( 0 === strpos( $argv[ $i ], '--checkout=' ) ) { + return substr( $argv[ $i ], strlen( '--checkout=' ) ); + } + } + return null; +} + +/** + * Returns the short git head of a checkout, or null. + * + * @param string $checkout Checkout root. + * @return string|null + */ +function benchmark_worker_git_head( string $checkout ): ?string { + $output = array(); + $status = 1; + exec( 'git -C ' . escapeshellarg( $checkout ) . ' rev-parse --short HEAD 2>/dev/null', $output, $status ); + if ( 0 !== $status || empty( $output ) ) { + return null; + } + return trim( $output[0] ); +} + +/** + * Reports the JIT mode actually in effect, or null when the JIT is off. + * + * @param bool $opcache_enabled Whether opcache is enabled for this process. + * @return string|null + */ +function benchmark_worker_jit_mode( bool $opcache_enabled ): ?string { + if ( ! $opcache_enabled ) { + return null; + } + $buffer = (string) ini_get( 'opcache.jit_buffer_size' ); + $jit = (string) ini_get( 'opcache.jit' ); + if ( '' === $buffer || '0' === $buffer || '' === $jit || '0' === $jit || 'off' === $jit || 'disable' === $jit ) { + return null; + } + return $jit; +} + +$checkout = benchmark_worker_checkout_arg( $argv ); +if ( null === $checkout ) { + benchmark_worker_log( 'usage: php worker.php --checkout ' ); + exit( 2 ); +} + +try { + Benchmark_Bootstrap::load( $checkout ); +} catch ( Throwable $e ) { + benchmark_worker_log( $e->getMessage() ); + exit( 2 ); +} + +$documents = array(); + +// phpcs:ignore WordPress.CodeAnalysis.AssignmentInCondition.FoundInWhileCondition +while ( false !== ( $line = fgets( STDIN ) ) ) { + $line = trim( $line ); + if ( '' === $line ) { + continue; + } + + $request = json_decode( $line, true ); + if ( ! is_array( $request ) || ! isset( $request['op'] ) ) { + benchmark_worker_reply( + array( + 'ok' => false, + 'error' => 'malformed request', + ) + ); + continue; + } + + try { + switch ( $request['op'] ) { + case 'hello': + $opcache_enabled = extension_loaded( 'Zend OPcache' ) && filter_var( ini_get( 'opcache.enable_cli' ), FILTER_VALIDATE_BOOLEAN ) && filter_var( ini_get( 'opcache.enable' ), FILTER_VALIDATE_BOOLEAN ); + benchmark_worker_reply( + array( + 'ok' => true, + 'php' => PHP_VERSION, + 'opcache' => $opcache_enabled, + 'jit' => benchmark_worker_jit_mode( $opcache_enabled ), + 'checkout' => false === realpath( $checkout ) ? $checkout : realpath( $checkout ), + 'head' => benchmark_worker_git_head( $checkout ), + ) + ); + break; + + case 'load': + if ( ! isset( $request['id'] ) ) { + throw new InvalidArgumentException( 'load requires id' ); + } + if ( isset( $request['path'] ) ) { + $html = file_get_contents( $request['path'] ); + if ( false === $html ) { + throw new RuntimeException( "could not read {$request['path']}" ); + } + } elseif ( isset( $request['html'] ) ) { + $html = (string) $request['html']; + } else { + throw new InvalidArgumentException( 'load requires path or html' ); + } + $documents[ $request['id'] ] = $html; + benchmark_worker_reply( + array( + 'ok' => true, + 'id' => $request['id'], + 'bytes' => strlen( $html ), + ) + ); + break; + + case 'calibrate': + case 'run': + if ( ! isset( $request['id'], $documents[ $request['id'] ] ) ) { + throw new InvalidArgumentException( 'unknown document id' ); + } + $html = $documents[ $request['id'] ]; + $parse = benchmark_worker_parser( (string) ( $request['parser'] ?? '' ) ); + $iterations = 'run' === $request['op'] ? max( 1, (int) ( $request['iterations'] ?? 1 ) ) : 1; + + gc_collect_cycles(); + if ( function_exists( 'memory_reset_peak_usage' ) ) { + memory_reset_peak_usage(); + } + $memory_before = memory_get_peak_usage( true ); + $alloc_before = memory_get_peak_usage( false ); + + $result = null; + $start = hrtime( true ); + for ( $i = 0; $i < $iterations; $i++ ) { + $result = $parse( $html ); + } + $ns = hrtime( true ) - $start; + + $memory_after = memory_get_peak_usage( true ); + $alloc_after = memory_get_peak_usage( false ); + + $reply = array( + 'ok' => true, + 'ns' => $ns, + 'tokens' => $result['tokens'], + 'bailed' => $result['bailed'], + ); + if ( 'run' === $request['op'] ) { + $reply['iterations'] = $iterations; + $reply['peak_bytes'] = max( 0, $memory_after - $memory_before ); + $reply['peak_alloc_bytes'] = max( 0, $alloc_after - $alloc_before ); + } + benchmark_worker_reply( $reply ); + break; + + case 'quit': + exit( 0 ); + + default: + throw new InvalidArgumentException( "unknown op: {$request['op']}" ); + } + } catch ( Throwable $e ) { + benchmark_worker_reply( + array( + 'ok' => false, + 'error' => get_class( $e ) . ': ' . $e->getMessage(), + ) + ); + } +} + +exit( 0 ); From 4c8643810ebbfbbdbf4fe8fde1fd1d3b8788e384 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 2 Oct 2026 08:23:32 +0400 Subject: [PATCH 2/6] HTML API benchmark: ignore the fetched corpus directory. --- tests/performance/html-api/corpus/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/performance/html-api/corpus/.gitignore diff --git a/tests/performance/html-api/corpus/.gitignore b/tests/performance/html-api/corpus/.gitignore new file mode 100644 index 0000000000000..70b9cafea0558 --- /dev/null +++ b/tests/performance/html-api/corpus/.gitignore @@ -0,0 +1 @@ +real/ From 453b0a84191e5d95b7eed43ec39dd7af4730aac0 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 2 Oct 2026 08:23:39 +0400 Subject: [PATCH 3/6] HTML API benchmark: ignore the fetched corpus from the root .gitignore, which this repository uses instead of nested ignore files. --- .gitignore | 1 + tests/performance/html-api/corpus/.gitignore | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 tests/performance/html-api/corpus/.gitignore diff --git a/.gitignore b/.gitignore index f96e379fbb304..c1858d7cc5c09 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ wp-tests-config.php /tests/phpunit/data/.trac-ticket-cache* /tests/qunit/compiled.html /tests/performance/**/*.test.results.json +/tests/performance/html-api/corpus/real/ /src/.wp-tests-version /node_modules /npm-debug.log diff --git a/tests/performance/html-api/corpus/.gitignore b/tests/performance/html-api/corpus/.gitignore deleted file mode 100644 index 70b9cafea0558..0000000000000 --- a/tests/performance/html-api/corpus/.gitignore +++ /dev/null @@ -1 +0,0 @@ -real/ From d62a68457b9ec918af13aa1ef9922552feb5e485 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 9 Oct 2026 20:33:19 +0400 Subject: [PATCH 4/6] HTML API benchmark: shuffle each round's order, warm up longer under the JIT, and serialize timed runs. Under the tracing JIT, the code paths a process compiles first set the speed of everything it runs afterwards: a worker that parsed wordpress.org's home page first ran it and later documents 10% to 16% slower than one that started with another document. With one fixed order every round shared that history, so the interval over rounds did not include it. Each round now runs the (document, parser) pairs in an order shuffled from the seed and round number, the same for both trees, and the saved JSON records the orders. The HTML Processor needs about six parses before its traces settle. With --jit the warmup is now at least 10 parses and 200 ms (--warmup-ms), which took its CV on wporg-home from 12% to 3%. bench.php now holds a lock on /tmp/wp-html-api-bench.lock for the whole run and waits while another run holds it; with-lock.php runs any other command under the same lock, for profiling and ad hoc timing. A worker with a profiler extension loaded stops the run. --- tests/performance/html-api/README.md | 21 +- tests/performance/html-api/bench.php | 255 +++++++++++------- tests/performance/html-api/compare.php | 2 +- .../html-api/lib/class-benchmark-lock.php | 51 ++++ .../html-api/lib/class-benchmark-report.php | 3 +- tests/performance/html-api/with-lock.php | 28 ++ tests/performance/html-api/worker.php | 48 +++- 7 files changed, 302 insertions(+), 106 deletions(-) create mode 100644 tests/performance/html-api/lib/class-benchmark-lock.php create mode 100644 tests/performance/html-api/with-lock.php diff --git a/tests/performance/html-api/README.md b/tests/performance/html-api/README.md index 2a2ae9fbf57f4..d575655615fca 100644 --- a/tests/performance/html-api/README.md +++ b/tests/performance/html-api/README.md @@ -53,7 +53,8 @@ Options (`--name value` or `--name=value`): | `--filter ` | none | Keep ids containing the text, or matching it when written as `/regex/`. | | `--samples ` | 10 | Timed samples per document, parser and tree, per round. | | `--min-sample-ms ` | 25 | Each sample runs enough parses to last at least this long on the slower tree. | -| `--warmup ` | 2 | Untimed parses before sampling. | +| `--warmup ` | 2; 10 with `--jit` | Untimed parses before sampling. | +| `--warmup-ms ` | 0; 200 with `--jit` | Keep warming up until this much time has passed as well. | | `--rounds ` | 4 | Repeat the whole run with a fresh pair of worker processes each time; see below. | | `--format table\|markdown\|json` | `table` | Report format. | | `--save ` | none | Write the full JSON (every sample, config, environment) whatever the format. | @@ -64,8 +65,8 @@ Progress goes to stderr, one line per document and parser; the report goes to st ## How a run proceeds -1. `bench.php` starts one worker per tree (`php [ini flags] worker.php --checkout `). Both workers stay alive for the whole run. -2. For each document and parser: the document is loaded into both workers; each runs `--warmup` untimed parses; each runs one calibration parse, and the slower of the two sets the number of parses per sample so that a sample lasts at least `--min-sample-ms`. +1. `bench.php` takes the benchmark lock (see below), then starts one worker per tree (`php [ini flags] worker.php --checkout `). Both workers stay alive for the whole round. A worker with a profiler extension loaded (Xdebug, Excimer, SPX, XHProf and similar) stops the run. +2. Each round runs the (document, parser) pairs in a shuffled order, seeded from `--seed` and the round number; both trees use the same order. For each pair: the document is loaded into both workers if it is not already; each runs at least `--warmup` untimed parses for at least `--warmup-ms`; each runs one calibration parse, and the slower of the two sets the number of parses per sample so that a sample lasts at least `--min-sample-ms`. 3. The samples are taken alternating between trees, and the tree that goes first alternates each sample (A B, B A, A B, ...). Each sample calls `gc_collect_cycles()` first, then times its parses with `hrtime( true )`. 4. The per-sample measure is nanoseconds per parse. Per document, parser and tree: median, min, mean, standard deviation, coefficient of variation (CV), MB/s and tokens/s from the median. @@ -89,6 +90,10 @@ The bootstrap only sees the spread between samples of one process. Two PHP proce `--rounds N` with N of 2 or more (the default is 4) starts a fresh pair of workers for each round and repeats the whole schedule. Each round then has its own ratio of medians per document, and the interval is a t-interval over the per-round log ratios (N minus 1 degrees of freedom); the overall interval is a t-interval over the per-round geometric means. The point estimate becomes the geometric mean of the per-round ratios, and the per-tree columns show the mean of the per-round medians. With 3 rounds the interval covers changes of about 2%; with 5 rounds about 1%. Use rounds when the claim is below a few percent. +### The tracing JIT + +With `--jit`, a process compiles traces for the code paths it runs first, and those traces set the speed of everything it runs afterwards. On this corpus, a process that parsed wordpress.org's home page first ran it and every later document 10% to 16% slower than a process that started with a Wikipedia article or RFC 9110. That is why each round shuffles its order: rounds then differ in history, and the interval over rounds includes the effect. The HTML Processor needs about six parses before its traces settle, against three for the Tag Processor, which is why `--jit` raises the warmup. The saved JSON records each round's order under `orders`. + ### A/A validation Before trusting a result on a given machine, run with the same checkout on both sides: @@ -140,15 +145,25 @@ php tests/performance/html-api/compare.php before.json after.json The head tree of each file is used, matched by document id and parser. Prefer a live A/B when both checkouts are available. +## Timed runs take a lock + +Two timed runs at once slow each other and corrupt both results. `bench.php` holds an exclusive lock on `/tmp/wp-html-api-bench.lock` from the first worker to the report, and waits, saying so on stderr, while another run holds it. Run any other timing or profiling through the same lock: + +``` +php tests/performance/html-api/with-lock.php php -d extension=excimer.so profile-script.php +``` + ## Files - `bench.php`: the orchestrator. - `worker.php`: a subprocess that loads one checkout's HTML API and runs timed jobs read as JSON lines on stdin, answering on stdout. - `compare.php`: compares two `--save` files. +- `with-lock.php`: runs a command while holding the benchmark lock. - `fetch-corpus.php`: restores `corpus/real/` from `corpus/manifest.json`. - `lib/class-benchmark-bootstrap.php`: loads the HTML API from a checkout root with stubs for the WordPress functions it calls. - `lib/class-benchmark-synthetic.php`: the shape generator. - `lib/class-benchmark-stats.php`: median, bootstrap, t-interval, geometric mean, formatting. - `lib/class-benchmark-report.php`: table, markdown and JSON output. +- `lib/class-benchmark-lock.php`: the lock shared by `bench.php` and `with-lock.php`. `bench.php` and `compare.php` need PHP 8.1 or newer; the workers run on whatever `--php` names, as long as the checkout's HTML API loads there. diff --git a/tests/performance/html-api/bench.php b/tests/performance/html-api/bench.php index e721f8914393b..3570db5eaeccc 100644 --- a/tests/performance/html-api/bench.php +++ b/tests/performance/html-api/bench.php @@ -19,6 +19,7 @@ require_once __DIR__ . '/lib/class-benchmark-stats.php'; require_once __DIR__ . '/lib/class-benchmark-report.php'; +require_once __DIR__ . '/lib/class-benchmark-lock.php'; /** * Prints a message to stderr and exits. @@ -42,7 +43,7 @@ function benchmark_fail( string $message, int $code = 1 ): void { function benchmark_parse_options( array $argv ): array { $flags = array( 'opcache', 'no-opcache', 'jit', 'synthetic', 'no-synthetic', 'no-corpus', 'include-optional', 'quiet', 'list', 'help' ); $repeatable = array( 'corpus' ); - $valued = array( 'head', 'base', 'php', 'php-args', 'parser', 'synthetic-size', 'seed', 'filter', 'samples', 'min-sample-ms', 'warmup', 'rounds', 'format', 'save' ); + $valued = array( 'head', 'base', 'php', 'php-args', 'parser', 'synthetic-size', 'seed', 'filter', 'samples', 'min-sample-ms', 'warmup', 'warmup-ms', 'rounds', 'format', 'save' ); $options = array( 'corpus' => array() ); $count = count( $argv ); @@ -111,7 +112,8 @@ function benchmark_usage(): void { --filter Restrict document ids. --samples Timed samples per document, parser and tree, per round (default: 10). --min-sample-ms Minimum duration of one sample (default: 25). - --warmup Untimed parses before sampling (default: 2). + --warmup Untimed parses before sampling (default: 2; 10 with --jit). + --warmup-ms Keep warming up for at least this long (default: 0; 200 with --jit). --rounds Repeat the whole run with a fresh pair of workers each time (default: 4). Each PHP process runs the same code at a slightly different speed; more rounds turn that per-process offset into measured variance. @@ -534,6 +536,9 @@ function benchmark_start_workers( array $config, array $worker_args, int $round $workers = array(); foreach ( $labels as $label ) { $workers[ $label ] = new Benchmark_Worker( $label, $config[ $label ], $config['php'], $worker_args ); + if ( ! empty( $workers[ $label ]->hello['profilers'] ) ) { + benchmark_fail( "The {$label} worker has profiler extensions loaded, which change timings: " . implode( ', ', $workers[ $label ]->hello['profilers'] ) . '. Run the benchmark without them.' ); + } } return $workers; } @@ -545,130 +550,179 @@ function benchmark_start_workers( array $config, array $worker_args, int $round * @param array $documents Documents. * @param Benchmark_Worker[] $workers Workers keyed by label. * @param string[] $parsers Parser ids. + * @param array[] $order Jobs in run order, from benchmark_round_order(). * @param int $round Round number, from 1. * @return array Raw rows keyed by "id\0parser": id, parser, bytes, iterations, trees => label => samples_ns, tokens, bailed, peak_bytes, peak_alloc_bytes. */ -function benchmark_run_round( array $config, array $documents, array $workers, array $parsers, int $round ): array { +function benchmark_run_round( array $config, array $documents, array $workers, array $parsers, array $order, int $round ): array { $labels = array_keys( $workers ); $is_ab = isset( $workers['base'] ); $rows = array(); + $loaded = array(); $temp_files = array(); $total = count( $documents ) * count( $parsers ); $done = 0; $prefix = $config['rounds'] > 1 ? sprintf( '[round %d/%d] ', $round, $config['rounds'] ) : ''; - foreach ( $documents as $document ) { - foreach ( $workers as $worker ) { - benchmark_load_document( $worker, $document, $temp_files ); - } + foreach ( $order as $job ) { + $document = $documents[ $job['document'] ]; + $parser = $job['parser']; + $id = $document['id']; + ++$done; - foreach ( $parsers as $parser ) { - ++$done; - $id = $document['id']; - - // Warm up. - if ( $config['warmup'] > 0 ) { - foreach ( $workers as $worker ) { - $worker->request( - array( - 'op' => 'run', - 'id' => $id, - 'parser' => $parser, - 'iterations' => $config['warmup'], - ) - ); - } + if ( ! isset( $loaded[ $id ] ) ) { + foreach ( $workers as $worker ) { + benchmark_load_document( $worker, $document, $temp_files ); } + $loaded[ $id ] = true; + } - // Calibrate: one parse per tree, iterations from the slower tree. - $slowest_ns = 0; + // Warm up. + if ( $config['warmup'] > 0 || $config['warmup_ms'] > 0 ) { foreach ( $workers as $worker ) { - $reply = $worker->request( + $worker->request( array( - 'op' => 'calibrate', - 'id' => $id, - 'parser' => $parser, + 'op' => 'warmup', + 'id' => $id, + 'parser' => $parser, + 'iterations' => $config['warmup'], + 'min_ms' => $config['warmup_ms'], ) ); - $slowest_ns = max( $slowest_ns, (int) $reply['ns'] ); } - $iterations = max( 1, (int) ceil( ( $config['min_sample_ms'] * 1e6 ) / max( 1, $slowest_ns ) ) ); + } - // Sample with ABBA interleaving. - $trees = array(); - foreach ( $labels as $label ) { - $trees[ $label ] = array( - 'samples_ns' => array(), - 'tokens' => 0, - 'bailed' => null, - 'peak_bytes' => 0, - 'peak_alloc_bytes' => 0, - ); + // Calibrate: one parse per tree, iterations from the slower tree. + $slowest_ns = 0; + foreach ( $workers as $worker ) { + $reply = $worker->request( + array( + 'op' => 'calibrate', + 'id' => $id, + 'parser' => $parser, + ) + ); + $slowest_ns = max( $slowest_ns, (int) $reply['ns'] ); + } + $iterations = max( 1, (int) ceil( ( $config['min_sample_ms'] * 1e6 ) / max( 1, $slowest_ns ) ) ); + + // Sample with ABBA interleaving. + $trees = array(); + foreach ( $labels as $label ) { + $trees[ $label ] = array( + 'samples_ns' => array(), + 'tokens' => 0, + 'bailed' => null, + 'peak_bytes' => 0, + 'peak_alloc_bytes' => 0, + ); + } + for ( $sample = 1; $sample <= $config['samples']; $sample++ ) { + $order = $labels; + if ( 0 === $sample % 2 ) { + $order = array_reverse( $order ); } - for ( $sample = 1; $sample <= $config['samples']; $sample++ ) { - $order = $labels; - if ( 0 === $sample % 2 ) { - $order = array_reverse( $order ); - } - foreach ( $order as $label ) { - $reply = $workers[ $label ]->request( - array( - 'op' => 'run', - 'id' => $id, - 'parser' => $parser, - 'iterations' => $iterations, - ) - ); + foreach ( $order as $label ) { + $reply = $workers[ $label ]->request( + array( + 'op' => 'run', + 'id' => $id, + 'parser' => $parser, + 'iterations' => $iterations, + ) + ); - $trees[ $label ]['samples_ns'][] = (int) $reply['ns'] / $iterations; - $trees[ $label ]['tokens'] = (int) $reply['tokens']; - $trees[ $label ]['bailed'] = $reply['bailed']; - $trees[ $label ]['peak_bytes'] = max( $trees[ $label ]['peak_bytes'], (int) $reply['peak_bytes'] ); - $trees[ $label ]['peak_alloc_bytes'] = max( $trees[ $label ]['peak_alloc_bytes'], (int) ( $reply['peak_alloc_bytes'] ?? 0 ) ); - } + $trees[ $label ]['samples_ns'][] = (int) $reply['ns'] / $iterations; + $trees[ $label ]['tokens'] = (int) $reply['tokens']; + $trees[ $label ]['bailed'] = $reply['bailed']; + $trees[ $label ]['peak_bytes'] = max( $trees[ $label ]['peak_bytes'], (int) $reply['peak_bytes'] ); + $trees[ $label ]['peak_alloc_bytes'] = max( $trees[ $label ]['peak_alloc_bytes'], (int) ( $reply['peak_alloc_bytes'] ?? 0 ) ); } + } - $rows[ "{$id}\0{$parser}" ] = array( - 'id' => $id, - 'parser' => $parser, - 'bytes' => $document['bytes'], - 'iterations' => $iterations, - 'trees' => $trees, - ); + $rows[ "{$id}\0{$parser}" ] = array( + 'id' => $id, + 'parser' => $parser, + 'bytes' => $document['bytes'], + 'iterations' => $iterations, + 'trees' => $trees, + ); - $status = sprintf( '%s[%d/%d] %s %s: %d x %d parses', $prefix, $done, $total, $id, $parser, $config['samples'], $iterations ); - if ( $is_ab ) { - $base_median = Benchmark_Stats::median( $trees['base']['samples_ns'] ); - $head_median = Benchmark_Stats::median( $trees['head']['samples_ns'] ); - $status .= sprintf( - ', base %s ms, head %s ms, %s', - Benchmark_Stats::format_ms( $base_median ), - Benchmark_Stats::format_ms( $head_median ), - Benchmark_Stats::format_pct( ( $head_median / $base_median - 1.0 ) * 100.0 ) - ); - } else { - $stats = Benchmark_Stats::summarize( $trees['head']['samples_ns'], $document['bytes'], $trees['head']['tokens'] ); - $status .= sprintf( - ', median %s ms, %s MB/s, CV %s', - Benchmark_Stats::format_ms( $stats['median_ns'] ), - Benchmark_Stats::format_rate( $stats['mb_per_s'] ), - Benchmark_Stats::format_cv( $stats['cv'] ) - ); - } - foreach ( $labels as $label ) { - if ( null !== $trees[ $label ]['bailed'] ) { - $status .= " [{$label} bailed]"; - } + $status = sprintf( '%s[%d/%d] %s %s: %d x %d parses', $prefix, $done, $total, $id, $parser, $config['samples'], $iterations ); + if ( $is_ab ) { + $base_median = Benchmark_Stats::median( $trees['base']['samples_ns'] ); + $head_median = Benchmark_Stats::median( $trees['head']['samples_ns'] ); + $status .= sprintf( + ', base %s ms, head %s ms, %s', + Benchmark_Stats::format_ms( $base_median ), + Benchmark_Stats::format_ms( $head_median ), + Benchmark_Stats::format_pct( ( $head_median / $base_median - 1.0 ) * 100.0 ) + ); + } else { + $stats = Benchmark_Stats::summarize( $trees['head']['samples_ns'], $document['bytes'], $trees['head']['tokens'] ); + $status .= sprintf( + ', median %s ms, %s MB/s, CV %s', + Benchmark_Stats::format_ms( $stats['median_ns'] ), + Benchmark_Stats::format_rate( $stats['mb_per_s'] ), + Benchmark_Stats::format_cv( $stats['cv'] ) + ); + } + foreach ( $labels as $label ) { + if ( null !== $trees[ $label ]['bailed'] ) { + $status .= " [{$label} bailed]"; } - benchmark_progress( $status ); } + benchmark_progress( $status ); } foreach ( $temp_files as $temp ) { @unlink( $temp ); } - return $rows; + // Report rows in document order whatever order they ran in. + $ordered = array(); + foreach ( $documents as $document ) { + foreach ( $parsers as $parser ) { + $key = "{$document['id']}\0{$parser}"; + $ordered[ $key ] = $rows[ $key ]; + } + } + return $ordered; +} + +/** + * Returns the order in which one round runs its (document, parser) jobs. + * + * Under the tracing JIT, the code paths a process compiles first change the + * speed of everything it runs afterwards, by 10% to 16% on this corpus. A + * fixed order would give every round the same history and keep that effect + * out of the interval, so each round shuffles the jobs, seeded from the seed + * and the round number. Both trees run the same order within a round. + * + * @param array $documents Documents. + * @param string[] $parsers Parser ids. + * @param int $seed Seed. + * @param int $round Round number, from 1. + * @return array[] Jobs: document => index into $documents, parser => parser id. + */ +function benchmark_round_order( array $documents, array $parsers, int $seed, int $round ): array { + $jobs = array(); + foreach ( array_keys( $documents ) as $index ) { + foreach ( $parsers as $parser ) { + $jobs[] = array( + 'document' => $index, + 'parser' => $parser, + ); + } + } + mt_srand( crc32( "order {$seed} {$round}" ), MT_RAND_MT19937 ); + for ( $i = count( $jobs ) - 1; $i > 0; $i-- ) { + $j = mt_rand( 0, $i ); + $swap = $jobs[ $i ]; + $jobs[ $i ] = $jobs[ $j ]; + $jobs[ $j ] = $swap; + } + return $jobs; } /** @@ -684,6 +738,7 @@ function benchmark_run( array $config, array $documents, array $worker_args ): a $parsers = 'both' === $config['parser'] ? array( 'tag', 'html' ) : array( $config['parser'] ); $trees_info = array(); $merged = array(); + $orders = array(); for ( $round = 1; $round <= $config['rounds']; $round++ ) { $workers = benchmark_start_workers( $config, $worker_args, $round ); @@ -704,7 +759,14 @@ function benchmark_run( array $config, array $documents, array $worker_args ): a } } - $rows = benchmark_run_round( $config, $documents, $workers, $parsers, $round ); + $order = benchmark_round_order( $documents, $parsers, $config['seed'], $round ); + $rows = benchmark_run_round( $config, $documents, $workers, $parsers, $order, $round ); + $orders[] = array_map( + static function ( $job ) use ( $documents ) { + return $documents[ $job['document'] ]['id'] . ' ' . $job['parser']; + }, + $order + ); foreach ( $workers as $worker ) { $worker->quit(); @@ -767,6 +829,7 @@ static function ( $document ) { $documents ), 'results' => $results, + 'orders' => $orders, 'summary' => benchmark_summarize( $results, $labels, $is_ab, $config['seed'] ), 'notes' => array(), ); @@ -911,7 +974,8 @@ static function ( $row ) use ( $parser ) { 'include_optional' => isset( $options['include-optional'] ), 'samples' => (int) ( $options['samples'] ?? 10 ), 'min_sample_ms' => (int) ( $options['min-sample-ms'] ?? 25 ), - 'warmup' => (int) ( $options['warmup'] ?? 2 ), + 'warmup' => (int) ( $options['warmup'] ?? ( $jit ? 10 : 2 ) ), + 'warmup_ms' => (int) ( $options['warmup-ms'] ?? ( $jit ? 200 : 0 ) ), 'rounds' => (int) ( $options['rounds'] ?? 4 ), 'format' => $format, 'head' => $head, @@ -952,6 +1016,7 @@ static function ( $row ) use ( $parser ) { $worker_args = array_merge( $worker_args, preg_split( '/\s+/', trim( $config['php_args'] ) ) ); } +$lock = Benchmark_Lock::acquire( 'bench.php ' . implode( ' ', array_slice( $argv, 1 ) ) ); $result = benchmark_run( $config, $documents, $worker_args ); if ( isset( $options['save'] ) ) { diff --git a/tests/performance/html-api/compare.php b/tests/performance/html-api/compare.php index 1dbfc09ccf79f..4720251212f95 100644 --- a/tests/performance/html-api/compare.php +++ b/tests/performance/html-api/compare.php @@ -95,7 +95,7 @@ function compare_load( string $path ): array { $notes[] = sprintf( 'The runs differ in %s: base %s, head %s.', $key, json_encode( $base_result['trees']['head'][ $key ] ?? null ), json_encode( $head_result['trees']['head'][ $key ] ?? null ) ); } } -foreach ( array( 'samples', 'min_sample_ms', 'warmup', 'rounds', 'synthetic_size' ) as $key ) { +foreach ( array( 'samples', 'min_sample_ms', 'warmup', 'warmup_ms', 'rounds', 'synthetic_size' ) as $key ) { if ( ( $base_result['config'][ $key ] ?? null ) !== ( $head_result['config'][ $key ] ?? null ) ) { $notes[] = sprintf( 'The runs differ in %s: base %s, head %s.', $key, json_encode( $base_result['config'][ $key ] ?? null ), json_encode( $head_result['config'][ $key ] ?? null ) ); } diff --git a/tests/performance/html-api/lib/class-benchmark-lock.php b/tests/performance/html-api/lib/class-benchmark-lock.php new file mode 100644 index 0000000000000..48f72efc81886 --- /dev/null +++ b/tests/performance/html-api/lib/class-benchmark-lock.php @@ -0,0 +1,51 @@ + [args...] + * + * Use it for any timing or profiling outside bench.php, so that it never runs + * at the same time as a benchmark run. Exits with the command's exit code. + * + * @package WordPress + * @subpackage HTML-API + */ + +require_once __DIR__ . '/lib/class-benchmark-lock.php'; + +$command = array_slice( $argv, 1 ); +if ( empty( $command ) ) { + fwrite( STDERR, "usage: php with-lock.php [args...]\n" ); + exit( 2 ); +} + +$lock = Benchmark_Lock::acquire( 'with-lock.php ' . implode( ' ', $command ) ); +$process = proc_open( $command, array( STDIN, STDOUT, STDERR ), $pipes ); +if ( false === $process ) { + fwrite( STDERR, 'with-lock.php: could not start ' . $command[0] . "\n" ); + exit( 2 ); +} +exit( proc_close( $process ) ); diff --git a/tests/performance/html-api/worker.php b/tests/performance/html-api/worker.php index ef4b01a430614..424c8ef5a5082 100644 --- a/tests/performance/html-api/worker.php +++ b/tests/performance/html-api/worker.php @@ -147,6 +147,18 @@ function benchmark_worker_git_head( string $checkout ): ?string { return trim( $output[0] ); } +/** + * Lists loaded extensions that hook execution to profile or trace it. + * + * They change timings, so bench.php refuses to run a worker that has any. + * + * @return string[] Extension names. + */ +function benchmark_worker_profilers(): array { + $names = array( 'xdebug', 'excimer', 'spx', 'tideways_xhprof', 'tideways', 'xhprof', 'blackfire', 'pcov', 'memprof', 'meminfo', 'opentelemetry', 'newrelic', 'ddtrace' ); + return array_values( array_filter( $names, 'extension_loaded' ) ); +} + /** * Reports the JIT mode actually in effect, or null when the JIT is off. * @@ -204,12 +216,13 @@ function benchmark_worker_jit_mode( bool $opcache_enabled ): ?string { $opcache_enabled = extension_loaded( 'Zend OPcache' ) && filter_var( ini_get( 'opcache.enable_cli' ), FILTER_VALIDATE_BOOLEAN ) && filter_var( ini_get( 'opcache.enable' ), FILTER_VALIDATE_BOOLEAN ); benchmark_worker_reply( array( - 'ok' => true, - 'php' => PHP_VERSION, - 'opcache' => $opcache_enabled, - 'jit' => benchmark_worker_jit_mode( $opcache_enabled ), - 'checkout' => false === realpath( $checkout ) ? $checkout : realpath( $checkout ), - 'head' => benchmark_worker_git_head( $checkout ), + 'ok' => true, + 'php' => PHP_VERSION, + 'opcache' => $opcache_enabled, + 'jit' => benchmark_worker_jit_mode( $opcache_enabled ), + 'profilers' => benchmark_worker_profilers(), + 'checkout' => false === realpath( $checkout ) ? $checkout : realpath( $checkout ), + 'head' => benchmark_worker_git_head( $checkout ), ) ); break; @@ -238,6 +251,29 @@ function benchmark_worker_jit_mode( bool $opcache_enabled ): ?string { ); break; + case 'warmup': + // Untimed parses: at least `iterations`, and for at least `min_ms` milliseconds, + // so that the tracing JIT has compiled the hot paths before sampling. + if ( ! isset( $request['id'], $documents[ $request['id'] ] ) ) { + throw new InvalidArgumentException( 'unknown document id' ); + } + $html = $documents[ $request['id'] ]; + $parse = benchmark_worker_parser( (string) ( $request['parser'] ?? '' ) ); + $min = max( 0, (int) ( $request['iterations'] ?? 0 ) ); + $until_ns = hrtime( true ) + (int) ( max( 0, (float) ( $request['min_ms'] ?? 0 ) ) * 1e6 ); + $parses = 0; + while ( $parses < $min || hrtime( true ) < $until_ns ) { + $parse( $html ); + ++$parses; + } + benchmark_worker_reply( + array( + 'ok' => true, + 'parses' => $parses, + ) + ); + break; + case 'calibrate': case 'run': if ( ! isset( $request['id'], $documents[ $request['id'] ] ) ) { From 3b499098ad24cbf265effa71c789a290b7e8fd92 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 9 Oct 2026 21:03:36 +0400 Subject: [PATCH 5/6] HTML API benchmark: add a differential test of parsing behavior between two checkouts. A performance change must not change what the parser reports. difftrace.php loads the base and head Tag Processors into one process under renamed classes and compares the return value, every public accessor and every _doing_it_wrong() call after every step of six walks: next_token(), next_tag(), next_tag() visiting closers, next_tag() with queries, with modifications, and with bookmarks and seeks. For the HTML Processor it compares a hash of the token stream per input, one process per checkout. Inputs are the corpus, the synthetic shapes at two sizes, edge cases with every prefix, and 20,000 mutated corpus slices. Base against itself reports no differences over 4.6 million steps; dropping "\r" from the whitespace skipped around "=" or the XMP case from the special-element switch each produce hundreds of differing inputs. --- tests/performance/html-api/README.md | 13 + tests/performance/html-api/difftrace.php | 715 +++++++++++++++++++++++ 2 files changed, 728 insertions(+) create mode 100644 tests/performance/html-api/difftrace.php diff --git a/tests/performance/html-api/README.md b/tests/performance/html-api/README.md index d575655615fca..88dc668ed2cea 100644 --- a/tests/performance/html-api/README.md +++ b/tests/performance/html-api/README.md @@ -145,6 +145,18 @@ php tests/performance/html-api/compare.php before.json after.json The head tree of each file is used, matched by document id and parser. Prefer a live A/B when both checkouts are available. +## Checking that a change keeps behavior + +A faster parser is only useful if it parses the same way. `difftrace.php` compares a head checkout with a base checkout: + +``` +php tests/performance/html-api/difftrace.php --base ~/a8c/wordpress-develop/trunk --head . +``` + +For the Tag Processor it loads both checkouts' `class-wp-html-tag-processor.php` into one process under renamed classes (the other HTML API files come from base), walks each input six ways (`next_token()`, `next_tag()`, `next_tag()` visiting closers, `next_tag()` with queries, with attribute, class and text modifications, and with bookmarks and seeks), applies the same action to both, and compares the return value, every public accessor and every `_doing_it_wrong()` call after every step, then the final `get_updated_html()`. For the HTML Processor it runs each checkout in its own process and compares a hash of the token stream (type, name, breadcrumbs, namespace, attributes, modifiable text, final error) per input, and shows the first differing token for up to five inputs. + +The inputs are the real corpus, the synthetic shapes at 5 KB and 100 KB, about 250 hand-written edge cases with every prefix of each (which exercises incomplete input), and 20,000 mutated slices of the corpus (`--mutations`). A run takes about a minute and a half and exits 1 when anything differs. Removing `\r` from the whitespace skipped around `=`, or the `XMP` case from the special-element switch, each produce hundreds of differing inputs. + ## Timed runs take a lock Two timed runs at once slow each other and corrupt both results. `bench.php` holds an exclusive lock on `/tmp/wp-html-api-bench.lock` from the first worker to the report, and waits, saying so on stderr, while another run holds it. Run any other timing or profiling through the same lock: @@ -159,6 +171,7 @@ php tests/performance/html-api/with-lock.php php -d extension=excimer.so profile - `worker.php`: a subprocess that loads one checkout's HTML API and runs timed jobs read as JSON lines on stdin, answering on stdout. - `compare.php`: compares two `--save` files. - `with-lock.php`: runs a command while holding the benchmark lock. +- `difftrace.php`: checks that a head checkout parses every input the same way as a base checkout. - `fetch-corpus.php`: restores `corpus/real/` from `corpus/manifest.json`. - `lib/class-benchmark-bootstrap.php`: loads the HTML API from a checkout root with stubs for the WordPress functions it calls. - `lib/class-benchmark-synthetic.php`: the shape generator. diff --git a/tests/performance/html-api/difftrace.php b/tests/performance/html-api/difftrace.php new file mode 100644 index 0000000000000..1722fed76de0d --- /dev/null +++ b/tests/performance/html-api/difftrace.php @@ -0,0 +1,715 @@ + --head [options] + * + * --mutations Mutated slices of the corpus to add to the inputs (default: 20000). + * --seed Seed for the synthetic documents and the mutations (default: 1). + * --tag-only Only the Tag Processor check. + * --html-only Only the HTML Processor check. + * + * Tag Processor: the `class-wp-html-tag-processor.php` of each checkout is loaded into + * one process under a renamed class; every other HTML API file comes from base. Each + * input is walked several ways (next_token(), next_tag() with and without closers and + * with queries, with modifications, with bookmarks and seeks), the same action is + * applied to both processors, and the return value, every public accessor and every + * `_doing_it_wrong()` call are compared after every step. + * + * HTML Processor: each checkout runs in its own process and prints a hash of the token + * stream of every input (token type and name, breadcrumbs, namespace, attributes, + * modifiable text, and the error at the end). Inputs whose hashes differ are rerun to + * show the first differing token. + * + * Inputs: the real corpus, the synthetic shapes at two sizes, hand-written edge cases and + * every prefix of each, and mutated slices of the corpus. Exits 1 when anything differs. + * + * @package WordPress + * @subpackage HTML-API + */ + +require_once __DIR__ . '/lib/class-benchmark-bootstrap.php'; +require_once __DIR__ . '/lib/class-benchmark-synthetic.php'; + +$difftrace_log = array(); + +/** + * Records the call, so that both processors can be checked for the same warnings. + * + * @param string $function_name Function name. + * @param string $message Message. + * @param string $version Version. + */ +function _doing_it_wrong( $function_name, $message, $version ) { + $GLOBALS['difftrace_log'][] = "doing_it_wrong {$function_name}: {$message}"; +} + +/** + * Records the call, so that both processors can be checked for the same warnings. + * + * @param string $function_name Function name. + * @param string $message Message. + * @param int $error_level Level. + */ +function wp_trigger_error( $function_name, $message, $error_level = E_USER_NOTICE ) { + $GLOBALS['difftrace_log'][] = "trigger_error {$function_name}: {$message}"; +} + +/** + * Parses options. + * + * @param string[] $argv Arguments. + * @return array Options. + */ +function difftrace_options( array $argv ): array { + $options = array( + 'mutations' => 20000, + 'seed' => 1, + ); + $count = count( $argv ); + for ( $i = 1; $i < $count; $i++ ) { + $name = substr( $argv[ $i ], 2 ); + if ( in_array( $name, array( 'tag-only', 'html-only' ), true ) ) { + $options[ $name ] = true; + } elseif ( in_array( $name, array( 'base', 'head', 'mutations', 'seed', 'emit-html', 'emit-html-detail' ), true ) && isset( $argv[ $i + 1 ] ) ) { + $options[ $name ] = $argv[ ++$i ]; + } else { + fwrite( STDERR, "difftrace.php: unexpected argument {$argv[ $i ]}\n" ); + exit( 2 ); + } + } + return $options; +} + +/** + * Hand-written inputs at the edges of the tokenizer: truncated syntax, special + * elements, attribute syntax, comments, character references, and bytes that need care. + * + * @return string[] Inputs. + */ +function difftrace_edge_cases(): array { + $cases = array( + '', + '<', + '', + "', + '', + '', + 'x', + '', + '', + '

      ', + '
      ', + '      ', + '', + '', + '', + '', + '-->', + ' -->', + '', + 'x-->', + '', + '', + '', + '', + '', + '', + '', + '-->', + '', + '-->', + '', + "", + '', + "", + '&x', + '<b>', + '<b>', + '', + '<b>', + '<b>', + '', + '<b>', + '<plaintext>', + '<a href=x>', + "<a href='x'>", + '<a href=x/>', + '<a/b/c>', + '<a =b>', + '<a ==b>', + '<a b c d>', + '<a b=c b=d>', + '<a B=c b=d>', + "<a\tb\nc\fd\re>", + '<a b="&amp;&lt;&#x41;&#65;&notin;&notit;&amp">', + '<a b=&amp;c>', + '<a class="x y z x">', + '<a class>', + '<a class="">', + '<a CLASS="A a">', + '<a class="x" class="y">', + '<a title="a>b">', + '<a title=a>b>', + '<a b="c"d="e">', + '<a b=">"c>', + '<a "b"=c>', + "<a 'b'>", + '<a <b>', + '<a<b>', + '<a b<c>', + '<div>text &amp; more</div>', + "a\0b", + "<a\0b>", + "<\0>", + "<a b=\"\0\">", + '<1>', + '<-a>', + '< a>', + '<a.b>', + '<A-B:C>', + '<br/>', + '<br />', + '<br/ >', + '<img src=x/>', + '<a b/>c', + '<div/>', + '<input disabled/>', + 'x&#0;y', + '&#x110000;', + '&', + '&amp', + '&;', + '&#', + '&#x', + '&#x;', + '&#99999999999;', + '&notin', + '&NotNestedLessLess;', + '&AMP;', + '&amp;amp;', + '<p>a<p>b', + "\xEF\xBB\xBF<p>", + "<p>\xC3\xA9\xF0\x9F\x98\x80</p>", + "<p>\xFF\xFE</p>", + "<p title=\"\xC3\">x</p>", + "<\xC3\xA9>", + '<template><td>x</template>', + '<table><tr><td>x</table>', + '<table>x<tr>', + '<select><option>x</select>', + '<ul><li>a<li>b</ul>', + '<b><i>x</b></i>', + '<a><a>x', + '<p><div>x</p></div>', + '<html><head><title>t</title></head><body><p class="x" id=y>Hello<br>world</p></body></html>', + str_repeat( '<div a=b>', 50 ) . str_repeat( '</div>', 50 ), + '<a ' . str_repeat( 'b ', 200 ) . '>', + '<p>' . str_repeat( ' ', 300 ) . '</p>', + ); + + // Every whitespace byte at every place the tokenizer skips or ends on whitespace. + foreach ( array( ' ', "\t", "\f", "\r", "\n", "\r\n" ) as $ws ) { + $cases[] = "<a{$ws}b{$ws}={$ws}c{$ws}d{$ws}={$ws}\"e\"{$ws}f{$ws}={$ws}'g'{$ws}h{$ws}/{$ws}>"; + $cases[] = "<a b{$ws}>x</a{$ws}>"; + $cases[] = "<a{$ws}/{$ws}>"; + $cases[] = "<a b={$ws}>"; + $cases[] = "<a b{$ws}={$ws}"; + $cases[] = "<!DOCTYPE{$ws}html{$ws}PUBLIC{$ws}\"x\"{$ws}\"y\"{$ws}>"; + $cases[] = "<script>x</script{$ws}>y"; + $cases[] = "<textarea>{$ws}x</textarea{$ws}>"; + $cases[] = "<pre>{$ws}x</pre>"; + $cases[] = "<title{$ws}a=b>x</title{$ws}>"; + } + return $cases; +} + +/** + * Yields every input, as id => HTML, in the same order on every run with the same seed. + * + * @param int $seed Seed. + * @param int $mutations Number of mutated slices. + * @return Generator + */ +function difftrace_inputs( int $seed, int $mutations ): Generator { + $corpus = glob( __DIR__ . '/corpus/real/*.{html,htm}', GLOB_BRACE ); + sort( $corpus ); + $documents = array(); + foreach ( $corpus as $path ) { + $documents[ basename( $path ) ] = file_get_contents( $path ); + } + foreach ( $documents as $id => $html ) { + yield "corpus {$id}" => $html; + } + + foreach ( Benchmark_Synthetic::shapes() as $shape ) { + foreach ( array( 5000, 100000 ) as $size ) { + yield "synthetic {$shape} {$size}" => Benchmark_Synthetic::generate( $shape, $size, $seed ); + } + } + + foreach ( difftrace_edge_cases() as $n => $html ) { + $length = strlen( $html ); + for ( $i = 0; $i <= $length; $i++ ) { + yield "edge {$n} prefix {$i}" => substr( $html, 0, $i ); + } + } + + if ( empty( $documents ) || $mutations <= 0 ) { + return; + } + $sources = array_values( $documents ); + $alphabet = array( '<', '>', '/', '!', '-', '-', '=', '"', "'", ' ', "\t", "\n", "\0", '&', ';', '#', '?', '[', ']', 'a', 'x', 'S', 'T', '<!--', '-->', '</', '/>', '<script>', '</script>', '<textarea>', '<style>', '<title>', '<svg>', '<math>', '<![CDATA[', ']]>', '<!DOCTYPE html>', '&amp;', '&lt', "\xC3", "\xC3\xA9", "\r", "\f", "\r\n", ' = ', "\t" ); + $prefixes = array( '', '', '', '<script>', '<textarea>', '<style>', '<title>', '<!--', '<svg>', '<plaintext>', '<xmp>', '<a b="' ); + mt_srand( crc32( "mutations {$seed}" ), MT_RAND_MT19937 ); + for ( $m = 0; $m < $mutations; $m++ ) { + $source = $sources[ mt_rand( 0, count( $sources ) - 1 ) ]; + $length = mt_rand( 20, 600 ); + $start = mt_rand( 0, max( 0, strlen( $source ) - $length ) ); + $html = $prefixes[ mt_rand( 0, count( $prefixes ) - 1 ) ] . substr( $source, $start, $length ); + $edits = mt_rand( 1, 4 ); + for ( $e = 0; $e < $edits; $e++ ) { + $at = mt_rand( 0, strlen( $html ) ); + switch ( mt_rand( 0, 4 ) ) { + case 0: // Insert. + $html = substr( $html, 0, $at ) . $alphabet[ mt_rand( 0, count( $alphabet ) - 1 ) ] . substr( $html, $at ); + break; + case 1: // Replace one byte. + $html = substr( $html, 0, $at ) . $alphabet[ mt_rand( 0, count( $alphabet ) - 1 ) ] . substr( $html, $at + 1 ); + break; + case 2: // Delete a few bytes. + $html = substr( $html, 0, $at ) . substr( $html, $at + mt_rand( 1, 8 ) ); + break; + case 3: // Truncate. + $html = substr( $html, 0, $at ); + break; + default: // Whitespace beside an equals sign. + $equals = strpos( $html, '=', $at ); + if ( false !== $equals ) { + $where = $equals + mt_rand( 0, 1 ); + $html = substr( $html, 0, $where ) . substr( " \t\f\r\n", mt_rand( 0, 4 ), 1 ) . substr( $html, $where ); + } + } + } + yield "mutation {$m}" => $html; + } +} + +/** + * Loads a checkout's Tag Processor source under another class name. + * + * @param string $checkout Checkout root. + * @param string $class_name New class name. + */ +function difftrace_load_renamed( string $checkout, string $class_name ): void { + $source = file_get_contents( "{$checkout}/src/wp-includes/html-api/class-wp-html-tag-processor.php" ); + if ( false === $source ) { + fwrite( STDERR, "difftrace.php: no Tag Processor in {$checkout}\n" ); + exit( 2 ); + } + $source = preg_replace( '~\bWP_HTML_Tag_Processor\b~', $class_name, $source ); + $path = sys_get_temp_dir() . "/difftrace-{$class_name}-" . getmypid() . '.php'; + file_put_contents( $path, $source ); + require $path; + unlink( $path ); +} + +/** + * Everything observable about the current token through the public API. + * + * @param WP_HTML_Tag_Processor $p Processor. + * @return array Snapshot. + */ +function difftrace_snapshot( $p ): array { + $doctype = $p->get_doctype_info(); + $snapshot = array( + 'type' => $p->get_token_type(), + 'name' => $p->get_token_name(), + 'tag' => $p->get_tag(), + 'qualified' => $p->get_qualified_tag_name(), + 'namespace' => $p->get_namespace(), + 'closer' => $p->is_tag_closer(), + 'self_closing' => $p->has_self_closing_flag(), + 'comment_type' => $p->get_comment_type(), + 'comment_text' => $p->get_full_comment_text(), + 'text' => $p->get_modifiable_text(), + 'incomplete' => $p->paused_at_incomplete_token(), + 'doctype' => null === $doctype ? null : get_object_vars( $doctype ), + 'attributes' => null, + ); + $names = $p->get_attribute_names_with_prefix( '' ); + if ( null !== $names ) { + $snapshot['attributes'] = array(); + foreach ( $names as $name ) { + $snapshot['attributes'][ $name ] = array( $p->get_attribute( $name ), $p->get_qualified_attribute_name( $name ) ); + } + $snapshot['data_prefix'] = $p->get_attribute_names_with_prefix( 'data-' ); + $snapshot['probes'] = array( $p->get_attribute( 'CLASS' ), $p->get_attribute( 'missing' ), $p->has_class( 'x' ), $p->has_class( 'X' ) ); + $snapshot['classes'] = array(); + foreach ( $p->class_list() as $class_name ) { + $snapshot['classes'][] = $class_name; + } + } + return $snapshot; +} + +/** + * Applies one action to a processor and returns what it returned, any exception, and the warnings it raised. + * + * @param WP_HTML_Tag_Processor $p Processor. + * @param array $action Method name and arguments. + * @return array Outcome. + */ +function difftrace_apply( $p, array $action ): array { + $GLOBALS['difftrace_log'] = array(); + try { + $result = $p->{$action[0]}( ...array_slice( $action, 1 ) ); + } catch ( Throwable $e ) { + $result = 'threw ' . get_class( $e ) . ': ' . $e->getMessage(); + } + + // Messages name the renamed class through __METHOD__, and errors name a file and line; compare without them. + $renamed = array( 'WP_HTML_Tag_Processor_Base', 'WP_HTML_Tag_Processor_Head' ); + if ( is_string( $result ) && 0 === strpos( $result, 'threw ' ) ) { + $result = preg_replace( '~,? (?:called )?in \S+ on line \d+~', '', str_replace( $renamed, 'WP_HTML_Tag_Processor', $result ) ); + } + return array( + 'return' => $result, + 'log' => str_replace( $renamed, 'WP_HTML_Tag_Processor', $GLOBALS['difftrace_log'] ), + ); +} + +/** + * Walks one input with both Tag Processors in lockstep and returns the first difference, or null. + * + * @param string $id Input id. + * @param string $html Input. + * @param string $walk Walk name. + * @param int $seed Seed. + * @param int $steps Steps taken, incremented. + * @return array|null Difference. + */ +function difftrace_tag_walk( string $id, string $html, string $walk, int $seed, int &$steps ): ?array { + $a = new WP_HTML_Tag_Processor_Base( $html ); + $b = new WP_HTML_Tag_Processor_Head( $html ); + mt_srand( crc32( "{$seed} {$id} {$walk}" ), MT_RAND_MT19937 ); + + $limit = in_array( $walk, array( 'modify', 'bookmarks' ), true ) ? 20000 : PHP_INT_MAX; + $snapshot = null; + $bookmarks = array(); + $seeks = 0; + for ( $step = 0; $step < $limit; $step++ ) { + $action = array( 'next_token' ); + switch ( $walk ) { + case 'tag': + $action = array( 'next_tag' ); + break; + case 'closers': + $action = array( 'next_tag', array( 'tag_closers' => 'visit' ) ); + break; + case 'query': + $queries = array( + 'div', + 'A', + array( 'class_name' => 'x' ), + array( + 'tag_name' => 'p', + 'match_offset' => 2, + ), + array( + 'tag_name' => 'script', + 'tag_closers' => 'visit', + ), + ); + $action = array( 'next_tag', $queries[ mt_rand( 0, count( $queries ) - 1 ) ] ); + break; + case 'modify': + $roll = mt_rand( 0, 99 ); + if ( null !== $snapshot && '#tag' === $snapshot['type'] && $roll < 30 ) { + $names = array_keys( (array) $snapshot['attributes'] ); + switch ( $roll % 5 ) { + case 0: + $action = array( 'set_attribute', 'data-x', mt_rand( 0, 1 ) ? 'a"b&c<d' : true ); + break; + case 1: + $action = array( 'remove_attribute', empty( $names ) ? 'id' : $names[ mt_rand( 0, count( $names ) - 1 ) ] ); + break; + case 2: + $action = array( 'add_class', 'x' ); + break; + case 3: + $action = array( 'remove_class', empty( $snapshot['classes'] ) ? 'x' : $snapshot['classes'][0] ); + break; + default: + $action = array( 'set_modifiable_text', 'new </script> & <!-- text' ); + } + } elseif ( null !== $snapshot && $roll < 40 && '#tag' !== $snapshot['type'] ) { + $action = array( 'set_modifiable_text', 'x & y' ); + } elseif ( $roll < 43 ) { + $action = array( 'get_updated_html' ); + } + break; + case 'bookmarks': + $roll = mt_rand( 0, 99 ); + if ( $roll < 10 ) { + $name = 'b' . mt_rand( 0, 4 ); + $action = array( 'set_bookmark', $name ); + $bookmarks[ $name ] = true; + } elseif ( $roll < 13 && ! empty( $bookmarks ) && $seeks < 100 ) { + $names = array_keys( $bookmarks ); + $action = array( 'seek', $names[ mt_rand( 0, count( $names ) - 1 ) ] ); + ++$seeks; + } elseif ( $roll < 15 && ! empty( $bookmarks ) ) { + $names = array_keys( $bookmarks ); + $name = $names[ mt_rand( 0, count( $names ) - 1 ) ]; + $action = array( 'release_bookmark', $name ); + unset( $bookmarks[ $name ] ); + } + break; + } + + $outcome_a = difftrace_apply( $a, $action ); + $outcome_b = difftrace_apply( $b, $action ); + $snapshot = difftrace_snapshot( $a ); + $state_a = array( $outcome_a, $snapshot ); + $state_b = array( $outcome_b, difftrace_snapshot( $b ) ); + ++$steps; + if ( $state_a !== $state_b ) { + return array( + 'walk' => $walk, + 'step' => $step, + 'action' => $action, + 'base' => $state_a, + 'head' => $state_b, + ); + } + if ( 'next_token' === $action[0] || 'next_tag' === $action[0] ) { + if ( false === $outcome_a['return'] ) { + break; + } + } + } + + $final_a = difftrace_apply( $a, array( 'get_updated_html' ) ); + $final_b = difftrace_apply( $b, array( 'get_updated_html' ) ); + if ( $final_a !== $final_b ) { + return array( + 'walk' => $walk, + 'step' => 'final get_updated_html()', + 'action' => array( 'get_updated_html' ), + 'base' => $final_a, + 'head' => $final_b, + ); + } + return null; +} + +/** + * Serializes the HTML Processor's token stream for one input. + * + * @param string $html Input. + * @return string[] One line per token, then the final state. + */ +function difftrace_html_stream( string $html ): array { + $lines = array(); + $p = WP_HTML_Processor::create_full_parser( $html ); + if ( null === $p ) { + return array( 'create_full_parser returned null' ); + } + while ( $p->next_token() ) { + $attributes = array(); + foreach ( (array) $p->get_attribute_names_with_prefix( '' ) as $name ) { + $attributes[] = $name . '=' . var_export( $p->get_attribute( $name ), true ); + } + $lines[] = json_encode( + array( + $p->get_token_type(), + $p->get_token_name(), + $p->is_tag_closer(), + $p->get_namespace(), + implode( ' ', (array) $p->get_breadcrumbs() ), + $attributes, + $p->get_modifiable_text(), + ), + JSON_INVALID_UTF8_SUBSTITUTE + ); + } + $exception = $p->get_unsupported_exception(); + $lines[] = 'end ' . var_export( $p->get_last_error(), true ) . ' ' . ( null === $exception ? '' : $exception->getMessage() ) . ' ' . var_export( $p->paused_at_incomplete_token(), true ); + return $lines; +} + +/** + * Runs this script in a child process for one checkout and returns its stdout lines. + * + * @param string $checkout Checkout root. + * @param array $options Options. + * @param string[] $extra Extra arguments. + * @return string[] Lines. + */ +function difftrace_child( string $checkout, array $options, array $extra ): array { + $command = array_merge( + array( PHP_BINARY, '-d', 'memory_limit=-1', __FILE__, '--seed', (string) $options['seed'], '--mutations', (string) $options['mutations'] ), + $extra + ); + $process = proc_open( + $command, + array( + 1 => array( 'pipe', 'w' ), + 2 => STDERR, + ), + $pipes + ); + $output = stream_get_contents( $pipes[1] ); + fclose( $pipes[1] ); + if ( 0 !== proc_close( $process ) ) { + fwrite( STDERR, "difftrace.php: the child for {$checkout} failed\n" ); + exit( 2 ); + } + return explode( "\n", rtrim( $output, "\n" ) ); +} + +$options = difftrace_options( $argv ); +$seed = (int) $options['seed']; + +// Child modes: load one checkout and print the HTML Processor stream. +if ( isset( $options['emit-html'] ) ) { + Benchmark_Bootstrap::load( $options['emit-html'] ); + foreach ( difftrace_inputs( $seed, (int) $options['mutations'] ) as $id => $html ) { + echo $id, "\t", sha1( implode( "\n", difftrace_html_stream( $html ) ) ), "\n"; + } + exit( 0 ); +} +if ( isset( $options['emit-html-detail'] ) ) { + list( $checkout, $wanted ) = explode( "\t", $options['emit-html-detail'], 2 ); + Benchmark_Bootstrap::load( $checkout ); + foreach ( difftrace_inputs( $seed, (int) $options['mutations'] ) as $id => $html ) { + if ( $id === $wanted ) { + echo implode( "\n", difftrace_html_stream( $html ) ), "\n"; + exit( 0 ); + } + } + exit( 1 ); +} + +if ( ! isset( $options['base'], $options['head'] ) ) { + fwrite( STDERR, "usage: php difftrace.php --base <checkout> --head <checkout> [--mutations <n>] [--seed <n>] [--tag-only|--html-only]\n" ); + exit( 2 ); +} +$base = rtrim( $options['base'], '/' ); +$head = rtrim( $options['head'], '/' ); + +$failures = 0; +$started = microtime( true ); + +$other_changes = array(); +foreach ( glob( "{$base}/src/wp-includes/html-api/*.php" ) as $path ) { + $file = basename( $path ); + if ( 'class-wp-html-tag-processor.php' !== $file && file_get_contents( $path ) !== @file_get_contents( "{$head}/src/wp-includes/html-api/{$file}" ) ) { + $other_changes[] = $file; + } +} +if ( ! empty( $other_changes ) ) { + fwrite( STDERR, 'Note: head also changes ' . implode( ', ', $other_changes ) . "; the Tag Processor check loads those files from base, the HTML Processor check covers them.\n" ); +} + +if ( ! isset( $options['html-only'] ) ) { + Benchmark_Bootstrap::load( $base ); + difftrace_load_renamed( $base, 'WP_HTML_Tag_Processor_Base' ); + difftrace_load_renamed( $head, 'WP_HTML_Tag_Processor_Head' ); + + $inputs = 0; + $steps = 0; + $shown = 0; + foreach ( difftrace_inputs( $seed, (int) $options['mutations'] ) as $id => $html ) { + ++$inputs; + foreach ( array( 'token', 'tag', 'closers', 'query', 'modify', 'bookmarks' ) as $walk ) { + $difference = difftrace_tag_walk( $id, $html, $walk, $seed, $steps ); + if ( null === $difference ) { + continue; + } + ++$failures; + if ( $shown++ < 5 ) { + echo "Tag Processor differs: {$id}, walk {$difference['walk']}, step {$difference['step']}\n"; + if ( strlen( $html ) <= 300 ) { + echo ' input: ', var_export( $html, true ), "\n"; + } + echo ' action: ', json_encode( $difference['action'], JSON_INVALID_UTF8_SUBSTITUTE ), "\n"; + echo ' base: ', json_encode( $difference['base'], JSON_INVALID_UTF8_SUBSTITUTE | JSON_PARTIAL_OUTPUT_ON_ERROR ), "\n"; + echo ' head: ', json_encode( $difference['head'], JSON_INVALID_UTF8_SUBSTITUTE | JSON_PARTIAL_OUTPUT_ON_ERROR ), "\n"; + } + } + } + printf( "Tag Processor: %d inputs, 6 walks each, %d steps compared, %d walks differ.\n", $inputs, $steps, $failures ); +} + +if ( ! isset( $options['tag-only'] ) ) { + $base_lines = difftrace_child( $base, $options, array( '--emit-html', $base ) ); + $head_lines = difftrace_child( $head, $options, array( '--emit-html', $head ) ); + $differing = array(); + foreach ( $base_lines as $n => $line ) { + if ( ( $head_lines[ $n ] ?? null ) !== $line ) { + $differing[] = explode( "\t", $line )[0]; + } + } + foreach ( array_slice( $differing, 0, 5 ) as $id ) { + $a = difftrace_child( $base, $options, array( '--emit-html-detail', "{$base}\t{$id}" ) ); + $b = difftrace_child( $head, $options, array( '--emit-html-detail', "{$head}\t{$id}" ) ); + $n = 0; + while ( $n < max( count( $a ), count( $b ) ) && ( $a[ $n ] ?? null ) === ( $b[ $n ] ?? null ) ) { + ++$n; + } + echo "HTML Processor differs: {$id}, token {$n}\n"; + echo ' base: ', $a[ $n ] ?? '(none)', "\n"; + echo ' head: ', $b[ $n ] ?? '(none)', "\n"; + } + $failures += count( $differing ); + printf( "HTML Processor: %d inputs, %d differ.\n", count( $base_lines ), count( $differing ) ); +} + +printf( "%s in %.0f s.\n", 0 === $failures ? 'No differences' : 'DIFFERENCES FOUND', microtime( true ) - $started ); +exit( 0 === $failures ? 0 : 1 ); From 8422879c17d93ab8949049b1b5265512e93b90ee Mon Sep 17 00:00:00 2001 From: Jon Surrell <sirreal@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:00:32 +0400 Subject: [PATCH 6/6] HTML API benchmark: widen the differential test and show per-round results. difftrace.php gains a walk that mixes modifications with bookmarks and seeks, special-element edge cases (case, near-miss names, NUL bytes, leading newlines, virtual closers), and an HTML Processor pass that calls get_updated_html() and then sets an attribute while walking, so that an update is pending when the processor parses a virtual token or an error again. It now finds the eight inputs on which matching special elements by raw bytes, instead of through get_tag(), changes HTML Processor output. get_updated_html() calls are capped at 20 per input, since each one copies the whole document. proc_open() no longer receives STDOUT or STDERR: PHP seeks a redirected file's shared offset to the stream's own position before handing it on, and the outputs overwrote each other. The report prints each round's overall change under the overall line, so one slow round is visible instead of only widening the interval. --- tests/performance/html-api/README.md | 4 +- tests/performance/html-api/difftrace.php | 82 +++++++++++++++---- .../html-api/lib/class-benchmark-report.php | 12 +++ tests/performance/html-api/with-lock.php | 6 +- 4 files changed, 84 insertions(+), 20 deletions(-) diff --git a/tests/performance/html-api/README.md b/tests/performance/html-api/README.md index 88dc668ed2cea..acf97c3c0e015 100644 --- a/tests/performance/html-api/README.md +++ b/tests/performance/html-api/README.md @@ -153,9 +153,9 @@ A faster parser is only useful if it parses the same way. `difftrace.php` compar php tests/performance/html-api/difftrace.php --base ~/a8c/wordpress-develop/trunk --head . ``` -For the Tag Processor it loads both checkouts' `class-wp-html-tag-processor.php` into one process under renamed classes (the other HTML API files come from base), walks each input six ways (`next_token()`, `next_tag()`, `next_tag()` visiting closers, `next_tag()` with queries, with attribute, class and text modifications, and with bookmarks and seeks), applies the same action to both, and compares the return value, every public accessor and every `_doing_it_wrong()` call after every step, then the final `get_updated_html()`. For the HTML Processor it runs each checkout in its own process and compares a hash of the token stream (type, name, breadcrumbs, namespace, attributes, modifiable text, final error) per input, and shows the first differing token for up to five inputs. +For the Tag Processor it loads both checkouts' `class-wp-html-tag-processor.php` into one process under renamed classes (the other HTML API files come from base), walks each input seven ways (`next_token()`, `next_tag()`, `next_tag()` visiting closers, `next_tag()` with queries, with attribute, class and text modifications, with bookmarks and seeks, and with both mixed), applies the same action to both, and compares the return value, every public accessor and every `_doing_it_wrong()` call after every step, then the final `get_updated_html()`. For the HTML Processor it runs each checkout in its own process and compares a hash of the token stream (type, name, breadcrumbs, namespace, attributes, modifiable text, final error) per input, walked once plain and once setting attributes and calling `get_updated_html()` along the way, and shows the first differing token for up to five inputs. -The inputs are the real corpus, the synthetic shapes at 5 KB and 100 KB, about 250 hand-written edge cases with every prefix of each (which exercises incomplete input), and 20,000 mutated slices of the corpus (`--mutations`). A run takes about a minute and a half and exits 1 when anything differs. Removing `\r` from the whitespace skipped around `=`, or the `XMP` case from the special-element switch, each produce hundreds of differing inputs. +The inputs are the real corpus, the synthetic shapes at 5 KB and 100 KB, about 270 hand-written edge cases with every prefix of each (which exercises incomplete input), and 20,000 mutated slices of the corpus (`--mutations`). A run takes about a minute and a half and exits 1 when anything differs. Removing `\r` from the whitespace skipped around `=`, or the `XMP` case from the special-element switch, each produce hundreds of differing inputs. ## Timed runs take a lock diff --git a/tests/performance/html-api/difftrace.php b/tests/performance/html-api/difftrace.php index 1722fed76de0d..084f73484538b 100644 --- a/tests/performance/html-api/difftrace.php +++ b/tests/performance/html-api/difftrace.php @@ -252,6 +252,35 @@ function difftrace_edge_cases(): array { '<p>' . str_repeat( ' ', 300 ) . '</p>', ); + // Special elements: case, near-miss names, NUL bytes, and leading newlines. + array_push( + $cases, + '<TITLE>a</title>', + '<titlex>y</titlex>', + "<listing>\nx</listing>", + "<LISTING>\r\nx", + '<svg><title>a</title></svg>', + '<textarea/>a</textarea>', + '</script>', + "<scri\0pt>x</script>", + "<title\0>x</title>", + "<t\xC4\xB1tle>x</title>", + '<IFRAME>x</iframe>', + '<NoEmBeD>x</noembed>', + '<NOFRAMES>x</noframes>', + '<Xmp>x</xmp>', + '<scripts>x</scripts>', + '<styles>x', + "<pre>\n\nx</pre>", + "<pre a=b>\nx</pre>", + "<textarea>\n\nx</textarea>", + '<p><xmp>b</xmp>c', + '<p><title>b</title>c', + '<table data-a=1><textarea>x</textarea>', + "<table><tr><td><pre><br><td>\ny</table>", + "<pre>\nx</pre><listing>\ny</listing>" + ); + // Every whitespace byte at every place the tokenizer skips or ends on whitespace. foreach ( array( ' ', "\t", "\f", "\r", "\n", "\r\n" ) as $ws ) { $cases[] = "<a{$ws}b{$ws}={$ws}c{$ws}d{$ws}={$ws}\"e\"{$ws}f{$ws}={$ws}'g'{$ws}h{$ws}/{$ws}>"; @@ -438,13 +467,17 @@ function difftrace_tag_walk( string $id, string $html, string $walk, int $seed, $b = new WP_HTML_Tag_Processor_Head( $html ); mt_srand( crc32( "{$seed} {$id} {$walk}" ), MT_RAND_MT19937 ); - $limit = in_array( $walk, array( 'modify', 'bookmarks' ), true ) ? 20000 : PHP_INT_MAX; + $limit = in_array( $walk, array( 'modify', 'bookmarks', 'mixed' ), true ) ? 20000 : PHP_INT_MAX; $snapshot = null; $bookmarks = array(); $seeks = 0; for ( $step = 0; $step < $limit; $step++ ) { $action = array( 'next_token' ); - switch ( $walk ) { + $kind = $walk; + if ( 'mixed' === $walk ) { + $kind = mt_rand( 0, 1 ) ? 'modify' : 'bookmarks'; + } + switch ( $kind ) { case 'tag': $action = array( 'next_tag' ); break; @@ -551,16 +584,39 @@ function difftrace_tag_walk( string $id, string $html, string $walk, int $seed, /** * Serializes the HTML Processor's token stream for one input. * - * @param string $html Input. + * With `$modify`, it also sets an attribute on some opening tags and calls + * get_updated_html() on some tokens (on every token of a small input, at most + * 20 times per input), which makes the processor parse the current token + * again, and records what those calls return. + * + * @param string $html Input. + * @param bool $modify Whether to modify while walking. * @return string[] One line per token, then the final state. */ -function difftrace_html_stream( string $html ): array { +function difftrace_html_stream( string $html, bool $modify = false ): array { $lines = array(); $p = WP_HTML_Processor::create_full_parser( $html ); if ( null === $p ) { return array( 'create_full_parser returned null' ); } + mt_srand( crc32( $html ), MT_RAND_MT19937 ); + $flushes = 0; + // Small inputs set an attribute on half the opening tags and flush on every token, so that a flush + // lands on virtual tokens and after errors; large inputs do both rarely. + $small = strlen( $html ) < 2000; + $set_below = $small ? 50 : 20; while ( $p->next_token() ) { + if ( $modify ) { + // Flush before modifying, so that an update set on one token is still pending at the next. + $roll = mt_rand( 0, 99 ); + if ( ( $small || ( $roll >= 20 && $roll < 25 ) ) && ++$flushes <= 20 ) { + // Each call copies and reparses the whole document; a cap keeps large inputs linear. + $lines[] = 'get_updated_html ' . sha1( $p->get_updated_html() ); + } + if ( $roll < $set_below && '#tag' === $p->get_token_type() && ! $p->is_tag_closer() ) { + $lines[] = 'set_attribute ' . var_export( $p->set_attribute( 'data-x', 'y' ), true ); + } + } $attributes = array(); foreach ( (array) $p->get_attribute_names_with_prefix( '' ) as $name ) { $attributes[] = $name . '=' . var_export( $p->get_attribute( $name ), true ); @@ -596,14 +652,8 @@ function difftrace_child( string $checkout, array $options, array $extra ): arra array( PHP_BINARY, '-d', 'memory_limit=-1', __FILE__, '--seed', (string) $options['seed'], '--mutations', (string) $options['mutations'] ), $extra ); - $process = proc_open( - $command, - array( - 1 => array( 'pipe', 'w' ), - 2 => STDERR, - ), - $pipes - ); + // Stderr is left out of the spec so the child inherits it: passing STDERR makes PHP seek the shared offset. + $process = proc_open( $command, array( 1 => array( 'pipe', 'w' ) ), $pipes ); $output = stream_get_contents( $pipes[1] ); fclose( $pipes[1] ); if ( 0 !== proc_close( $process ) ) { @@ -620,7 +670,7 @@ function difftrace_child( string $checkout, array $options, array $extra ): arra if ( isset( $options['emit-html'] ) ) { Benchmark_Bootstrap::load( $options['emit-html'] ); foreach ( difftrace_inputs( $seed, (int) $options['mutations'] ) as $id => $html ) { - echo $id, "\t", sha1( implode( "\n", difftrace_html_stream( $html ) ) ), "\n"; + echo $id, "\t", sha1( implode( "\n", array_merge( difftrace_html_stream( $html ), array( 'modified' ), difftrace_html_stream( $html, true ) ) ) ), "\n"; } exit( 0 ); } @@ -629,7 +679,7 @@ function difftrace_child( string $checkout, array $options, array $extra ): arra Benchmark_Bootstrap::load( $checkout ); foreach ( difftrace_inputs( $seed, (int) $options['mutations'] ) as $id => $html ) { if ( $id === $wanted ) { - echo implode( "\n", difftrace_html_stream( $html ) ), "\n"; + echo implode( "\n", array_merge( difftrace_html_stream( $html ), array( 'modified' ), difftrace_html_stream( $html, true ) ) ), "\n"; exit( 0 ); } } @@ -667,7 +717,7 @@ function difftrace_child( string $checkout, array $options, array $extra ): arra $shown = 0; foreach ( difftrace_inputs( $seed, (int) $options['mutations'] ) as $id => $html ) { ++$inputs; - foreach ( array( 'token', 'tag', 'closers', 'query', 'modify', 'bookmarks' ) as $walk ) { + foreach ( array( 'token', 'tag', 'closers', 'query', 'modify', 'bookmarks', 'mixed' ) as $walk ) { $difference = difftrace_tag_walk( $id, $html, $walk, $seed, $steps ); if ( null === $difference ) { continue; @@ -684,7 +734,7 @@ function difftrace_child( string $checkout, array $options, array $extra ): arra } } } - printf( "Tag Processor: %d inputs, 6 walks each, %d steps compared, %d walks differ.\n", $inputs, $steps, $failures ); + printf( "Tag Processor: %d inputs, 7 walks each, %d steps compared, %d walks differ.\n", $inputs, $steps, $failures ); } if ( ! isset( $options['tag-only'] ) ) { diff --git a/tests/performance/html-api/lib/class-benchmark-report.php b/tests/performance/html-api/lib/class-benchmark-report.php index 56f07cade0147..a4ae58ef8d48f 100644 --- a/tests/performance/html-api/lib/class-benchmark-report.php +++ b/tests/performance/html-api/lib/class-benchmark-report.php @@ -217,6 +217,18 @@ private static function render_summary( array $summary, string $mode, bool $mark $overall['documents'], $overall['method'] ?? 'bootstrap' ); + if ( ! empty( $overall['round_geomeans'] ) ) { + // One slow round (a worker stuck on a slow core) widens the interval; show where it came from. + $lines[] = 'Per round: ' . implode( + ' ', + array_map( + static function ( $ratio ) { + return Benchmark_Stats::format_pct( ( $ratio - 1.0 ) * 100.0 ); + }, + $overall['round_geomeans'] + ) + ); + } } } diff --git a/tests/performance/html-api/with-lock.php b/tests/performance/html-api/with-lock.php index 1a40233a5bd07..11ed19ef73754 100644 --- a/tests/performance/html-api/with-lock.php +++ b/tests/performance/html-api/with-lock.php @@ -19,8 +19,10 @@ exit( 2 ); } -$lock = Benchmark_Lock::acquire( 'with-lock.php ' . implode( ' ', $command ) ); -$process = proc_open( $command, array( STDIN, STDOUT, STDERR ), $pipes ); +$lock = Benchmark_Lock::acquire( 'with-lock.php ' . implode( ' ', $command ) ); +// An empty spec lets the child inherit stdin, stdout and stderr; passing STDOUT or STDERR makes +// PHP seek a redirected file's shared offset, and the outputs then overwrite each other. +$process = proc_open( $command, array(), $pipes ); if ( false === $process ) { fwrite( STDERR, 'with-lock.php: could not start ' . $command[0] . "\n" ); exit( 2 );