diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..1fe880a --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,5 @@ +# getrandom 0.3 (via lopdf → rand) requires an explicit backend opt-in for +# wasm32-unknown-unknown. This cfg matches the `wasm_js` feature enabled on +# the getrandom dependency in batdoc-core/Cargo.toml. +[target.wasm32-unknown-unknown] +rustflags = ["--cfg", "getrandom_backend=\"wasm_js\""] diff --git a/.gitignore b/.gitignore index f96a67d..46e60e0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ .worktrees/ docs/ .vexp +.pi-lens +# wasm demo build artifacts (regenerated by web/build.sh and web/make_samples.py) +web/pkg/ +web/samples/ diff --git a/Cargo.lock b/Cargo.lock index 82a4bf3..2970473 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,6 +134,7 @@ dependencies = [ "base64", "cfb", "encoding_rs", + "getrandom 0.3.4", "image", "lopdf", "ocrs", @@ -142,6 +143,7 @@ dependencies = [ "rten", "thiserror 2.0.18", "ureq", + "wasm-bindgen", "zip", ] @@ -637,9 +639,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasip2", + "wasm-bindgen", ] [[package]] @@ -1137,8 +1141,6 @@ dependencies = [ [[package]] name = "pdf-extract" version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28ba1758a3d3f361459645780e09570b573fc3c82637449e9963174c813a98" dependencies = [ "adobe-cmap-parser", "cff-parser", @@ -1147,6 +1149,7 @@ dependencies = [ "log", "lopdf", "postscript", + "ttf-parser", "type1-encoding-parser", "unicode-normalization", ] diff --git a/Cargo.toml b/Cargo.toml index a5df981..a714e05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,11 @@ [workspace] -members = [".", "batdoc-core"] +members = [".", "batdoc-core", "crates/pdf-extract"] + +# Local fork of pdf-extract (spec D1/D6): local builds get panic-free decode +# + glyph recovery; downstream crates-io consumers of batdoc-core resolve +# upstream pdf-extract 0.10 (a documented, safe degradation). +[patch.crates-io] +pdf-extract = { path = "crates/pdf-extract" } [package] name = "batdoc" diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..b264e19 --- /dev/null +++ b/NOTICE @@ -0,0 +1,10 @@ +batdoc includes or derives from: + +- pdf-extract 0.10 (https://github.com/jrmuizel/pdf-extract), MIT license, + Jeff Muizelaar and Mozilla contributors — vendored with modifications + under crates/pdf-extract (panic-free decoding, glyph recovery). +- liteparse (https://github.com/run-llama/liteparse), Apache-2.0 — PDF + layout algorithms (reading order, columns, headings, tables, OCR merge) + reimplemented in batdoc-core/src/pdf_layout.rs, pdf_ocr.rs, and + crates/pdf-extract/src/font_cmap.rs. No liteparse code or data tables + were copied. diff --git a/README.md b/README.md index fd94be5..6e743c1 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ too. ## When to use -- PDF, text layer or scanned → `batdoc --ocr paper.pdf` +- PDF, text layer or scanned → `batdoc paper.pdf` (textless pages auto-OCR) - Office documents (doc/docx/xls/xlsx/pptx) → `batdoc report.docx` - Images or scans of text → `batdoc photo.png` (image files are always OCR'd) @@ -52,15 +52,16 @@ too. batdoc report.docx # markdown, highlighted, paged on a tty batdoc --plain legacy.doc > out.txt # plain text - batdoc --ocr scanned.pdf # OCR pages with no text layer + batdoc scanned.pdf # textless PDF pages auto-OCR + batdoc --ocr report.docx # OCR docx/pptx embedded images batdoc --images report.docx # embed images as base64 data URIs cat mystery.bin | batdoc # format detected from magic bytes ## Notes - `--plain` for text-only output; markdown is the default on a tty. -- OCR needs no flag for image files; `--ocr` also covers docx/pptx embedded - images and textless PDF pages. +- OCR needs no flag for PDFs or image files; `--ocr` covers docx/pptx + embedded images and is a no-op for PDF/image input. - OCR models (~12 MB) download on first use; pre-seed `$BATDOC_MODELS_DIR` for offline or package-managed installs. - Use a release build — OCR is much slower in debug builds. @@ -69,26 +70,31 @@ too. ## Install **Arch Linux (AUR):** + ``` yay -S batdoc ``` **Homebrew:** + ``` brew install daemonp/tap/batdoc ``` **Linux (x86_64, static musl):** + ``` curl -sL https://github.com/daemonp/batdoc/releases/latest/download/batdoc-linux-x86_64.zst | zstd -d > batdoc && chmod +x batdoc ``` **macOS (Apple Silicon):** + ``` curl -sL https://github.com/daemonp/batdoc/releases/latest/download/batdoc-darwin-aarch64.zst | zstd -d > batdoc && chmod +x batdoc ``` **From source:** + ``` cargo build --release cp target/release/batdoc ~/.local/bin/ @@ -119,11 +125,21 @@ markdown links. Multi-slide decks get `## Slide N` headings. Speaker notes are appended after the deck under `## Notes` when present. `.pdf` extracts text from text-based PDFs using `pdf-extract`. Multi-page -documents get `## Page N` headings in markdown mode. With `--ocr`, pages -whose text layer is empty are OCR'd from their embedded images (scanned -documents). Scanned/image-only PDFs that yield no text at all get a clean -error message. Malformed PDFs that would crash the underlying library are -caught and reported as errors rather than panics. +documents get `## Page N` headings in markdown mode. When a PDF has no text +layer at all (a scan), batdoc automatically falls back to OCR'ing its embedded +page images — no `--ocr` flag needed. A scanned PDF whose OCR also finds +nothing gets a clean error message. Malformed PDFs that would crash the +underlying library are caught and reported as errors rather than panics. + +### PDF extraction notes + +PDFs with broken or missing font mappings (garbled CID fonts) are recovered +via a vendored fork of `pdf-extract` under `crates/pdf-extract`, wired in with +`[patch.crates-io]`. Builds from this repository — releases, AUR, deb/rpm, +Homebrew — include the fix. `cargo install batdoc` resolves `pdf-extract` from +crates.io instead and gets upstream behavior: still safe (panics are caught), but +garbled documents stay garbled. Publishing the fork to close this gap is a +deferred follow-up. ## Options @@ -134,8 +150,7 @@ cat FILE | batdoc [OPTIONS] -p, --plain plain text, no highlighting -m, --markdown force markdown (default on tty) -i, --images embed images as inline base64 data URIs - --ocr OCR text from images: docx/pptx embedded images and - textless PDF pages; image files are always OCR'd + --ocr OCR embedded images (docx/pptx); textless PDFs auto-OCR -h, --help help ``` @@ -168,23 +183,22 @@ and for formats without OOXML image support (`.doc`, `.xls`, `.pdf`). ## OCR -`--ocr` runs the [ocrs](https://github.com/robertknight/ocrs) engine (pure +OCR runs the [ocrs](https://github.com/robertknight/ocrs) engine (pure Rust, rten backend) over: - **Image files** — `batdoc photo.png` always OCRs (no flag needed; text is the only output an image can produce). PNG, JPEG, GIF, WebP, and BMP. - **DOCX/PPTX embedded images** — `batdoc --ocr report.docx` renders OCR'd text as a blockquote after each image (paragraphs in `--plain` mode). -- **PDF pages without a text layer** — `batdoc --ocr scan.pdf` OCRs the - embedded page images (typical one-image-per-page scans). Text-bearing - pages are never OCR'd. +- **PDF pages without a text layer** — automatic: a scanned PDF whose pages + have no text is OCR'd from its embedded page images (typical one-image-per- + page scans). Text-bearing pages and text-bearing documents are never OCR'd. Models (~12 MB, two `.rten` files) are downloaded on first OCR use from the ocrs upstream distribution and cached in `$BATDOC_MODELS_DIR`, else `$XDG_CACHE_HOME/batdoc/models`, else `~/.cache/batdoc/models`. Set `BATDOC_MODELS_DIR` to a pre-seeded directory for offline or -package-managed installs. Without `--ocr`, output is identical to older -versions. OCR takes roughly 0.5–2 s per image on CPU. +package-managed installs. OCR takes roughly 0.5–2 s per image on CPU. Note: ocrs runs much slower in debug builds; always use a release build (`cargo build --release`) when testing OCR. diff --git a/WASM.md b/WASM.md new file mode 100644 index 0000000..229dff2 --- /dev/null +++ b/WASM.md @@ -0,0 +1,203 @@ +# Compiling batdoc to WebAssembly + +Status: **implemented.** `batdoc-core` builds a browser-loadable +`cdylib` for `wasm32-unknown-unknown` with `--no-default-features`, exposes +`wasm-bindgen` entry points (`detect`, `to_plain`, `to_markdown`), and ships a +working browser demo in `web/` (see [Demo](#demo)). Native builds are +unchanged. + +## Why you'd care + +`batdoc-core` is a pure-Rust document-to-text/OCR library (docx/xlsx/pptx/doc/xls/ +pdf → plain text or markdown). The entire *document* pipeline (via `lopdf`, +`pdf-extract`, `zip`, `cfb`, `image`) is wasm-clean. Only the OCR model +**download** path and the terminal-oriented **CLI binary** are host-only. That +means a wasm build of the library — e.g. an in-browser doc viewer/extractor — is +very achievable with modest, well-scoped changes. + +## Findings (empirically verified) + +All of the following came from actually running +`cargo check --target wasm32-unknown-unknown` against the workspace, not from +reading dependency READMEs. + +### State (2026-08-17) + +1. **Done — model download is feature-gated behind `net`.** `ureq` is now an + `optional` dependency behind a `net` feature (default-on). Building with + `--no-default-features` drops `ureq`/`rustls`/`ring`, and the download + branch of `ocr.rs::ensure_file` is compiled out — it only checks for + pre-seeded model files and returns a descriptive error otherwise. Model + loading/inference (`ocrs`/`rten`) are untouched. +2. **Done — `getrandom` wasm backend configured.** `.cargo/config.toml` sets + `getrandom_backend="wasm_js"` for `wasm32-unknown-unknown`, and the + `batdoc-core` manifest enables `getrandom/0.3`'s `wasm_js` feature on the + wasm target only. `getrandom 0.2` disappears with `ring` (state 1). +3. **Done — wasm-facing entry points + browser demo.** `batdoc-core` is now a + `rlib`+`cdylib`; `src/wasm.rs` (wasm32-only) exports `detect`, `to_plain`, + and `to_markdown` via `wasm-bindgen`. The `web/` demo builds the `.wasm`, + generates the JS glue, and runs entirely in the browser (see [Demo](#demo)). + +### Good news + +- **`rayon` compiles for `wasm32-unknown-unknown`.** Tested standalone: a tiny + crate using `rayon::prelude::ParallelIterator` (`.par_iter().sum()`) builds and + type-checks for wasm. rayon-core now ships a single-threaded wasm path. +- **`ocrs` is wasm-aware by design.** Its `Cargo.toml` declares a dedicated + `[target.'cfg(target_arch = "wasm32")'.dependencies.wasm-bindgen]` and the + author ships in-browser builds, so `ocrs` + `rten` (the OCR engine) are expected + to build on wasm32 once the broader graph is unblocked. +- **`getrandom` is a config issue, not a rewrite** (see blockers). + +### Blockers to a plain `cargo check --target wasm32-unknown-unknown` + +1. **`ureq`/`rustls`/`ring` → startup/clock failure (hard blocker) — RESOLVED.** + `rustls` fails to compile for wasm because it needs `UnixTime::now()` / + `std::time::SystemTime` (needs a clock). This whole stack exists **only** for + `batdoc-core/src/ocr.rs::ensure_file`, which downloads the two OCR model + files (`.rten`) over HTTP on first use. On wasm there is no socket/clock, so + this path is gated out by the `net` feature (see State 1). + +2. **`getrandom` (two versions) needs the wasm backend config — RESOLVED.** + - `getrandom 0.2` — pulled via `ring → rustls → ureq` (gone with `net` off). + - `getrandom 0.3` — pulled via `rand → lopdf`. + Both reject wasm unless you pass a config flag *and* enable a feature: + ```toml + [target.wasm32-unknown-unknown] + rustflags = "--cfg getrandom_backend=\"wasm_js\"" + ``` + plus the `wasm_js` feature (0.3) / `js` feature (0.2) enabled on the + dependency. Now wired up in `.cargo/config.toml` and the `batdoc-core` + manifest (see State 2). + +3. **The CLI crate is host-shaped.** `src/main.rs` uses `std::fs`, stdin/stdout, + `is-terminal`, and the `bat` pager; `build.rs` synthesizes a man page. A + `batdoc` **binary** on wasm is meaningless. Only `batdoc-core` the **library** + is a realistic target, and it currently exposes no wasm-friendly entry point. + +4. **The OCR model download is network-at-runtime.** On wasm the model files are + not download-and-cache; they must be embedded at build time or supplied + out-of-band (mirror the existing `BATDOC_MODELS_DIR` seeding, but baked in). + +## Reproduction + +```sh +# add the wasm target once +rustup target add wasm32-unknown-unknown + +# compile the library for wasm (model download is feature-gated off) +cargo check --target wasm32-unknown-unknown -p batdoc-core --no-default-features + +# produce the rlib (links; verifies codegen end-to-end) +cargo build --target wasm32-unknown-unknown --release -p batdoc-core --no-default-features + +# build the browser demo (wasm-bindgen glue + .wasm into web/pkg/) +./web/build.sh +``` + +## Porting plan (whenever someone picks this up) + +Ordered so each step is independently shippable and keeps the native CLI intact. + +1. **Make networking optional — DONE.** Added the `net` feature (default-on) + in `batdoc-core`: + - `ureq` is `optional` behind `net = ["dep:ureq"]`, so + `--no-default-features` removes `ureq` (and `rustls`/`ring`), and + - `#[cfg(feature = "net")]` gates the download branch of `ocr.rs::ensure_file`; + without it the path only looks up pre-seeded model files (or returns a + "models must be provided/embedded" error). Model *loading* and inference + (`ocrs`/`rten`) remain wasm-safe. + +2. **Fix `getrandom` for wasm — DONE.** Added `.cargo/config.toml` with the + `getrandom_backend="wasm_js"` rustflag and enabled `getrandom/0.3`'s + `wasm_js` feature on the wasm target only. The 0.2 instance disappeared with + `ring`; only the 0.3 one (via `lopdf`) remains. + +3. **Expose a library entry point — DONE.** `batdoc-core` is now + `crate-type = ["rlib", "cdylib"]`, and `src/wasm.rs` (compiled only for + `wasm32`) exports `detect`, `to_plain`, and `to_markdown` via `wasm-bindgen`, + each delegating to the existing `detect_format` / `extract_plain_with` / + `extract_markdown_with` API. Native builds never compile `wasm-bindgen`. + +4. **Verify + demo — DONE.** `web/build.sh` builds the wasm cdylib and runs + `wasm-bindgen --target web`; `web/index.html` + `web/demo.js` load it and + convert a chosen file (or the generated `samples/*.docx`, `*.pdf`) to + markdown/plain text entirely in the browser. See [Demo](#demo). + +## Scope notes + +- The **document/OCR accuracy** code paths are cross-platform and need no changes. +- Only the **model acquisition** (`ensure_file`, `ureq`) and the **CLI shell** + (`src/main.rs`, `build.rs`) are wasm-hostile. +- Native builds (Linux/macOS/Windows) are unchanged: `net` is a **default-on** + feature, so `ureq` and the download path are present exactly as before. Only + `--no-default-features` (or wasm, which needs it) opts out. The `getrandom` + `wasm_js` feature is enabled on the wasm target only, so native is untouched. + +### Memory (streaming extract) + +Non-OCR extract now streams through `ExtractSink` (`extract_*_to`); the old +`extract_*` functions are collecting wrappers, and `batdoc` (the CLI) pipes +through `extract_*_to(IoSink(stdout))`. Measured on a synthetic 13.45 MB XLSX +(300,000 rows × 8 cols = 2,400,000 cells, 20,000 shared strings, deflate-9), +peak RSS via `VmHWM` (`/proc/self/status`): + +| path | plain RSS | markdown RSS | +|------|----------:|-------------:| +| pre-streaming CLI (dense grid + `String`) | 326.5 MiB | 694.1 MiB | +| `extract_*` → `String` (library, streaming internals) | 109 MiB | 114 MiB | +| CLI / `extract_*_to` → `IoSink` | 19.9 MiB | 19.8 MiB | + +The `String` API's ~110 MiB is the output text itself (unavoidable when the +caller asks for a `String`). The sink path holds only the input file, the +shared-string arena, and a row buffer — flat across document size (7 MiB at +240 K cells, 19 MiB at 2.4 M), comfortably under the 64 MiB goal. + +For Workers (128 MiB default memory limit), use `extract_*_to` with an +incremental sink rather than the `String` API, and set +`ExtractOptions.max_output_bytes` to bound output (the budget error is +`"output exceeded {n} bytes"`). + +Known streaming behavior changes (vs. the buffered path, both non-OCR only): + +- A present-but-corrupt primary XML part (`word/document.xml`, a slide, or a + sheet) now truncates silently to the text read so far instead of erroring — + the pull parser treats a decompression/parse error as end-of-stream. + Unreachable on valid files; error *wording* for other failures is unchanged. +- With an `IoSink`, a late error (e.g. a later sheet exceeding `MAX_COLS` or + the output budget) leaves earlier output already written. Invisible on the + `String`/`extract_*` wrapper path, which discards the buffer on error. + +## Demo + +The browser demo lives in `web/`. It builds `batdoc-core` as a wasm cdylib, +emits the `wasm-bindgen` glue, and serves a single static page that converts a +file to markdown/plain text client-side. + +Prereqs: `rustup target add wasm32-unknown-unknown` and +`cargo install wasm-bindgen-cli --version 0.2.108`. + +```sh +./web/make_samples.py # once: write web/samples/{sample.docx,sample.pdf} +./web/build.sh # build wasm + JS glue into web/pkg/ + +# serve (wasm must be fetched over http, not file://) and open the browser +cd web && python3 -m http.server 8000 # then open http://localhost:8000/ +``` + +Notes: + +- `web/pkg/` and `web/samples/` are generated and git-ignored. +- The exported functions are `detect(data)`, `to_plain(data)`, and + `to_markdown(data, images, ocr)` — `data` is a `Uint8Array`; errors throw a + JS `Error` with the `BatdocError` message. +- This build disables `net`, so OCR model files are **not** downloaded — they + must be seeded (e.g. preloaded into the cache dir the library resolves). The + document/text pipeline (DOCX/XLSX/PPTX/DOC/XLS/PDF-text) needs no models and + works out of the box. + +## Related + +- Source of the full dependency graph: `cargo tree -p batdoc-core`, + `cargo tree -i getrandom`, `cargo tree -i rayon`. +- OCR engine: [ocrs](https://github.com/robertknight/ocrs), [rten](https://github.com/robertknight/rten). diff --git a/batdoc-core/Cargo.toml b/batdoc-core/Cargo.toml index eb78612..7317639 100644 --- a/batdoc-core/Cargo.toml +++ b/batdoc-core/Cargo.toml @@ -10,6 +10,7 @@ readme = "README.md" [lib] name = "batdoc_core" path = "src/lib.rs" +crate-type = ["rlib", "cdylib"] [dependencies] base64 = "0.22" @@ -22,9 +23,23 @@ pdf-extract = "0.10" quick-xml = "0.37" rten = { version = "0.24", default-features = false } thiserror = "2" -ureq = "2" +ureq = { version = "2", optional = true } zip = { version = "2", default-features = false, features = ["deflate"] } +[features] +default = ["net"] +# Model download over HTTP. Disable (--no-default-features) for offline, +# embedded, or wasm targets where there is no socket/clock. +net = ["dep:ureq"] + +[target.'cfg(target_arch = "wasm32")'.dependencies] +# lopdf pulls getrandom 0.3 transitively (via rand); enable its wasm backend +# (see .cargo/config.toml for the matching `getrandom_backend="wasm_js"` cfg). +getrandom = { version = "0.3", features = ["wasm_js"] } +# wasm-bindgen is only needed for the browser-facing entry points in +# src/wasm.rs; it is compiled out of native builds. +wasm-bindgen = "0.2" + [dev-dependencies] image = { version = "0.25.10", default-features = false, features = ["png", "jpeg"] } lopdf = { version = "0.38", default-features = false } diff --git a/batdoc-core/src/arena.rs b/batdoc-core/src/arena.rs new file mode 100644 index 0000000..af1e95b --- /dev/null +++ b/batdoc-core/src/arena.rs @@ -0,0 +1,63 @@ +pub(crate) struct StringArena { + buf: Vec, + spans: Vec<(u32, u32)>, +} + +impl StringArena { + pub(crate) const fn new() -> Self { + Self { + buf: Vec::new(), + spans: Vec::new(), + } + } + + #[allow(clippy::cast_possible_truncation)] + pub(crate) fn push(&mut self, s: &str) -> usize { + // Offsets are stored as u32 by design; the arena is bounded by the + // 32-bit string table lengths used in the legacy formats we parse. + let start = self.buf.len() as u32; + self.buf.extend_from_slice(s.as_bytes()); + self.spans.push((start, s.len() as u32)); + self.spans.len() - 1 + } + + pub(crate) fn get(&self, i: usize) -> Option<&str> { + let (start, len) = *self.spans.get(i)?; + let bytes = &self.buf[start as usize..][..len as usize]; + Some(std::str::from_utf8(bytes).unwrap_or("")) + } + + #[allow(dead_code)] + pub(crate) const fn len(&self) -> usize { + self.spans.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn push_get_roundtrip() { + let mut a = StringArena::new(); + let i = a.push("hello"); + let j = a.push("world"); + assert_eq!(a.get(i), Some("hello")); + assert_eq!(a.get(j), Some("world")); + assert_eq!(a.len(), 2); + } + + #[test] + fn get_oob_is_none() { + let a = StringArena::new(); + assert_eq!(a.get(0), None); + } + + #[test] + fn empty_string_is_distinct_slot() { + let mut a = StringArena::new(); + let i = a.push(""); + assert_eq!(a.get(i), Some("")); + assert_eq!(a.len(), 1); + } +} diff --git a/batdoc-core/src/doc.rs b/batdoc-core/src/doc.rs index c6b5c08..3ac1356 100644 --- a/batdoc-core/src/doc.rs +++ b/batdoc-core/src/doc.rs @@ -12,6 +12,7 @@ use std::io::{Cursor, Read}; use crate::codepage; use crate::error::BatdocError; use crate::heuristic; +use crate::ExtractSink; // FIB flag bits const F_ENCRYPTED: u16 = 0x0100; @@ -32,6 +33,21 @@ pub(crate) fn extract_markdown(data: &[u8]) -> crate::error::Result { /// Extract plain text from an OLE2 .doc file. /// Returns the document text as a String with paragraph separation. pub(crate) fn extract_plain(data: &[u8]) -> crate::error::Result { + let mut out = String::new(); + extract_plain_to(data, &mut out)?; + Ok(out) +} + +/// Stream plain text from an OLE2 .doc file into `sink`. +/// +/// The `WordDocument` stream is still read fully into memory (the compound +/// file stream API is not chunked), but each 256-byte block is decoded into +/// the sink as it is scanned, instead of building a full `Vec` then a +/// full output `String`. +pub(crate) fn extract_plain_to( + data: &[u8], + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { let cursor = Cursor::new(data); let mut cfb = CompoundFile::open(cursor)?; @@ -75,18 +91,21 @@ pub(crate) fn extract_plain(data: &[u8]) -> crate::error::Result { let text_data = &buf[text_start..text_end]; let is_unicode = flags & F_EXT_CHAR != 0; - let chars = if is_unicode { - extract_word8_text(text_data, cp) + let mut writer = DocWriter::new(sink); + if is_unicode { + extract_word8_text_to(text_data, cp, &mut writer)?; } else { - extract_8bit_text(text_data, cp) - }; - - Ok(chars_to_text(&chars)) + extract_8bit_text_to(text_data, cp, &mut writer)?; + } + writer.finish() } -/// Extract text from Word 97+ format using the 256-byte block heuristic. -fn extract_word8_text(data: &[u8], codepage: u16) -> Vec { - let mut result = Vec::new(); +/// Stream Word 97+ text out block by block using the 256-byte heuristic. +fn extract_word8_text_to( + data: &[u8], + codepage: u16, + writer: &mut DocWriter<'_, S>, +) -> crate::error::Result<()> { let mut offset = 0; while offset < data.len() { @@ -95,24 +114,43 @@ fn extract_word8_text(data: &[u8], codepage: u16) -> Vec { if detect_unicode_block(block) { for pair in block.as_chunks::<2>().0 { - result.push(u16::from_le_bytes(*pair)); + writer.push_char(u16::from_le_bytes(*pair))?; } } else { for &b in block { let ch = codepage::decode_byte(b, codepage); - // Convert char to u16 for the existing pipeline. + // Convert char to u16 for the shared pipeline. // BMP characters fit in u16; supplementary plane chars // (unlikely in 8-bit codepages) get two u16 surrogates. let mut buf = [0u16; 2]; let encoded = ch.encode_utf16(&mut buf); - result.extend_from_slice(encoded); + for &cu in encoded.iter() { + writer.push_char(cu)?; + } } } offset = block_end; } - result + Ok(()) +} + +/// Stream pre-Word97 8-bit text out byte by byte. +fn extract_8bit_text_to( + data: &[u8], + codepage: u16, + writer: &mut DocWriter<'_, S>, +) -> crate::error::Result<()> { + for &b in data { + let ch = codepage::decode_byte(b, codepage); + let mut buf = [0u16; 2]; + let encoded = ch.encode_utf16(&mut buf); + for &cu in encoded.iter() { + writer.push_char(cu)?; + } + } + Ok(()) } /// Detect if a 256-byte block is UTF-16LE encoded. @@ -123,7 +161,11 @@ fn detect_unicode_block(block: &[u8]) -> bool { }) } -/// Extract text from pre-Word97 8-bit encoded stream. +/// Extract text from pre-Word97 8-bit encoded stream into a `Vec`. +/// +/// Only used by tests: the streaming path decodes into the sink via +/// [`extract_8bit_text_to`]. +#[cfg(test)] fn extract_8bit_text(data: &[u8], codepage: u16) -> Vec { let mut result = Vec::new(); for &b in data { @@ -135,83 +177,65 @@ fn extract_8bit_text(data: &[u8], codepage: u16) -> Vec { result } -/// Flush a paragraph buffer into the output string if non-empty. -fn flush_paragraph(paragraph: &mut String, output: &mut String, first: &mut bool) { - let text = paragraph.trim_end(); - if !text.is_empty() { - if !*first { - output.push('\n'); - } - output.push_str(text); - output.push('\n'); - *first = false; - } - paragraph.clear(); +/// Incremental writer that turns decoded u16 code units into paragraph text +/// and emits completed paragraphs to a sink. +/// +/// Shared by the streaming [`extract_plain_to`] path and, via a `String` +/// sink, by the buffered `chars_to_text` used in tests. +struct DocWriter<'a, S: ExtractSink> { + sink: &'a mut S, + paragraph: String, + first: bool, + field_depth: i32, + field_stack: Vec, + pending_high_surrogate: Option, } -/// Tracks what we're collecting during a field: instruction or display text. -#[derive(Clone)] -enum FieldState { - /// Collecting field instruction text (between 0x0013 and 0x0014). - Instruction(String), - /// Collecting display text (between 0x0014 and 0x0015), with the URL - /// if this is a HYPERLINK field. - Display { url: Option, text: String }, -} +impl<'a, S: ExtractSink> DocWriter<'a, S> { + const fn new(sink: &'a mut S) -> Self { + Self { + sink, + paragraph: String::new(), + first: true, + field_depth: 0, + field_stack: Vec::new(), + pending_high_surrogate: None, + } + } -/// Process the u16 character stream into paragraphs, extracting hyperlinks -/// from field codes and suppressing other field codes. -/// -/// Word field codes use three markers: -/// - `0x0013` — field begin (instruction text follows) -/// - `0x0014` — field separator (display text follows) -/// - `0x0015` — field end -/// -/// For `HYPERLINK` fields, we capture the URL from the instruction and -/// emit `[display text](url)` inline. Other field types (TOC, PAGE, etc.) -/// are suppressed as before. -/// -/// Handles UTF-16 surrogate pairs: a high surrogate (0xD800-0xDBFF) followed -/// by a low surrogate (0xDC00-0xDFFF) is decoded into the correct supplementary -/// plane character. Unpaired surrogates are replaced with U+FFFD. -fn chars_to_text(chars: &[u16]) -> String { - let mut output = String::new(); - let mut paragraph = String::new(); - let mut first = true; - let mut field_depth: i32 = 0; - let mut field_stack: Vec = Vec::new(); - let mut pending_high_surrogate: Option = None; - - for &c in chars { + /// Feed a single u16 code unit through the field/surrogate/paragraph + /// state machine, emitting completed paragraphs to the sink. + fn push_char(&mut self, c: u16) -> crate::error::Result<()> { // Handle surrogate pair completion - if let Some(hi) = pending_high_surrogate.take() { + if let Some(hi) = self.pending_high_surrogate.take() { if (0xDC00..=0xDFFF).contains(&c) { // Valid surrogate pair → supplementary plane character let code = 0x10000 + ((u32::from(hi) - 0xD800) << 10) + (u32::from(c) - 0xDC00); if let Some(ch) = char::from_u32(code) { - push_char_to_field_or_para(ch, &mut field_stack, &mut paragraph); + push_char_to_field_or_para(ch, &mut self.field_stack, &mut self.paragraph); } - continue; + return Ok(()); } // Unpaired high surrogate — emit replacement and process `c` normally - push_char_to_field_or_para('\u{FFFD}', &mut field_stack, &mut paragraph); + push_char_to_field_or_para('\u{FFFD}', &mut self.field_stack, &mut self.paragraph); } // Buffer high surrogates for the next iteration if (0xD800..=0xDBFF).contains(&c) { - pending_high_surrogate = Some(c); - continue; + self.pending_high_surrogate = Some(c); + return Ok(()); } match c { 0x0013 => { // Field begin — start capturing instruction text - field_depth += 1; - field_stack.push(FieldState::Instruction(String::new())); + self.field_depth += 1; + self.field_stack + .push(FieldState::Instruction(String::new())); } 0x0014 => { // Field separator — switch from instruction to display text - if let Some(state) = field_stack.last_mut() { + if let Some(state) = self.field_stack.last_mut() { let url = if let FieldState::Instruction(ref instr) = state { extract_hyperlink_url(instr) } else { @@ -224,16 +248,16 @@ fn chars_to_text(chars: &[u16]) -> String { } } 0x0015 => { - emit_field_end(&mut field_stack, &mut paragraph); - if field_depth > 0 { - field_depth -= 1; + emit_field_end(&mut self.field_stack, &mut self.paragraph); + if self.field_depth > 0 { + self.field_depth -= 1; } } - _ if field_depth > 0 => { + _ if self.field_depth > 0 => { // Inside a field — accumulate into the appropriate buffer if c == 0x000D || c == 0x000B { // Paragraph break inside field — clear current field text - if let Some(state) = field_stack.last_mut() { + if let Some(state) = self.field_stack.last_mut() { match state { FieldState::Instruction(ref mut s) => s.clear(), FieldState::Display { ref mut text, .. } => text.clear(), @@ -241,7 +265,7 @@ fn chars_to_text(chars: &[u16]) -> String { } } else if let Some(ch) = char::from_u32(u32::from(c)) { if ch >= ' ' { - if let Some(state) = field_stack.last_mut() { + if let Some(state) = self.field_stack.last_mut() { match state { FieldState::Instruction(ref mut s) => s.push(ch), FieldState::Display { ref mut text, .. } => text.push(ch), @@ -251,35 +275,89 @@ fn chars_to_text(chars: &[u16]) -> String { } } 0x000B..=0x000D => { - flush_paragraph(&mut paragraph, &mut output, &mut first); + self.flush_paragraph()?; } 0x0007 | 0x0009 => { - paragraph.push('\t'); + self.paragraph.push('\t'); } 0x001E => { - paragraph.push('-'); + self.paragraph.push('-'); } 0x001F | 0x0002 | 0xFEFF => {} // Lone low surrogate (not preceded by high) — replace c if (0xDC00..=0xDFFF).contains(&c) => { - paragraph.push('\u{FFFD}'); + self.paragraph.push('\u{FFFD}'); } c if c < 0x0020 => {} c => { if let Some(ch) = char::from_u32(u32::from(c)) { - paragraph.push(ch); + self.paragraph.push(ch); } } } + Ok(()) } - // Flush any trailing unpaired high surrogate - if pending_high_surrogate.is_some() { - paragraph.push('\u{FFFD}'); + /// Flush the current paragraph buffer into the sink if non-empty. + fn flush_paragraph(&mut self) -> crate::error::Result<()> { + let text = self.paragraph.trim_end(); + if !text.is_empty() { + if !self.first { + self.sink.write_str("\n")?; + } + self.sink.write_str(text)?; + self.sink.write_str("\n")?; + self.first = false; + } + self.paragraph.clear(); + Ok(()) } - flush_paragraph(&mut paragraph, &mut output, &mut first); - output + /// Flush any trailing unpaired surrogate and the final paragraph. + fn finish(&mut self) -> crate::error::Result<()> { + if self.pending_high_surrogate.is_some() { + self.paragraph.push('\u{FFFD}'); + } + self.flush_paragraph() + } +} + +/// Tracks what we're collecting during a field: instruction or display text. +#[derive(Clone)] +enum FieldState { + /// Collecting field instruction text (between 0x0013 and 0x0014). + Instruction(String), + /// Collecting display text (between 0x0014 and 0x0015), with the URL + /// if this is a HYPERLINK field. + Display { url: Option, text: String }, +} + +/// Process the u16 character stream into paragraphs, extracting hyperlinks +/// from field codes and suppressing other field codes. +/// +/// Word field codes use three markers: +/// - `0x0013` — field begin (instruction text follows) +/// - `0x0014` — field separator (display text follows) +/// - `0x0015` — field end +/// +/// For `HYPERLINK` fields, we capture the URL from the instruction and +/// emit `[display text](url)` inline. Other field types (TOC, PAGE, etc.) +/// are suppressed as before. +/// +/// Handles UTF-16 surrogate pairs: a high surrogate (0xD800-0xDBFF) followed +/// by a low surrogate (0xDC00-0xDFFF) is decoded into the correct supplementary +/// plane character. Unpaired surrogates are replaced with U+FFFD. +#[cfg(test)] +fn chars_to_text(chars: &[u16]) -> String { + let mut out = String::new(); + { + let mut writer = DocWriter::new(&mut out); + for &c in chars { + writer.push_char(c).expect("String sink cannot fail"); + } + writer.finish().expect("String sink cannot fail"); + } + out } /// Process a field-end marker (0x0015): pop the field state and emit @@ -617,4 +695,62 @@ mod tests { let result = extract_8bit_text(&data, 1251); assert_eq!(result, vec![0x0410]); } + + // ── extract_plain_to parity ────────────────────────────────── + + /// Build a minimal OLE2 `.doc` whose `WordDocument` stream is a 32-byte + /// FIB header (lid=0x0409, `flags`, text boundaries) followed by `text_data`. + fn build_doc(flags: u16, text_data: &[u8]) -> Vec { + use std::io::Write; + + let header_len = 32usize; + let mut doc = vec![0u8; header_len]; + doc[6..8].copy_from_slice(&0x0409u16.to_le_bytes()); // lid (US English) + doc[10..12].copy_from_slice(&flags.to_le_bytes()); + let text_start = header_len as u32; + let text_end = text_start + u32::try_from(text_data.len()).unwrap(); + doc[24..28].copy_from_slice(&text_start.to_le_bytes()); + doc[28..32].copy_from_slice(&text_end.to_le_bytes()); + doc.extend_from_slice(text_data); + + let mut cfb = CompoundFile::create(Cursor::new(Vec::new())).unwrap(); + { + let mut stream = cfb.create_stream("/WordDocument").unwrap(); + stream.write_all(&doc).unwrap(); + } + cfb.flush().unwrap(); + cfb.into_inner().into_inner() + } + + #[test] + fn extract_plain_to_matches_extract_plain_unicode() { + let mut chars: Vec = Vec::new(); + chars.extend("Hello".encode_utf16()); + chars.push(0x0013); + chars.extend("HYPERLINK \"http://example.com\"".encode_utf16()); + chars.push(0x0014); + chars.extend("there".encode_utf16()); + chars.push(0x0015); + chars.push(0x000D); + chars.extend("Second".encode_utf16()); + chars.push(0x000D); + chars.extend("\u{1F600}".encode_utf16()); // surrogate pair, no trailing CR + let bytes: Vec = chars.iter().flat_map(|c| c.to_le_bytes()).collect(); + + let data = build_doc(F_EXT_CHAR, &bytes); + let expected = extract_plain(&data).unwrap(); + let mut out = String::new(); + extract_plain_to(&data, &mut out).unwrap(); + assert_eq!(out, expected); + } + + #[test] + fn extract_plain_to_matches_extract_plain_8bit() { + let text = b"Hello world\rSecond line\r"; + let data = build_doc(0, text); + let expected = extract_plain(&data).unwrap(); + let mut out = String::new(); + extract_plain_to(&data, &mut out).unwrap(); + assert_eq!(out, expected); + } } diff --git a/batdoc-core/src/docx.rs b/batdoc-core/src/docx.rs index 0dba895..8a81202 100644 --- a/batdoc-core/src/docx.rs +++ b/batdoc-core/src/docx.rs @@ -1,20 +1,23 @@ //! OOXML `.docx` (Office Open XML) format parser. //! -//! Unzips the `.docx` archive, parses `word/document.xml` with `quick-xml` -//! into structured [`Block`] types (paragraphs with heading/list styles and -//! runs with bold/italic/hyperlink, tables with rows and cells), then renders -//! to either plain text or markdown. +//! Unzips the `.docx` archive and streams `word/document.xml` with `quick-xml`. +//! Paragraphs and table rows are rendered as they close and then dropped. +//! Comments, footnotes, and endnotes are still parsed into [`Block`] trees +//! for the trailers. use quick_xml::events::Event; use quick_xml::reader::Reader; use std::collections::{HashMap, HashSet}; use std::fmt::Write as _; -use std::io::{Cursor, Read}; +#[cfg(test)] +use std::io::Read; +use std::io::{BufRead, Cursor}; use zip::ZipArchive; use crate::markup; use crate::xml_util::{self, get_attr, Rels}; use crate::ExtractOptions; +use crate::ExtractSink; /// Extracted document structure for rich output. #[derive(Debug)] @@ -211,16 +214,26 @@ fn assign_display_indexes(notes: Vec, index: &NoteIndex) -> Vec { /// Extract plain text from a .docx file. pub(crate) fn extract_plain(data: &[u8], opts: ExtractOptions) -> crate::error::Result { - let (blocks, _, comments, footnotes, endnotes) = parse_docx( + let mut out = String::new(); + extract_plain_to(data, opts, &mut out)?; + Ok(out) +} + +/// Stream plain text from a .docx file into `sink`. +pub(crate) fn extract_plain_to( + data: &[u8], + opts: ExtractOptions, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + extract_to( data, ExtractOptions { images: false, ..opts }, - )?; - let mut out = render_plain(&blocks); - append_extras_plain(&mut out, &comments, &footnotes, &endnotes); - Ok(out) + Mode::Plain, + sink, + ) } /// Extract markdown-formatted text from a .docx file. @@ -230,16 +243,608 @@ pub(crate) fn extract_plain(data: &[u8], opts: ExtractOptions) -> crate::error:: /// appended at the end of the document. When `opts.ocr` is set, embedded images /// are OCR'd and their text is rendered as a blockquote after the image. pub(crate) fn extract_markdown(data: &[u8], opts: ExtractOptions) -> crate::error::Result { + let mut out = String::new(); + extract_markdown_to(data, opts, &mut out)?; + Ok(out) +} + +/// Stream markdown from a .docx file into `sink`. +pub(crate) fn extract_markdown_to( + data: &[u8], + opts: ExtractOptions, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + extract_to(data, opts, Mode::Markdown, sink) +} + +#[derive(Clone, Copy)] +enum Mode { + Plain, + Markdown, +} + +fn xml_error(err: quick_xml::Error) -> crate::error::BatdocError { + match err { + quick_xml::Error::Io(io) => { + crate::error::BatdocError::Io(std::io::Error::new(io.kind(), io.to_string())) + } + other => crate::error::BatdocError::Document(other.to_string()), + } +} + +fn extract_to( + data: &[u8], + opts: ExtractOptions, + mode: Mode, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + if opts.ocr { + return extract_to_buffered(data, opts, mode, sink); + } + extract_to_streaming(data, opts, mode, sink) +} + +fn extract_to_buffered( + data: &[u8], + opts: ExtractOptions, + mode: Mode, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { let (blocks, image_defs, comments, footnotes, endnotes) = parse_docx(data, opts)?; - let mut md = render_markdown(&blocks); - append_extras_markdown(&mut md, &comments, &footnotes, &endnotes); - if !image_defs.is_empty() { - for def in &image_defs { - md.push_str(def); - md.push('\n'); + let out = match mode { + Mode::Plain => { + let mut out = render_plain(&blocks); + append_extras_plain(&mut out, &comments, &footnotes, &endnotes); + out + } + Mode::Markdown => { + let mut out = render_markdown(&blocks); + append_extras_markdown(&mut out, &comments, &footnotes, &endnotes); + if !image_defs.is_empty() { + for def in &image_defs { + out.push_str(def); + out.push('\n'); + } + } + out + } + }; + sink.write_str(&out) +} + +struct StreamOut<'a, S: ExtractSink> { + sink: &'a mut S, + trailing_newlines: u8, + wrote: bool, + plain_first: bool, + image_counter: usize, + image_queue: Vec<(String, String)>, +} + +impl<'a, S: ExtractSink> StreamOut<'a, S> { + const fn new(sink: &'a mut S) -> Self { + Self { + sink, + trailing_newlines: 0, + wrote: false, + plain_first: true, + image_counter: 0, + image_queue: Vec::new(), + } + } + + #[allow(clippy::cast_possible_truncation)] + fn write_str(&mut self, s: &str) -> crate::error::Result<()> { + if s.is_empty() { + return Ok(()); + } + self.sink.write_str(s)?; + self.wrote = true; + let bytes = s.as_bytes(); + let mut n = 0u8; + for &b in bytes.iter().rev() { + if b == b'\n' { + n = n.saturating_add(1); + } else { + break; + } + } + // `bytes.len() < 256` makes the cast safe; n is already u8 and + // bounded by the trailing-newline count of the input string. + if n == bytes.len() as u8 && bytes.len() < 256 { + self.trailing_newlines = self.trailing_newlines.saturating_add(n); + } else { + self.trailing_newlines = n; + } + Ok(()) + } + + fn ensure_trailer_blank_line(&mut self) -> crate::error::Result<()> { + if !self.wrote || self.trailing_newlines >= 2 { + return Ok(()); + } + if self.trailing_newlines == 1 { + self.write_str("\n") + } else { + self.write_str("\n\n") + } + } +} + +fn extract_to_streaming( + data: &[u8], + opts: ExtractOptions, + mode: Mode, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + let markdown = matches!(mode, Mode::Markdown); + let cursor = Cursor::new(data); + let mut archive = ZipArchive::new(cursor)?; + + let rels = xml_util::load_rels(&mut archive, "word/_rels/document.xml.rels"); + let image_rels = if opts.images { + xml_util::load_image_rels(&mut archive, "word/_rels/document.xml.rels") + } else { + xml_util::Rels::new() + }; + + let comments = parse_comments_part(&mut archive)?; + let footnotes = parse_notes_part(&mut archive, "word/footnotes.xml", b"footnote")?; + let endnotes = parse_notes_part(&mut archive, "word/endnotes.xml", b"endnote")?; + + let mut footnotes_idx = NoteIndex::new(); + let mut endnotes_idx = NoteIndex::new(); + for note in &footnotes { + if blocks_have_text(¬e.blocks) { + footnotes_idx.add_defined(¬e.id); + } + } + for note in &endnotes { + if blocks_have_text(¬e.blocks) { + endnotes_idx.add_defined(¬e.id); + } + } + + let mut images = ZipArchive::new(Cursor::new(data))?; + let mut out = StreamOut::new(sink); + { + let mut reader = xml_util::open_xml(&mut archive, "word/document.xml")?; + let mut buf = Vec::new(); + emit_body( + &mut reader, + &mut buf, + &rels, + &image_rels, + &mut footnotes_idx, + &mut endnotes_idx, + markdown, + opts, + &mut images, + &mut out, + )?; + } + + let footnotes = assign_display_indexes(footnotes, &footnotes_idx); + let endnotes = assign_display_indexes(endnotes, &endnotes_idx); + + let mut extras = String::new(); + match mode { + Mode::Plain => append_extras_plain(&mut extras, &comments, &footnotes, &endnotes), + Mode::Markdown => append_extras_markdown(&mut extras, &comments, &footnotes, &endnotes), + } + if !extras.is_empty() { + out.ensure_trailer_blank_line()?; + out.write_str(&extras)?; + } + + if opts.images { + let queue = std::mem::take(&mut out.image_queue); + write_image_defs(&mut images, &queue, &mut out)?; + } + + Ok(()) +} + +fn write_image_defs( + archive: &mut ZipArchive>, + queue: &[(String, String)], + out: &mut StreamOut<'_, S>, +) -> crate::error::Result<()> { + for (id, path) in queue { + if let Some(data) = xml_util::read_image_from_zip(archive, path, "") { + if let Some(img_ref) = crate::markup::image_to_base64_ref(&data, id) { + out.write_str(&img_ref.definition)?; + out.write_str("\n")?; + } + } + } + Ok(()) +} + +fn parse_notes_part( + archive: &mut ZipArchive>, + path: &str, + note_tag: &[u8], +) -> crate::error::Result> { + match xml_util::open_xml(archive, path) { + Ok(mut reader) => { + let mut buf = Vec::new(); + parse_notes_reader(&mut reader, &mut buf, note_tag) + } + Err(crate::error::BatdocError::Zip(zip::result::ZipError::FileNotFound)) => Ok(Vec::new()), + Err(e) => Err(e), + } +} + +fn parse_comments_part( + archive: &mut ZipArchive>, +) -> crate::error::Result> { + match xml_util::open_xml(archive, "word/comments.xml") { + Ok(mut reader) => { + let mut buf = Vec::new(); + parse_comments_reader(&mut reader, &mut buf) + } + Err(crate::error::BatdocError::Zip(zip::result::ZipError::FileNotFound)) => Ok(Vec::new()), + Err(e) => Err(e), + } +} + +#[allow(clippy::too_many_arguments)] +fn emit_body( + reader: &mut Reader, + buf: &mut Vec, + rels: &Rels, + image_rels: &Rels, + footnotes: &mut NoteIndex, + endnotes: &mut NoteIndex, + markdown: bool, + opts: ExtractOptions, + images: &mut ZipArchive>, + out: &mut StreamOut<'_, S>, +) -> crate::error::Result<()> { + loop { + let kind = match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"body" => 1u8, + Ok(Event::Eof) | Err(_) => 2, + _ => 0, + }; + buf.clear(); + match kind { + 1 => { + emit_block_children( + reader, buf, b"body", rels, image_rels, footnotes, endnotes, markdown, opts, + images, out, + )?; + } + 2 => break, + _ => {} + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn emit_block_children( + reader: &mut Reader, + buf: &mut Vec, + stop: &[u8], + rels: &Rels, + image_rels: &Rels, + footnotes: &mut NoteIndex, + endnotes: &mut NoteIndex, + markdown: bool, + opts: ExtractOptions, + images: &mut ZipArchive>, + out: &mut StreamOut<'_, S>, +) -> crate::error::Result<()> { + loop { + let mut start_p = false; + let mut start_tbl = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) => match e.local_name().as_ref() { + b"p" => start_p = true, + b"tbl" => start_tbl = true, + _ => {} + }, + Ok(Event::End(ref e)) if e.local_name().as_ref() == stop => done = true, + Ok(Event::Eof) | Err(_) => done = true, + _ => {} + } + buf.clear(); + if done { + break; + } + if start_p { + let blocks = parse_paragraph(reader, buf, rels, image_rels, footnotes, endnotes); + emit_parsed_blocks(blocks, markdown, opts, images, out)?; + } else if start_tbl { + emit_table(reader, buf, rels, footnotes, endnotes, markdown, out)?; + } + } + Ok(()) +} + +fn emit_parsed_blocks( + blocks: Vec, + markdown: bool, + opts: ExtractOptions, + images: &mut ZipArchive>, + out: &mut StreamOut<'_, S>, +) -> crate::error::Result<()> { + for block in blocks { + match block { + Block::Paragraph { style, runs } => { + emit_paragraph(&style, &runs, markdown, out)?; + } + Block::Image { path, .. } => { + emit_image(&path, opts, images, out)?; + } + Block::Table { .. } => {} + } + } + Ok(()) +} + +fn emit_paragraph( + style: &ParaStyle, + runs: &[Run], + markdown: bool, + out: &mut StreamOut<'_, S>, +) -> crate::error::Result<()> { + if markdown { + let text = render_runs_markdown(runs); + let text = text.trim_end(); + if text.is_empty() { + return Ok(()); + } + if style.heading_level > 0 && style.heading_level <= 6 { + for _ in 0..style.heading_level { + out.write_str("#")?; + } + out.write_str(" ")?; + out.write_str(text)?; + out.write_str("\n\n")?; + } else if let Some(level) = style.list_level { + let indent = " ".repeat(usize::from(level)); + out.write_str(&indent)?; + out.write_str("- ")?; + out.write_str(text)?; + out.write_str("\n")?; + } else { + out.write_str(text)?; + out.write_str("\n\n")?; + } + } else { + let text: String = runs.iter().map(run_plain_text).collect(); + let text = text.trim_end(); + if text.is_empty() { + return Ok(()); + } + if !out.plain_first { + out.write_str("\n")?; + } + out.write_str(text)?; + out.write_str("\n")?; + out.plain_first = false; + } + Ok(()) +} + +fn emit_image( + path: &str, + opts: ExtractOptions, + images: &mut ZipArchive>, + out: &mut StreamOut<'_, S>, +) -> crate::error::Result<()> { + let Some(data) = xml_util::read_image_from_zip(images, path, "") else { + return Ok(()); + }; + if opts.images { + out.image_counter += 1; + let id = format!("image{}", out.image_counter); + if crate::markup::image_to_base64_ref(&data, &id).is_some() { + out.write_str("![][")?; + out.write_str(&id)?; + out.write_str("]\n\n")?; + out.image_queue.push((id, path.to_string())); + } + } + Ok(()) +} + +fn emit_table( + reader: &mut Reader, + buf: &mut Vec, + rels: &Rels, + footnotes: &mut NoteIndex, + endnotes: &mut NoteIndex, + markdown: bool, + out: &mut StreamOut<'_, S>, +) -> crate::error::Result<()> { + if markdown { + // Buffer just this table's rows (a table is a bounded unit), compute + // the maximum width, then emit all rows padded to that width — the + // same shape the buffered `render_block_markdown` produces for ragged + // tables (later rows wider than the header). + let mut rows: Vec> = Vec::new(); + loop { + let mut start_tr = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"tr" => start_tr = true, + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"tbl" => done = true, + Ok(Event::Eof) | Err(_) => done = true, + _ => {} + } + buf.clear(); + if done { + break; + } + if start_tr { + rows.push(parse_table_row_text( + reader, buf, rels, footnotes, endnotes, markdown, + )); + } + } + if rows.is_empty() { + return Ok(()); } + let ncols = rows.iter().map(Vec::len).max().unwrap_or(0); + if ncols == 0 { + return Ok(()); + } + + let mut md_rows: Vec> = Vec::new(); + for row in rows { + let mut md_row: Vec = row + .into_iter() + .map(|cell| cell.replace('|', "\\|")) + .collect(); + while md_row.len() < ncols { + md_row.push(String::new()); + } + md_rows.push(md_row); + } + + if let Some(header) = md_rows.first() { + out.write_str("| ")?; + out.write_str(&header.join(" | "))?; + out.write_str(" |\n")?; + + out.write_str("|")?; + for _ in 0..ncols { + out.write_str(" --- |")?; + } + out.write_str("\n")?; + + for row in md_rows.iter().skip(1) { + out.write_str("| ")?; + out.write_str(&row.join(" | "))?; + out.write_str(" |\n")?; + } + out.write_str("\n")?; + } + return Ok(()); } - Ok(md) + + loop { + let mut start_tr = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"tr" => start_tr = true, + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"tbl" => done = true, + Ok(Event::Eof) | Err(_) => done = true, + _ => {} + } + buf.clear(); + if done { + break; + } + if start_tr { + let cells = parse_table_row_text(reader, buf, rels, footnotes, endnotes, markdown); + emit_table_row(&cells, out)?; + } + } + Ok(()) +} + +fn emit_table_row( + cells: &[String], + out: &mut StreamOut<'_, S>, +) -> crate::error::Result<()> { + let line = cells.join("\t"); + let line = line.trim_end(); + if !line.is_empty() { + if !out.plain_first { + out.write_str("\n")?; + } + out.write_str(line)?; + out.write_str("\n")?; + out.plain_first = false; + } + Ok(()) +} + +fn parse_table_row_text( + reader: &mut Reader, + buf: &mut Vec, + rels: &Rels, + footnotes: &mut NoteIndex, + endnotes: &mut NoteIndex, + markdown: bool, +) -> Vec { + let mut cells = Vec::new(); + loop { + let mut start_tc = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"tc" => start_tc = true, + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"tr" => done = true, + Ok(Event::Eof) | Err(_) => done = true, + _ => {} + } + buf.clear(); + if done { + break; + } + if start_tc { + cells.push(parse_table_cell_text( + reader, buf, rels, footnotes, endnotes, markdown, + )); + } + } + cells +} + +fn parse_table_cell_text( + reader: &mut Reader, + buf: &mut Vec, + rels: &Rels, + footnotes: &mut NoteIndex, + endnotes: &mut NoteIndex, + markdown: bool, +) -> String { + let empty_image_rels = xml_util::Rels::new(); + let mut parts = Vec::new(); + loop { + let mut start_p = false; + let mut start_tbl = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) => match e.local_name().as_ref() { + b"p" => start_p = true, + b"tbl" => start_tbl = true, + _ => {} + }, + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"tc" => done = true, + Ok(Event::Eof) | Err(_) => done = true, + _ => {} + } + buf.clear(); + if done { + break; + } + if start_p { + let blocks = parse_paragraph(reader, buf, rels, &empty_image_rels, footnotes, endnotes); + for block in blocks { + if let Block::Paragraph { runs, .. } = block { + let t = if markdown { + render_runs_markdown(&runs) + } else { + runs.iter().map(run_plain_text).collect::() + }; + let t = t.trim().to_string(); + if !t.is_empty() { + parts.push(t); + } + } + } + } else if start_tbl { + skip_element(reader, buf); + } + } + parts.join(" ") } /// What [`parse_docx`] produces: body blocks, image reference definitions, @@ -254,6 +859,7 @@ type DocxOutput = (Vec, Vec, Vec, Vec, Vec); /// (corrupt deflate data, truncated entry, archive I/O failure) fails the /// extract with a [`crate::error::BatdocError`]; it is NOT treated as /// missing. +#[cfg(test)] fn read_optional_part( archive: &mut ZipArchive>, name: &str, @@ -295,22 +901,10 @@ fn parse_docx(data: &[u8], opts: ExtractOptions) -> crate::error::Result crate::error::Result crate::error::Result` are collected via the shared /// [`parse_block_children`] walker; anything outside the body is ignored. -fn parse_body( - reader: &mut Reader<&[u8]>, +fn parse_body( + reader: &mut Reader, + buf: &mut Vec, blocks: &mut Vec, rels: &Rels, image_rels: &Rels, @@ -372,13 +972,19 @@ fn parse_body( endnotes: &mut NoteIndex, ) { loop { - match &reader.read_event() { - Ok(Event::Start(e)) if e.local_name().as_ref() == b"body" => { + let kind = match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"body" => 1u8, + Ok(Event::Eof) | Err(_) => 2, + _ => 0, + }; + buf.clear(); + match kind { + 1 => { blocks.extend(parse_block_children( - reader, b"body", rels, image_rels, footnotes, endnotes, + reader, buf, b"body", rels, image_rels, footnotes, endnotes, )); } - Ok(Event::Eof) | Err(_) => break, + 2 => break, _ => {} } } @@ -393,8 +999,9 @@ fn parse_body( /// paragraph/table machinery. Extra parts pass empty `Rels` and empty note /// indexes: hyperlinks are not resolved inside definitions and nested note /// references produce no markers there. -fn parse_block_children( - reader: &mut Reader<&[u8]>, +fn parse_block_children( + reader: &mut Reader, + buf: &mut Vec, stop: &[u8], rels: &Rels, image_rels: &Rels, @@ -403,22 +1010,30 @@ fn parse_block_children( ) -> Vec { let mut blocks = Vec::new(); loop { - match &reader.read_event() { - Ok(Event::Start(e)) => match e.local_name().as_ref() { - b"p" => { - let mut para_blocks = - parse_paragraph(reader, rels, image_rels, footnotes, endnotes); - blocks.append(&mut para_blocks); - } - b"tbl" => { - blocks.push(parse_table(reader, rels, footnotes, endnotes)); - } + let mut start_p = false; + let mut start_tbl = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) => match e.local_name().as_ref() { + b"p" => start_p = true, + b"tbl" => start_tbl = true, _ => {} }, - Ok(Event::End(e)) if e.local_name().as_ref() == stop => break, - Ok(Event::Eof) | Err(_) => break, + Ok(Event::End(ref e)) if e.local_name().as_ref() == stop => done = true, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_p { + let mut para_blocks = + parse_paragraph(reader, buf, rels, image_rels, footnotes, endnotes); + blocks.append(&mut para_blocks); + } else if start_tbl { + blocks.push(parse_table(reader, buf, rels, footnotes, endnotes)); + } } blocks } @@ -431,20 +1046,25 @@ fn parse_block_children( /// ends only when the opened element itself closes, so nested elements /// reusing the same local name are swallowed as part of the subtree instead /// of ending the skip early. -fn skip_element(reader: &mut Reader<&[u8]>) { +fn skip_element(reader: &mut Reader, buf: &mut Vec) { let mut depth = 1u32; loop { - match &reader.read_event() { + match reader.read_event_into(buf) { Ok(Event::Start(_)) => depth += 1, Ok(Event::End(_)) => { depth -= 1; if depth == 0 { + buf.clear(); break; } } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => { + buf.clear(); + break; + } _ => {} } + buf.clear(); } } @@ -459,8 +1079,18 @@ fn skip_element(reader: &mut Reader<&[u8]>) { /// (`w:footnoteRef`/`w:endnoteRef`) are skipped by `parse_run` so they /// never leak text, and nested note references produce no markers (the /// indexes passed here are empty by design). +#[cfg(test)] fn parse_notes_xml(xml: &str, note_tag: &[u8]) -> Vec { let mut reader = Reader::from_str(xml); + let mut buf = Vec::new(); + parse_notes_reader(&mut reader, &mut buf, note_tag).unwrap_or_default() +} + +fn parse_notes_reader( + reader: &mut Reader, + buf: &mut Vec, + note_tag: &[u8], +) -> crate::error::Result> { let mut notes = Vec::new(); let mut seen = HashSet::new(); // Empty rels and empty note indexes: no hyperlink resolution and no @@ -470,48 +1100,59 @@ fn parse_notes_xml(xml: &str, note_tag: &[u8]) -> Vec { let mut endnotes = NoteIndex::new(); loop { - match &reader.read_event() { - Ok(Event::Start(e)) if e.local_name().as_ref() == note_tag => { - let Some(id) = get_attr(e, b"w:id").or_else(|| get_attr(e, b"id")) else { - // Missing id: consume the element without interpreting it. - skip_element(&mut reader); - continue; - }; - if id == "-1" || id == "0" || !seen.insert(id.clone()) { - // Separator (-1), continuation (0), or duplicate id - // (first wins): skip. - skip_element(&mut reader); - continue; + let mut id_opt = None; + let mut skip = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == note_tag => { + match get_attr(e, b"w:id").or_else(|| get_attr(e, b"id")) { + None => skip = true, + Some(id) if id == "-1" || id == "0" || !seen.insert(id.clone()) => { + skip = true; + } + Some(id) => id_opt = Some(id), } - let blocks = parse_block_children( - &mut reader, - note_tag, - &empty_rels, - &empty_rels, - &mut footnotes, - &mut endnotes, - ); - notes.push(Note { - id, - display_index: 0, - blocks, - }); - // parse_block_children consumed through End(note_tag). - } - Ok(Event::Eof) | Err(_) => break, + } + Ok(Event::Eof) => { + buf.clear(); + break; + } + Err(e) => return Err(xml_error(e)), _ => {} } + buf.clear(); + if skip { + skip_element(reader, buf); + continue; + } + if let Some(id) = id_opt { + let blocks = parse_block_children( + reader, + buf, + note_tag, + &empty_rels, + &empty_rels, + &mut footnotes, + &mut endnotes, + ); + notes.push(Note { + id, + display_index: 0, + blocks, + }); + } } - notes + Ok(notes) } /// Parse `word/footnotes.xml` into note definitions in document order. +#[cfg(test)] fn parse_footnotes_xml(xml: &str) -> Vec { parse_notes_xml(xml, b"footnote") } /// Parse `word/endnotes.xml` into note definitions (same shape and rules as /// [`parse_footnotes_xml`]). +#[cfg(test)] fn parse_endnotes_xml(xml: &str) -> Vec { parse_notes_xml(xml, b"endnote") } @@ -522,8 +1163,17 @@ fn parse_endnotes_xml(xml: &str) -> Vec { /// whose body has no non-whitespace text are dropped; duplicate ids keep the /// first occurrence. The `w:annotationRef` glyph inside comment bodies is /// skipped by `parse_run`. +#[cfg(test)] fn parse_comments_xml(xml: &str) -> Vec { let mut reader = Reader::from_str(xml); + let mut buf = Vec::new(); + parse_comments_reader(&mut reader, &mut buf).unwrap_or_default() +} + +fn parse_comments_reader( + reader: &mut Reader, + buf: &mut Vec, +) -> crate::error::Result> { let mut comments = Vec::new(); let mut seen = HashSet::new(); let empty_rels = Rels::new(); @@ -531,42 +1181,53 @@ fn parse_comments_xml(xml: &str) -> Vec { let mut endnotes = NoteIndex::new(); loop { - match &reader.read_event() { - Ok(Event::Start(e)) if e.local_name().as_ref() == b"comment" => { - let Some(id) = get_attr(e, b"w:id").or_else(|| get_attr(e, b"id")) else { - // Comment without an id: nothing to key it by, drop it. - skip_element(&mut reader); - continue; - }; - if !seen.insert(id.clone()) { - // Duplicate id: first wins; consume the body unparsed so - // it can never leak text or markers. - skip_element(&mut reader); - continue; - } - let mut author = get_attr(e, b"w:author") - .or_else(|| get_attr(e, b"author")) - .unwrap_or_default(); - if author.trim().is_empty() { - author = "Anonymous".to_string(); - } - let blocks = parse_block_children( - &mut reader, - b"comment", - &empty_rels, - &empty_rels, - &mut footnotes, - &mut endnotes, - ); - if blocks_have_text(&blocks) { - comments.push(Comment { id, author, blocks }); + let mut id_opt = None; + let mut author = String::new(); + let mut skip = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"comment" => { + match get_attr(e, b"w:id").or_else(|| get_attr(e, b"id")) { + None => skip = true, + Some(id) if !seen.insert(id.clone()) => skip = true, + Some(id) => { + author = get_attr(e, b"w:author") + .or_else(|| get_attr(e, b"author")) + .unwrap_or_default(); + if author.trim().is_empty() { + author = "Anonymous".to_string(); + } + id_opt = Some(id); + } } } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) => { + buf.clear(); + break; + } + Err(e) => return Err(xml_error(e)), _ => {} } + buf.clear(); + if skip { + skip_element(reader, buf); + continue; + } + if let Some(id) = id_opt { + let blocks = parse_block_children( + reader, + buf, + b"comment", + &empty_rels, + &empty_rels, + &mut footnotes, + &mut endnotes, + ); + if blocks_have_text(&blocks) { + comments.push(Comment { id, author, blocks }); + } + } } - comments + Ok(comments) } /// Parse a `` element into blocks. @@ -575,8 +1236,9 @@ fn parse_comments_xml(xml: &str) -> Vec { /// is active, any `` elements (which live inside `` runs) /// produce additional `Block::Image` entries. The paragraph is always first, /// followed by any images found. -fn parse_paragraph( - reader: &mut Reader<&[u8]>, +fn parse_paragraph( + reader: &mut Reader, + buf: &mut Vec, rels: &Rels, image_rels: &Rels, footnotes: &mut NoteIndex, @@ -587,61 +1249,72 @@ fn parse_paragraph( let mut image_blocks: Vec = Vec::new(); loop { - match reader.read_event() { - Ok(Event::Start(ref e)) => { - let name = e.local_name(); - match name.as_ref() { - b"pPr" => parse_para_props(reader, &mut style), - b"r" => { - let (run_runs, img_opt) = - parse_run(reader, image_rels, footnotes, endnotes); - runs.extend(run_runs); - if let Some(blk) = img_opt { - image_blocks.push(blk); - } - } - b"hyperlink" => { - // Resolve the hyperlink URL from r:id → rels map - let url = get_attr(e, b"r:id").and_then(|rid| rels.get(&rid).cloned()); - parse_hyperlink_runs( - reader, - &mut runs, - url.as_deref(), - image_rels, - &mut image_blocks, - footnotes, - endnotes, - ); - } - _ => {} - } - } - Ok(Event::End(ref e)) if e.local_name().as_ref() == b"p" => { - break; - } - Ok(Event::Empty(ref e)) => { - let name = e.local_name(); - if name.as_ref() == b"tab" { - runs.push(Run { - text: "\t".into(), - bold: false, - italic: false, - link_url: None, - marker: None, - }); - } else if name.as_ref() == b"br" { - runs.push(Run { - text: "\n".into(), - bold: false, - italic: false, - link_url: None, - marker: None, - }); + let mut start_ppr = false; + let mut start_r = false; + let mut start_hyperlink = false; + let mut hyperlink_url: Option = None; + let mut empty_tab = false; + let mut empty_br = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) => match e.local_name().as_ref() { + b"pPr" => start_ppr = true, + b"r" => start_r = true, + b"hyperlink" => { + start_hyperlink = true; + hyperlink_url = get_attr(e, b"r:id").and_then(|rid| rels.get(&rid).cloned()); } - } - Ok(Event::Eof) | Err(_) => break, + _ => {} + }, + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"p" => done = true, + Ok(Event::Empty(ref e)) => match e.local_name().as_ref() { + b"tab" => empty_tab = true, + b"br" => empty_br = true, + _ => {} + }, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_ppr { + parse_para_props(reader, buf, &mut style); + } else if start_r { + let (run_runs, img_opt) = parse_run(reader, buf, image_rels, footnotes, endnotes); + runs.extend(run_runs); + if let Some(blk) = img_opt { + image_blocks.push(blk); + } + } else if start_hyperlink { + parse_hyperlink_runs( + reader, + buf, + &mut runs, + hyperlink_url.as_deref(), + image_rels, + &mut image_blocks, + footnotes, + endnotes, + ); + } else if empty_tab { + runs.push(Run { + text: "\t".into(), + bold: false, + italic: false, + link_url: None, + marker: None, + }); + } else if empty_br { + runs.push(Run { + text: "\n".into(), + bold: false, + italic: false, + link_url: None, + marker: None, + }); + } } let mut result = vec![Block::Paragraph { style, runs }]; @@ -650,10 +1323,10 @@ fn parse_paragraph( } /// Parse `` to extract heading level and list info. -fn parse_para_props(reader: &mut Reader<&[u8]>, style: &mut ParaStyle) { +fn parse_para_props(reader: &mut Reader, buf: &mut Vec, style: &mut ParaStyle) { let mut depth = 1u32; loop { - match reader.read_event() { + match reader.read_event_into(buf) { Ok(Event::Start(_)) => { depth += 1; } @@ -680,12 +1353,17 @@ fn parse_para_props(reader: &mut Reader<&[u8]>, style: &mut ParaStyle) { Ok(Event::End(_)) => { depth -= 1; if depth == 0 { + buf.clear(); break; } } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => { + buf.clear(); + break; + } _ => {} } + buf.clear(); } } @@ -746,8 +1424,10 @@ fn push_note_marker( /// bodies) are skipped entirely. When `image_rels` is non-empty and a /// `` is found inside the run, the image reference is extracted /// and returned as a `Block::Image`. -fn parse_run( - reader: &mut Reader<&[u8]>, +#[allow(clippy::too_many_lines)] +fn parse_run( + reader: &mut Reader, + buf: &mut Vec, image_rels: &Rels, footnotes: &mut NoteIndex, endnotes: &mut NoteIndex, @@ -770,89 +1450,106 @@ fn parse_run( }; loop { - match reader.read_event() { + let mut start_rpr = false; + let mut start_t = false; + let mut start_drawing = false; + let mut start_note: Option<(Vec, Option)> = None; + let mut start_glyph = false; + let mut empty_tab = false; + let mut empty_br = false; + let mut empty_note: Option<(Vec, Option)> = None; + let mut empty_b = false; + let mut empty_i = false; + let mut done = false; + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) => { let name = e.local_name(); match name.as_ref() { - b"rPr" => parse_run_props(reader, &mut bold, &mut italic), - b"t" => { - // Read text content - if let Ok(Event::Text(t)) = reader.read_event() { - if let Ok(s) = t.unescape() { - text.push_str(&s); - } - } - // Note: the end tag will be consumed below - } - b"drawing" if !image_rels.is_empty() => { - if let Some(blk) = parse_drawing(reader, image_rels) { - image_block = Some(blk); - } - } - // Defensive Start form: note references are empty - // elements in practice, but consume through the end tag - // if a non-empty one appears. + b"rPr" => start_rpr = true, + b"t" => start_t = true, + b"drawing" if !image_rels.is_empty() => start_drawing = true, b"footnoteReference" | b"endnoteReference" => { - if let Some(id) = get_attr(e, b"w:id").or_else(|| get_attr(e, b"id")) { - flush_text(&mut runs, &mut text, bold, italic); - push_note_marker(&mut runs, name.as_ref(), &id, footnotes, endnotes); - } - loop { - match reader.read_event() { - Ok(Event::End(ref e)) - if e.local_name().as_ref() == name.as_ref() => - { - break; - } - Ok(Event::Eof) | Err(_) => break, - _ => {} - } - } - } - // Ref glyphs (`w:footnoteRef`, `w:endnoteRef`, - // `w:annotationRef`): empty elements without an id that - // mark a reference site inside the note/comment bodies. - // Skipped (also in the Empty arm below) so they never - // leak text or produce marker runs. The Empty form needs - // no consumption; this defensive Start form swallows the - // whole subtree if a non-empty one ever appears. - b"footnoteRef" | b"endnoteRef" | b"annotationRef" => { - skip_element(reader); + start_note = Some(( + name.as_ref().to_vec(), + get_attr(e, b"w:id").or_else(|| get_attr(e, b"id")), + )); } + b"footnoteRef" | b"endnoteRef" | b"annotationRef" => start_glyph = true, _ => {} } } Ok(Event::Empty(ref e)) => { let name = e.local_name(); if name.as_ref() == b"tab" { - text.push('\t'); + empty_tab = true; } else if name.as_ref() == b"br" { - text.push('\n'); + empty_br = true; } else if name.as_ref() == b"footnoteReference" || name.as_ref() == b"endnoteReference" { - if let Some(id) = get_attr(e, b"w:id").or_else(|| get_attr(e, b"id")) { - flush_text(&mut runs, &mut text, bold, italic); - push_note_marker(&mut runs, name.as_ref(), &id, footnotes, endnotes); - } + empty_note = Some(( + name.as_ref().to_vec(), + get_attr(e, b"w:id").or_else(|| get_attr(e, b"id")), + )); } else if name.as_ref() == b"b" || name.as_ref() == b"bCs" { - // Self-closing in rPr means bold on - bold = true; + empty_b = true; } else if name.as_ref() == b"i" || name.as_ref() == b"iCs" { - italic = true; - } else if name.as_ref() == b"footnoteRef" - || name.as_ref() == b"endnoteRef" - || name.as_ref() == b"annotationRef" - { - // Ref glyph (empty form): skipped; never text or marker. + empty_i = true; } } - Ok(Event::End(ref e)) if e.local_name().as_ref() == b"r" => { - break; - } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"r" => done = true, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_rpr { + parse_run_props(reader, buf, &mut bold, &mut italic); + } else if start_t { + if let Ok(Event::Text(t)) = reader.read_event_into(buf) { + if let Ok(s) = t.unescape() { + text.push_str(&s); + } + } + buf.clear(); + } else if start_drawing { + if let Some(blk) = parse_drawing(reader, buf, image_rels) { + image_block = Some(blk); + } + } else if let Some((name, id)) = start_note { + if let Some(id) = id { + flush_text(&mut runs, &mut text, bold, italic); + push_note_marker(&mut runs, &name, &id, footnotes, endnotes); + } + loop { + let ended = match reader.read_event_into(buf) { + Ok(Event::End(ref e)) if e.local_name().as_ref() == name.as_slice() => true, + Ok(Event::Eof) | Err(_) => true, + _ => false, + }; + buf.clear(); + if ended { + break; + } + } + } else if start_glyph { + skip_element(reader, buf); + } else if empty_tab { + text.push('\t'); + } else if empty_br { + text.push('\n'); + } else if let Some((name, id)) = empty_note { + if let Some(id) = id { + flush_text(&mut runs, &mut text, bold, italic); + push_note_marker(&mut runs, &name, &id, footnotes, endnotes); + } + } else if empty_b { + bold = true; + } else if empty_i { + italic = true; + } } flush_text(&mut runs, &mut text, bold, italic); @@ -861,10 +1558,15 @@ fn parse_run( } /// Parse to extract bold/italic. -fn parse_run_props(reader: &mut Reader<&[u8]>, bold: &mut bool, italic: &mut bool) { +fn parse_run_props( + reader: &mut Reader, + buf: &mut Vec, + bold: &mut bool, + italic: &mut bool, +) { let mut depth = 1u32; loop { - match reader.read_event() { + match reader.read_event_into(buf) { Ok(Event::Start(_)) => { depth += 1; } @@ -872,7 +1574,6 @@ fn parse_run_props(reader: &mut Reader<&[u8]>, bold: &mut bool, italic: &mut boo let name = e.local_name(); match name.as_ref() { b"b" | b"bCs" => { - // Check for val="false" or val="0" let val = get_val_attr(e); *bold = !matches!(val.as_deref(), Some("false" | "0")); } @@ -886,18 +1587,25 @@ fn parse_run_props(reader: &mut Reader<&[u8]>, bold: &mut bool, italic: &mut boo Ok(Event::End(_)) => { depth -= 1; if depth == 0 { + buf.clear(); break; } } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => { + buf.clear(); + break; + } _ => {} } + buf.clear(); } } /// Parse runs inside a `` element, tagging each run with the URL. -fn parse_hyperlink_runs( - reader: &mut Reader<&[u8]>, +#[allow(clippy::too_many_arguments)] +fn parse_hyperlink_runs( + reader: &mut Reader, + buf: &mut Vec, runs: &mut Vec, url: Option<&str>, image_rels: &Rels, @@ -906,22 +1614,27 @@ fn parse_hyperlink_runs( endnotes: &mut NoteIndex, ) { loop { - match reader.read_event() { - Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"r" => { - let (run_runs, img_opt) = parse_run(reader, image_rels, footnotes, endnotes); - for mut run in run_runs { - run.link_url = url.map(String::from); - runs.push(run); - } - if let Some(blk) = img_opt { - image_blocks.push(blk); - } + let mut start_r = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"r" => start_r = true, + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"hyperlink" => done = true, + Ok(Event::Eof) | Err(_) => done = true, + _ => {} + } + buf.clear(); + if done { + break; + } + if start_r { + let (run_runs, img_opt) = parse_run(reader, buf, image_rels, footnotes, endnotes); + for mut run in run_runs { + run.link_url = url.map(String::from); + runs.push(run); } - Ok(Event::End(ref e)) if e.local_name().as_ref() == b"hyperlink" => { - break; + if let Some(blk) = img_opt { + image_blocks.push(blk); } - Ok(Event::Eof) | Err(_) => break, - _ => {} } } } @@ -932,12 +1645,16 @@ fn parse_hyperlink_runs( /// `` → `` → ``. /// Returns a `Block::Image` with the image's ZIP path (to be resolved later) /// stored in the `path` field. -fn parse_drawing(reader: &mut Reader<&[u8]>, image_rels: &Rels) -> Option { +fn parse_drawing( + reader: &mut Reader, + buf: &mut Vec, + image_rels: &Rels, +) -> Option { let mut embed_rid: Option = None; let mut depth = 1u32; loop { - match reader.read_event() { + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) => { depth += 1; if e.local_name().as_ref() == b"blip" { @@ -953,16 +1670,22 @@ fn parse_drawing(reader: &mut Reader<&[u8]>, image_rels: &Rels) -> Option } Ok(Event::End(ref e)) => { if e.local_name().as_ref() == b"drawing" { + buf.clear(); break; } depth -= 1; if depth == 0 { + buf.clear(); break; } } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => { + buf.clear(); + break; + } _ => {} } + buf.clear(); } let rid = embed_rid?; @@ -1032,8 +1755,9 @@ fn resolve_images( } /// Parse a `` element into a `Block::Table`. -fn parse_table( - reader: &mut Reader<&[u8]>, +fn parse_table( + reader: &mut Reader, + buf: &mut Vec, rels: &Rels, footnotes: &mut NoteIndex, endnotes: &mut NoteIndex, @@ -1041,28 +1765,30 @@ fn parse_table( let mut rows: Vec = Vec::new(); loop { - match reader.read_event() { - Ok(Event::Start(ref e)) => { - let name = e.local_name(); - if name.as_ref() == b"tr" { - let row = parse_table_row(reader, rels, footnotes, endnotes); - rows.push(row); - } - } - Ok(Event::End(ref e)) if e.local_name().as_ref() == b"tbl" => { - break; - } - Ok(Event::Eof) | Err(_) => break, + let mut start_tr = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"tr" => start_tr = true, + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"tbl" => done = true, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_tr { + rows.push(parse_table_row(reader, buf, rels, footnotes, endnotes)); + } } Block::Table { rows } } /// Parse a `` element into a row of cells. -fn parse_table_row( - reader: &mut Reader<&[u8]>, +fn parse_table_row( + reader: &mut Reader, + buf: &mut Vec, rels: &Rels, footnotes: &mut NoteIndex, endnotes: &mut NoteIndex, @@ -1070,20 +1796,21 @@ fn parse_table_row( let mut cells: Row = Vec::new(); loop { - match reader.read_event() { - Ok(Event::Start(ref e)) => { - let name = e.local_name(); - if name.as_ref() == b"tc" { - let cell = parse_table_cell(reader, rels, footnotes, endnotes); - cells.push(cell); - } - } - Ok(Event::End(ref e)) if e.local_name().as_ref() == b"tr" => { - break; - } - Ok(Event::Eof) | Err(_) => break, + let mut start_tc = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"tc" => start_tc = true, + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"tr" => done = true, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_tc { + cells.push(parse_table_cell(reader, buf, rels, footnotes, endnotes)); + } } cells @@ -1093,8 +1820,9 @@ fn parse_table_row( /// /// Images inside table cells are not extracted (impractical in markdown /// tables), so an empty `image_rels` is used for paragraph parsing. -fn parse_table_cell( - reader: &mut Reader<&[u8]>, +fn parse_table_cell( + reader: &mut Reader, + buf: &mut Vec, rels: &Rels, footnotes: &mut NoteIndex, endnotes: &mut NoteIndex, @@ -1103,28 +1831,30 @@ fn parse_table_cell( let mut blocks = Vec::new(); loop { - match reader.read_event() { - Ok(Event::Start(ref e)) => { - let name = e.local_name(); - match name.as_ref() { - b"p" => { - let mut para_blocks = - parse_paragraph(reader, rels, &empty_image_rels, footnotes, endnotes); - blocks.append(&mut para_blocks); - } - b"tbl" => { - blocks.push(parse_table(reader, rels, footnotes, endnotes)); - // nested table - } - _ => {} - } - } - Ok(Event::End(ref e)) if e.local_name().as_ref() == b"tc" => { - break; - } - Ok(Event::Eof) | Err(_) => break, + let mut start_p = false; + let mut start_tbl = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) => match e.local_name().as_ref() { + b"p" => start_p = true, + b"tbl" => start_tbl = true, + _ => {} + }, + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"tc" => done = true, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_p { + let mut para_blocks = + parse_paragraph(reader, buf, rels, &empty_image_rels, footnotes, endnotes); + blocks.append(&mut para_blocks); + } else if start_tbl { + blocks.push(parse_table(reader, buf, rels, footnotes, endnotes)); + } } blocks @@ -1928,8 +2658,13 @@ mod tests { ) -> (Vec, Option) { let full = format!("{xml}"); let mut reader = Reader::from_str(&full); - assert!(matches!(reader.read_event(), Ok(Event::Start(_)))); // consume - parse_run(&mut reader, &Rels::new(), footnotes, endnotes) + let mut buf = Vec::new(); + assert!(matches!( + reader.read_event_into(&mut buf), + Ok(Event::Start(_)) + )); + buf.clear(); + parse_run(&mut reader, &mut buf, &Rels::new(), footnotes, endnotes) } #[test] @@ -2036,9 +2771,15 @@ mod tests { let xml = "X"; let full = format!("{xml}"); let mut reader = Reader::from_str(&full); - assert!(matches!(reader.read_event(), Ok(Event::Start(_)))); // consume + let mut buf = Vec::new(); + assert!(matches!( + reader.read_event_into(&mut buf), + Ok(Event::Start(_)) + )); + buf.clear(); let blocks = parse_paragraph( &mut reader, + &mut buf, &Rels::new(), &Rels::new(), &mut footnotes, @@ -2069,11 +2810,17 @@ mod tests { "; let full = format!("{xml}"); let mut reader = Reader::from_str(&full); - assert!(matches!(reader.read_event(), Ok(Event::Start(_)))); // consume + let mut buf = Vec::new(); + assert!(matches!( + reader.read_event_into(&mut buf), + Ok(Event::Start(_)) + )); + buf.clear(); let mut rels = Rels::new(); rels.insert("rId9".into(), "https://example.com".into()); let blocks = parse_paragraph( &mut reader, + &mut buf, &rels, &Rels::new(), &mut footnotes, @@ -2108,8 +2855,19 @@ mod tests { note"; let full = format!("{xml}"); let mut reader = Reader::from_str(&full); - assert!(matches!(reader.read_event(), Ok(Event::Start(_)))); // consume - let table = parse_table(&mut reader, &Rels::new(), &mut footnotes, &mut endnotes); + let mut buf = Vec::new(); + assert!(matches!( + reader.read_event_into(&mut buf), + Ok(Event::Start(_)) + )); + buf.clear(); + let table = parse_table( + &mut reader, + &mut buf, + &Rels::new(), + &mut footnotes, + &mut endnotes, + ); assert!(matches!(&table, Block::Table { rows } if rows.len() == 1)); let md = render_markdown(std::slice::from_ref(&table)); @@ -2706,6 +3464,62 @@ mod tests { ); } + #[test] + fn extract_markdown_ragged_table_matches_block_renderer() { + // A ragged table (a later row has more cells than the header) must + // match the buffered block renderer byte-for-byte: `ncols` is the max + // row width and every row (including the header) is padded with empty + // cells instead of the streaming path truncating to the first row. + let body_tbl = r#" + + + Name + Age + + + Alice + 30 + Extra + +"#; + + let data = minimal_docx(&[("word/document.xml", &document_xml(body_tbl))]); + + let table = Block::Table { + rows: vec![ + vec![ + vec![Block::Paragraph { + style: ParaStyle::default(), + runs: vec![run("Name", false, false)], + }], + vec![Block::Paragraph { + style: ParaStyle::default(), + runs: vec![run("Age", false, false)], + }], + ], + vec![ + vec![Block::Paragraph { + style: ParaStyle::default(), + runs: vec![run("Alice", false, false)], + }], + vec![Block::Paragraph { + style: ParaStyle::default(), + runs: vec![run("30", false, false)], + }], + vec![Block::Paragraph { + style: ParaStyle::default(), + runs: vec![run("Extra", false, false)], + }], + ], + ], + }; + let mut reference = String::new(); + render_block_markdown(&table, &mut reference); + + let streamed = extract_markdown(&data, crate::ExtractOptions::default()).unwrap(); + assert_eq!(streamed, reference); + } + #[test] fn extract_markdown_empty_defined_note_no_marker_no_section() { // A defined note whose blocks render empty must not be seeded: the @@ -3052,4 +3866,63 @@ mod tests { "a present-but-unreadable part must fail the extract" ); } + + #[test] + fn extract_markdown_to_equals_extract_markdown_on_locked_strings() { + let opts = crate::ExtractOptions::default(); + let data = minimal_docx(&[( + "word/document.xml", + &document_xml("Body only"), + )]); + let a = extract_markdown(&data, opts).unwrap(); + assert_eq!(a, "Body only\n\n"); + let mut b = String::new(); + extract_markdown_to(&data, opts, &mut b).unwrap(); + assert_eq!(a, b); + + let plain_a = extract_plain(&data, opts).unwrap(); + assert_eq!(plain_a, "Body only\n"); + let mut plain_b = String::new(); + extract_plain_to(&data, opts, &mut plain_b).unwrap(); + assert_eq!(plain_a, plain_b); + } + + #[test] + fn extract_markdown_to_equals_extract_markdown_on_image_locked_string() { + let data = image_docx( + r#"Pic"#, + &[], + ); + let opts = crate::ExtractOptions { + images: true, + ..crate::ExtractOptions::default() + }; + let a = extract_markdown(&data, opts).unwrap(); + assert_eq!( + a, + "Pic\n\n![][image1]\n\n[image1]: \n" + ); + let mut b = String::new(); + extract_markdown_to(&data, opts, &mut b).unwrap(); + assert_eq!(a, b); + } + + #[test] + fn extract_to_equals_extract_on_hi_locked_strings() { + let data = minimal_docx(&[( + "word/document.xml", + &document_xml(r#"Hi"#), + )]); + let opts = crate::ExtractOptions::default(); + let a = extract_markdown(&data, opts).unwrap(); + assert_eq!(a, "Hi\n\n"); + let mut b = String::new(); + extract_markdown_to(&data, opts, &mut b).unwrap(); + assert_eq!(a, b); + let plain_a = extract_plain(&data, opts).unwrap(); + assert_eq!(plain_a, "Hi\n"); + let mut plain_b = String::new(); + extract_plain_to(&data, opts, &mut plain_b).unwrap(); + assert_eq!(plain_a, plain_b); + } } diff --git a/batdoc-core/src/lib.rs b/batdoc-core/src/lib.rs index d8e9c4f..1312d6d 100644 --- a/batdoc-core/src/lib.rs +++ b/batdoc-core/src/lib.rs @@ -7,6 +7,7 @@ #![allow(clippy::redundant_pub_crate)] +mod arena; mod codepage; mod dateconv; mod doc; @@ -16,14 +17,22 @@ mod heuristic; mod markup; mod ocr; mod pdf; +mod pdf_geometry; +mod pdf_layout; +mod pdf_ocr; +mod pdf_text; mod pptx; mod sheet; +mod sink; +#[cfg(target_arch = "wasm32")] +mod wasm; mod xls; mod xlsx; mod xml_util; pub use error::{BatdocError, Result}; pub use ocr::models_present; +pub use sink::{BudgetSink, ExtractSink, IoSink}; use std::io::Cursor; @@ -148,9 +157,12 @@ pub fn detect_format(data: &[u8]) -> Result { pub struct ExtractOptions { /// Include embedded images as base64 markdown (markdown mode only). pub images: bool, - /// OCR embedded images (DOCX/PPTX) and textless PDF pages. Has no effect - /// on `Format::Image` — image input is always OCR'd. + /// OCR embedded images (DOCX/PPTX). Textless PDF pages are OCR'd + /// automatically as a fallback regardless of this flag. Has no effect on + /// `Format::Image` — image input is always OCR'd. pub ocr: bool, + /// Stop writing after this many output bytes. `None` means unlimited. + pub max_output_bytes: Option, } /// Extract plain text from a document. @@ -165,9 +177,10 @@ pub fn extract_plain(data: &[u8], format: Format) -> Result { /// Extract plain text with explicit options. /// -/// `opts.ocr` enables OCR for DOCX/PPTX embedded images and textless PDF -/// pages; the `images` option is ignored in plain mode. `Format::Image` -/// input is always OCR'd regardless of options and returns plain OCR text. +/// `opts.ocr` enables OCR for DOCX/PPTX embedded images; the `images` option +/// is ignored in plain mode. Textless PDF pages are OCR'd automatically as +/// a fallback, with or without `opts.ocr`. `Format::Image` input is always +/// OCR'd regardless of options and returns plain OCR text. /// /// # Errors /// @@ -196,15 +209,24 @@ pub fn extract_plain_with(data: &[u8], format: Format, opts: ExtractOptions) -> /// Returns [`BatdocError::Io`] or [`BatdocError::Document`] if the /// document is malformed, encrypted, or cannot be parsed. pub fn extract_markdown(data: &[u8], format: Format, images: bool) -> Result { - extract_markdown_with(data, format, ExtractOptions { images, ocr: false }) + extract_markdown_with( + data, + format, + ExtractOptions { + images, + ocr: false, + ..Default::default() + }, + ) } /// Extract Markdown with explicit options. /// /// `opts.images` embeds DOCX/XLSX/PPTX images as base64 markdown; -/// `opts.ocr` OCRs DOCX/PPTX embedded images and textless PDF pages, -/// rendered as blockquotes. `Format::Image` input is always OCR'd -/// regardless of options and returns plain OCR text (no markdown). +/// `opts.ocr` OCRs DOCX/PPTX embedded images, rendered as blockquotes. +/// Textless PDF pages are OCR'd automatically as a fallback. `Format::Image` +/// input is always OCR'd regardless of options and returns plain OCR text +/// (no markdown). /// /// # Errors /// @@ -222,6 +244,93 @@ pub fn extract_markdown_with(data: &[u8], format: Format, opts: ExtractOptions) } } +/// Extract plain text into a sink. +/// +/// When `opts.max_output_bytes` is `Some`, writing stops with +/// [`BatdocError::Document`] once that many bytes would be exceeded. +/// +/// # Errors +/// +/// Returns any error from [`extract_plain_with`], or +/// [`BatdocError::Document`] if the output budget is exceeded. +pub fn extract_plain_to( + data: &[u8], + format: Format, + opts: ExtractOptions, + sink: &mut impl ExtractSink, +) -> Result<()> { + match opts.max_output_bytes { + Some(max) => { + let mut limited = BudgetSink::new(sink, max); + write_plain(data, format, opts, &mut limited) + } + None => write_plain(data, format, opts, sink), + } +} + +fn write_plain( + data: &[u8], + format: Format, + opts: ExtractOptions, + sink: &mut impl ExtractSink, +) -> Result<()> { + match format { + Format::Xlsx => xlsx::extract_plain_to(data, sink), + Format::Xls => xls::extract_plain_to(data, sink), + Format::Docx => docx::extract_plain_to(data, opts, sink), + Format::Pptx => pptx::extract_plain_to(data, opts, sink), + Format::Doc => doc::extract_plain_to(data, sink), + Format::Pdf => pdf::extract_plain_to(data, opts, sink), + _ => { + let text = extract_plain_with(data, format, opts)?; + sink.write_str(&text) + } + } +} + +/// Extract Markdown into a sink. +/// +/// When `opts.max_output_bytes` is `Some`, writing stops with +/// [`BatdocError::Document`] once that many bytes would be exceeded. +/// +/// # Errors +/// +/// Returns any error from [`extract_markdown_with`], or +/// [`BatdocError::Document`] if the output budget is exceeded. +pub fn extract_markdown_to( + data: &[u8], + format: Format, + opts: ExtractOptions, + sink: &mut impl ExtractSink, +) -> Result<()> { + match opts.max_output_bytes { + Some(max) => { + let mut limited = BudgetSink::new(sink, max); + write_markdown(data, format, opts, &mut limited) + } + None => write_markdown(data, format, opts, sink), + } +} + +fn write_markdown( + data: &[u8], + format: Format, + opts: ExtractOptions, + sink: &mut impl ExtractSink, +) -> Result<()> { + match format { + Format::Xlsx => xlsx::extract_markdown_to(data, opts.images, sink), + Format::Xls => xls::extract_markdown_to(data, sink), + Format::Docx => docx::extract_markdown_to(data, opts, sink), + Format::Pptx => pptx::extract_markdown_to(data, opts, sink), + Format::Pdf => pdf::extract_markdown_to(data, opts, sink), + _ => { + let text = extract_markdown_with(data, format, opts)?; + sink.write_str(&text) + } + } +} + /// Convenience: detect format and extract plain text in one call. /// /// # Errors @@ -294,4 +403,20 @@ mod tests { .to_string(); assert!(err.contains("no text found in image")); } + + #[test] + fn extract_plain_to_equals_extract_plain_on_image_garbage() { + let data = b"garbage"; + let format = Format::Image; + let opts = ExtractOptions::default(); + let a = extract_plain_with(data, format, opts) + .unwrap_err() + .to_string(); + let mut out = String::new(); + let b = extract_plain_to(data, format, opts, &mut out) + .unwrap_err() + .to_string(); + assert_eq!(a, b); + assert!(out.is_empty()); + } } diff --git a/batdoc-core/src/markup.rs b/batdoc-core/src/markup.rs index b9f198d..23fad78 100644 --- a/batdoc-core/src/markup.rs +++ b/batdoc-core/src/markup.rs @@ -24,6 +24,47 @@ pub(crate) trait InlineRun { fn link_url(&self) -> Option<&str>; } +/// Escape markdown inline specials in extracted text: `\` `` ` `` `*` `_` +/// `[` `]`. Applied to run text and link text so document content cannot +/// break out of the markdown we generate. +pub(crate) fn escape_md_inline(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match c { + '\\' | '`' | '*' | '_' | '[' | ']' => { + out.push('\\'); + out.push(c); + } + _ => out.push(c), + } + } + out +} + +/// Make a URL safe to embed in `[text](url)`, or `None` to render the link +/// text without link syntax. Schemes are allowlisted to http/https/mailto/ +/// ftp (scheme-less relative/fragment URLs pass), and `(`, `)`, and space +/// are percent-encoded so they cannot terminate the destination early. +pub(crate) fn safe_link_url(url: &str) -> Option { + let trimmed = url.trim(); + let lower = trimmed.to_ascii_lowercase(); + if let Some(i) = lower.find(':') { + if !matches!(&lower[..i], "http" | "https" | "mailto" | "ftp") { + return None; + } + } + let mut out = String::with_capacity(trimmed.len()); + for c in trimmed.chars() { + match c { + '(' => out.push_str("%28"), + ')' => out.push_str("%29"), + ' ' => out.push_str("%20"), + _ => out.push(c), + } + } + Some(out) +} + /// Render a slice of runs as markdown with inline formatting and grouped /// hyperlinks. /// @@ -51,11 +92,17 @@ pub(crate) fn render_runs_markdown(runs: &[R]) -> String { } let link_text = link_text.trim(); if !link_text.is_empty() { - out.push('['); - out.push_str(link_text); - out.push_str("]("); - out.push_str(url); - out.push(')'); + match safe_link_url(url) { + Some(url) => { + out.push('['); + out.push_str(link_text); + out.push_str("]("); + out.push_str(&url); + out.push(')'); + } + // Unsafe scheme: emit the text without link syntax. + None => out.push_str(link_text), + } } continue; } @@ -82,24 +129,25 @@ pub(crate) fn format_run_inline(run: &R, out: &mut String) { return; } + let text = escape_md_inline(run.text()); match (run.bold(), run.italic()) { (true, true) => { out.push_str("***"); - out.push_str(run.text()); + out.push_str(&text); out.push_str("***"); } (true, false) => { out.push_str("**"); - out.push_str(run.text()); + out.push_str(&text); out.push_str("**"); } (false, true) => { out.push('*'); - out.push_str(run.text()); + out.push_str(&text); out.push('*'); } (false, false) => { - out.push_str(run.text()); + out.push_str(&text); } } } @@ -451,4 +499,52 @@ mod tests { push_ocr_plain(&mut out, "c", &mut first); assert_eq!(out, "a\n\nb\n\nc\n"); } + + // ── escaping / URL safety ───────────────────────────────── + + #[test] + fn escape_md_inline_escapes_specials() { + assert_eq!( + escape_md_inline("a*b_c[d]e\\f`g"), + "a\\*b\\_c\\[d\\]e\\\\f\\`g" + ); + assert_eq!(escape_md_inline("plain text"), "plain text"); + } + + #[test] + fn format_escapes_text_but_not_markers() { + let mut out = String::new(); + format_run_inline(&run("5 * 3", true, false), &mut out); + assert_eq!(out, "**5 \\* 3**"); + } + + #[test] + fn link_text_is_escaped() { + let runs = vec![link_run("a [b]", false, false, "https://example.com")]; + assert_eq!( + render_runs_markdown(&runs), + "[a \\[b\\]](https://example.com)" + ); + } + + #[test] + fn unsafe_scheme_renders_as_plain_text() { + let runs = vec![link_run("click", false, false, "javascript:alert(1)")]; + assert_eq!(render_runs_markdown(&runs), "click"); + } + + #[test] + fn url_parens_and_spaces_are_encoded() { + let runs = vec![link_run("t", false, false, "https://x.org/a (b)")]; + assert_eq!( + render_runs_markdown(&runs), + "[t](https://x.org/a%20%28b%29)" + ); + } + + #[test] + fn scheme_less_urls_are_kept() { + let runs = vec![link_run("rel", false, false, "/docs/page#frag")]; + assert_eq!(render_runs_markdown(&runs), "[rel](/docs/page#frag)"); + } } diff --git a/batdoc-core/src/ocr.rs b/batdoc-core/src/ocr.rs index 40b9350..7a9e982 100644 --- a/batdoc-core/src/ocr.rs +++ b/batdoc-core/src/ocr.rs @@ -6,9 +6,11 @@ use crate::error::{BatdocError, Result}; use ocrs::{ImageSource, OcrEngine, OcrEngineParams}; +#[cfg(feature = "net")] use std::io::Read; use std::path::{Path, PathBuf}; use std::sync::OnceLock; +#[cfg(feature = "net")] use std::time::Duration; /// Detection model (text regions). 2.4 MB. @@ -21,11 +23,14 @@ const DETECTION_MODEL_FILE: &str = "text-detection.rten"; const RECOGNITION_MODEL_FILE: &str = "text-recognition.rten"; /// Network timeout for model downloads. +#[cfg(feature = "net")] const DOWNLOAD_TIMEOUT: Duration = Duration::from_mins(2); /// Maximum accepted model download size (the real models are 2.4 MB and /// 9.3 MB; a larger response means a compromised/redirected URL). +#[cfg(feature = "net")] const MAX_MODEL_DOWNLOAD_BYTES: u64 = 64 * 1024 * 1024; /// Age after which leftover `*.tmp.` download files are swept. +#[cfg(feature = "net")] const STALE_TMP_AGE: Duration = Duration::from_hours(1); /// Resolve the model cache directory, in priority order. Injectable for testing. @@ -55,6 +60,7 @@ fn cache_dir() -> PathBuf { /// /// Writes to a `.tmp` sibling and renames, so concurrent processes cannot /// observe a partially written model. +#[cfg(feature = "net")] fn ensure_file(path: &Path, url: &str) -> Result<()> { if path.exists() { return Ok(()); @@ -107,9 +113,25 @@ fn ensure_file(path: &Path, url: &str) -> Result<()> { Ok(()) } +/// Without network support (no `net` feature), model files must already be +/// present locally — e.g. pre-seeded via `$BATDOC_MODELS_DIR` or embedded by +/// the caller. Loading and inference still work; only acquisition is gated. +#[cfg(not(feature = "net"))] +fn ensure_file(path: &Path, _url: &str) -> Result<()> { + if path.exists() { + return Ok(()); + } + Err(BatdocError::Document(format!( + "OCR model {} not found; this build has no network support — \ + seed it via BATDOC_MODELS_DIR or embed it before use", + path.display() + ))) +} + /// Remove stale `*.tmp.` download leftovers in `dir` (from interrupted /// runs). Only files older than [`STALE_TMP_AGE`] are touched, so an active /// concurrent download is never disturbed. Best-effort: errors ignored. +#[cfg(feature = "net")] fn sweep_stale_tmp_files(dir: &Path) { let Ok(entries) = std::fs::read_dir(dir) else { return; @@ -127,6 +149,7 @@ fn sweep_stale_tmp_files(dir: &Path) { } /// `true` for `*.tmp.` download leftovers older than [`STALE_TMP_AGE`]. +#[cfg(feature = "net")] fn is_stale_tmp_file(name: &str, modified: Option) -> bool { name.contains(".tmp.") && modified @@ -143,6 +166,7 @@ struct ModelPaths { /// Ensure both model files exist locally, downloading when needed. fn ensure_models() -> Result { let dir = cache_dir(); + #[cfg(feature = "net")] sweep_stale_tmp_files(&dir); let detection = dir.join(DETECTION_MODEL_FILE); let recognition = dir.join(RECOGNITION_MODEL_FILE); @@ -214,6 +238,106 @@ pub(crate) fn ocr_rgb_image(img: &image::RgbImage) -> Result> { } } +/// One recognized OCR line with its pixel-space bounding box (top-left +/// origin, y-down, same orientation as the input image). +pub(crate) struct OcrTextLine { + pub text: String, + pub x: i32, + pub y: i32, + pub width: u32, + pub height: u32, +} + +/// Minimum glyphs for an OCR line to be kept (junk filter replacing the +/// confidence gate ocrs 0.12 doesn't provide, spec §6). +const MIN_OCR_LINE_CHARS: usize = 2; +/// Minimum pixel area of an OCR line's bounding box. +const MIN_OCR_LINE_AREA: u64 = 16; + +/// One glyph and its (x, y, width, height) pixel box, as extracted from +/// an ocrs `TextChar.rect`. A plain tuple keeps the pure core testable +/// without depending on rten-imageproc types. +type GlyphRun = Vec<(char, (i32, i32, u32, u32))>; + +/// Pure core of [`ocr_rgb_image_lines`]: glyph runs → boxed, filtered +/// lines. Takes tuples so tests need no rten-imageproc types. +fn lines_to_boxes(lines: Vec) -> Vec { + let mut out = Vec::new(); + for line in lines { + let text: String = line.iter().map(|(c, _)| c).collect(); + if text.trim().chars().count() < MIN_OCR_LINE_CHARS { + continue; + } + let (mut x0, mut y0, mut x1, mut y1) = (i32::MAX, i32::MAX, i32::MIN, i32::MIN); + for (_, (x, y, w, h)) in &line { + x0 = x0.min(*x); + y0 = y0.min(*y); + // `saturating_add_unsigned` keeps w u32 without wrap semantics. + x1 = x1.max(x.saturating_add_unsigned(*w)); + y1 = y1.max(y.saturating_add_unsigned(*h)); + } + // Bounds are `>= 0` for a valid box; a defensive `0` on an + // impossible inverted box fails the area filter below. + let (w, h) = ( + u32::try_from(x1 - x0).unwrap_or_default(), + u32::try_from(y1 - y0).unwrap_or_default(), + ); + if u64::from(w) * u64::from(h) < MIN_OCR_LINE_AREA { + continue; + } + out.push(OcrTextLine { + text: text.trim().to_string(), + x: x0, + y: y0, + width: w, + height: h, + }); + } + out +} + +/// OCR an already-decoded RGB image, returning recognized lines with +/// pixel-space bounding boxes (in reading order, as `find_text_lines` +/// orders them). Empty when no text was detected. +pub(crate) fn ocr_rgb_image_lines(img: &image::RgbImage) -> Result> { + use ocrs::TextItem as _; + let source = ImageSource::from_bytes(img.as_raw(), img.dimensions()) + .map_err(|e| BatdocError::Document(format!("OCR input preparation failed: {e}")))?; + let engine = engine()?; + let input = engine + .prepare_input(source) + .map_err(|e| BatdocError::Document(format!("OCR preprocessing failed: {e}")))?; + let word_rects = engine + .detect_words(&input) + .map_err(|e| BatdocError::Document(format!("OCR failed: {e}")))?; + let line_rects = engine.find_text_lines(&input, &word_rects); + let lines = engine + .recognize_text(&input, &line_rects) + .map_err(|e| BatdocError::Document(format!("OCR failed: {e}")))?; + let glyph_runs = lines + .into_iter() + .flatten() + .map(|line| { + line.chars() + .iter() + .map(|c| { + let r = &c.rect; + ( + c.char, + ( + r.left(), + r.top(), + u32::try_from(r.width()).unwrap_or_default(), + u32::try_from(r.height()).unwrap_or_default(), + ), + ) + }) + .collect::>() + }) + .collect(); + Ok(lines_to_boxes(glyph_runs)) +} + /// Maximum width/height accepted when decoding images for OCR. Strict /// decoder-side guard (shared with PDF embedded-image decoding) so a /// crafted image whose header claims huge dimensions cannot allocate past @@ -332,6 +456,7 @@ mod tests { } #[test] + #[cfg(feature = "net")] fn stale_tmp_file_detection() { use std::time::SystemTime; // Fresh download in progress: recent mtime → keep. @@ -364,4 +489,22 @@ mod tests { assert!(models_present_in(&tmp)); std::fs::remove_dir_all(&tmp).ok(); } + + // ── ocr_rgb_image_lines / lines_to_boxes ──────────────────────── + + #[test] + fn lines_to_boxes_filters_and_bounds() { + let lines = vec![ + // (char, (x, y, w, h)) per glyph + vec![('H', (10, 20, 8, 12)), ('i', (19, 20, 4, 12))], + vec![('x', (0, 0, 5, 5))], // 1 char — too short + vec![(' ', (0, 0, 2, 2)), (' ', (3, 0, 2, 2))], // whitespace-only + ]; + let out = super::lines_to_boxes(lines); + assert_eq!(out.len(), 1); + assert_eq!(out[0].text, "Hi"); + assert_eq!((out[0].x, out[0].y), (10, 20)); + assert_eq!(out[0].width, 13); // union of glyph boxes (10..18)+(19..23) + assert_eq!(out[0].height, 12); + } } diff --git a/batdoc-core/src/pdf.rs b/batdoc-core/src/pdf.rs index c17337d..08e68f0 100644 --- a/batdoc-core/src/pdf.rs +++ b/batdoc-core/src/pdf.rs @@ -6,11 +6,8 @@ //! [`BatdocError::Document`] errors. use crate::error::{BatdocError, Result}; -use crate::ocr::MAX_OCR_IMAGE_DIM; use crate::ExtractOptions; -use lopdf::xobject::PdfImage; -use std::fmt::Write as _; -use std::io::Cursor; +use crate::ExtractSink; use std::panic::{self, AssertUnwindSafe}; /// Extract pages of text from a PDF byte slice, returning one `String` per @@ -18,9 +15,8 @@ use std::panic::{self, AssertUnwindSafe}; /// /// Panics from the underlying library are caught and converted to errors. fn extract_pages(data: &[u8]) -> Result> { - let data = data.to_vec(); // owned copy for the unwind boundary let result = panic::catch_unwind(AssertUnwindSafe(|| { - pdf_extract::extract_text_from_mem_by_pages(&data) + pdf_extract::extract_text_from_mem_by_pages(data) })); match result { Ok(Ok(pages)) => Ok(pages), @@ -63,50 +59,6 @@ fn clean_page(raw: &str) -> String { } } -/// Maximum number of embedded images OCR'd per page (largest first). -const MAX_OCR_IMAGES_PER_PAGE: usize = 4; -/// Maximum decoded pixels per embedded image (~100 MP ≈ 300 MB RGB). -/// Larger images are skipped so a single image cannot exhaust memory. -const MAX_OCR_IMAGE_PIXELS: u64 = 100_000_000; - -/// Select up to [`MAX_OCR_IMAGES_PER_PAGE`] OCR candidates, largest by -/// declared pixel area first, skipping images whose decoded size would -/// exceed [`MAX_OCR_IMAGE_PIXELS`]. The cap must bound peak memory, which -/// the declared dimensions determine (compressed images expand when decoded). -fn ocr_candidates<'a>(images: &'a [PdfImage<'a>]) -> Vec<&'a PdfImage<'a>> { - let mut candidates: Vec<(&PdfImage, u64)> = images - .iter() - .map(|img| { - ( - img, - u64::try_from(img.width.max(0)) - .unwrap_or(0) - .saturating_mul(u64::try_from(img.height.max(0)).unwrap_or(0)), - ) - }) - .filter(|(_, area)| *area > 0 && *area <= MAX_OCR_IMAGE_PIXELS) - .collect(); - candidates.sort_by_key(|(_, area)| std::cmp::Reverse(*area)); - candidates - .into_iter() - .take(MAX_OCR_IMAGES_PER_PAGE) - .map(|(img, _)| img) - .collect() -} - -/// Decode JPEG bytes with strict dimension limits, so a stream whose real -/// dimensions exceed the PDF dictionary's claim cannot allocate beyond the -/// OCR pixel budget. -fn decode_jpeg_bounded(bytes: &[u8]) -> Option { - let mut reader = image::ImageReader::new(Cursor::new(bytes)); - reader.set_format(image::ImageFormat::Jpeg); - let mut limits = image::Limits::default(); - limits.max_image_width = Some(MAX_OCR_IMAGE_DIM); - limits.max_image_height = Some(MAX_OCR_IMAGE_DIM); - reader.limits(limits); - reader.decode().ok().map(image::DynamicImage::into_rgb8) -} - /// Produce the per-page text, OCR'ing empty pages when `ocr` is set. fn extract_pages_with_ocr(data: &[u8], ocr: bool) -> Result> { let pages = extract_pages(data)?; @@ -134,6 +86,23 @@ fn extract_pages_with_ocr(data: &[u8], ocr: bool) -> Result> { Ok(out) } +/// Extract per-page text, transparently falling back to OCR when the user did +/// not pass `--ocr` but the document has no text layer at all. +/// +/// Returns the per-page text and whether OCR was actually performed (which +/// drives the wording of the no-text error if OCR also finds nothing). +fn extract_pages_with_fallback(data: &[u8], ocr: bool) -> Result<(Vec, bool)> { + let pages = extract_pages_with_ocr(data, ocr)?; + if ocr || pages.iter().any(|p| !p.is_empty()) { + return Ok((pages, ocr)); + } + // Every page is textless and OCR wasn't requested: this is a scan, so + // retry with OCR. A textless-but-empty document (no images) costs only a + // re-parse here; `ocr_page` finds nothing and reports it at the call site. + let ocr_pages = extract_pages_with_ocr(data, true)?; + Ok((ocr_pages, true)) +} + /// OCR the embedded images of one page (0-based index into `page_ids`). /// `None` when the page has no OCR-able images or no text was detected. /// @@ -153,8 +122,8 @@ fn ocr_page( return Ok(None); }; let mut texts = Vec::new(); - for img in ocr_candidates(&images) { - let Some(rgb) = decode_pdf_image(img) else { + for img in crate::pdf_ocr::ocr_candidates(&images) { + let Some(rgb) = crate::pdf_ocr::decode_pdf_image(img) else { continue; }; if let Some(text) = crate::ocr::ocr_rgb_image(&rgb)? { @@ -168,52 +137,6 @@ fn ocr_page( } } -/// Decode a `lopdf::PdfImage` into an RGB image usable by the OCR engine. -/// -/// Handles `DCTDecode` (JPEG bytes) and raw/`FlateDecode` 8-bit `DeviceGray`/`DeviceRGB`. -/// Returns `None` for other encodings (`CCITT`, `JPX`, `JBIG2`, `Indexed`, `CMYK`). -fn decode_pdf_image(img: &PdfImage<'_>) -> Option { - let filters: Vec<&str> = img.filters.iter().flatten().map(String::as_str).collect(); - if filters.contains(&"DCTDecode") { - return decode_jpeg_bounded(img.content); - } - if filters.iter().any(|f| *f != "FlateDecode") { - return None; // unknown encoding - } - let width = u32::try_from(img.width).ok()?; - let height = u32::try_from(img.height).ok()?; - if width == 0 || height == 0 { - return None; - } - let content = if filters.contains(&"FlateDecode") { - let mut stream = lopdf::Stream::new(img.origin_dict.clone(), img.content.to_vec()); - stream.decompress().ok()?; - stream.content - } else { - img.content.to_vec() - }; - // Validate the exact buffer length so `ImageSource::from_bytes` never - // receives a mis-sized buffer (it errors on non-exact lengths). - let channels: u64 = match ( - img.color_space.as_deref(), - img.bits_per_component.unwrap_or(8), - ) { - (Some("DeviceRGB"), 8) => 3, - (Some("DeviceGray"), 8) => 1, - _ => return None, - }; - let expected = u64::from(width) * u64::from(height) * channels; - if u64::try_from(content.len()).ok() != Some(expected) { - return None; - } - if channels == 3 { - image::RgbImage::from_raw(width, height, content) - } else { - let gray = image::GrayImage::from_raw(width, height, content)?; - Some(image::DynamicImage::ImageLuma8(gray).into_rgb8()) - } -} - fn no_text_error(ocr: bool) -> BatdocError { let message = if ocr { "PDF contains no extractable text (no text layer; OCR found nothing)" @@ -225,62 +148,268 @@ fn no_text_error(ocr: bool) -> BatdocError { /// Extract plain text from a PDF. pub(crate) fn extract_plain(data: &[u8], opts: ExtractOptions) -> Result { - let pages = extract_pages_with_ocr(data, opts.ocr)?; - let nonempty: Vec<&str> = pages - .iter() - .map(String::as_str) - .filter(|s| !s.is_empty()) - .collect(); + let mut out = String::new(); + extract_plain_to(data, opts, &mut out)?; + Ok(out) +} + +/// Stream plain text from a PDF into `sink`, one cleaned page at a time. +/// +/// Non-empty pages are joined with `\n` (no trailing newline); if no page has +/// text, [`no_text_error`] is returned with the OCR-attempt wording. +pub(crate) fn extract_plain_to( + data: &[u8], + opts: ExtractOptions, + sink: &mut impl ExtractSink, +) -> Result<()> { + let (pages, ocr_attempted) = extract_pages_with_fallback(data, opts.ocr)?; - if nonempty.is_empty() { - return Err(no_text_error(opts.ocr)); + let mut first = true; + let mut wrote = false; + for page in pages { + if page.is_empty() { + continue; + } + if !first { + sink.write_str("\n")?; + } + sink.write_str(&page)?; + first = false; + wrote = true; } - Ok(nonempty.join("\n")) + if !wrote { + return Err(no_text_error(ocr_attempted)); + } + Ok(()) } /// Extract markdown from a PDF. /// -/// Each page gets a `## Page N` heading. Single-page documents omit the -/// heading since it would be redundant. +/// Positioned layout pipeline: pages are classified into headings and +/// paragraphs (see [`extract_markdown_to`]). Multiple emitted pages get a +/// `## Page N` heading; a single emitted page omits it as redundant. pub(crate) fn extract_markdown(data: &[u8], opts: ExtractOptions) -> Result { - let pages = extract_pages_with_ocr(data, opts.ocr)?; - let nonempty: Vec<(usize, &str)> = pages - .iter() - .enumerate() - .filter_map(|(i, s)| { - if s.is_empty() { - None - } else { - Some((i + 1, s.as_str())) + let mut out = String::new(); + extract_markdown_to(data, opts, &mut out)?; + Ok(out) +} + +/// Result of rendering all pages with a given set of document signals. +struct RenderedPages { + emitted: Vec<(u32, String)>, + ocr_attempted: bool, + had_native_lines: bool, +} + +/// Render every page into a markdown buffer using the supplied `signals`. +/// Returns the emitted pages, whether OCR was attempted, and whether any +/// page had non-empty native assembled lines (the safety-net trigger). +fn render_pages( + doc: &lopdf::Document, + page_ids: &[lopdf::ObjectId], + page_count: u32, + signals: &crate::pdf_layout::DocSignals, + opts: ExtractOptions, +) -> Result { + let mut emitted: Vec<(u32, String)> = Vec::new(); + let mut ocr_attempted = opts.ocr; + let mut had_native_lines = false; + for (page_num, page_id) in (1..=page_count).zip(page_ids) { + let page = crate::pdf_text::extract_positioned_page(doc, page_num)?; + let garbled = page_looks_garbled(&page); + let mut native = crate::pdf_layout::assemble(&page); + if !native.is_empty() { + had_native_lines = true; + } + if garbled { + native.clear(); // garbage native layer: OCR replaces it + } + // OCR when asked, when the page assembled no non-empty lines + // (auto-fallback, mirroring `extract_pages_with_fallback`), or when + // the native layer is garbage. + let need_ocr = opts.ocr || garbled || native.iter().all(|l| l.text.trim().is_empty()); + let mut ocr_lines = Vec::new(); + let mut unplaced_text = String::new(); + if need_ocr { + ocr_attempted = true; + let placed = crate::pdf_geometry::placed_images(doc, *page_id); + if let Ok(images) = doc.get_page_images(*page_id) { + for img in crate::pdf_ocr::ocr_candidates(&images) { + let Some(rgb) = crate::pdf_ocr::decode_pdf_image(img) else { + continue; + }; + let lines = crate::ocr::ocr_rgb_image_lines(&rgb)?; + if lines.is_empty() { + continue; + } + match placed.iter().find(|p| p.object_id == img.id) { + Some(p) => ocr_lines.extend(crate::pdf_ocr::map_ocr_lines( + p, + rgb.width(), + rgb.height(), + &lines, + )), + None => { + // Inline (BI/EI) or otherwise unplaceable image: + // page-end append (today's fallback, spec §4.1). + for l in &lines { + unplaced_text.push_str(&l.text); + unplaced_text.push('\n'); + } + } + } + } + } + } + let merged = crate::pdf_ocr::merge(native, ocr_lines); + // Tables are detected on the assembly-order stream (row fragments + // are adjacent there), then regions join the xy-cut as opaque + // full-width bands. + let items = crate::pdf_layout::find_tables(&merged, signals); + let ordered = crate::pdf_layout::reading_order_items(items); + let blocks = crate::pdf_layout::classify_items(ordered, signals); + let mut page_md = String::new(); + crate::pdf_layout::render(&blocks, &mut page_md)?; + if !unplaced_text.trim().is_empty() { + if !page_md.is_empty() { + page_md.push('\n'); + } + page_md.push_str(unplaced_text.trim_end()); + page_md.push('\n'); + } + if !page_md.trim().is_empty() { + emitted.push((page.page_num, page_md)); + } + } + Ok(RenderedPages { + emitted, + ocr_attempted, + had_native_lines, + }) +} + +/// Stream markdown from a PDF into `sink` via the two-pass positioned-layout +/// driver (spec §4.2): +/// +/// - **Pass 1** collects document-wide signals (modal body size, repeated +/// first/last-line signatures); each page is dropped immediately. +/// - **Pass 2** keeps one positioned page in memory at a time: assemble → +/// garbled-page detection → OCR merge (region-aware when the image has a +/// placement, page-end append otherwise) → reading order → classify → +/// render into a page-local buffer. +/// +/// Pages whose rendered text is empty/whitespace are skipped. A single +/// emitted page is written bare; multiple pages each get a +/// `## Page N\n\n` heading (N = ORIGINAL 1-based page number, not the +/// filtered index) with `\n` between pages. If nothing is emitted, +/// [`no_text_error`] is returned with the OCR-attempt wording. +pub(crate) fn extract_markdown_to( + data: &[u8], + opts: ExtractOptions, + sink: &mut impl ExtractSink, +) -> Result<()> { + let mut doc = match lopdf::Document::load_mem(data) { + Ok(d) => d, + Err(e) => { + return Err(BatdocError::Document(format!("PDF extraction failed: {e}"))); + } + }; + if doc.is_encrypted() { + // Mirror pdf-extract's `maybe_decrypt`: empty-password attempt, + // and the same error string the plain path produces on failure. + if let Err(e) = doc.decrypt("") { + use lopdf::encryption::DecryptionError; + if matches!( + e, + lopdf::Error::Decryption(DecryptionError::IncorrectPassword) + ) { + return Err(BatdocError::Document(format!( + "PDF extraction failed: {}", + pdf_extract::OutputError::PdfError(e) + ))); } - }) - .collect(); + // Other decrypt failures: log-and-continue, per the plan ruling. + } + } + let page_ids: Vec = doc.get_pages().into_values().collect(); + // Absurd-page-count guard: > u32::MAX pages cannot be numbered anyway. + let page_count = u32::try_from(page_ids.len()).unwrap_or(u32::MAX); - if nonempty.is_empty() { - return Err(no_text_error(opts.ocr)); + // Pass 1: doc-level signals (tiny aggregates; pages dropped immediately). + let mut sig_builder = crate::pdf_layout::DocSignalsBuilder::new(); + for page_num in 1..=page_count { + let page = crate::pdf_text::extract_positioned_page(&doc, page_num)?; + sig_builder.add_lines(&crate::pdf_layout::assemble(&page)); } + let signals = sig_builder.finish(page_ids.len()); - let mut out = String::new(); + // Pass 2: one positioned page in memory at a time. Rendered per-page + // markdown is buffered (bounded by output size, same memory profile as + // today's Vec of all page text; the single-page heading rule + // requires not emitting page 1 before knowing whether page 2 exists). + let rendered = render_pages(&doc, &page_ids, page_count, &signals, opts)?; + let mut emitted = rendered.emitted; - if nonempty.len() == 1 { + // Safety net: if furniture stripping dropped every native line, re-run + // pass 2 with empty header/footer sets (body_size preserved) so real + // documents whose only text was misclassified as furniture still emit. + if emitted.is_empty() && rendered.had_native_lines { + let fallback_signals = crate::pdf_layout::DocSignals { + body_size: signals.body_size, + headers: std::collections::HashSet::new(), + footers: std::collections::HashSet::new(), + }; + let fallback = render_pages(&doc, &page_ids, page_count, &fallback_signals, opts)?; + if !fallback.emitted.is_empty() { + emitted = fallback.emitted; + } + } + + if emitted.is_empty() { + return Err(no_text_error(rendered.ocr_attempted)); + } + if emitted.len() == 1 { // Single page — no heading needed - out.push_str(nonempty[0].1); + sink.write_str(&emitted[0].1)?; } else { - for (i, (page_num, text)) in nonempty.iter().enumerate() { + for (i, (page_num, page_md)) in emitted.iter().enumerate() { if i > 0 { - out.push('\n'); + sink.write_str("\n")?; } - let _ = write!(out, "## Page {page_num}\n\n"); - out.push_str(text); + sink.write_str(&format!("## Page {page_num}\n\n"))?; + sink.write_str(page_md)?; } } + Ok(()) +} - Ok(out) +/// ≥10% replacement/PUA chars means the native text layer is garbage +/// (matches the fork's recovery-flip threshold, spec §5). +/// Driver-level garble check: a page is garbled when ≥10% of its chars are +/// U+FFFD or PUA (U+E000–U+F8FF). +/// +/// Deliberately has no 20-character minimum sample floor, unlike the fork's +/// font-level prescan (`FontRecovery::observe` flips only after ≥20 decoded +/// chars): a very short page of pure garbage must still trip the check and +/// fall back to OCR. The flip side — partial-page garble goes undetected — +/// is a documented v1 limitation (plan ruling #6). +fn page_looks_garbled(page: &crate::pdf_text::PositionedPage) -> bool { + let total = page.chars.len(); + if total == 0 { + return false; + } + let bad = page + .chars + .iter() + .filter(|c| c.ch == '\u{FFFD}' || ('\u{E000}'..='\u{F8FF}').contains(&c.ch)) + .count(); + bad * 10 >= total } #[cfg(test)] mod tests { + use super::*; #[test] @@ -335,7 +464,7 @@ mod tests { } #[test] - fn extract_plain_ocr_flag_errors_differ() { + fn extract_plain_textless_pdf_auto_ocrs_and_reports() { // A structurally valid PDF with no page tree (so text extraction succeeds // but yields zero pages); large enough for `%%EOF`/`startxref` detection. let data = b"%PDF-1.4\n\ @@ -355,156 +484,578 @@ trailer\n\ startxref\n\ 110\n\ %%EOF\n"; - let plain_err = extract_plain(data, crate::ExtractOptions::default()) - .unwrap_err() - .to_string(); - assert!(plain_err.contains("scanned/image-only")); - assert!(!plain_err.contains("OCR")); - let ocr_err = extract_plain( - data, + // The PDF falls back to OCR even without --ocr, so a textless-but-empty + // document reports the OCR-failed message in both the explicit and the + // implicit (auto-fallback) OCR cases. + for opts in [ + crate::ExtractOptions::default(), crate::ExtractOptions { ocr: true, ..crate::ExtractOptions::default() }, - ) - .unwrap_err() - .to_string(); - assert!(ocr_err.contains("OCR found nothing")); + ] { + let err = extract_plain(data, opts).unwrap_err().to_string(); + assert!(err.contains("OCR found nothing"), "got: {err}"); + } } #[test] - fn ocr_candidates_caps_and_ranks() { - let dict = lopdf::Dictionary::new(); - let mk = |width: i64, height: i64| PdfImage { - id: (1, 0), - width, - height, - color_space: Some("DeviceRGB".to_string()), - filters: Some(Vec::new()), - bits_per_component: Some(8), - content: &[], - origin_dict: &dict, - }; - let images = vec![ - mk(100, 100), // 10k px - mk(20_000, 20_000), // 400 MP — over budget, excluded - mk(1000, 1000), // 1 MP — largest valid - mk(500, 500), - mk(200, 200), - mk(50, 50), - mk(0, 100), // zero area — excluded - ]; - let picked = ocr_candidates(&images); - // 5 valid images → capped at 4, largest first; oversized/zero excluded. - assert_eq!(picked.len(), 4); - assert_eq!((picked[0].width, picked[0].height), (1000, 1000)); - assert_eq!((picked[1].width, picked[1].height), (500, 500)); - assert_eq!((picked[2].width, picked[2].height), (200, 200)); - assert_eq!((picked[3].width, picked[3].height), (100, 100)); + fn pdf_extract_pages_does_not_require_owned_copy() { + // Behavior lock: empty/malformed PDF error wording unchanged. + let err = crate::extract_plain(b"%PDF-not-really", crate::Format::Pdf) + .unwrap_err() + .to_string(); + assert!( + err.contains("PDF") + || err.contains("extract") + || err.contains("malformed") + || err.contains("no extractable"), + "unexpected error wording: {err}" + ); } #[test] - fn decode_pdf_image_raw_rgb() { - let mut dict = lopdf::Dictionary::new(); - dict.set("Subtype", lopdf::Object::Name(b"Image".to_vec())); - let img = PdfImage { - id: (1, 0), - width: 2, - height: 2, - color_space: Some("DeviceRGB".to_string()), - filters: Some(Vec::new()), - bits_per_component: Some(8), - content: &[255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255], - origin_dict: &dict, - }; - let rgb = decode_pdf_image(&img).unwrap(); - assert_eq!(rgb.dimensions(), (2, 2)); - assert_eq!(rgb.get_pixel(0, 0), &image::Rgb([255, 0, 0])); - assert_eq!(rgb.get_pixel(1, 1), &image::Rgb([255, 255, 255])); + fn extract_plain_to_matches_extract_plain_error() { + for data in [&b"not a pdf at all"[..], &b"%PDF-1.4\n%%EOF\n"[..]] { + let expected = extract_plain(data, crate::ExtractOptions::default()) + .unwrap_err() + .to_string(); + let mut out = String::new(); + let actual = extract_plain_to(data, crate::ExtractOptions::default(), &mut out) + .unwrap_err() + .to_string(); + assert_eq!(actual, expected); + assert!(out.is_empty()); + } } #[test] - fn decode_pdf_image_flate_gray() { - let mut dict = lopdf::Dictionary::new(); - dict.set("Subtype", lopdf::Object::Name(b"Image".to_vec())); - dict.set("Filter", lopdf::Object::Name(b"FlateDecode".to_vec())); - // zlib-compressed `[0, 255, 128, 64]` (stored deflate block + adler-32). - // Inlined because `Stream::compress()` will not compress a 4-byte payload. - let compressed = [ - 0x78, 0x9C, 0x01, 0x04, 0x00, 0xFB, 0xFF, 0x00, 0xFF, 0x80, 0x40, 0x04, 0x41, 0x01, - 0xC0, + fn extract_markdown_to_matches_extract_markdown_error() { + for data in [&b"not a pdf at all"[..], &b"%PDF-1.4\n%%EOF\n"[..]] { + let expected = extract_markdown(data, crate::ExtractOptions::default()) + .unwrap_err() + .to_string(); + let mut out = String::new(); + let actual = extract_markdown_to(data, crate::ExtractOptions::default(), &mut out) + .unwrap_err() + .to_string(); + assert_eq!(actual, expected); + assert!(out.is_empty()); + } + } + + #[test] + fn render_blocks_blank_line_separated() { + let blocks = vec![ + crate::pdf_layout::Block::Heading { + level: 2, + text: "Title".into(), + }, + crate::pdf_layout::Block::Paragraph("Body text.".into()), ]; - let mut origin = lopdf::Dictionary::new(); - origin.set("Width", 2); - origin.set("Height", 2); - origin.set("ColorSpace", lopdf::Object::Name(b"DeviceGray".to_vec())); - origin.set("BitsPerComponent", 8); - origin.set("Filter", lopdf::Object::Name(b"FlateDecode".to_vec())); - let img = PdfImage { - id: (1, 0), - width: 2, - height: 2, - color_space: Some("DeviceGray".to_string()), - filters: Some(vec!["FlateDecode".to_string()]), - bits_per_component: Some(8), - content: &compressed, - origin_dict: &origin, - }; - let rgb = decode_pdf_image(&img).unwrap(); - assert_eq!(rgb.dimensions(), (2, 2)); - assert_eq!(rgb.get_pixel(1, 0), &image::Rgb([255, 255, 255])); // Luma 255 - assert_eq!(rgb.get_pixel(0, 1), &image::Rgb([128, 128, 128])); // Luma 128 + let mut out = String::new(); + crate::pdf_layout::render(&blocks, &mut out).unwrap(); + assert_eq!(out, "## Title\n\nBody text.\n"); } #[test] - fn decode_pdf_image_overlong_buffer_skipped() { - let mut dict = lopdf::Dictionary::new(); - dict.set("Subtype", lopdf::Object::Name(b"Image".to_vec())); - let img = PdfImage { - id: (1, 0), - width: 2, - height: 2, - color_space: Some("DeviceRGB".to_string()), - filters: Some(Vec::new()), - bits_per_component: Some(8), - content: &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 15 bytes ≠ 12 - origin_dict: &dict, + fn page_looks_garbled_threshold() { + use crate::pdf_text::{PositionedChar, PositionedPage}; + let mk = |bad: usize, total: usize| { + let chars = (0..total) + .map(|i| PositionedChar { + ch: if i < bad { '\u{FFFD}' } else { 'a' }, + x: 0.0, + y: 0.0, + font_size: 12.0, + advance: 6.0, + rotation: 0, + }) + .collect(); + PositionedPage { + page_num: 1, + media_box: (0.0, 0.0, 612.0, 792.0), + chars, + } }; - assert!(decode_pdf_image(&img).is_none()); + assert!(page_looks_garbled(&mk(2, 10))); // 20% bad ≥ 10% + assert!(!page_looks_garbled(&mk(1, 20))); // 5% bad + assert!(!page_looks_garbled(&mk(0, 0))); // empty page is not garbled } #[test] - fn unknown_pdf_image_encoding_is_skipped() { - let mut dict = lopdf::Dictionary::new(); - dict.set("Subtype", lopdf::Object::Name(b"Image".to_vec())); - let img = PdfImage { - id: (1, 0), - width: 2, - height: 2, - color_space: Some("DeviceCMYK".to_string()), - filters: Some(vec!["DCTDecode".to_string()]), - bits_per_component: Some(8), - content: &[0xFF, 0xD8, 0xFF, 0x00], // JPEG magic but not decodable - origin_dict: &dict, - }; - assert!(decode_pdf_image(&img).is_none()); + fn markdown_single_page_omits_page_heading() { + let data = build_text_pdf(&["Hello World"]); + let md = extract_markdown(&data, crate::ExtractOptions::default()).unwrap(); + assert_eq!(md, "Hello World\n"); + assert!(!md.contains("## Page")); + } + + #[test] + fn markdown_multi_page_keeps_page_headings() { + let data = build_text_pdf(&["PageOne", "", "PageThree"]); + let md = extract_markdown(&data, crate::ExtractOptions::default()).unwrap(); + assert!( + md.contains("## Page 1") && md.contains("## Page 3") && !md.contains("## Page 2"), + "unexpected markdown: {md:?}" + ); + assert!(md.contains("PageOne") && md.contains("PageThree")); } #[test] - fn decode_jpeg_bounded_enforces_dim_limits() { - let encode = |w: u32, h: u32| { - let img = image::RgbImage::from_pixel(w, h, image::Rgb([128, 128, 128])); - let mut buf = std::io::Cursor::new(Vec::new()); - image::DynamicImage::ImageRgb8(img) - .write_to(&mut buf, image::ImageFormat::Jpeg) - .unwrap(); - buf.into_inner() + fn markdown_textless_pdf_reports_no_text_error() { + // For the zero-page fixture, the markdown path emits the no-text + // error (not the OCR wording the plain path uses after auto-OCR). + let data = b"%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\nxref\n0 3\n0000000000 65535 f \n0000000009 00000 n \n0000000058 00000 n \ntrailer\n<< /Size 3 /Root 1 0 R >>\nstartxref\n110\n%%EOF\n"; + let err = extract_markdown(data, crate::ExtractOptions::default()) + .unwrap_err() + .to_string(); + assert!(err.contains("no extractable text"), "got: {err}"); + } + + #[test] + fn markdown_heading_from_font_size_e2e() { + // 24pt title (short) then 12pt body (longer, so the modal size is + // 12 and 24/12 = 2.0 → H1) — from a real PDF, end to end. + let data = build_text_pdf_content( + "BT /F1 24 Tf 72 700 Td (Big) Tj ET\nBT /F1 12 Tf 72 660 Td (some body text goes here) Tj ET", + ); + let md = extract_markdown(&data, crate::ExtractOptions::default()).unwrap(); + assert_eq!(md, "# Big\n\nsome body text goes here\n", "got: {md:?}"); + } + + #[test] + fn markdown_two_column_reading_order_e2e() { + let data = build_text_pdf_content( + "BT /F1 12 Tf 72 700 Td (L1) Tj ET\nBT /F1 12 Tf 320 700 Td (R1) Tj ET\nBT /F1 12 Tf 72 680 Td (L2) Tj ET", + ); + let md = extract_markdown(&data, crate::ExtractOptions::default()).unwrap(); + let l1 = md.find("L1").unwrap(); + let l2 = md.find("L2").unwrap(); + let r1 = md.find("R1").unwrap(); + assert!(l1 < l2 && l2 < r1, "reading order wrong: {md:?}"); + } + + #[test] + fn narrow_gutter_columns_read_in_column_order() { + // The sharp version of the fused-gutter characterization: two + // rows per column at a 3em-ish gutter. Assembly splits the rows + // (gap > 2.5x font size) and xy-cut orders column by column — + // "L1 R1 L2 R2" interleaving would mean the gutter fused. + let data = build_text_pdf_content( + "BT /F1 12 Tf 72 700 Td (L1) Tj ET\n\ + BT /F1 12 Tf 180 700 Td (R1) Tj ET\n\ + BT /F1 12 Tf 72 680 Td (L2) Tj ET\n\ + BT /F1 12 Tf 180 680 Td (R2) Tj ET", + ); + let md = extract_markdown(&data, crate::ExtractOptions::default()).unwrap(); + let (l1, l2, r1, r2) = ( + md.find("L1").unwrap(), + md.find("L2").unwrap(), + md.find("R1").unwrap(), + md.find("R2").unwrap(), + ); + assert!(l1 < l2 && l2 < r1 && r1 < r2, "column order wrong: {md:?}"); + } + #[test] + fn furniture_stripping_safety_net_keeps_body_text() { + // Without the safety net, the repeated first line is a header and the + // repeated last line is a footer, so both are stripped and the output + // is empty ("no extractable text"). The net re-runs with empty + // header/footer sets and emits the body text. + let content = + "BT /F1 12 Tf 72 720 Td (Report page) Tj ET\nBT /F1 12 Tf 72 700 Td (Body text here) Tj ET"; + let data = build_text_pdf_pages(&[content, content]); + let md = extract_markdown(&data, crate::ExtractOptions::default()).unwrap(); + assert!(md.contains("Body text here"), "safety net failed: {md:?}"); + assert!(md.contains("Report page"), "safety net failed: {md:?}"); + } + + /// Build a minimal one-or-more page PDF with a WinAnsi Helvetica text layer, + /// so `pdf_extract` returns non-empty per-page text. + fn build_text_pdf(page_texts: &[&str]) -> Vec { + use lopdf::{dictionary, Document as LopdfDoc, Object, Stream}; + + let mut doc = LopdfDoc::with_version("1.4"); + let pages_id = doc.new_object_id(); + + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + "Encoding" => "WinAnsiEncoding", + }); + let resources_id = doc.add_object(dictionary! { + "Font" => dictionary! { "F1" => font_id }, + }); + + let mut kids = Vec::new(); + for text in page_texts { + let content = format!("BT /F1 12 Tf 72 720 Td ({text}) Tj ET"); + let content_id = + doc.add_object(Stream::new(lopdf::Dictionary::new(), content.into_bytes())); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + kids.push(Object::Reference(page_id)); + } + + let pages = dictionary! { + "Type" => "Pages", + "Kids" => kids, + "Count" => page_texts.len() as i64, }; - // Within limits: decodes. - let ok = encode(64, 64); - assert_eq!(decode_jpeg_bounded(&ok).unwrap().dimensions(), (64, 64)); - // Width beyond MAX_OCR_IMAGE_DIM: rejected by the strict dim limit. - let wide = encode(MAX_OCR_IMAGE_DIM + 1, 2); - assert!(decode_jpeg_bounded(&wide).is_none()); + doc.objects.insert(pages_id, Object::Dictionary(pages)); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + buf + } + + #[test] + fn extract_plain_to_matches_extract_plain_multipage() { + let data = build_text_pdf(&["PageOne", "PageTwo"]); + let expected = extract_plain(&data, crate::ExtractOptions::default()).unwrap(); + assert!( + expected.contains("PageOne"), + "no extracted text: {expected:?}" + ); + let mut out = String::new(); + extract_plain_to(&data, crate::ExtractOptions::default(), &mut out).unwrap(); + assert_eq!(out, expected); + } + + #[test] + fn extract_markdown_to_matches_extract_markdown_multipage() { + let data = build_text_pdf(&["PageOne", "", "PageThree"]); + let expected = extract_markdown(&data, crate::ExtractOptions::default()).unwrap(); + assert!( + expected.contains("## Page 1") && expected.contains("## Page 3"), + "unexpected markdown: {expected:?}" + ); + let mut out = String::new(); + extract_markdown_to(&data, crate::ExtractOptions::default(), &mut out).unwrap(); + assert_eq!(out, expected); + } + + /// Golden byte-stability corpus (spec D2): plain output for these + /// synthetic clean documents must never change, across the pdf-extract + /// fork and all later PDF work. Garbled-document output may change; + /// clean-document output may not. + #[test] + fn golden_plain_output_clean_pdfs() { + let cases: Vec<(Vec, &str)> = vec![ + (build_text_pdf(&["PageOne"]), "PageOne\n"), + ( + build_text_pdf(&["PageOne", "PageTwo"]), + "PageOne\n\nPageTwo\n", + ), + ( + build_text_pdf(&["Hello World", "", "PageThree"]), + "Hello World\n\nPageThree\n", + ), + ( + build_text_pdf_content("BT /F1 12 Tf 72 720 Td (caf\\351) Tj ET"), + "café\n", + ), + ( + build_text_pdf_content("BT /F1 12 Tf 72 720 Td [(A) -120 (B)] TJ ET"), + "A B\n", + ), + ]; + for (data, expected) in cases { + let actual = crate::extract_plain(&data, crate::Format::Pdf).unwrap(); + assert_eq!(actual, expected); + } + } + + /// Like `build_text_pdf`, but with a caller-supplied content stream + /// (for escapes/TJ arrays the simple builder can't express). + fn build_text_pdf_content(content: &str) -> Vec { + use lopdf::{dictionary, Document as LopdfDoc, Object, Stream}; + let mut doc = LopdfDoc::with_version("1.4"); + let pages_id = doc.new_object_id(); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + "Encoding" => "WinAnsiEncoding", + }); + let resources_id = doc.add_object(dictionary! { + "Font" => dictionary! { "F1" => font_id }, + }); + let content_id = doc.add_object(Stream::new( + lopdf::Dictionary::new(), + content.as_bytes().to_vec(), + )); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => 1_i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + buf + } + + /// Like `build_text_pdf_content`, but one caller-supplied content stream + /// per page (for multi-page fixtures with custom layout per page). + fn build_text_pdf_pages(contents: &[&str]) -> Vec { + use lopdf::{dictionary, Document as LopdfDoc, Object, Stream}; + let mut doc = LopdfDoc::with_version("1.4"); + let pages_id = doc.new_object_id(); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + "Encoding" => "WinAnsiEncoding", + }); + let resources_id = doc.add_object(dictionary! { + "Font" => dictionary! { "F1" => font_id }, + }); + let mut kids = Vec::new(); + for content in contents { + let content_id = doc.add_object(Stream::new( + lopdf::Dictionary::new(), + content.as_bytes().to_vec(), + )); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + kids.push(Object::Reference(page_id)); + } + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => kids, + "Count" => contents.len() as i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + buf + } + + #[test] + fn markdown_table_e2e() { + // Three aligned rows at fixed x positions → pipe table in output. + let data = build_text_pdf_content( + "BT /F1 12 Tf 72 700 Td (Name) Tj ET\n\ + BT /F1 12 Tf 200 700 Td (Age) Tj ET\n\ + BT /F1 12 Tf 72 680 Td (Alice) Tj ET\n\ + BT /F1 12 Tf 200 680 Td (30) Tj ET\n\ + BT /F1 12 Tf 72 660 Td (Bob) Tj ET\n\ + BT /F1 12 Tf 200 660 Td (25) Tj ET", + ); + let md = extract_markdown(&data, crate::ExtractOptions::default()).unwrap(); + assert!(md.contains("| Name | Age |"), "got: {md:?}"); + assert!(md.contains("| Alice | 30 |"), "got: {md:?}"); + assert!(md.contains("| --- | --- |"), "got: {md:?}"); + } + + #[test] + #[ignore = "benchmark fixture generator — writes /tmp/batdoc-bench.pdf"] + fn make_benchmark_pdf() { + use lopdf::{dictionary, Document as LopdfDoc, Object, Stream}; + // 100+ words; the first/last line on each page rotates through this + // list so no single normalized signature repeats enough times to be + // classified as a header/footer (digits would normalize away, so we + // use letter words). + let words: &[&str] = &[ + "alpha", + "bravo", + "charlie", + "delta", + "echo", + "foxtrot", + "golf", + "hotel", + "india", + "juliett", + "kilo", + "lima", + "mike", + "november", + "oscar", + "papa", + "quebec", + "romeo", + "sierra", + "tango", + "uniform", + "victor", + "whiskey", + "xray", + "yankee", + "zulu", + "apple", + "banana", + "cherry", + "date", + "elderberry", + "fig", + "grape", + "honeydew", + "kiwi", + "lemon", + "mango", + "nectarine", + "orange", + "papaya", + "quince", + "raspberry", + "strawberry", + "tangerine", + "ugli", + "vanilla", + "watermelon", + "xigua", + "yam", + "zucchini", + "amber", + "blue", + "crimson", + "emerald", + "fuchsia", + "gold", + "indigo", + "jade", + "khaki", + "lavender", + "magenta", + "navy", + "olive", + "purple", + "rose", + "silver", + "teal", + "umber", + "violet", + "wheat", + "azure", + "beige", + "coral", + "denim", + "ebony", + "ivory", + "jet", + "lime", + "maroon", + "mustard", + "ochre", + "peach", + "plum", + "rust", + "sage", + "tan", + "turquoise", + "wine", + "ash", + "birch", + "cedar", + "dogwood", + "elm", + "fir", + "ginkgo", + "hickory", + "ironwood", + "juniper", + "koa", + "larch", + "maple", + "oak", + "pine", + "redwood", + "spruce", + "teak", + "walnut", + "yew", + "zebrawood", + ]; + assert!( + words.len() >= 100, + "word list too small to avoid header/footer folding" + ); + let mut doc = LopdfDoc::with_version("1.4"); + let pages_id = doc.new_object_id(); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + "Encoding" => "WinAnsiEncoding", + }); + let resources_id = doc.add_object(dictionary! { + "Font" => dictionary! { "F1" => font_id }, + }); + let mut kids = Vec::new(); + for i in 0..200 { + let word = words[i % words.len()]; + let body = "lorem ipsum dolor sit amet consectetur adipiscing elit ".repeat(3); + let content = format!( + "BT /F1 18 Tf 72 720 Td (Topic {word} overview) Tj ET\n\ + BT /F1 11 Tf 72 690 Td ({body}) Tj ET\n\ + BT /F1 11 Tf 72 660 Td (End {word} summary) Tj ET" + ); + let content_id = + doc.add_object(Stream::new(lopdf::Dictionary::new(), content.into_bytes())); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + kids.push(Object::Reference(page_id)); + } + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => kids, + "Count" => 200_i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + std::fs::write("/tmp/batdoc-bench.pdf", buf).unwrap(); } } diff --git a/batdoc-core/src/pdf_geometry.rs b/batdoc-core/src/pdf_geometry.rs new file mode 100644 index 0000000..784cef5 --- /dev/null +++ b/batdoc-core/src/pdf_geometry.rs @@ -0,0 +1,650 @@ +//! Image placement extraction from PDF content streams. +//! +//! Walks the page content stream (and any nested Form `XObjects`) to record +//! every image `XObject` painted on the page, with its page-space bounding +//! rectangle expressed in top-down points (same coordinate system as +//! [`pdf_text::PositionedChar`]). This is the join key for region-aware OCR +//! merge in later phases. +//! +//! PDF coordinate rules followed here: +//! - Image `XObjects` paint the unit square `[0,0,1,1]` (PDF 32000 §8.9.5), +//! transformed by the current `CTM`. +//! - `cm` concatenates as `ctm_new = mul(ctm_old, cm_matrix)` because the +//! matrix maps *new* user space into *current* user space (PDF 32000 §8.3.4). +//! - Final rects are flipped to top-down page coordinates using the effective +//! `MediaBox` height. +//! +//! Malformed streams are skipped; the function never panics on bad input. + +use lopdf::content::Content; +use lopdf::{Dictionary, Object, ObjectId, Stream}; + +/// Axis-aligned rectangle in top-down page points (same space as +/// `pdf_text::PositionedChar`: y measured from the page top). +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct PtRect { + pub x0: f64, + pub y0: f64, + pub x1: f64, + pub y1: f64, +} + +impl PtRect { + /// Intersection test expanded by `tol` on all sides. + pub(crate) fn intersects_expanded(&self, other: &Self, tol: f64) -> bool { + self.x0 <= other.x1 + tol + && self.x1 >= other.x0 - tol + && self.y0 <= other.y1 + tol + && self.y1 >= other.y0 - tol + } +} + +/// An image `XObject` painted on a page, keyed by the same object id +/// `lopdf::Document::get_page_images` reports (the OCR join key, spec §6). +#[allow(dead_code)] // consumed by pdf_ocr/pdf_layout in later phases +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct PlacedImage { + pub object_id: lopdf::ObjectId, + pub rect: PtRect, +} + +/// Find every image `XObject` placement on `page_id`, including placements +/// inside nested `Form` `XObjects`. +/// +/// Never panics on malformed content; parse failures are skipped and the +/// function returns the placements that could be decoded. +pub(crate) fn placed_images(doc: &lopdf::Document, page_id: lopdf::ObjectId) -> Vec { + let Some(page_height) = page_height(doc, page_id) else { + return Vec::new(); + }; + let Ok((direct, resource_ids)) = doc.get_page_resources(page_id) else { + return Vec::new(); + }; + let mut merged = Dictionary::new(); + // Inheritance order: root → leaf, so leaf entries override ancestors. + for id in resource_ids.iter().rev() { + if let Ok(dict) = doc.get_dictionary(*id) { + for (k, v) in dict { + merged.set(k.clone(), v.clone()); + } + } + } + if let Some(dict) = direct { + for (k, v) in dict { + merged.set(k.clone(), v.clone()); + } + } + if merged.is_empty() { + return Vec::new(); + } + let Ok(content) = doc.get_page_content(page_id) else { + return Vec::new(); + }; + + let mut walker = Walker { + doc, + page_height, + out: Vec::new(), + }; + walker.walk(&content, &merged, IDENTITY, 0); + walker.out +} + +const IDENTITY: [f64; 6] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]; +const MAX_FORM_DEPTH: usize = 8; + +struct Walker<'a> { + doc: &'a lopdf::Document, + page_height: f64, + out: Vec, +} + +impl Walker<'_> { + fn walk(&mut self, content: &[u8], resources: &Dictionary, ctm: [f64; 6], depth: usize) { + let Ok(content) = Content::decode(content) else { + return; + }; + + let mut stack: Vec<[f64; 6]> = vec![ctm]; + for op in &content.operations { + match op.operator.as_str() { + "q" => stack.push(*stack.last().unwrap_or(&IDENTITY)), + "Q" => { + stack.pop(); + if stack.is_empty() { + stack.push(IDENTITY); + } + } + "cm" => { + if let Some(cur) = stack.last_mut() { + if let Some(m) = operands_to_matrix(&op.operands) { + *cur = mul(*cur, m); + } + } + } + "Do" => { + let cur = *stack.last().unwrap_or(&IDENTITY); + self.handle_do(resources, &op.operands, cur, depth); + } + _ => {} + } + } + } + + fn handle_do( + &mut self, + resources: &Dictionary, + operands: &[Object], + ctm: [f64; 6], + depth: usize, + ) { + let Some(Object::Name(name)) = operands.first() else { + return; + }; + let xobject_dict = match resources.get(b"XObject") { + Ok(obj) => match self.doc.dereference(obj) { + Ok((_, Object::Dictionary(dict))) => dict, + _ => return, + }, + Err(_) => return, + }; + let obj_id = match xobject_dict.get(name) { + Ok(Object::Reference(id)) => *id, + Ok(obj) => match self.doc.dereference(obj) { + Ok((_, Object::Reference(id))) => *id, + _ => return, + }, + Err(_) => return, + }; + let Ok(Object::Stream(stream)) = self.doc.get_object(obj_id) else { + return; + }; + let subtype = stream + .dict + .get(b"Subtype") + .ok() + .and_then(|o| self.doc.dereference(o).ok()) + .and_then(|(_, o)| o.as_name().ok()) + .and_then(|name| std::str::from_utf8(name).ok()) + .unwrap_or(""); + match subtype { + "Image" => { + self.out.push(PlacedImage { + object_id: obj_id, + rect: unit_rect_in_page_space(ctm, self.page_height), + }); + } + "Form" => { + if depth >= MAX_FORM_DEPTH { + return; + } + let form_matrix = matrix_from_stream(self.doc, stream); + let ctm2 = mul(ctm, form_matrix); + let form_resources = stream + .dict + .get(b"Resources") + .ok() + .and_then(|o| self.doc.dereference(o).ok()) + .and_then(|(_, o)| o.as_dict().ok()) + .unwrap_or(resources); + let form_content = match stream.decompressed_content() { + Ok(bytes) => bytes, + Err(_) => { + // lopdf's `decompressed_content` returns Err when the + // stream dict has no /Filter (i.e. raw content). Treat + // missing Filter as identity and use the raw bytes. + if stream.dict.get(b"Filter").is_err() { + stream.content.clone() + } else { + return; + } + } + }; + self.walk(&form_content, form_resources, ctm2, depth + 1); + } + _ => {} + } + } +} + +/// Extract the form `/Matrix` from a Form stream dictionary, defaulting to +/// identity. +fn matrix_from_stream(doc: &lopdf::Document, stream: &Stream) -> [f64; 6] { + let Ok(obj) = stream.dict.get(b"Matrix") else { + return IDENTITY; + }; + match doc.dereference(obj) { + Ok((_, o)) => operands_to_matrix_from_object(o).unwrap_or(IDENTITY), + Err(_) => operands_to_matrix_from_object(obj).unwrap_or(IDENTITY), + } +} + +/// Transform the unit square by `ctm` and convert to a top-down `PtRect`. +fn unit_rect_in_page_space(ctm: [f64; 6], page_height: f64) -> PtRect { + let corners = [ + transform_point(ctm, (0.0, 0.0)), + transform_point(ctm, (1.0, 0.0)), + transform_point(ctm, (0.0, 1.0)), + transform_point(ctm, (1.0, 1.0)), + ]; + let mut min_x = f64::INFINITY; + let mut max_x = f64::NEG_INFINITY; + let mut min_y = f64::INFINITY; + let mut max_y = f64::NEG_INFINITY; + for (x, y) in corners { + min_x = min_x.min(x); + max_x = max_x.max(x); + min_y = min_y.min(y); + max_y = max_y.max(y); + } + PtRect { + x0: min_x, + y0: page_height - max_y, + x1: max_x, + y1: page_height - min_y, + } +} + +#[allow(clippy::suboptimal_flops)] // Brief specifies the exact affine formula; reordering is not desired. +fn transform_point(m: [f64; 6], (x, y): (f64, f64)) -> (f64, f64) { + (m[0] * x + m[2] * y + m[4], m[1] * x + m[3] * y + m[5]) +} + +/// Convert six numeric operands into an affine matrix `[a b c d e f]`. +#[allow(clippy::cast_precision_loss)] // PDF integer operands are within f64 precision for coordinates. +fn operands_to_matrix(operands: &[Object]) -> Option<[f64; 6]> { + if operands.len() < 6 { + return None; + } + let mut out = [0.0; 6]; + for (i, obj) in operands.iter().enumerate().take(6) { + out[i] = match obj { + Object::Integer(n) => *n as f64, + Object::Real(f) => f64::from(*f), + Object::Reference(_) => { + // Indirect operands are invalid in a content stream; skip. + return None; + } + _ => return None, + }; + } + Some(out) +} + +/// Convert six numeric objects (e.g. an array) into an affine matrix. +fn operands_to_matrix_from_object(obj: &Object) -> Option<[f64; 6]> { + match obj { + Object::Array(arr) => operands_to_matrix(arr), + _ => None, + } +} + +/// Multiply two affine matrices: `mul(m1, m2)` means apply `m2` first, then `m1`. +#[allow(clippy::suboptimal_flops)] // Brief specifies the exact affine formula; reordering is not desired. +fn mul(m1: [f64; 6], m2: [f64; 6]) -> [f64; 6] { + [ + m1[0] * m2[0] + m1[2] * m2[1], + m1[1] * m2[0] + m1[3] * m2[1], + m1[0] * m2[2] + m1[2] * m2[3], + m1[1] * m2[2] + m1[3] * m2[3], + m1[0] * m2[4] + m1[2] * m2[5] + m1[4], + m1[1] * m2[4] + m1[3] * m2[5] + m1[5], + ] +} + +/// Effective `MediaBox` height for `page_id`, walking the `/Parent` chain. +fn page_height(doc: &lopdf::Document, page_id: ObjectId) -> Option { + let dict = doc.get_dictionary(page_id).ok()?; + let media_box = media_box_for_dict(doc, dict)?; + Some(media_box[3] - media_box[1]) +} + +const MAX_MEDIABOX_PARENT_DEPTH: u32 = 32; + +/// Resolve the effective `MediaBox` array for a dictionary, walking `/Parent`. +fn media_box_for_dict(doc: &lopdf::Document, dict: &Dictionary) -> Option<[f64; 4]> { + media_box_for_dict_inner(doc, dict, 0) +} + +fn media_box_for_dict_inner( + doc: &lopdf::Document, + dict: &Dictionary, + depth: u32, +) -> Option<[f64; 4]> { + if depth > MAX_MEDIABOX_PARENT_DEPTH { + // Cyclic or absurdly deep parent chain: fall back to the default. + return Some([0.0, 0.0, 612.0, 792.0]); + } + if let Ok(obj) = dict.get(b"MediaBox") { + if let Ok(rect) = rect_array(doc, obj) { + return Some(rect); + } + } + if let Ok(parent) = dict.get(b"Parent") { + let parent_id = parent.as_reference().ok()?; + let parent_dict = doc.get_dictionary(parent_id).ok()?; + return media_box_for_dict_inner(doc, parent_dict, depth + 1); + } + // PDF spec default: US Letter. + Some([0.0, 0.0, 612.0, 792.0]) +} + +/// Convert an indirect or direct array into `[llx, lly, urx, ury]`. +#[allow(clippy::cast_precision_loss)] // PDF MediaBox coordinates are within f64 precision. +fn rect_array(doc: &lopdf::Document, obj: &Object) -> Result<[f64; 4], ()> { + let (_, obj) = doc.dereference(obj).map_err(|_| ())?; + let arr = obj.as_array().map_err(|_| ())?; + if arr.len() < 4 { + return Err(()); + } + let mut out = [0.0; 4]; + for (i, item) in arr.iter().enumerate().take(4) { + out[i] = match item { + Object::Integer(n) => *n as f64, + Object::Real(f) => f64::from(*f), + _ => return Err(()), + }; + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use lopdf::{dictionary, Document, Object, Stream}; + + #[test] + fn placed_image_rect_from_cm_do() { + // q 200 0 0 50 72 600 cm /Im0 Do Q → PDF-space rect + // x: 72..272, y: 600..650; page height 792 → top-down y: 142..192. + let (doc, page_id) = image_pdf("q 200 0 0 50 72 600 cm /Im0 Do Q"); + let placed = placed_images(&doc, page_id); + assert_eq!(placed.len(), 1); + let r = &placed[0].rect; + assert!((r.x0 - 72.0).abs() < 0.01, "{r:?}"); + assert!((r.x1 - 272.0).abs() < 0.01, "{r:?}"); + assert!((r.y0 - 142.0).abs() < 0.01, "{r:?}"); + assert!((r.y1 - 192.0).abs() < 0.01, "{r:?}"); + } + + #[test] + fn object_id_matches_get_page_images() { + let (doc, page_id) = image_pdf("q 100 0 0 100 0 0 cm /Im0 Do Q"); + let placed = placed_images(&doc, page_id); + let images = doc.get_page_images(page_id).unwrap(); + assert_eq!(placed.len(), 1); + assert_eq!(placed[0].object_id, images[0].id); + } + + #[test] + fn nested_form_xobject_recursion() { + // A Form XObject (own resources, /Matrix translate by 10,20) + // containing "q 50 0 0 50 1 2 cm /Im1 Do Q"; page content does + // "q 2 0 0 2 100 100 cm /Fm0 Do Q". Expected PDF-space unit-square + // mapping: ctm = [2 0 0 2 100 100] ∘ [1 0 0 1 10 20] ∘ [50 0 0 50 1 2] + // → x = 2*(1*50x ... ) — compute in the test via the same corner + // math, but assert the final top-down rect numerically: + // inner point (0,0) → form: (1,2) → +matrix (11,22) → page ctm: + // (122, 144); inner (1,1) → form (51,52) → (61,72) → (222, 244). + // top-down (h=792): y0 = 792-244 = 548, y1 = 792-144 = 648. + let (doc, page_id) = form_pdf(); + let placed = placed_images(&doc, page_id); + assert_eq!(placed.len(), 1); + let r = &placed[0].rect; + assert!((r.x0 - 122.0).abs() < 0.01, "{r:?}"); + assert!((r.x1 - 222.0).abs() < 0.01, "{r:?}"); + assert!((r.y0 - 548.0).abs() < 0.01, "{r:?}"); + assert!((r.y1 - 648.0).abs() < 0.01, "{r:?}"); + } + + #[test] + fn resources_inherited_from_pages_node() { + // Page dict omits /Resources; the /Pages node carries them. + // Same content as the first test → same rect. + let (doc, page_id) = image_pdf_inherited_resources("q 200 0 0 50 72 600 cm /Im0 Do Q"); + let placed = placed_images(&doc, page_id); + assert_eq!(placed.len(), 1); + assert!((placed[0].rect.x0 - 72.0).abs() < 0.01); + } + + #[test] + fn rect_intersection_with_tolerance() { + let a = PtRect { + x0: 0.0, + y0: 0.0, + x1: 10.0, + y1: 10.0, + }; + let b = PtRect { + x0: 11.0, + y0: 0.0, + x1: 20.0, + y1: 10.0, + }; + assert!(!a.intersects_expanded(&b, 0.5)); + assert!(a.intersects_expanded(&b, 2.0)); + } + + #[test] + fn cyclic_parent_chain_does_not_stack_overflow() { + // A cyclic /Parent chain must not hang or crash. get_page_resources + // detects the cycle and returns Err, so placed_images returns empty. + // The key property is that the call terminates without a stack overflow. + let (doc, page_id) = image_pdf_cyclic_parent("q 200 0 0 50 72 600 cm /Im0 Do Q"); + let placed = placed_images(&doc, page_id); + assert!(placed.is_empty()); + } + + #[test] + fn media_box_for_dict_guards_against_cyclic_parent() { + // Directly exercise media_box_for_dict with a Pages node whose Parent + // is itself. Without the depth cap it would recurse until stack overflow. + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + let pages_dict = dictionary! { + "Type" => "Pages", + "Kids" => Vec::::new(), + "Count" => 0_i64, + "Parent" => pages_id, + }; + doc.objects.insert(pages_id, Object::Dictionary(pages_dict)); + let pages_dict = doc + .get_dictionary(pages_id) + .expect("pages dict we just inserted"); + let rect = media_box_for_dict(&doc, pages_dict); + // Falls back to the PDF default US-Letter MediaBox. + assert_eq!(rect, Some([0.0, 0.0, 612.0, 792.0])); + } + + /// Build a one-page PDF with an image XObject in resources and a + /// caller-supplied content stream. + fn image_pdf(content: &str) -> (Document, ObjectId) { + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + + let image_id = doc.add_object(image_stream()); + let resources_id = doc.add_object(dictionary! { + "XObject" => dictionary! { "Im0" => image_id }, + }); + let content_id = + doc.add_object(Stream::new(Dictionary::new(), content.as_bytes().to_vec())); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => 1_i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + let loaded = Document::load_mem(&buf).unwrap(); + (loaded, page_id) + } + + /// Build a one-page PDF where the /Pages node carries /Resources and the + /// page dict omits it. + fn image_pdf_inherited_resources(content: &str) -> (Document, ObjectId) { + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + + let image_id = doc.add_object(image_stream()); + let resources_id = doc.add_object(dictionary! { + "XObject" => dictionary! { "Im0" => image_id }, + }); + let content_id = + doc.add_object(Stream::new(Dictionary::new(), content.as_bytes().to_vec())); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => 1_i64, + "Resources" => resources_id, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + let loaded = Document::load_mem(&buf).unwrap(); + (loaded, page_id) + } + + /// Build a one-page PDF with a cyclic /Parent chain. The page has direct + /// /Resources so resource lookup succeeds, but it has no /MediaBox and the + /// Pages node's /Parent points to itself, so media_box_for_dict must guard + /// against infinite recursion. + fn image_pdf_cyclic_parent(content: &str) -> (Document, ObjectId) { + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + + let image_id = doc.add_object(image_stream()); + let resources_id = doc.add_object(dictionary! { + "XObject" => dictionary! { "Im0" => image_id }, + }); + let content_id = + doc.add_object(Stream::new(Dictionary::new(), content.as_bytes().to_vec())); + // Page intentionally omits /MediaBox so the parent walk is exercised. + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => resources_id, + }); + // Cyclic Pages node: its /Parent points to itself and it has no /MediaBox. + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => 1_i64, + "Parent" => pages_id, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + let loaded = Document::load_mem(&buf).unwrap(); + (loaded, page_id) + } + + /// Build a one-page PDF with a Form XObject that contains an image. + fn form_pdf() -> (Document, ObjectId) { + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + + let image_id = doc.add_object(image_stream_named("Im1")); + let form_resources_id = doc.add_object(dictionary! { + "XObject" => dictionary! { "Im1" => image_id }, + }); + let form_content = b"q 50 0 0 50 1 2 cm /Im1 Do Q".to_vec(); + let form_id = doc.add_object(Stream::new( + dictionary! { + "Type" => "XObject", + "Subtype" => "Form", + "BBox" => vec![0.into(), 0.into(), 100.into(), 100.into()], + "Matrix" => vec![1.into(), 0.into(), 0.into(), 1.into(), 10.into(), 20.into()], + "Resources" => form_resources_id, + }, + form_content, + )); + let page_resources_id = doc.add_object(dictionary! { + "XObject" => dictionary! { "Fm0" => form_id }, + }); + let page_content = b"q 2 0 0 2 100 100 cm /Fm0 Do Q".to_vec(); + let content_id = doc.add_object(Stream::new(Dictionary::new(), page_content)); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => page_resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => 1_i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + let loaded = Document::load_mem(&buf).unwrap(); + (loaded, page_id) + } + + fn image_stream() -> Stream { + image_stream_named("Im0") + } + + fn image_stream_named(_name: &str) -> Stream { + // 2x2 DeviceRGB image, raw (no compression needed for tests). + Stream::new( + dictionary! { + "Type" => "XObject", + "Subtype" => "Image", + "Width" => 2_i64, + "Height" => 2_i64, + "ColorSpace" => "DeviceRGB", + "BitsPerComponent" => 8_i64, + }, + vec![255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255], + ) + } +} diff --git a/batdoc-core/src/pdf_layout.rs b/batdoc-core/src/pdf_layout.rs new file mode 100644 index 0000000..9a6448a --- /dev/null +++ b/batdoc-core/src/pdf_layout.rs @@ -0,0 +1,2379 @@ +//! Layout algorithms derived from run-llama/liteparse (Apache-2.0); +//! reimplemented for batdoc under MIT — no code or tables copied. +//! +//! Assembles the positioned per-char stream from [`crate::pdf_text`] into +//! words and lines: canonical rotation → rotation partition → baseline +//! clustering → word breaks (space glyphs / median-relative advance gaps) → horizontal (column) line split. +//! liteparse runs the same pipeline over whole text items with PDFium-sized +//! boxes; here it runs per char with `font_size`/`advance` from pdf-extract, +//! so the thresholds are restated in those units (see the constants below). +//! +//! Coordinate frames: chars arrive in top-down page space. [`assemble`] +//! canonicalizes the page rotation (majority text becomes horizontal), +//! assembles each rotation group in its own frame, and transforms every +//! emitted [`Line::rect`] back to top-down page space — the join key for +//! region-aware OCR merge. [`Word`] x-extents stay in the line's assembly +//! frame (identity unless the page was canonicalized). + +use crate::pdf_geometry::PtRect; +use crate::pdf_text::{PositionedChar, PositionedPage}; + +/// Where a line's text came from. `Ocr` lines are synthesized by the +/// region-aware merge (later phase) rather than from native PDF glyphs. +/// +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum LineSource { + Native, + Ocr, +} + +/// A word with its x-extent, rebuilt from per-char advance gaps (spec §7 — +/// NOT naive space splitting). +/// +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct Word { + pub text: String, + pub x0: f64, + pub x1: f64, +} + +/// A baseline cluster of chars assembled into text, with the union of its +/// char boxes in top-down page space. +/// +#[derive(Clone, Debug)] +pub(crate) struct Line { + pub text: String, + /// Per-word x-extents in the line's ASSEMBLY frame (see the module + /// docs): the table detector (Task 14) clusters word x0s across lines + /// of a run — all canonical-group lines share one frame, so that is + /// safe — but a word's x0/x1 must never be compared against its own + /// line's page-space `rect`. + pub words: Vec, + pub rect: PtRect, + /// Median transformed font size of the line's chars. + pub font_size: f64, + pub source: LineSource, +} + +/// Word break: gap between consecutive chars exceeds a quarter of the line +/// font size, with a 1pt floor so tiny fonts don't shatter words. +/// Word break when a gap exceeds all of: 1.5× the line's median char +/// advance, 0.5× the line's font size, and 2.0pt. PDF producers that +/// position glyphs with TJ kerning report understated per-char advances +/// (the real advance is visible only in the next glyph's origin); the +/// median-relative term keeps those small phantom gaps from splitting +/// words, while explicit space glyphs (which exist in such files) still +/// break words directly. +const WORD_GAP_MEDIAN_RATIO: f64 = 1.5; +const WORD_GAP_SIZE_RATIO: f64 = 0.5; +const WORD_GAP_FLOOR: f64 = 2.0; +/// Horizontal line split: a same-baseline gap this many times the font size +/// means two columns, not two words (liteparse `form_lines` equivalent — +/// its column gap fires at 2× median char width). Table rows DO shatter +/// into one-word fragments at this threshold — that is fine: +/// `merge_baseline_fragments` rejoins them for table detection, and xy-cut +/// reading order handles the rest. (v1 note: this was briefly 10.0 to keep +/// table rows intact, which fused narrow-gutter columns — the fragment +/// merge made the low value safe again.) +const LINE_SPLIT_RATIO: f64 = 2.5; +/// One rotation value must cover this share of the page's chars before the +/// whole page is canonicalized to it. +const CANONICAL_SHARE_PCT: usize = 70; + +// Rotation math runs in `u16` (`PositionedChar.rotation` is u16 degrees), +// and the module must model the full 0/90/180/270 set (spec-pinned +// transforms). +const R90: u16 = 90; +const R180: u16 = 180; +const R270: u16 = 270; +const R360: u16 = 360; + +/// Assemble one page of positioned chars into lines (words, rects, sizes). +/// +pub(crate) fn assemble(page: &PositionedPage) -> Vec { + if page.chars.is_empty() { + return Vec::new(); + } + let page_w = page.media_box.2 - page.media_box.0; + let page_h = page.media_box.3 - page.media_box.1; + let (chars, page_rot) = canonicalize(&page.chars, page_w, page_h); + let (frame_w, frame_h) = frame_dims(page_rot, page_w, page_h); + + // Partition by rotation relative to the canonical frame: the majority + // group is rotation 0; minority rotations land in their own groups. The + // relative rotation is kept in a local u16 (matching the `PositionedChar` + // field width) and never written back into the original `PositionedChar`. + let mut groups: [Vec; 4] = [Vec::new(), Vec::new(), Vec::new(), Vec::new()]; + for c in chars { + let rel = (c.rotation + R360 - page_rot) % R360; + groups[rot_index(rel)].push(c); + } + + let back = (R360 - page_rot) % R360; + let mut out = assemble_group(&groups[0]); + for line in &mut out { + line.rect = rotate_rect(line.rect, back, frame_w, frame_h); + } + // Minority groups (rotated side-labels on an otherwise-canonical page): + // assemble each in its own unrotated frame so they are not silently + // dropped, then map rects group frame → canonical frame → page space. + for (i, group) in groups.iter().enumerate().skip(1) { + if group.is_empty() { + continue; + } + let g = u16::try_from(i * 90).unwrap_or_default(); // i ∈ 1..=3 → ≤ 270 + let unrotated: Vec = group + .iter() + .map(|c| { + let (x, y) = unrotate_point(g, c.x, c.y, frame_w, frame_h); + PositionedChar { + x, + y, + rotation: 0, + ..*c + } + }) + .collect(); + let (gw, gh) = frame_dims(g, frame_w, frame_h); + let mut lines = assemble_group(&unrotated); + for line in &mut lines { + let canonical = rotate_rect(line.rect, (R360 - g) % R360, gw, gh); + line.rect = rotate_rect(canonical, back, frame_w, frame_h); + } + out.extend(lines); + } + out +} + +/// If one non-zero rotation covers ≥ 70% of chars, rotate every char into +/// that majority's reading frame and return `(chars, applied_rotation)`. +/// Char rotations are left untouched; the caller combines them with the +/// applied rotation (in u16) to partition minority groups. Font size and +/// advance are lengths and survive rotation unchanged. +fn canonicalize(chars: &[PositionedChar], w: f64, h: f64) -> (Vec, u16) { + let mut counts = [0usize; 4]; + for c in chars { + counts[rot_index(c.rotation)] += 1; + } + let mut best: u16 = 0; + let mut best_count = 0; + for (i, &n) in counts.iter().enumerate().skip(1) { + if n > best_count { + best_count = n; + best = u16::try_from(i * 90).unwrap_or_default(); // i ∈ 1..=3 → ≤ 270 + } + } + if best_count * 100 < chars.len() * CANONICAL_SHARE_PCT { + return (chars.to_vec(), 0); + } + let out = chars + .iter() + .map(|c| { + let (x, y) = unrotate_point(best, c.x, c.y, w, h); + PositionedChar { x, y, ..*c } + }) + .collect(); + (out, best) +} + +/// Cluster chars into baseline groups: y snapped to an absolute grid of +/// max(median char size × 0.5, 5.0) pt — same bucket = same line, x-sorted +/// within. Absolute buckets, not a running band: a tall line bridging two +/// tight lines (e.g. a sidebar line between two body lines) must not +/// chain-merge them into one interleaved cluster. liteparse `form_lines` +/// (`projection.rs`) uses the same snap-to-grid approach. +fn cluster_lines(chars: &[PositionedChar]) -> Vec> { + if chars.is_empty() { + return Vec::new(); + } + let tol = (median_size(chars) * 0.5).max(5.0); + // y is bounded by the page box (thousands of points); the cast cannot + // truncate a real value. + #[allow(clippy::cast_possible_truncation)] + let bucket = |c: &PositionedChar| (c.y / tol).round() as i64; + let mut sorted = chars.to_vec(); + sorted.sort_by(|a, b| bucket(a).cmp(&bucket(b)).then_with(|| a.x.total_cmp(&b.x))); + let mut lines: Vec> = Vec::new(); + let mut cur_bucket = i64::MIN; + for c in sorted { + let b = bucket(&c); + if b != cur_bucket { + lines.push(Vec::new()); + cur_bucket = b; + } + lines.last_mut().expect("bucket change pushes").push(c); + } + lines +} + +/// Assemble one rotation group (all rotation 0 in its own frame) into +/// lines: baseline clusters, then split each cluster horizontally where a +/// gap exceeds the column threshold. +fn assemble_group(chars: &[PositionedChar]) -> Vec { + let mut out = Vec::new(); + for cluster in cluster_lines(chars) { + let mut by_x = cluster; + by_x.sort_by(|a, b| a.x.total_cmp(&b.x)); + let split_gap = LINE_SPLIT_RATIO * median_size(&by_x); + let mut cur: Vec = Vec::new(); + for c in by_x { + if let Some(prev) = cur.last() { + if c.x - (prev.x + prev.advance) > split_gap { + out.push(build_line(&cur)); + cur.clear(); + } + } + cur.push(c); + } + if !cur.is_empty() { + out.push(build_line(&cur)); + } + } + out +} + +/// Median char advance within a line (chars with zero advance — +/// ligature-expansion phantoms sharing a glyph origin — excluded). Falls +/// back to 0 for an all-zero line, where the size/floor terms rule. +fn median_advance(chars: &[PositionedChar]) -> f64 { + let mut advs: Vec = chars + .iter() + .map(|c| c.advance) + .filter(|a| *a > 0.0) + .collect(); + if advs.is_empty() { + return 0.0; + } + advs.sort_by(f64::total_cmp); + advs[advs.len() / 2] +} + +/// Build one line from its x-sorted chars: word breaks on explicit space +/// glyphs and on advance gaps, rect = union of char boxes. +fn build_line(chars: &[PositionedChar]) -> Line { + let size = median_size(chars); + let word_gap = (WORD_GAP_MEDIAN_RATIO * median_advance(chars)) + .max(WORD_GAP_SIZE_RATIO * size) + .max(WORD_GAP_FLOOR); + let mut words: Vec = Vec::new(); + let mut text = String::new(); + let (mut wx0, mut wx1) = (0.0, 0.0); + let mut rect = PtRect { + x0: f64::INFINITY, + y0: f64::INFINITY, + x1: f64::NEG_INFINITY, + y1: f64::NEG_INFINITY, + }; + let mut prev: Option<&PositionedChar> = None; + for c in chars { + rect.x0 = rect.x0.min(c.x); + rect.y0 = rect.y0.min(c.y - c.font_size); + rect.x1 = rect.x1.max(c.x + c.advance); + rect.y1 = rect.y1.max(c.y); + // A real space glyph always breaks the word (and is not itself + // emitted); so does an advance gap past the threshold. + let break_before = + prev.is_some_and(|p| p.ch == ' ' || c.ch == ' ' || c.x - (p.x + p.advance) > word_gap); + if break_before && !text.is_empty() { + words.push(Word { + text: std::mem::take(&mut text), + x0: wx0, + x1: wx1, + }); + } + if c.ch != ' ' { + if text.is_empty() { + wx0 = c.x; + wx1 = c.x + c.advance; + } else { + wx1 = wx1.max(c.x + c.advance); + } + text.push(c.ch); + } + prev = Some(c); + } + if !text.is_empty() { + words.push(Word { + text, + x0: wx0, + x1: wx1, + }); + } + let text = words + .iter() + .map(|w| w.text.as_str()) + .collect::>() + .join(" "); + Line { + text, + words, + rect, + font_size: size, + source: LineSource::Native, + } +} + +/// Median char size of a non-empty slice (mean of the two middles when even). +fn median_size(chars: &[PositionedChar]) -> f64 { + let mut sizes: Vec = chars.iter().map(|c| c.font_size).collect(); + sizes.sort_by(f64::total_cmp); + let n = sizes.len(); + f64::midpoint(sizes[(n - 1) / 2], sizes[n / 2]) +} + +/// Rotation bucket 0..=3 for a quantized rotation value. +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] // rot < 360 → rot/90 ∈ 0..=3 +const fn rot_index(rot: u16) -> usize { + (rot / 90 % 4) as usize +} + +/// Inverse-rotate a point by `rot` (0/90/180/270) within a `w`×`h` frame so +/// that text written at `rot` reads horizontally. Pinned by +/// `rotated_page_is_canonicalized`: 90° text reads bottom-to-top, so the +/// first glyph (largest top-down y) must map to the smallest x'. +fn unrotate_point(rot: u16, x: f64, y: f64, w: f64, h: f64) -> (f64, f64) { + match rot { + R90 => (h - y, x), + R180 => (w - x, h - y), + R270 => (y, w - x), + _ => (x, y), + } +} + +/// Frame dimensions after `unrotate_point`: quarter turns swap the axes. +const fn frame_dims(rot: u16, w: f64, h: f64) -> (f64, f64) { + if rot == R90 || rot == R270 { + (h, w) + } else { + (w, h) + } +} + +/// Rotate an axis-aligned rect by `rot` within a `w`×`h` frame (the inverse +/// of [`unrotate_point`] when `rot = 360 - applied`), re-normalizing corners. +fn rotate_rect(rect: PtRect, rot: u16, w: f64, h: f64) -> PtRect { + if rot == 0 { + return rect; + } + let (ax, ay) = unrotate_point(rot, rect.x0, rect.y0, w, h); + let (bx, by) = unrotate_point(rot, rect.x1, rect.y1, w, h); + PtRect { + x0: ax.min(bx), + y0: ay.min(by), + x1: ax.max(bx), + y1: ay.max(by), + } +} + +// --------------------------------------------------------------------------- +// Reading order: recursive XY-cut (liteparse `xy_cut`, reimplemented). +// +// liteparse builds a bucketed density projection per axis and searches for +// valleys; here lines are whole baseline clusters, so the same idea collapses +// to interval merging: merge each axis's coverage intervals, and the widest +// surviving gap between merged clusters is a cut no line crosses. The axis +// with the LARGER absolute gap wins (classic xy-cut: a column gutter dwarfs +// inter-line y-gaps, while a full-width spanning line blocks the vertical +// cut entirely, letting the horizontal band break fire first). +// --------------------------------------------------------------------------- + +/// Column gutter: a gap in x-coverage this many times the median font size. +const XCUT_COLUMN_RATIO: f64 = 2.0; +/// Band break: a gap in y-coverage this many times the median font size +/// (half the column ratio — stacked paragraphs sit closer than columns). +const XCUT_BAND_RATIO: f64 = 1.0; +/// Recursion backstop: adversarial geometry peeling one line per cut must +/// not blow the stack (repo rule: no panics on malformed input). +const XCUT_MAX_DEPTH: u32 = 32; + +/// A qualifying cut: `gap` is its absolute width (the larger-gap axis wins), +/// `at` the gap midpoint to compare line centers against. +struct Cut { + gap: f64, + at: f64, + vertical: bool, +} + +/// Order `lines` for reading: recursive XY-cut, columns left→right inside +/// bands top→bottom. Runs on the merged (native + OCR) line set. +/// Thin line-only wrapper over [`reading_order_items`] — used by the +/// xy-cut unit tests; the production driver passes a mixed stream. +#[allow(dead_code)] // test-only wrapper +pub(crate) fn reading_order(lines: Vec) -> Vec { + let items: Vec = lines.into_iter().map(Item::Line).collect(); + reading_order_items(items) + .into_iter() + .map(|it| match it { + Item::Line(l) => l, + Item::Table(_) => unreachable!("lines never become tables"), + }) + .collect() +} + +/// Reading-order a mixed line/table stream: table regions are opaque +/// full-width bands in the xy-cut geometry — they sort between text +/// regions by position and are never column-cut. +pub(crate) fn reading_order_items(items: Vec) -> Vec { + let mut out = Vec::with_capacity(items.len()); + let median = median_font_size_items(&items); + xy_cut(items, median, 0, &mut out); + out +} + +/// Median `font_size` of the item stream. +fn median_font_size_items(items: &[Item]) -> f64 { + let mut sizes: Vec = items.iter().map(Item::font_size).collect(); + sizes.sort_by(f64::total_cmp); + if sizes.is_empty() { + return 0.0; + } + let n = sizes.len(); + f64::midpoint(sizes[(n - 1) / 2], sizes[n / 2]) +} + +/// Median `font_size` of the line set (mean of the two middles when even). +fn median_font_size(lines: &[Line]) -> f64 { + let mut sizes: Vec = lines.iter().map(|l| l.font_size).collect(); + sizes.sort_by(f64::total_cmp); + if sizes.is_empty() { + return 0.0; + } + let n = sizes.len(); + f64::midpoint(sizes[(n - 1) / 2], sizes[n / 2]) +} + +/// Append `lines` to `out` in reading order. The median is computed once at +/// the top level and passed down so nested regions use page-consistent +/// thresholds. +fn xy_cut(lines: Vec, median: f64, depth: u32, out: &mut Vec) { + if lines.len() <= 1 || depth >= XCUT_MAX_DEPTH { + return push_sorted(lines, out); + } + let Some(cut) = find_cut(&lines, median) else { + return push_sorted(lines, out); + }; + // Split every line by which side of the gap midpoint its rect center + // falls on. No line can straddle the gap (a straddler would have merged + // the coverage intervals), so both sides are non-empty and recursion + // strictly shrinks. First = top for horizontal cuts, left for vertical. + let (mut first, mut second) = (Vec::new(), Vec::new()); + for l in lines { + let r = l.rect(); + let center = if cut.vertical { + f64::midpoint(r.x0, r.x1) + } else { + f64::midpoint(r.y0, r.y1) + }; + if center < cut.at { + first.push(l); + } else { + second.push(l); + } + } + xy_cut(first, median, depth + 1, out); + xy_cut(second, median, depth + 1, out); +} + +/// No qualifying cut: a leaf sorts top→bottom, then left→right. +fn push_sorted(mut lines: Vec, out: &mut Vec) { + lines.sort_by(|a, b| { + a.rect() + .y0 + .total_cmp(&b.rect().y0) + .then_with(|| a.rect().x0.total_cmp(&b.rect().x0)) + }); + out.extend(lines); +} + +/// Pick the cut on the axis with the larger absolute qualifying gap; `None` +/// when neither axis has one. +fn find_cut(lines: &[Item], median: f64) -> Option { + let v = widest_gap(lines, XCUT_COLUMN_RATIO * median, true); + let h = widest_gap(lines, XCUT_BAND_RATIO * median, false); + match (v, h) { + (Some(v), Some(h)) => Some(if v.gap > h.gap { v } else { h }), + (Some(v), None) => Some(v), + (None, h) => h, + } +} + +/// Widest gap between consecutive merged coverage intervals along one axis, +/// if it clears `threshold`. Merging first is what makes a spanning line +/// block the cut: it overlaps both neighbors, so no gap survives between +/// them (liteparse `xy_find_best_cut`, restated as interval merging). +fn widest_gap(lines: &[Item], threshold: f64, vertical: bool) -> Option { + let mut ivals: Vec<(f64, f64)> = lines + .iter() + .map(|l| { + let r = l.rect(); + if vertical { + (r.x0, r.x1) + } else { + (r.y0, r.y1) + } + }) + .collect(); + ivals.sort_by(|a, b| a.0.total_cmp(&b.0)); + // Sole caller (`xy_cut`) guarantees ≥ 2 lines, so `ivals` is non-empty. + let mut best: Option = None; + let mut end = ivals[0].1; + for &(n0, n1) in &ivals[1..] { + if n0 <= end { + // Overlapping/adjacent: same coverage cluster, extend its end. + end = end.max(n1); + continue; + } + let gap = n0 - end; + if gap >= threshold && best.as_ref().is_none_or(|b| gap > b.gap) { + best = Some(Cut { + gap, + at: f64::midpoint(end, n0), + vertical, + }); + } + end = n1; + } + best +} + +// --------------------------------------------------------------------------- +// Document-level signals + heading/paragraph classification +// (liteparse `markdown_layout/{headings,repetition,classify}`, minimal port). +// +// Pass 1 of the Task 13 driver feeds each page's assembled lines to +// [`DocSignalsBuilder`] and drops the page; pass 2 runs [`classify`] over the +// merged, reading-ordered line set. liteparse gates headings on `size > body +// + 0.5pt` and derives levels from a doc-wide rank of distinct heading sizes +// (its headings.rs:578/:644) — the plan deliberately trimmed that to a pure +// size-ratio scheme so signals stay three small fields (spec §7). +// --------------------------------------------------------------------------- + +/// Heading gate: a line at least this much larger than the body size. +const HEADING_MIN_RATIO: f64 = 1.2; +/// Size-ratio bands → heading level: ≥1.8 → H1, ≥1.5 → H2, ≥1.3 → H3, and +/// anything still over [`HEADING_MIN_RATIO`] → H4. +const HEADING_H1_RATIO: f64 = 1.8; +const HEADING_H2_RATIO: f64 = 1.5; +const HEADING_H3_RATIO: f64 = 1.3; +/// Paragraph split: a top-to-top vertical gap larger than this many times +/// the previous line's font size starts a new paragraph. +const PARAGRAPH_GAP_RATIO: f64 = 1.5; +/// List grouping: a marker line joins the open list while its top-to-top +/// gap stays within this many times the previous item's font size (same +/// ratio as paragraph joining — list items sit at body line spacing). +const LIST_GAP_RATIO: f64 = 1.5; +/// Bullet glyphs that open an unordered list item when followed by +/// whitespace (spec-pinned set; liteparse recognizes a few more PUA/Symbol +/// glyphs that this pipeline never produces). +const BULLET_CHARS: &[char] = &[ + '\u{2022}', // • + '\u{25e6}', // ◦ + '\u{2023}', // ‣ + '-', '*', '\u{2013}', // – + '\u{2014}', // — + '\u{25a0}', // ■ + '\u{25aa}', // ▪ + '\u{25c6}', // ◆ + '\u{25ba}', // ► + '\u{25b8}', // ▸ + '\u{25cf}', // ● + '\u{25cb}', // ○ +]; +/// Body size when the document yields no measurable text (image-only PDF). +const DEFAULT_BODY_SIZE: f64 = 12.0; +/// A normalized first/last-line signature must repeat on at least this many +/// pages to count as a header/footer… +const HEADER_FOOTER_MIN_PAGES: usize = 2; +/// …or on one page in `HEADER_FOOTER_PAGE_DIV`, whichever is larger. +const HEADER_FOOTER_PAGE_DIV: usize = 3; + +/// Document-wide aggregates collected in driver pass 1 (tiny — no pages +/// retained, spec §4.2/§11). +#[derive(Debug)] +pub(crate) struct DocSignals { + /// Modal font size across the document (quarter-point buckets) — the + /// body text size. + pub body_size: f64, + /// Normalized signatures of lines repeated as a page's first line. + pub headers: std::collections::HashSet, + /// Same for last lines (footers). + pub footers: std::collections::HashSet, +} + +/// Pass-1 accumulator for [`DocSignals`]. +#[derive(Default)] +pub(crate) struct DocSignalsBuilder { + /// Quarter-point bucket key → summed text length at that size. + size_weights: std::collections::HashMap, + /// Normalized first-line signature → pages seen on. + first_counts: std::collections::HashMap, + /// Normalized last-line signature → pages seen on. + last_counts: std::collections::HashMap, +} + +/// A classified block of body content. +#[derive(Debug)] +pub(crate) enum Block { + Heading { + level: u8, + text: String, + }, + Paragraph(String), + /// Borderless table (Task 14): rows of cells, empty string = no word + /// landed in that column for that row. + Table(Vec>), + /// List items (Task 16) with their markers stripped; ordered and + /// unordered items both render as markdown `- ` bullets. + List(Vec), +} + +impl DocSignalsBuilder { + pub(crate) fn new() -> Self { + Self::default() + } + + /// Fold one page's assembled lines into the aggregates. Lines are + /// unordered here: the page's "first line" is the one with min + /// `rect.y0`, its "last line" the one with max `rect.y0`. + /// Whitespace-only lines carry no signal and are skipped entirely. + pub(crate) fn add_lines(&mut self, lines: &[Line]) { + let mut first: Option<&Line> = None; + let mut last: Option<&Line> = None; + for l in lines { + let text = l.text.trim(); + if text.is_empty() { + continue; + } + if l.font_size > 0.0 { + *self + .size_weights + .entry(size_bucket(l.font_size)) + .or_insert(0) += text.chars().count(); + } + if first.is_none_or(|f| l.rect.y0 < f.rect.y0) { + first = Some(l); + } + if last.is_none_or(|f| l.rect.y0 > f.rect.y0) { + last = Some(l); + } + } + if let Some(f) = first { + *self + .first_counts + .entry(normalize_signature(&f.text)) + .or_insert(0) += 1; + } + if let Some(l) = last { + *self + .last_counts + .entry(normalize_signature(&l.text)) + .or_insert(0) += 1; + } + } + + /// Emit the aggregates. `body_size` is the bucket with the most text + /// (ties go to the larger size — the body is never smaller than its own + /// footnotes); signatures under the repetition threshold are dropped. + pub(crate) fn finish(self, page_count: usize) -> DocSignals { + let body_size = self + .size_weights + .iter() + .max_by(|a, b| a.1.cmp(b.1).then_with(|| a.0.cmp(b.0))) + .map_or(DEFAULT_BODY_SIZE, |(&key, _)| bucket_size(key)); + let threshold = HEADER_FOOTER_MIN_PAGES.max(page_count / HEADER_FOOTER_PAGE_DIV); + let collect = |counts: std::collections::HashMap| { + counts + .into_iter() + .filter(|(_, n)| *n >= threshold) + .map(|(sig, _)| sig) + .collect() + }; + DocSignals { + body_size, + headers: collect(self.first_counts), + footers: collect(self.last_counts), + } + } +} + +/// Quarter-point histogram bucket for a font size. Positive sizes only — +/// callers skip `<= 0.0` (a malformed-PDF guard, not a real case). +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] // size > 0 and ≤ ~1e3pt +fn size_bucket(size: f64) -> u64 { + (size * 4.0).round() as u64 +} + +/// Inverse of [`size_bucket`]. +#[allow(clippy::cast_precision_loss)] // bucket keys are small integers +fn bucket_size(key: u64) -> f64 { + key as f64 / 4.0 +} + +/// Normalize a line for header/footer matching: lowercase, every RUN of +/// ASCII digits collapses to one `#`, whitespace runs collapse to one space +/// (liteparse `normalize_for_repetition`). Run-collapse — not per-digit — +/// so "Page 3 of 9" and "Page 12 of 9" normalize together. +fn normalize_signature(text: &str) -> String { + let mut out = String::new(); + let mut in_digits = false; + let mut pending_space = false; + for c in text.trim().chars().flat_map(char::to_lowercase) { + if c.is_whitespace() { + pending_space = !out.is_empty(); + in_digits = false; + continue; + } + if pending_space { + out.push(' '); + pending_space = false; + } + if c.is_ascii_digit() { + if !in_digits { + out.push('#'); + in_digits = true; + } + } else { + out.push(c); + in_digits = false; + } + } + out +} + +/// Open paragraph under construction: text plus the last line's font +/// size and top edge (the gap rule needs both). +struct Para { + text: String, + font_size: f64, + y0: f64, +} + +/// Open list under construction: stripped items plus the last item line's +/// font size and top edge (the [`LIST_GAP_RATIO`] rule needs both). +struct ListState { + items: Vec, + font_size: f64, + y0: f64, + /// x0 of the marker lines — a wrapped item's continuation is more + /// indented than this. + indent: f64, +} + +/// If `text` starts with a list marker, return the item text (marker +/// stripped, trimmed). Unordered: a [`BULLET_CHARS`] glyph followed by +/// whitespace. Ordered: 1–3 digits or one lowercase letter, then `.` or +/// `)`, then whitespace. The remainder must be non-empty — a bare marker +/// carries no item (liteparse `parse_list_marker`, trimmed to the +/// spec-pinned marker set; its sequence-confirmed lettered/roman runs are +/// deliberately not ported). +fn parse_list_marker(text: &str) -> Option<&str> { + let t = text.trim_start(); + let mut chars = t.chars(); + let first = chars.next()?; + if BULLET_CHARS.contains(&first) { + return marker_item(chars.as_str()); + } + let body_len = if first.is_ascii_digit() { + let digits = t.bytes().take_while(u8::is_ascii_digit).count(); + // ^\d{1,3}: four or more leading digits is a number, not a marker. + if digits > 3 { + return None; + } + digits + } else if first.is_ascii_lowercase() { + 1 // ^[a-z]: exactly one letter + } else { + return None; + }; + let mut rest = t[body_len..].chars(); + let punct = rest.next()?; + if punct != '.' && punct != ')' { + return None; + } + marker_item(rest.as_str()) +} + +/// Item text after a marker: the next char must be whitespace (so +/// "well-known" and "1.5x" never match) and the trimmed remainder must +/// carry content. +fn marker_item(rest: &str) -> Option<&str> { + if !rest.chars().next().is_some_and(char::is_whitespace) { + return None; + } + let item = rest.trim(); + (!item.is_empty()).then_some(item) +} + +/// Incremental heading/list/paragraph classifier: [`detect_tables`] feeds +/// it the non-table lines interleaved with detected table blocks, so the +/// paragraph and list state must survive across calls. [`classify`] is +/// the thin all-lines wrapper; the block rules live here exactly once. +struct Classifier<'a> { + signals: &'a DocSignals, + blocks: Vec, + para: Option, + list: Option, + /// Position of the most recent heading line, for multi-line heading + /// merging (reset by any intervening non-heading line). + last_heading: Option<(u8, f64, f64)>, +} + +impl Classifier<'_> { + const fn new(signals: &DocSignals) -> Classifier<'_> { + Classifier { + signals, + blocks: Vec::new(), + para: None, + list: None, + last_heading: None, + } + } + + fn flush_para(&mut self) { + if let Some(p) = self.para.take() { + self.blocks.push(Block::Paragraph(p.text)); + } + } + + fn flush_list(&mut self) { + if let Some(l) = self.list.take() { + self.blocks.push(Block::List(l.items)); + } + } + + fn flush(&mut self) { + self.flush_para(); + self.flush_list(); + } + + /// Fold one line into the block stream: header/footer drop, heading + /// gate, then paragraph merge/split — see [`classify`] for the rules. + fn feed(&mut self, line: &Line) { + let text = line.text.trim(); + if text.is_empty() { + return; + } + let sig = normalize_signature(text); + if self.signals.headers.contains(&sig) || self.signals.footers.contains(&sig) { + self.flush(); + return; + } + let ratio = if self.signals.body_size > 0.0 { + line.font_size / self.signals.body_size + } else { + 0.0 + }; + let prev_heading = self.last_heading.take(); + // List items (Task 16): marker detection runs on text, so OCR + // lines (no words) can join lists. Marker lines interrupt + // paragraph runs and vice versa. + if let Some(item) = parse_list_marker(text) { + self.flush_para(); + match self.list.as_mut() { + Some(l) if line.rect.y0 - l.y0 <= LIST_GAP_RATIO * l.font_size => { + l.items.push(item.to_string()); + l.font_size = line.font_size; + l.y0 = line.rect.y0; + } + _ => { + self.flush_list(); + self.list = Some(ListState { + items: vec![item.to_string()], + font_size: line.font_size, + y0: line.rect.y0, + indent: line.rect.x0, + }); + } + } + return; + } + // Wrapped list item: an indented non-marker line right after an + // item continues it (liteparse `lists.rs` continuation lines). + if let Some(l) = self.list.as_mut() { + if line.rect.x0 > l.indent && line.rect.y0 - l.y0 <= LIST_GAP_RATIO * l.font_size { + if let Some(item) = l.items.last_mut() { + item.push(' '); + item.push_str(text); + } + l.font_size = line.font_size; + l.y0 = line.rect.y0; + return; + } + } + self.flush_list(); + if ratio >= HEADING_MIN_RATIO { + self.flush(); + let level = if ratio >= HEADING_H1_RATIO { + 1 + } else if ratio >= HEADING_H2_RATIO { + 2 + } else if ratio >= HEADING_H3_RATIO { + 3 + } else { + 4 + }; + // Multi-line headings (deck-style display-sized paragraphs) + // merge consecutive same-level heading lines into one block. + if let Some((prev_level, prev_y0, prev_size)) = prev_heading { + if prev_level == level && line.rect.y0 - prev_y0 <= PARAGRAPH_GAP_RATIO * prev_size + { + if let Some(Block::Heading { text: t, .. }) = self.blocks.last_mut() { + t.push(' '); + t.push_str(text); + } + self.last_heading = Some((level, line.rect.y0, line.font_size)); + return; + } + } + self.blocks.push(Block::Heading { + level, + text: text.to_string(), + }); + self.last_heading = Some((level, line.rect.y0, line.font_size)); + return; + } + self.flush_list(); + match self.para.as_mut() { + Some(p) if line.rect.y0 - p.y0 <= PARAGRAPH_GAP_RATIO * p.font_size => { + if p.text.ends_with('-') { + p.text.pop(); + } else { + p.text.push(' '); + } + p.text.push_str(text); + p.font_size = line.font_size; + p.y0 = line.rect.y0; + } + _ => { + self.flush(); + self.para = Some(Para { + text: text.to_string(), + font_size: line.font_size, + y0: line.rect.y0, + }); + } + } + } + + /// A detected table interrupts any open paragraph/list. + fn feed_table(&mut self, rows: Vec>) { + self.flush(); + self.blocks.push(Block::Table(rows)); + } + + fn finish(mut self) -> Vec { + self.flush(); + self.blocks + } +} + +/// Classify an ordered mixed stream (tables already detected) into blocks. +pub(crate) fn classify_items(items: Vec, signals: &DocSignals) -> Vec { + let mut c = Classifier::new(signals); + for item in items { + match item { + Item::Line(line) => c.feed(&line), + Item::Table(t) => c.feed_table(t.rows), + } + } + c.finish() +} + +/// Classify reading-ordered `lines` (native + OCR merged — OCR lines +/// participate exactly like native ones) into blocks: +/// +/// 1. Lines whose normalized text is a known header/footer are dropped +/// (and break any open paragraph — chrome never glues text together). +/// 2. A line at ≥ [`HEADING_MIN_RATIO`]× the body size is a heading; the +/// ratio band picks the level. +/// 3. A body-size line starting with a list marker (see +/// [`parse_list_marker`]) becomes a list item; consecutive items within +/// [`LIST_GAP_RATIO`]× the previous item's font size group into one +/// [`Block::List`]. Marker lines interrupt paragraph runs and vice +/// versa. +/// 4. Remaining consecutive lines merge into paragraphs, joined with a +/// space — except a trailing `-` dehyphenates ("exam-" + "ple" → +/// "example"). A top-to-top gap > [`PARAGRAPH_GAP_RATIO`]× the previous +/// line's font size splits the paragraph. +/// +/// Whitespace-only lines (all-space clusters) are dropped, never emitted +/// as empty paragraphs. +// Pass-by-value is the established Task 13 signature (mirrors the driver, +// which hands over the owned line vec); the lines are only borrowed now +// that classification is incremental. +#[allow(dead_code, clippy::needless_pass_by_value)] +pub(crate) fn classify(lines: Vec, signals: &DocSignals) -> Vec { + let mut c = Classifier::new(signals); + for line in &lines { + c.feed(line); + } + c.finish() +} + +// --------------------------------------------------------------------------- +// Borderless table detection (spec §7 v1.5; liteparse `try_detect_table*`, +// adapted: no ruled-line graphics, words rebuilt from advance gaps, word x0s +// clustered across the run instead of PDFium span pieces). +// --------------------------------------------------------------------------- + +/// A table needs at least this many consecutive candidate lines. +const TABLE_MIN_ROWS: usize = 3; +/// …and each candidate line needs at least this many words (single-word +/// lines break runs — they carry no column signal). +const TABLE_MIN_WORDS: usize = 2; +/// Candidate lines sit within this fraction of the body size (heading-size +/// lines break runs). +const TABLE_BODY_SIZE_TOLERANCE: f64 = 0.2; +/// Run continuity: the bottom-to-top vertical gap between consecutive +/// candidate lines stays within this many times the previous line's font +/// size (liteparse `table_rows_adjacent`, restated in `Line` geometry). +const TABLE_ROW_GAP_RATIO: f64 = 2.5; +/// Column-anchor clustering tolerance in points (liteparse +/// `TABLE_TRACK_TOLERANCE_PT`, same role): absolute, not font-relative — +/// a header label centered over its data column sits a fixed few points +/// off the body anchor regardless of font size. +const TABLE_TRACK_TOLERANCE_PT: f64 = 6.0; +/// Gutter check: between consecutive anchor columns a line occupies, the +/// whitespace gap must reach this many times the line's font size. Prose +/// word spacing (~0.25em) fails; table gutters pass. +const TABLE_GUTTER_RATIO: f64 = 0.75; +/// Occupancy (liteparse `count_text_table_runs` criterion): ≥60% of a +/// line's words start at an anchor, and each anchor column is occupied by +/// ≥60% of the run's lines. Exact 3/5 comparisons, so a 2/3 sparse row +/// (67%) passes. +const TABLE_OCCUPANCY_NUM: usize = 3; +const TABLE_OCCUPANCY_DEN: usize = 5; +/// Fewer than two columns is a definition list, not a table. +const TABLE_MIN_COLUMNS: usize = 2; + +/// Detect borderless tables in reading-ordered `lines` and classify the +/// rest: a run of [`TABLE_MIN_ROWS`]+ consecutive body-size lines whose +/// word x0s cluster into ≥ [`TABLE_MIN_COLUMNS`] anchor columns — with a +/// real gutter between occupied column pairs and 60% line/column +/// occupancy — becomes [`Block::Table`]; every other line goes through the +/// same heading/paragraph rules as [`classify`]. Runs classification +/// internally and replaces the bare `classify` call in the driver +/// (Task 15 wires it in). +/// +/// OCR-source lines carry no word geometry (`words` is empty), so they +/// fail the word-count gate and never join a run. +// Pass-by-value is the binding Task 14 signature (mirrors `classify`). +#[allow(clippy::needless_pass_by_value)] +/// Line-in/block-out wrapper over `find_tables` + `classify_items` for +/// the table unit tests; the production driver interposes +/// `reading_order_items` between the two. +#[allow(dead_code)] // test-only wrapper +pub(crate) fn detect_tables(lines: Vec, signals: &DocSignals) -> Vec { + classify_items(find_tables(&lines, signals), signals) +} + +/// A detected borderless table as an opaque reading-order item. Its +/// full-width rect participates in xy-cut geometry — a table is a band, +/// never column-cut. +pub(crate) struct TableRegion { + pub rect: PtRect, + pub font_size: f64, + pub rows: Vec>, +} + +/// A reading-order item: a text line or a detected table region. +pub(crate) enum Item { + Line(Line), + Table(TableRegion), +} + +impl Item { + const fn rect(&self) -> &PtRect { + match self { + Self::Line(l) => &l.rect, + Self::Table(t) => &t.rect, + } + } + const fn font_size(&self) -> f64 { + match self { + Self::Line(l) => l.font_size, + Self::Table(t) => t.font_size, + } + } +} + +/// Walk lines in assembly order (row fragments are adjacent there — +/// after a column cut they would not be), detect table runs, and return +/// a mixed item stream in the same relative order: a table occupies the +/// position of the lines it consumed. +pub(crate) fn find_tables(lines: &[Line], signals: &DocSignals) -> Vec { + let mut out = Vec::new(); + let mut i = 0; + while i < lines.len() { + if let Some((rows, rect, font_size, end)) = try_table_run(lines, i, signals) { + out.push(Item::Table(TableRegion { + rect, + font_size, + rows, + })); + i = end; + } else { + out.push(Item::Line(lines[i].clone())); + i += 1; + } + } + out +} + +/// If a table run starts at `lines[start]`, return its rows of cells and +/// the index one past the run's last line; `None` falls through to normal +/// classification of that one line (the walk retries at `start + 1`, so a +/// table whose run begins deeper inside a candidate band is still found). +fn try_table_run( + lines: &[Line], + start: usize, + signals: &DocSignals, +) -> Option<(Vec>, PtRect, f64, usize)> { + if !fragment_candidate(&lines[start], None, signals) { + return None; + } + let mut end = start + 1; + while end < lines.len() && fragment_candidate(&lines[end], Some(&lines[end - 1]), signals) { + end += 1; + } + // Same-baseline fragments (a row split across a wide column gutter) + // merge into logical rows before geometry is evaluated. + let run = &merge_baseline_fragments(&lines[start..end]); + if run.len() < TABLE_MIN_ROWS { + return None; + } + if run.iter().any(|r| r.words.len() < TABLE_MIN_WORDS) { + return None; + } + let anchors = column_anchors(run, TABLE_TRACK_TOLERANCE_PT); + if anchors.len() < TABLE_MIN_COLUMNS { + return None; + } + let assigned: Vec>> = run + .iter() + .map(|line| assign_columns(&anchors, line)) + .collect(); + // Gutter check: for every pair of consecutive anchor columns a line + // occupies, the whitespace between the right edge of the left cell's + // last word and the left edge of the right cell's first word must be + // a real gutter, not a prose word space. + for (line, cols) in run.iter().zip(&assigned) { + let gutter = TABLE_GUTTER_RATIO * line.font_size; + for pair in 0..anchors.len() - 1 { + let left = line + .words + .iter() + .zip(cols) + .filter(|(_, c)| **c == Some(pair)) + .map(|(w, _)| w.x1) + .fold(f64::NEG_INFINITY, f64::max); + let right = line + .words + .iter() + .zip(cols) + .filter(|(_, c)| **c == Some(pair + 1)) + .map(|(w, _)| w.x0) + .fold(f64::INFINITY, f64::min); + if left.is_finite() && right.is_finite() && right - left < gutter { + return None; + } + } + } + // Line occupancy: ≥60% of the line's words start at an anchor. + for (line, cols) in run.iter().zip(&assigned) { + let on_anchor = cols.iter().filter(|c| c.is_some()).count(); + if on_anchor * TABLE_OCCUPANCY_DEN < line.words.len() * TABLE_OCCUPANCY_NUM { + return None; + } + } + // Column occupancy: each anchor is used by ≥60% of the run's lines. + for col in 0..anchors.len() { + let occupied = assigned + .iter() + .filter(|cols| cols.contains(&Some(col))) + .count(); + if occupied * TABLE_OCCUPANCY_DEN < run.len() * TABLE_OCCUPANCY_NUM { + return None; + } + } + // Cell text: words assigned to a column join with spaces; a column + // with no word in a line yields an empty cell. + let rows = run + .iter() + .zip(&assigned) + .map(|(line, cols)| { + let mut cells = vec![String::new(); anchors.len()]; + for (w, c) in line.words.iter().zip(cols) { + if let Some(col) = c { + if !cells[*col].is_empty() { + cells[*col].push(' '); + } + cells[*col].push_str(&w.text); + } + } + cells + }) + .collect(); + let rect = run + .iter() + .map(|l| l.rect) + .reduce(|a, b| PtRect { + x0: a.x0.min(b.x0), + y0: a.y0.min(b.y0), + x1: a.x1.max(b.x1), + y1: a.y1.max(b.y1), + }) + .expect("run is non-empty (TABLE_MIN_ROWS)"); + let font_size = median_font_size(run); + Some((rows, rect, font_size, end)) +} + +/// A line that can extend a table-run candidate window: body-size, +/// non-empty, not a list-marker line, with ≥1 word, and either sharing +/// the previous line's baseline (a right-hand row fragment) or within +/// [`TABLE_ROW_GAP_RATIO`] × font size below it. Word-count candidacy is +/// evaluated AFTER baseline fragments merge ([`TABLE_MIN_WORDS`] on +/// logical rows), because a split row's left fragment is often a single +/// word. +fn fragment_candidate(line: &Line, prev: Option<&Line>, signals: &DocSignals) -> bool { + if line.text.trim().is_empty() || line.words.is_empty() { + return false; + } + // Binding rule (Task 16): bullets win over table geometry. + if parse_list_marker(&line.text).is_some() { + return false; + } + if !(signals.body_size > 0.0 + && (line.font_size - signals.body_size).abs() + <= TABLE_BODY_SIZE_TOLERANCE * signals.body_size) + { + return false; + } + prev.is_none_or(|p| { + let same_baseline = + (line.rect.y0 - p.rect.y0).abs() <= 0.5 * line.font_size && line.rect.x0 > p.rect.x1; + let next_row = line.rect.y0 - p.rect.y1 <= TABLE_ROW_GAP_RATIO * p.font_size; + same_baseline || next_row + }) +} + +/// Merge same-baseline right-hand fragments into logical rows: a line +/// whose baseline matches the previous line's (within half a font size) +/// and which starts right of it is the same row, split across a wide +/// gutter. Identity when no fragments exist. +fn merge_baseline_fragments(lines: &[Line]) -> Vec { + let mut rows: Vec = Vec::new(); + for line in lines { + let merge = rows.last().is_some_and(|prev| { + (line.rect.y0 - prev.rect.y0).abs() <= 0.5 * line.font_size + && line.rect.x0 > prev.rect.x1 + }); + if merge { + let prev = rows.last_mut().expect("checked above"); + prev.text.push(' '); + prev.text.push_str(&line.text); + prev.words.extend(line.words.iter().cloned()); + prev.rect.x1 = prev.rect.x1.max(line.rect.x1); + prev.rect.y0 = prev.rect.y0.min(line.rect.y0); + prev.rect.y1 = prev.rect.y1.max(line.rect.y1); + } else { + rows.push(line.clone()); + } + } + rows +} + +/// Word → column assignment (hybrid): coverage first — an anchor inside +/// the word's x-extent ± [`TABLE_TRACK_TOLERANCE_PT`] (handles header +/// labels centered over right-aligned data, and right-aligned numbers +/// under left-aligned labels); otherwise region — the rightmost anchor +/// left of the word (continuation words inside a prose cell). Words left +/// of the first anchor are out of band (unassigned) and count against +/// line occupancy. +fn assign_columns(anchors: &[f64], line: &Line) -> Vec> { + line.words + .iter() + .map(|w| { + anchors + .iter() + .enumerate() + .filter(|(_, a)| { + **a >= w.x0 - TABLE_TRACK_TOLERANCE_PT && **a <= w.x1 + TABLE_TRACK_TOLERANCE_PT + }) + .min_by(|(ai, a), (bi, b)| { + (w.x0 - *a) + .abs() + .total_cmp(&(w.x0 - *b).abs()) + .then_with(|| ai.cmp(bi)) + }) + .map(|(i, _)| i) + .or_else(|| { + anchors + .iter() + .enumerate() + .rev() + .find(|(_, a)| **a <= w.x0 - TABLE_TRACK_TOLERANCE_PT) + .map(|(i, _)| i) + }) + }) + .collect() +} + +/// Cluster the run's column-start positions into sorted column anchors. +/// A word seeds a column only when it is the line's first word or follows +/// a real gutter (≥ [`TABLE_GUTTER_RATIO`] × font size from the previous +/// word's right edge) — continuation words inside a prose cell never seed +/// (they would form phantom columns at scattered x positions). +/// +/// Seeds within `tol` of a cluster's running mean merge into it (the +/// mean keeps the anchor centered as left/right-aligned cells drift in). +/// Clusters witnessed by a single seed (a header label offset from its +/// data column) are pruned when a clear multi-row body exists (max +/// support ≥ 3) — liteparse's phantom-track pruning (`tables.rs` +/// `infer_tracks_from_raw_items`), reimplemented. +#[allow(clippy::cast_precision_loss)] // cluster sizes are tiny; exact in f64 +fn column_anchors(run: &[Line], tol: f64) -> Vec { + // A word seeds a column only when it is the line's first word or + // follows a real gutter (≥ TABLE_GUTTER_RATIO × font size from the + // previous word's right edge) — continuation words inside a prose + // cell never seed (they would form phantom columns at scattered x). + let mut xs: Vec = Vec::new(); + for line in run { + let gutter = TABLE_GUTTER_RATIO * line.font_size; + let mut prev_x1 = f64::NEG_INFINITY; + for w in &line.words { + if w.x0 - prev_x1 >= gutter { + xs.push(w.x0); + } + prev_x1 = w.x1; + } + } + xs.sort_by(f64::total_cmp); + // (sum, count) per open cluster; anchors are the final means. + let mut clusters: Vec<(f64, usize)> = Vec::new(); + for x in xs { + match clusters.last_mut() { + Some((sum, n)) if (x - *sum / *n as f64).abs() <= tol => { + *sum += x; + *n += 1; + } + _ => clusters.push((x, 1)), + } + } + let max_support = clusters.iter().map(|c| c.1).max().unwrap_or(0); + if max_support >= 3 { + clusters.retain(|c| c.1 >= 2); + } + clusters.iter().map(|(sum, n)| sum / *n as f64).collect() +} + +// --------------------------------------------------------------------------- +// Markdown render (Task 13 driver output). +// --------------------------------------------------------------------------- + +/// Render classified blocks to markdown: blocks separated by exactly one +/// blank line, document ends with a single trailing newline. +pub(crate) fn render( + blocks: &[Block], + sink: &mut impl crate::ExtractSink, +) -> crate::error::Result<()> { + for (i, block) in blocks.iter().enumerate() { + if i > 0 { + sink.write_str("\n")?; + } + match block { + Block::Heading { level, text } => { + sink.write_str(&"#".repeat(usize::from(*level)))?; + sink.write_str(" ")?; + sink.write_str(text)?; + sink.write_str("\n")?; + } + Block::Paragraph(text) => { + sink.write_str(text)?; + sink.write_str("\n")?; + } + Block::Table(rows) => { + render_table(rows, sink)?; + } + Block::List(items) => { + for item in items { + sink.write_str("- ")?; + sink.write_str(item)?; + sink.write_str("\n")?; + } + } + } + } + Ok(()) +} + +/// Escape a table cell for pipe-table markdown: backslashes first, then +/// pipes, so we don't double-escape the separator we just inserted. +fn escape_cell(s: &str) -> String { + s.replace('\\', "\\\\").replace('|', "\\|") +} + +/// Render a pipe table: header row, `| --- |` separator per column, then +/// body rows. The last row ends with a newline; the surrounding render loop +/// supplies the blank-line separation between blocks. +fn render_table( + rows: &[Vec], + sink: &mut impl crate::ExtractSink, +) -> crate::error::Result<()> { + let Some(header) = rows.first() else { + return Ok(()); + }; + render_table_row(header, sink)?; + for _ in header { + sink.write_str("| --- ")?; + } + sink.write_str("|\n")?; + for row in rows.iter().skip(1) { + render_table_row(row, sink)?; + } + Ok(()) +} + +fn render_table_row( + row: &[String], + sink: &mut impl crate::ExtractSink, +) -> crate::error::Result<()> { + for cell in row { + sink.write_str("| ")?; + sink.write_str(&escape_cell(cell))?; + sink.write_str(" ")?; + } + sink.write_str("|\n")?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pdf_text::{PositionedChar, PositionedPage}; + + fn pc(ch: char, x: f64, y: f64, size: f64, adv: f64) -> PositionedChar { + PositionedChar { + ch, + x, + y, + font_size: size, + advance: adv, + rotation: 0, + } + } + + /// "Hi" at 12pt: H at x=100 (adv 8), i at x=108 (adv 3). + fn hi_page() -> PositionedPage { + PositionedPage { + page_num: 1, + media_box: (0.0, 0.0, 612.0, 792.0), + chars: vec![ + pc('H', 100.0, 92.0, 12.0, 8.0), + pc('i', 108.0, 92.0, 12.0, 3.0), + ], + } + } + + #[test] + fn assembles_one_line() { + let lines = assemble(&hi_page()); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].text, "Hi"); + assert_eq!(lines[0].words.len(), 1); + assert!((lines[0].rect.x0 - 100.0).abs() < 0.01); + assert!((lines[0].font_size - 12.0).abs() < 0.01); + } + + #[test] + fn word_break_on_advance_gap() { + // "Hi there": gap of 10pt (> 1.5*5.5 median advance = 8.25) + // after "Hi". + let mut chars = hi_page().chars; + #[allow(clippy::cast_precision_loss)] // tiny synthetic index; exact in f64 + let mut rest = "there" + .chars() + .enumerate() + .map(|(i, c)| pc(c, 121.0 + i as f64 * 6.0, 92.0, 12.0, 5.5)) + .collect::>(); + chars.append(&mut rest); + let lines = assemble(&PositionedPage { + page_num: 1, + media_box: (0.0, 0.0, 612.0, 792.0), + chars, + }); + assert_eq!(lines[0].text, "Hi there"); + assert_eq!(lines[0].words.len(), 2); + } + + #[test] + fn explicit_space_glyph_does_not_double() { + let chars = vec![ + pc('a', 100.0, 92.0, 12.0, 6.0), + pc(' ', 106.0, 92.0, 12.0, 3.0), + pc('b', 109.0, 92.0, 12.0, 6.0), + ]; + let lines = assemble(&PositionedPage { + page_num: 1, + media_box: (0.0, 0.0, 612.0, 792.0), + chars, + }); + assert_eq!(lines[0].text, "a b"); + } + + #[test] + fn two_baselines_make_two_lines() { + let mut chars = hi_page().chars; + chars.push(pc('B', 100.0, 106.0, 12.0, 7.0)); + chars.push(pc('y', 107.0, 106.0, 12.0, 6.0)); + chars.push(pc('e', 113.0, 106.0, 12.0, 6.0)); + let lines = assemble(&PositionedPage { + page_num: 1, + media_box: (0.0, 0.0, 612.0, 792.0), + chars, + }); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0].text, "Hi"); + assert_eq!(lines[1].text, "Bye"); + } + + #[test] + fn tight_lines_do_not_chain_merge_through_a_tall_line() { + // Real layout: a sidebar line (taller font, x >= 440) between two + // body lines only 1.5 line-pitches apart. The running-band sweep + // used to let the sidebar's band bridge the two body lines into + // one cluster whose x-sorted chars interleaved into soup. + let mut chars = Vec::new(); + let mut emit = |text: &str, x: f64, y: f64, size: f64| { + for (i, ch) in text.chars().enumerate() { + #[allow(clippy::cast_precision_loss)] // synthetic positions + chars.push(PositionedChar { + ch, + x: x + i as f64 * 5.0, + y, + font_size: size, + advance: 4.5, + rotation: 0, + }); + } + }; + emit("guesswork", 42.0, 100.0, 9.4); // body line A + emit("How do we compare", 440.0, 109.8, 11.0); // sidebar S (bridging band) + emit("consulting", 42.0, 114.3, 9.4); // body line B + let lines = assemble(&PositionedPage { + page_num: 1, + media_box: (0.0, 0.0, 612.0, 792.0), + chars, + }); + let texts: Vec<&str> = lines.iter().map(|l| l.text.as_str()).collect(); + assert!(texts.contains(&"guesswork"), "{texts:?}"); + assert!(texts.contains(&"consulting"), "{texts:?}"); + assert!(!texts.iter().any(|t| t.contains("guces")), "{texts:?}"); + assert!(texts.contains(&"How do we compare"), "{texts:?}"); + } + + #[test] + fn rotated_page_is_canonicalized() { + // 4 of 5 chars at rotation 90 → page canonicalized; the line reads + // in ascending original-y order. "abc" written bottom-to-top: + // a at (50, 300), b at (50, 290), c at (50, 280), rotation 90. + let chars = vec![ + PositionedChar { + ch: 'a', + x: 50.0, + y: 300.0, + font_size: 12.0, + advance: 7.0, + rotation: 90, + }, + PositionedChar { + ch: 'b', + x: 50.0, + y: 290.0, + font_size: 12.0, + advance: 7.0, + rotation: 90, + }, + PositionedChar { + ch: 'c', + x: 50.0, + y: 280.0, + font_size: 12.0, + advance: 7.0, + rotation: 90, + }, + ]; + let lines = assemble(&PositionedPage { + page_num: 1, + media_box: (0.0, 0.0, 612.0, 792.0), + chars, + }); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0].text, "abc"); + } + + #[test] + fn wide_horizontal_gap_splits_line() { + // "L1" at x=72 and "R1" at x=320 on the same baseline → two lines + // (gap 234pt = 19.5·12, well above LINE_SPLIT_RATIO=10.0·12). This + // is the upper pin of the (8.0, 19.5) ratio window that brackets the + // horizontal line-split threshold; the lower pin keeps a 96pt gap on + // one line. + let chars = vec![ + pc('L', 72.0, 92.0, 12.0, 8.0), + pc('1', 80.0, 92.0, 12.0, 6.0), + pc('R', 320.0, 92.0, 12.0, 8.0), + pc('1', 328.0, 92.0, 12.0, 6.0), + ]; + let lines = assemble(&PositionedPage { + page_num: 1, + media_box: (0.0, 0.0, 612.0, 792.0), + chars, + }); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0].text, "L1"); + assert_eq!(lines[1].text, "R1"); + } + + fn line(text: &str, x0: f64, y0: f64, x1: f64) -> Line { + Line { + text: text.into(), + words: vec![], + rect: PtRect { + x0, + y0, + x1, + y1: y0 + 12.0, + }, + font_size: 12.0, + source: LineSource::Native, + } + } + + fn texts(lines: &[Line]) -> Vec<&str> { + lines.iter().map(|l| l.text.as_str()).collect() + } + + #[test] + fn single_column_sorts_by_y() { + let lines = vec![ + line("second", 72.0, 100.0, 200.0), + line("first", 72.0, 80.0, 200.0), + ]; + assert_eq!(texts(&reading_order(lines)), ["first", "second"]); + } + + #[test] + fn two_columns_left_then_right() { + // Left column x 72..250, right column x 320..500 (gap 70pt >= 2*12). + let lines = vec![ + line("R1", 320.0, 80.0, 500.0), + line("L2", 72.0, 100.0, 250.0), + line("L1", 72.0, 80.0, 250.0), + line("R2", 320.0, 100.0, 500.0), + ]; + assert_eq!(texts(&reading_order(lines)), ["L1", "L2", "R1", "R2"]); + } + + #[test] + fn full_width_title_then_columns() { + let lines = vec![ + line("R1", 320.0, 100.0, 500.0), + line("TITLE", 72.0, 60.0, 500.0), // spans both columns + line("L1", 72.0, 100.0, 250.0), + ]; + assert_eq!(texts(&reading_order(lines)), ["TITLE", "L1", "R1"]); + } + + #[test] + fn narrow_gap_is_not_a_column_cut() { + // 15pt gap < 2*12=24pt threshold → one column, y-sorted. + let lines = vec![line("b", 200.0, 90.0, 280.0), line("a", 72.0, 80.0, 185.0)]; + assert_eq!(texts(&reading_order(lines)), ["a", "b"]); + } + + #[test] + fn empty_page_assembles_empty() { + let lines = assemble(&PositionedPage { + page_num: 1, + media_box: (0.0, 0.0, 612.0, 792.0), + chars: vec![], + }); + assert!(lines.is_empty()); + } + + fn sig_lines(pages: &[Vec]) -> DocSignals { + let mut b = DocSignalsBuilder::new(); + for p in pages { + b.add_lines(p); + } + b.finish(pages.len()) + } + + #[test] + fn body_size_is_modal() { + let pages = vec![vec![ + line("body text here", 72.0, 100.0, 300.0), // 12pt (helper default) + Line { + font_size: 18.0, + ..line("H", 72.0, 60.0, 100.0) + }, + ]]; + assert!((sig_lines(&pages).body_size - 12.0).abs() < 0.01); + } + + #[test] + fn repeated_first_line_becomes_header() { + // The trailing "common closer" line is deliberate: without it the + // digit-normalized body line is every page's LAST line and would + // itself qualify as a footer (first/last-line signatures have no + // page-band gating — see task-12 report). It doubles as a footer + // fixture: classify must drop it too. + let mk = |n: usize| { + vec![ + line("CONFIDENTIAL", 72.0, 30.0, 200.0), + line(&format!("body {n}"), 72.0, 100.0, 300.0), + line("common closer", 72.0, 700.0, 300.0), + ] + }; + let pages: Vec> = (0..4).map(mk).collect(); + let sig = sig_lines(&pages); + assert!(sig.headers.contains("confidential")); + let blocks = classify(pages[0].clone(), &sig); + assert_eq!(blocks.len(), 1); + assert!(matches!(&blocks[0], Block::Paragraph(t) if t == "body 0")); + } + + #[test] + fn page_numbers_in_footers_normalize_together() { + let mk = |n: usize| { + vec![ + line("body", 72.0, 100.0, 300.0), + line(&format!("Page {n} of 9"), 250.0, 750.0, 350.0), + ] + }; + let pages: Vec> = (1..=3).map(mk).collect(); + let sig = sig_lines(&pages); + assert!(sig.footers.contains("page # of #"), "{:?}", sig.footers); + } + + #[test] + fn heading_levels_from_size_ratio() { + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let lines = vec![ + Line { + font_size: 24.0, + ..line("Big", 72.0, 50.0, 200.0) + }, // 2.0x → H1 + Line { + font_size: 18.0, + ..line("Mid", 72.0, 90.0, 200.0) + }, // 1.5x → H2 + Line { + font_size: 14.5, + ..line("Small", 72.0, 120.0, 200.0) + }, // 1.21x → H4 + line("body", 72.0, 150.0, 300.0), + ]; + let blocks = classify(lines, &sig); + assert!(matches!(&blocks[0], Block::Heading { level: 1, text } if text == "Big")); + assert!(matches!(&blocks[1], Block::Heading { level: 2, .. })); + assert!(matches!(&blocks[2], Block::Heading { level: 4, .. })); + assert!(matches!(&blocks[3], Block::Paragraph(t) if t == "body")); + } + + #[test] + fn paragraph_join_and_dehyphenation() { + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let lines = vec![ + line("This is an exam-", 72.0, 100.0, 300.0), + line("ple of joining.", 72.0, 114.0, 300.0), + line("New para.", 72.0, 140.0, 300.0), // 26pt gap > 1.5*12 → split + ]; + let blocks = classify(lines, &sig); + assert_eq!(blocks.len(), 2); + assert!(matches!(&blocks[0], Block::Paragraph(t) if t == "This is an example of joining.")); + assert!(matches!(&blocks[1], Block::Paragraph(t) if t == "New para.")); + } + + #[test] + fn whitespace_only_lines_are_dropped() { + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let lines = vec![ + line("real text", 72.0, 100.0, 300.0), + Line { + text: " ".into(), + ..line("", 72.0, 114.0, 300.0) + }, + ]; + let blocks = classify(lines, &sig); + assert_eq!(blocks.len(), 1); + assert!(matches!(&blocks[0], Block::Paragraph(t) if t == "real text")); + } + + /// Build a word-ful Line (detect_tables reads words, not just text). + #[allow(clippy::cast_precision_loss)] // tiny synthetic word lengths; exact in f64 + fn wline(words: &[&str], x0s: &[f64], y0: f64) -> Line { + assert_eq!(words.len(), x0s.len()); + let words_v: Vec = words + .iter() + .zip(x0s) + .map(|(t, x)| Word { + text: t.to_string(), + x0: *x, + x1: *x + t.len() as f64 * 6.0, + }) + .collect(); + let text = words.join(" "); + Line { + text, + rect: PtRect { + x0: x0s[0], + y0, + x1: 400.0, + y1: y0 + 12.0, + }, + font_size: 12.0, + source: LineSource::Native, + words: words_v, + } + } + + #[test] + fn detects_simple_grid() { + let lines = vec![ + wline(&["Name", "Age", "City"], &[72.0, 200.0, 300.0], 100.0), + wline(&["Alice", "30", "NYC"], &[72.0, 200.0, 300.0], 114.0), + wline(&["Bob", "25", "LA"], &[72.0, 200.0, 300.0], 128.0), + ]; + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let blocks = detect_tables(lines, &sig); + assert_eq!(blocks.len(), 1); + match &blocks[0] { + Block::Table(rows) => { + assert_eq!(rows.len(), 3); + assert_eq!(rows[0], vec!["Name", "Age", "City"]); + assert_eq!(rows[2], vec!["Bob", "25", "LA"]); + } + other => panic!("expected table, got {other:?}"), + } + } + + #[test] + fn prose_is_not_a_table() { + // Realistic prose spacing (6pt/char, ~3pt word gaps at 12pt): + // aligned-looking x0s but gutters of 3pt < 0.75*12=9pt → rejected + // by the gutter check. + let lines = vec![ + wline(&["The", "quick", "brown"], &[72.0, 93.0, 126.0], 100.0), + wline(&["fox", "jumps", "over"], &[72.0, 93.0, 126.0], 114.0), + wline(&["the", "lazy", "dog"], &[72.0, 93.0, 126.0], 128.0), + ]; + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let blocks = detect_tables(lines, &sig); + assert!(blocks.iter().all(|b| matches!(b, Block::Paragraph(_)))); + } + + #[test] + fn short_run_is_not_a_table() { + // Only 2 aligned rows — below the 3-line minimum. + let lines = vec![ + wline(&["a", "b"], &[72.0, 200.0], 100.0), + wline(&["c", "d"], &[72.0, 200.0], 114.0), + ]; + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let blocks = detect_tables(lines, &sig); + assert!(blocks.iter().all(|b| matches!(b, Block::Paragraph(_)))); + } + + #[test] + fn find_tables_preserves_stream_order() { + // Paragraph, then a shattered-row table (fragments adjacent in + // assembly order), then another paragraph: the item stream keeps + // relative order, with the table occupying its lines' position. + let lines = vec![ + line("Intro text.", 72.0, 60.0, 200.0), + frag(&["Class"], &[173.0], 100.0), + frag(&["Criteria", "Share"], &[328.0, 386.0], 100.0), + frag(&["Metric"], &[173.0], 124.0), + frag(&["89", "66%"], &[354.0, 392.0], 124.0), + frag(&["Document"], &[173.0], 148.0), + frag(&["28", "21%"], &[354.0, 392.0], 148.0), + frag(&["Judgment"], &[173.0], 172.0), + frag(&["18", "13%"], &[354.0, 392.0], 172.0), + line("Outro text.", 72.0, 220.0, 200.0), + ]; + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let items = find_tables(&lines, &sig); + assert_eq!(items.len(), 3); + assert!(matches!(&items[0], Item::Line(l) if l.text == "Intro text.")); + assert!(matches!(&items[1], Item::Table(_))); + assert!(matches!(&items[2], Item::Line(l) if l.text == "Outro text.")); + match &items[1] { + Item::Table(t) => { + assert_eq!(t.rows[0], vec!["Class", "Criteria", "Share"]); + assert_eq!(t.rows[1], vec!["Metric", "89", "66%"]); + } + Item::Line(_) => unreachable!(), + } + } + + #[test] + fn reading_order_items_keeps_table_regions_whole() { + // A full-width table band between two two-column text rows: the + // table must not be column-cut; it orders between the rows by y. + let lines = vec![ + line("L1", 72.0, 80.0, 120.0), + line("R1", 320.0, 80.0, 370.0), + line("L2", 72.0, 300.0, 120.0), + line("R2", 320.0, 300.0, 370.0), + ]; + let mut items: Vec = lines.into_iter().map(Item::Line).collect(); + items.push(Item::Table(TableRegion { + rect: PtRect { + x0: 72.0, + y0: 150.0, + x1: 500.0, + y1: 200.0, + }, + font_size: 12.0, + rows: vec![vec!["a".into(), "b".into()]], + })); + let ordered = reading_order_items(items); + let kinds: Vec = ordered + .iter() + .map(|it| match it { + Item::Line(l) => l.text.clone(), + Item::Table(_) => "TABLE".into(), + }) + .collect(); + assert_eq!(kinds, ["L1", "R1", "TABLE", "L2", "R2"]); + } + + #[test] + fn sidebar_orders_after_main_column() { + // Same-baseline main-column line + sidebar line: assembly splits + // them (gap > 2.5x size), xy-cut orders main column first. + let lines = vec![ + line("Main body text", 42.0, 100.0, 300.0), + line("Side note", 440.0, 103.0, 560.0), + line("More body", 42.0, 114.0, 300.0), + ]; + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let items = find_tables(&lines, &sig); + let ordered = reading_order_items(items); + let texts: Vec<&str> = ordered + .iter() + .map(|it| match it { + Item::Line(l) => l.text.as_str(), + Item::Table(_) => "TABLE", + }) + .collect(); + assert_eq!(texts, ["Main body text", "More body", "Side note"]); + } + + /// A fragment of a table row: one baseline's left or right piece. + fn frag(words: &[&str], x0s: &[f64], y0: f64) -> Line { + let mut l = wline(words, x0s, y0); + l.rect.x1 = l + .words + .iter() + .map(|w| w.x1) + .fold(f64::NEG_INFINITY, f64::max); + l + } + + #[test] + fn table_from_same_baseline_fragments() { + // Modeled on a real page: each row arrives as two lines (left + // label fragment, right numbers fragment) on the same baseline, + // split by a gap exceeding the line-split threshold. + let lines = vec![ + frag(&["Class"], &[173.0], 100.0), + frag(&["Criteria", "Share"], &[328.0, 386.0], 100.0), + frag(&["Metric"], &[173.0], 124.0), + frag(&["89", "66%"], &[354.0, 392.0], 124.0), + frag(&["Document"], &[173.0], 148.0), + frag(&["28", "21%"], &[354.0, 392.0], 148.0), + frag(&["Judgment"], &[173.0], 172.0), + frag(&["18", "13%"], &[354.0, 392.0], 172.0), + ]; + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let blocks = detect_tables(lines, &sig); + assert_eq!(blocks.len(), 1); + match &blocks[0] { + Block::Table(rows) => { + assert_eq!(rows[0], vec!["Class", "Criteria", "Share"]); + assert_eq!(rows[1], vec!["Metric", "89", "66%"]); + assert_eq!(rows[3], vec!["Judgment", "18", "13%"]); + } + other => panic!("expected table, got {other:?}"), + } + } + + #[test] + fn list_item_continuation_joins() { + // A wrapped list item: the indented non-marker line right after + // the item continues it instead of becoming its own paragraph. + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let lines = vec![ + line( + "- Metric. Evidenced by a tabular or API pull, scoreable against a", + 90.0, + 100.0, + 500.0, + ), + line("threshold.", 108.0, 114.0, 159.0), + ]; + let blocks = classify(lines, &sig); + assert_eq!(blocks.len(), 1); + assert!( + matches!(&blocks[0], Block::List(items) if items == &["Metric. Evidenced by a tabular or API pull, scoreable against a threshold."]), + "got {blocks:?}" + ); + } + + #[test] + fn non_indented_line_after_list_is_a_new_paragraph() { + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let lines = vec![ + line("- a", 90.0, 100.0, 120.0), + line("Following text.", 72.0, 114.0, 300.0), + ]; + let blocks = classify(lines, &sig); + assert_eq!(blocks.len(), 2); + assert!(matches!(&blocks[0], Block::List(items) if items == &["a"])); + assert!(matches!(&blocks[1], Block::Paragraph(t) if t == "Following text.")); + } + + #[test] + fn table_with_offset_header_and_prose_column() { + // Modeled on a real document's table: the header row's labels sit + // several points off the body column anchors (centered over + // right-aligned data), and the last column holds long prose. + let lines = vec![ + wline( + &["Tier", "Dips", "Share", "Meaning"], + &[55.0, 167.0, 234.0, 274.0], + 100.0, + ), + wline( + &["Full-auto", "33", "62%", "An", "agent", "pulls"], + &[55.0, 177.0, 244.0, 274.0, 289.0, 318.0], + 124.0, + ), + wline( + &["Partial-auto", "10", "19%", "Automatable", "with"], + &[55.0, 177.0, 244.0, 274.0, 334.0], + 148.0, + ), + wline( + &["Manual", "10", "19%", "Human,", "permanently,"], + &[55.0, 177.0, 244.0, 274.0, 313.0], + 172.0, + ), + ]; + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let blocks = detect_tables(lines, &sig); + assert_eq!(blocks.len(), 1); + match &blocks[0] { + Block::Table(rows) => { + assert_eq!(rows.len(), 4); + assert_eq!(rows[0], vec!["Tier", "Dips", "Share", "Meaning"]); + assert_eq!(rows[1], vec!["Full-auto", "33", "62%", "An agent pulls"]); + assert_eq!(rows[3], vec!["Manual", "10", "19%", "Human, permanently,"]); + } + other => panic!("expected table, got {other:?}"), + } + } + + #[test] + fn missing_cell_yields_empty_string() { + let lines = vec![ + wline(&["Name", "Age", "City"], &[72.0, 200.0, 300.0], 100.0), + wline(&["Alice", "NYC"], &[72.0, 300.0], 114.0), // no Age cell + wline(&["Bob", "25", "LA"], &[72.0, 200.0, 300.0], 128.0), + ]; + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let blocks = detect_tables(lines, &sig); + match &blocks[0] { + Block::Table(rows) => assert_eq!(rows[1], vec!["Alice", "", "NYC"]), + other => panic!("expected table, got {other:?}"), + } + } + + #[test] + fn ocr_lines_join_paragraphs() { + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let ocr = Line { + source: LineSource::Ocr, + ..line("from ocr", 72.0, 114.0, 300.0) + }; + let lines = vec![line("native", 72.0, 100.0, 300.0), ocr]; + let blocks = classify(lines, &sig); + assert_eq!(blocks.len(), 1); + assert!(matches!(&blocks[0], Block::Paragraph(t) if t == "native from ocr")); + } + + #[test] + fn square_bullet_lines_become_list_even_when_larger_than_body() { + // Consulting-deck bullets: U+25A0 glyphs in a font ~1.25x body. + // Marker lines must classify as list items BEFORE the heading + // gate, or every bullet becomes a heading. + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let lines = vec![ + line("\u{25a0} first item", 72.0, 100.0, 300.0), + line("\u{25a0} second item", 72.0, 118.0, 300.0), + ]; + let mut big = lines; + for l in &mut big { + l.font_size = 15.0; // 1.25x body — would be a heading otherwise + } + let blocks = classify(big, &sig); + assert_eq!(blocks.len(), 1); + assert!( + matches!(&blocks[0], Block::List(items) if items == &["first item", "second item"]), + "got {blocks:?}" + ); + } + + #[test] + fn consecutive_heading_lines_merge() { + // A multi-line heading (common in deck-style PDFs where a whole + // paragraph is display-sized) must not produce one heading per + // line. + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let mut h1 = line( + "Lenders value predictable businesses and data", + 72.0, + 100.0, + 500.0, + ); + h1.font_size = 18.0; + let mut h2 = line( + "maturity enables value creation with automation", + 72.0, + 114.0, + 500.0, + ); + h2.font_size = 18.0; + let blocks = classify(vec![h1, h2], &sig); + assert_eq!(blocks.len(), 1); + assert!( + matches!(&blocks[0], Block::Heading { level: 2, text } if text == "Lenders value predictable businesses and data maturity enables value creation with automation"), + "got {blocks:?}" + ); + } + + #[test] + fn large_bullet_continuation_joins() { + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let mut b = line( + "\u{25a0} Tightly coordinated to minimize", + 72.0, + 100.0, + 500.0, + ); + b.font_size = 15.0; + let mut cont = line("PortCo time required.", 90.0, 118.0, 300.0); + cont.font_size = 15.0; + let blocks = classify(vec![b, cont], &sig); + assert_eq!(blocks.len(), 1); + assert!( + matches!(&blocks[0], Block::List(items) if items == &["Tightly coordinated to minimize PortCo time required."]), + "got {blocks:?}" + ); + } + + #[test] + fn bullet_lines_become_list() { + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let lines = vec![ + line("\u{2022} first item", 72.0, 100.0, 300.0), + line("\u{2022} second item", 72.0, 114.0, 300.0), + line("trailing paragraph", 72.0, 140.0, 300.0), // 26pt gap → break + ]; + let blocks = classify(lines, &sig); + assert!( + matches!(&blocks[0], Block::List(items) if items == &["first item", "second item"]) + ); + assert!(matches!(&blocks[1], Block::Paragraph(t) if t == "trailing paragraph")); + } + + #[test] + fn ordered_markers_become_list() { + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let lines = vec![ + line("1. one", 72.0, 100.0, 300.0), + line("2. two", 72.0, 114.0, 300.0), + ]; + let blocks = classify(lines, &sig); + assert!(matches!(&blocks[0], Block::List(items) if items == &["one", "two"])); + } + + #[test] + fn lettered_markers_become_list() { + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let lines = vec![ + line("a. first", 72.0, 100.0, 300.0), + line("b. second", 72.0, 114.0, 300.0), + ]; + let blocks = classify(lines, &sig); + assert!( + matches!(&blocks[0], Block::List(items) if items == &["first", "second"]), + "got: {blocks:?}" + ); + } + + #[test] + fn hyphen_inside_prose_is_not_a_list() { + // "well-known" mid-line and a wrapped hyphenated line are not items. + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let lines = vec![line("a well-known fact", 72.0, 100.0, 300.0)]; + let blocks = classify(lines, &sig); + assert!(matches!(&blocks[0], Block::Paragraph(_))); + } + + #[test] + fn marker_lines_are_not_table_rows() { + // Binding rule: a bullet line with ≥2 words could pass the table + // geometry gates (3 aligned x-clusters, real gutters), but marker + // lines never join table runs — bullets win. + let lines = vec![ + wline(&["-", "alpha", "1"], &[72.0, 110.0, 200.0], 100.0), + wline(&["-", "beta", "2"], &[72.0, 110.0, 200.0], 114.0), + wline(&["-", "gamma", "3"], &[72.0, 110.0, 200.0], 128.0), + ]; + let sig = DocSignals { + body_size: 12.0, + headers: Default::default(), + footers: Default::default(), + }; + let blocks = detect_tables(lines, &sig); + assert_eq!(blocks.len(), 1); + assert!( + matches!(&blocks[0], Block::List(items) if items == &["alpha 1", "beta 2", "gamma 3"]) + ); + } + + #[test] + fn render_list_block() { + let blocks = vec![crate::pdf_layout::Block::List(vec![ + "one".into(), + "two".into(), + ])]; + let mut out = String::new(); + crate::pdf_layout::render(&blocks, &mut out).unwrap(); + assert_eq!(out, "- one\n- two\n"); + } + + #[test] + fn render_table_block() { + let blocks = vec![ + Block::Paragraph("Before.".into()), + Block::Table(vec![ + vec!["Name".into(), "Age".into()], + vec!["Alice".into(), "30".into()], + ]), + Block::Paragraph("After.".into()), + ]; + let mut out = String::new(); + render(&blocks, &mut out).unwrap(); + assert_eq!( + out, + "Before.\n\n| Name | Age |\n| --- | --- |\n| Alice | 30 |\n\nAfter.\n" + ); + } + + #[test] + fn render_table_escapes_pipes() { + let blocks = vec![Block::Table(vec![vec!["a|b".into()], vec!["c".into()]])]; + let mut out = String::new(); + render(&blocks, &mut out).unwrap(); + assert_eq!(out, "| a\\|b |\n| --- |\n| c |\n"); + } +} diff --git a/batdoc-core/src/pdf_ocr.rs b/batdoc-core/src/pdf_ocr.rs new file mode 100644 index 0000000..c00ab97 --- /dev/null +++ b/batdoc-core/src/pdf_ocr.rs @@ -0,0 +1,508 @@ +//! Region-aware OCR merge for PDF embedded images. +//! +//! Moved from `pdf.rs`: image decoding and OCR candidate selection helpers. +//! New: map OCR pixel-space lines into page points, and merge them with +//! native PDF text lines while dropping overlaps. + +use crate::ocr::MAX_OCR_IMAGE_DIM; +use crate::pdf_geometry::{PlacedImage, PtRect}; +use crate::pdf_layout::{Line, LineSource}; +use lopdf::xobject::PdfImage; +use std::io::Cursor; + +/// Maximum number of embedded images OCR'd per page (largest first). +const MAX_OCR_IMAGES_PER_PAGE: usize = 4; +/// Maximum decoded pixels per embedded image (~100 MP ≈ 300 MB RGB). +/// Larger images are skipped so a single image cannot exhaust memory. +const MAX_OCR_IMAGE_PIXELS: u64 = 100_000_000; + +/// Overlap tolerance in page points for OCR/native dedup (liteparse +/// `ocr_merge.rs:688` uses 2.0 pt). +const OVERLAP_TOLERANCE_PT: f64 = 2.0; + +/// Select up to [`MAX_OCR_IMAGES_PER_PAGE`] OCR candidates, largest by +/// declared pixel area first, skipping images whose decoded size would +/// exceed [`MAX_OCR_IMAGE_PIXELS`]. The cap must bound peak memory, which +/// the declared dimensions determine (compressed images expand when decoded). +pub(crate) fn ocr_candidates<'a>(images: &'a [PdfImage<'a>]) -> Vec<&'a PdfImage<'a>> { + let mut candidates: Vec<(&PdfImage, u64)> = images + .iter() + .map(|img| { + ( + img, + u64::try_from(img.width.max(0)) + .unwrap_or(0) + .saturating_mul(u64::try_from(img.height.max(0)).unwrap_or(0)), + ) + }) + .filter(|(_, area)| *area > 0 && *area <= MAX_OCR_IMAGE_PIXELS) + .collect(); + candidates.sort_by_key(|(_, area)| std::cmp::Reverse(*area)); + candidates + .into_iter() + .take(MAX_OCR_IMAGES_PER_PAGE) + .map(|(img, _)| img) + .collect() +} + +/// Decode JPEG bytes with strict dimension limits, so a stream whose real +/// dimensions exceed the PDF dictionary's claim cannot allocate beyond the +/// OCR pixel budget. +pub(crate) fn decode_jpeg_bounded(bytes: &[u8]) -> Option { + let mut reader = image::ImageReader::new(Cursor::new(bytes)); + reader.set_format(image::ImageFormat::Jpeg); + let mut limits = image::Limits::default(); + limits.max_image_width = Some(MAX_OCR_IMAGE_DIM); + limits.max_image_height = Some(MAX_OCR_IMAGE_DIM); + reader.limits(limits); + reader.decode().ok().map(image::DynamicImage::into_rgb8) +} + +/// Decode a `lopdf::PdfImage` into an RGB image usable by the OCR engine. +/// +/// Handles `DCTDecode` (JPEG bytes) and raw/`FlateDecode` 8-bit `DeviceGray`/`DeviceRGB`, +/// plus ICC-based spaces whose `ColorSpace` is an indirect reference (which lopdf +/// leaves as `None`) or `ICCBased` — component count is read from the decoded +/// buffer length. +/// Returns `None` for other encodings (`CCITT`, `JPX`, `JBIG2`, `Indexed`, `CMYK`). +pub(crate) fn decode_pdf_image(img: &PdfImage<'_>) -> Option { + let filters: Vec<&str> = img.filters.iter().flatten().map(String::as_str).collect(); + if filters.contains(&"DCTDecode") { + return decode_jpeg_bounded(img.content); + } + if filters.iter().any(|f| *f != "FlateDecode") { + return None; // unknown encoding + } + let width = u32::try_from(img.width).ok()?; + let height = u32::try_from(img.height).ok()?; + if width == 0 || height == 0 { + return None; + } + let content = if filters.contains(&"FlateDecode") { + let mut stream = lopdf::Stream::new(img.origin_dict.clone(), img.content.to_vec()); + stream.decompress().ok()?; + stream.content + } else { + img.content.to_vec() + }; + // Validate the exact buffer length so `ImageSource::from_bytes` never + // receives a mis-sized buffer (it errors on non-exact lengths). + let area = u64::from(width) * u64::from(height); + let len = u64::try_from(content.len()).unwrap_or(0); + let bpc = img.bits_per_component.unwrap_or(8); + let channels: u64 = match (img.color_space.as_deref(), bpc) { + (Some("DeviceRGB"), 8) => 3, + (Some("DeviceGray"), 8) => 1, + // Indirect / ICCBased color spaces surface as `None` (a `/ColorSpace N 0 R` + // reference is never dereferenced by lopdf) or `"ICCBased"`; there is no + // channel count on `PdfImage`, so the decoded buffer length disambiguates. + (Some("ICCBased") | None, 8) if area > 0 && len != 0 && len % area == 0 => { + match len / area { + 1 | 3 => len / area, + _ => return None, + } + } + _ => return None, + }; + if len != area * channels { + return None; + } + if channels == 3 { + image::RgbImage::from_raw(width, height, content) + } else { + let gray = image::GrayImage::from_raw(width, height, content)?; + Some(image::DynamicImage::ImageLuma8(gray).into_rgb8()) + } +} + +/// Map OCR pixel-space lines for one embedded image into top-down page +/// points via its placement (spec §6: `page_pt_per_pixel` = placed / pixels). +pub(crate) fn map_ocr_lines( + placed: &PlacedImage, + img_width_px: u32, + img_height_px: u32, + lines: &[crate::ocr::OcrTextLine], +) -> Vec { + if img_width_px == 0 || img_height_px == 0 { + return Vec::new(); + } + let sx = (placed.rect.x1 - placed.rect.x0) / f64::from(img_width_px); + let sy = (placed.rect.y1 - placed.rect.y0) / f64::from(img_height_px); + lines + .iter() + .map(|line| { + let x0 = f64::from(line.x).mul_add(sx, placed.rect.x0); + let x1 = (f64::from(line.x) + f64::from(line.width)).mul_add(sx, placed.rect.x0); + let y0 = f64::from(line.y).mul_add(sy, placed.rect.y0); + let y1 = (f64::from(line.y) + f64::from(line.height)).mul_add(sy, placed.rect.y0); + let rect = PtRect { x0, y0, x1, y1 }; + Line { + text: line.text.clone(), + words: vec![], + rect, + font_size: (y1 - y0) * 0.8, + source: LineSource::Ocr, + } + }) + .collect() +} + +/// Drop OCR lines that overlap native text (2.0 pt tolerance, spec §6) and +/// return the rest, concatenated after `native` (unordered — the driver's +/// reading-order pass sorts everything together). +pub(crate) fn merge(native: Vec, ocr: Vec) -> Vec { + let mut out = native; + for ocr_line in ocr { + let overlaps = out.iter().any(|line| { + matches!(line.source, LineSource::Native) + && line + .rect + .intersects_expanded(&ocr_line.rect, OVERLAP_TOLERANCE_PT) + }); + if !overlaps { + out.push(ocr_line); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ocr::OcrTextLine; + use crate::pdf_layout::LineSource; + + fn ocr_line(text: &str, x: i32, y: i32, w: u32, h: u32) -> OcrTextLine { + OcrTextLine { + text: text.into(), + x, + y, + width: w, + height: h, + } + } + + fn native_line(text: &str, x0: f64, y0: f64, x1: f64, y1: f64) -> Line { + Line { + text: text.into(), + words: vec![], + rect: PtRect { x0, y0, x1, y1 }, + font_size: 12.0, + source: LineSource::Native, + } + } + + #[test] + fn maps_pixel_lines_to_page_points() { + // 100x50 px image placed at page rect x 72..272 (200pt), y 142..192 (50pt) + // → 2 pt/px in x, 1 pt/px in y. + let placed = PlacedImage { + object_id: (5, 0), + rect: PtRect { + x0: 72.0, + y0: 142.0, + x1: 272.0, + y1: 192.0, + }, + }; + let lines = vec![ocr_line("Hi", 10, 20, 20, 10)]; + let mapped = map_ocr_lines(&placed, 100, 50, &lines); + assert_eq!(mapped.len(), 1); + let r = &mapped[0].rect; + assert!((r.x0 - 92.0).abs() < 0.01, "{r:?}"); // 72 + 10*2 + assert!((r.x1 - 132.0).abs() < 0.01, "{r:?}"); // 72 + 30*2 + assert!((r.y0 - 162.0).abs() < 0.01, "{r:?}"); // 142 + 20*1 + assert!((r.y1 - 172.0).abs() < 0.01, "{r:?}"); // 142 + 30*1 + assert_eq!(mapped[0].text, "Hi"); + assert!(matches!(mapped[0].source, LineSource::Ocr)); + assert!((mapped[0].font_size - 8.0).abs() < 0.01); // 10pt height * 0.8 + } + + #[test] + fn merge_drops_overlapping_ocr() { + let native = vec![native_line("Hello", 100.0, 90.0, 160.0, 104.0)]; + let ocr = vec![ + // Overlaps the native line within 2pt → dropped. + Line { + text: "Hello".into(), + words: vec![], + rect: PtRect { + x0: 101.0, + y0: 91.0, + x1: 159.0, + y1: 103.0, + }, + font_size: 11.0, + source: LineSource::Ocr, + }, + // Far away → kept. + Line { + text: "Caption".into(), + words: vec![], + rect: PtRect { + x0: 100.0, + y0: 300.0, + x1: 200.0, + y1: 314.0, + }, + font_size: 11.0, + source: LineSource::Ocr, + }, + ]; + let merged = merge(native, ocr); + assert_eq!(merged.len(), 2); + assert!(merged + .iter() + .any(|l| l.text == "Hello" && matches!(l.source, LineSource::Native))); + assert!(merged.iter().any(|l| l.text == "Caption")); + assert!(!merged + .iter() + .any(|l| matches!(l.source, LineSource::Ocr) && l.text == "Hello")); + } + + #[test] + fn merge_keeps_everything_when_no_overlap() { + let merged = merge( + vec![native_line("a", 0.0, 0.0, 10.0, 10.0)], + vec![Line { + text: "b".into(), + words: vec![], + rect: PtRect { + x0: 0.0, + y0: 50.0, + x1: 10.0, + y1: 60.0, + }, + font_size: 10.0, + source: LineSource::Ocr, + }], + ); + assert_eq!(merged.len(), 2); + } + + #[test] + fn merge_keeps_two_ocr_lines_that_only_overlap_each_other() { + let native = vec![native_line("Native", 0.0, 0.0, 10.0, 10.0)]; + let ocr = vec![ + Line { + text: "OcrOne".into(), + words: vec![], + rect: PtRect { + x0: 100.0, + y0: 100.0, + x1: 120.0, + y1: 114.0, + }, + font_size: 11.0, + source: LineSource::Ocr, + }, + // Overlaps the first OCR line but not the native line → must be kept. + Line { + text: "OcrTwo".into(), + words: vec![], + rect: PtRect { + x0: 110.0, + y0: 105.0, + x1: 130.0, + y1: 119.0, + }, + font_size: 11.0, + source: LineSource::Ocr, + }, + ]; + let merged = merge(native, ocr); + assert_eq!(merged.len(), 3); + assert!(merged + .iter() + .any(|l| l.text == "Native" && matches!(l.source, LineSource::Native))); + assert!(merged + .iter() + .any(|l| l.text == "OcrOne" && matches!(l.source, LineSource::Ocr))); + assert!(merged + .iter() + .any(|l| l.text == "OcrTwo" && matches!(l.source, LineSource::Ocr))); + } + + #[test] + fn ocr_candidates_caps_and_ranks() { + let dict = lopdf::Dictionary::new(); + let mk = |width: i64, height: i64| PdfImage { + id: (1, 0), + width, + height, + color_space: Some("DeviceRGB".to_string()), + filters: Some(Vec::new()), + bits_per_component: Some(8), + content: &[], + origin_dict: &dict, + }; + let images = vec![ + mk(100, 100), // 10k px + mk(20_000, 20_000), // 400 MP — over budget, excluded + mk(1000, 1000), // 1 MP — largest valid + mk(500, 500), + mk(200, 200), + mk(50, 50), + mk(0, 100), // zero area — excluded + ]; + let picked = ocr_candidates(&images); + // 5 valid images → capped at 4, largest first; oversized/zero excluded. + assert_eq!(picked.len(), 4); + assert_eq!((picked[0].width, picked[0].height), (1000, 1000)); + assert_eq!((picked[1].width, picked[1].height), (500, 500)); + assert_eq!((picked[2].width, picked[2].height), (200, 200)); + assert_eq!((picked[3].width, picked[3].height), (100, 100)); + } + + #[test] + fn decode_pdf_image_raw_rgb() { + let mut dict = lopdf::Dictionary::new(); + dict.set("Subtype", lopdf::Object::Name(b"Image".to_vec())); + let img = PdfImage { + id: (1, 0), + width: 2, + height: 2, + color_space: Some("DeviceRGB".to_string()), + filters: Some(Vec::new()), + bits_per_component: Some(8), + content: &[255, 0, 0, 0, 255, 0, 0, 0, 255, 255, 255, 255], + origin_dict: &dict, + }; + let rgb = decode_pdf_image(&img).unwrap(); + assert_eq!(rgb.dimensions(), (2, 2)); + assert_eq!(rgb.get_pixel(0, 0), &image::Rgb([255, 0, 0])); + assert_eq!(rgb.get_pixel(1, 1), &image::Rgb([255, 255, 255])); + } + + #[test] + fn decode_pdf_image_flate_gray() { + let mut dict = lopdf::Dictionary::new(); + dict.set("Subtype", lopdf::Object::Name(b"Image".to_vec())); + dict.set("Filter", lopdf::Object::Name(b"FlateDecode".to_vec())); + // zlib-compressed `[0, 255, 128, 64]` (stored deflate block + adler-32). + // Inlined because `Stream::compress()` will not compress a 4-byte payload. + let compressed = [ + 0x78, 0x9C, 0x01, 0x04, 0x00, 0xFB, 0xFF, 0x00, 0xFF, 0x80, 0x40, 0x04, 0x41, 0x01, + 0xC0, + ]; + let mut origin = lopdf::Dictionary::new(); + origin.set("Width", 2); + origin.set("Height", 2); + origin.set("ColorSpace", lopdf::Object::Name(b"DeviceGray".to_vec())); + origin.set("BitsPerComponent", 8); + origin.set("Filter", lopdf::Object::Name(b"FlateDecode".to_vec())); + let img = PdfImage { + id: (1, 0), + width: 2, + height: 2, + color_space: Some("DeviceGray".to_string()), + filters: Some(vec!["FlateDecode".to_string()]), + bits_per_component: Some(8), + content: &compressed, + origin_dict: &origin, + }; + let rgb = decode_pdf_image(&img).unwrap(); + assert_eq!(rgb.dimensions(), (2, 2)); + assert_eq!(rgb.get_pixel(1, 0), &image::Rgb([255, 255, 255])); // Luma 255 + assert_eq!(rgb.get_pixel(0, 1), &image::Rgb([128, 128, 128])); // Luma 128 + } + + #[test] + fn decode_pdf_image_indirect_iccbased_gray() { + let mut origin = lopdf::Dictionary::new(); + origin.set("Width", 2); + origin.set("Height", 2); + origin.set("BitsPerComponent", 8); + origin.set("Filter", lopdf::Object::Name(b"FlateDecode".to_vec())); + // The `/ColorSpace` here is an indirect reference to an ICCBased array, + // so lopdf reports `color_space = None` (reference never dereferenced). + let compressed = [ + 0x78, 0x9C, 0x01, 0x04, 0x00, 0xFB, 0xFF, 0x00, 0xFF, 0x80, 0x40, 0x04, 0x41, 0x01, + 0xC0, + ]; + let img = PdfImage { + id: (1, 0), + width: 2, + height: 2, + color_space: None, + filters: Some(vec!["FlateDecode".to_string()]), + bits_per_component: Some(8), + content: &compressed, + origin_dict: &origin, + }; + let rgb = decode_pdf_image(&img).unwrap(); + assert_eq!(rgb.dimensions(), (2, 2)); + assert_eq!(rgb.get_pixel(1, 0), &image::Rgb([255, 255, 255])); // Luma 255 + assert_eq!(rgb.get_pixel(0, 1), &image::Rgb([128, 128, 128])); // Luma 128 + } + + #[test] + fn decode_pdf_image_no_color_space_non_byte_multiple_skipped() { + let mut dict = lopdf::Dictionary::new(); + dict.set("Subtype", lopdf::Object::Name(b"Image".to_vec())); + dict.set("Filter", lopdf::Object::Name(b"FlateDecode".to_vec())); + let img = PdfImage { + id: (1, 0), + width: 2, + height: 2, + color_space: None, + filters: Some(vec!["FlateDecode".to_string()]), + bits_per_component: Some(1), // packed bits: length not a multiple of area + content: &[0xFF], + origin_dict: &dict, + }; + assert!(decode_pdf_image(&img).is_none()); + } + + #[test] + fn decode_pdf_image_overlong_buffer_skipped() { + let mut dict = lopdf::Dictionary::new(); + dict.set("Subtype", lopdf::Object::Name(b"Image".to_vec())); + let img = PdfImage { + id: (1, 0), + width: 2, + height: 2, + color_space: Some("DeviceRGB".to_string()), + filters: Some(Vec::new()), + bits_per_component: Some(8), + content: &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // 15 bytes ≠ 12 + origin_dict: &dict, + }; + assert!(decode_pdf_image(&img).is_none()); + } + + #[test] + fn unknown_pdf_image_encoding_is_skipped() { + let mut dict = lopdf::Dictionary::new(); + dict.set("Subtype", lopdf::Object::Name(b"Image".to_vec())); + let img = PdfImage { + id: (1, 0), + width: 2, + height: 2, + color_space: Some("DeviceCMYK".to_string()), + filters: Some(vec!["DCTDecode".to_string()]), + bits_per_component: Some(8), + content: &[0xFF, 0xD8, 0xFF, 0x00], // JPEG magic but not decodable + origin_dict: &dict, + }; + assert!(decode_pdf_image(&img).is_none()); + } + + #[test] + fn decode_jpeg_bounded_enforces_dim_limits() { + let encode = |w: u32, h: u32| { + let img = image::RgbImage::from_pixel(w, h, image::Rgb([128, 128, 128])); + let mut buf = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgb8(img) + .write_to(&mut buf, image::ImageFormat::Jpeg) + .unwrap(); + buf.into_inner() + }; + // Within limits: decodes. + let ok = encode(64, 64); + assert_eq!(decode_jpeg_bounded(&ok).unwrap().dimensions(), (64, 64)); + // Width beyond MAX_OCR_IMAGE_DIM: rejected by the strict dim limit. + let wide = encode(MAX_OCR_IMAGE_DIM + 1, 2); + assert!(decode_jpeg_bounded(&wide).is_none()); + } +} diff --git a/batdoc-core/src/pdf_text.rs b/batdoc-core/src/pdf_text.rs new file mode 100644 index 0000000..a6a03d7 --- /dev/null +++ b/batdoc-core/src/pdf_text.rs @@ -0,0 +1,362 @@ +//! Positioned per-character PDF text capture. +//! +//! Drives pdf-extract's public `OutputDev`/`output_doc_page` to collect one +//! [`PositionedPage`] at a time — every glyph with its page-space position, +//! font size, advance, and quantized rotation. Raw material for +//! `pdf_layout` (reading order, columns, headings) and `pdf_ocr` +//! (region-aware merge). Stock-API only (spec D1): the `catch_unwind` +//! wrapper stays as defense-in-depth for downstream consumers on +//! un-patched pdf-extract (spec D3). + +use crate::error::{BatdocError, Result}; +use pdf_extract::{MediaBox, OutputDev, Transform}; +use std::panic::{self, AssertUnwindSafe}; + +/// One decoded glyph in top-down page coordinates (y = distance from the +/// top of the page, matching markdown reading intuition). +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct PositionedChar { + /// The decoded character. + pub ch: char, + /// Page-point x of the glyph (left edge). + pub x: f64, + /// Page-point y, TOP-DOWN (distance from the page top). + pub y: f64, + /// Transformed (scaled) font size. + pub font_size: f64, + /// Horizontal advance: `width * font_size + spacing`, the same quantity + /// pdf-extract's `PlainTextOutput` tracks for its space threshold. + pub advance: f64, + /// Glyph rotation quantized to 0/90/180/270 degrees. `u16`, not `u8`: + /// 270° > `u8::MAX` and would overflow. + pub rotation: u16, +} + +/// All positioned characters of a single page. No all-pages `Vec` is ever +/// materialized (spec §4.1 / §11): one of these at a time, dropped after +/// emit. +#[derive(Debug)] +pub(crate) struct PositionedPage { + /// 1-based, matching `doc.get_pages()` keys. + pub page_num: u32, + /// `MediaBox` as (llx, lly, urx, ury). + pub media_box: (f64, f64, f64, f64), + /// Every glyph pdf-extract decoded for the page, in output order. + pub chars: Vec, +} + +/// `OutputDev` implementation for one-page positioned capture. +/// +/// y-flip math: pdf-extract's `PlainTextOutput` computes the top-down +/// position as `trm.post_transform(&flip_ctm)` where +/// `flip_ctm = row_major(1, 0, 0, -1, 0, ury - lly)`. With euclid 0.20's +/// `post_transform` definition this reduces element-wise to +/// `x = trm.m31`, `y = (ury - lly) - trm.m32` — see the plan's Task 6 +/// notes (verified against euclid 0.20.14 source). +struct PositionedOutputDev { + media_box: (f64, f64, f64, f64), + chars: Vec, +} + +impl OutputDev for PositionedOutputDev { + fn begin_page( + &mut self, + _page_num: u32, + media_box: &MediaBox, + _art_box: Option<(f64, f64, f64, f64)>, + ) -> std::result::Result<(), pdf_extract::OutputError> { + self.media_box = (media_box.llx, media_box.lly, media_box.urx, media_box.ury); + Ok(()) + } + + fn end_page(&mut self) -> std::result::Result<(), pdf_extract::OutputError> { + Ok(()) + } + + fn output_character( + &mut self, + trm: &Transform, + width: f64, + spacing: f64, + font_size: f64, + ch: &str, + ) -> std::result::Result<(), pdf_extract::OutputError> { + // y-flip post-transform reduces to (m31, height - m32) — see the + // struct-level note / plan for the euclid algebra. + let page_height = self.media_box.3 - self.media_box.1; + let x = trm.m31; + let y = page_height - trm.m32; + // Transformed font size: sqrt(|(m11+m21)·fs · (m12+m22)·fs|), the + // same area-preserving measure PlainTextOutput uses. `.abs()` guards + // against a reflected text matrix yielding a negative product (the + // upstream code produces NaN there; we degrade to a size instead). + let sx = (trm.m11 + trm.m21) * font_size; + let sy = (trm.m12 + trm.m22) * font_size; + let size = (sx * sy).abs().sqrt(); + let rotation = quantize_rotation(trm); + let char_count = ch.chars().count(); + for (idx, c) in ch.chars().enumerate() { + self.chars.push(PositionedChar { + ch: c, + x, + y, + font_size: size, + // `mul_add` is the fused equivalent of `width * size + + // spacing` (the quantity PlainTextOutput tracks); more + // accurate, same value within f64. + // + // A multi-char decode (e.g. a ligature glyph whose + // ToUnicode mapping is "fi") expands to several chars at + // ONE origin; only the last carries the glyph's advance, + // so downstream gap math measures from the glyph's true + // end instead of double-counting the advance per char. + advance: if idx + 1 == char_count { + width.mul_add(size, spacing) + } else { + 0.0 + }, + rotation, + }); + } + Ok(()) + } + + fn begin_word(&mut self) -> std::result::Result<(), pdf_extract::OutputError> { + Ok(()) + } + fn end_word(&mut self) -> std::result::Result<(), pdf_extract::OutputError> { + Ok(()) + } + fn end_line(&mut self) -> std::result::Result<(), pdf_extract::OutputError> { + Ok(()) + } +} + +/// Snap the text matrix's rotation to 0/90/180/270. +fn quantize_rotation(trm: &Transform) -> u16 { + let deg = trm.m12.atan2(trm.m11).to_degrees(); + // `.rem_euclid(4.0)` bounds the snapped quarter to [0, 4) before the + // narrowing cast, so no real value can truncate or lose sign. The + // multiplication is in u16: quarter=3 (270°) must not overflow. + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let quarter = (deg / 90.0).round().rem_euclid(4.0) as u16; + quarter * 90 +} + +/// Extract one page of positioned characters. `page_num` is 1-based and +/// matches `lopdf::Document::get_pages()` (the same page-tree order +/// `extract_pages` relies on for its `page_ids[i] ↔ pages[i]` invariant). +pub(crate) fn extract_positioned_page( + doc: &lopdf::Document, + page_num: u32, +) -> Result { + let result = panic::catch_unwind(AssertUnwindSafe(|| { + let mut dev = PositionedOutputDev { + media_box: (0.0, 0.0, 0.0, 0.0), + chars: Vec::new(), + }; + pdf_extract::output_doc_page(doc, &mut dev, page_num)?; + Ok::<_, pdf_extract::OutputError>(dev) + })); + match result { + Ok(Ok(dev)) => Ok(PositionedPage { + page_num, + media_box: dev.media_box, + chars: dev.chars, + }), + Ok(Err(e)) => Err(BatdocError::Document(format!("PDF extraction failed: {e}"))), + Err(_) => Err(BatdocError::Document( + "PDF extraction panicked (malformed document)".into(), + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use lopdf::dictionary; + + fn positioned(content: &str) -> PositionedPage { + let data = build_text_pdf_content(content); + let doc = lopdf::Document::load_mem(&data).unwrap(); + extract_positioned_page(&doc, 1).unwrap() + } + + #[test] + fn captures_positions_top_down() { + // MediaBox 612x792; Tm places baseline origin at (100, 700) PDF + // space → top-down y = 792 - 700 = 92. + let page = positioned("BT /F1 12 Tf 1 0 0 1 100 700 Tm (AB) Tj ET"); + assert_eq!(page.page_num, 1); + assert_eq!(page.media_box, (0.0, 0.0, 612.0, 792.0)); + assert_eq!(page.chars.len(), 2); + let a = page.chars[0]; + assert_eq!(a.ch, 'A'); + assert!((a.x - 100.0).abs() < 0.01, "x={}", a.x); + assert!((a.y - 92.0).abs() < 0.01, "y={}", a.y); + assert!((a.font_size - 12.0).abs() < 0.01); + assert!(a.advance > 0.0); + assert_eq!(a.rotation, 0); + let b = page.chars[1]; + assert_eq!(b.ch, 'B'); + assert!(b.x > a.x, "B right of A: {} vs {}", b.x, a.x); + assert!((b.y - a.y).abs() < 0.01); + } + + #[test] + fn captures_rotation() { + // Tm with a 90° rotation: 0 1 -1 0 x y. + let page = positioned("BT /F1 12 Tf 0 1 -1 0 100 700 Tm (R) Tj ET"); + assert_eq!(page.chars[0].rotation, 90); + } + + /// Build a one-page PDF whose font's ToUnicode maps char code 0x41 + /// ('A') to the two-char string "fi" (a ligature-style mapping) and + /// 0x42 ('B') to "x". + fn build_ligature_pdf() -> Vec { + use lopdf::{dictionary, Document as LopdfDoc, Object, Stream}; + let mut doc = LopdfDoc::with_version("1.4"); + let pages_id = doc.new_object_id(); + let cmap = b"/CIDInit /ProcSet findresource begin +12 dict begin +begincmap +/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def +/CMapName /Adobe-Identity-UCS def +/CMapType 2 def +1 begincodespacerange +<00> +endcodespacerange +2 beginbfchar +<41> <00660069> +<42> <0078> +endbfchar +endcmap +CMapName currentdict /CMap defineresource pop +end +end +" + .to_vec(); + let tounicode_id = doc.add_object(Stream::new(lopdf::Dictionary::new(), cmap)); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + "Encoding" => "WinAnsiEncoding", + "ToUnicode" => tounicode_id, + }); + let resources_id = doc.add_object(dictionary! { + "Font" => dictionary! { "F1" => font_id }, + }); + let content_id = doc.add_object(Stream::new( + lopdf::Dictionary::new(), + b"BT /F1 12 Tf 72 720 Td (AB) Tj ET".to_vec(), + )); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => 1_i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + buf + } + + #[test] + fn multi_char_glyph_shares_origin_last_carries_advance() { + // ToUnicode can decode one char code into several chars (e.g. a + // ligature glyph -> "fi"). The expanded chars share the glyph's + // origin; only the last carries its advance, so the char after + // the ligature measures its gap from the glyph's true end. + let data = build_ligature_pdf(); + let doc = lopdf::Document::load_mem(&data).unwrap(); + let page = extract_positioned_page(&doc, 1).unwrap(); + assert_eq!(page.chars.len(), 3, "{:?}", page.chars); + let (f, i, x) = (&page.chars[0], &page.chars[1], &page.chars[2]); + assert_eq!((f.ch, i.ch, x.ch), ('f', 'i', 'x')); + assert_eq!(f.x, i.x); + assert_eq!(f.advance, 0.0); + assert!(i.advance > 0.0); + assert!( + x.x >= i.x + i.advance - 0.01, + "x={} i.x={} adv={}", + x.x, + i.x, + i.advance + ); + } + + #[test] + fn captures_rotation_270() { + // Tm with a 270° rotation: 0 -1 1 0 x y. atan2(m12, m11) = + // atan2(-1, 0) = -90° → quarter 3 → 270°. 270 does not fit a u8 + // (overflow panics in debug), so the field must be u16. + let page = positioned("BT /F1 12 Tf 0 -1 1 0 100 700 Tm (R) Tj ET"); + assert_eq!(page.chars[0].rotation, 270); + } + + #[test] + fn missing_page_is_an_error_not_a_panic() { + let data = build_text_pdf_content("BT ET"); + let doc = lopdf::Document::load_mem(&data).unwrap(); + assert!(extract_positioned_page(&doc, 99).is_err()); + } + + /// Build a one-page PDF with a WinAnsi Helvetica font and a + /// caller-supplied content stream (duplicates the helper in pdf.rs's + /// tests deliberately — do not refactor across files). + fn build_text_pdf_content(content: &str) -> Vec { + let mut doc = lopdf::Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + "Encoding" => "WinAnsiEncoding", + }); + let resources_id = doc.add_object(dictionary! { + "Font" => dictionary! { "F1" => font_id }, + }); + let content_id = doc.add_object(lopdf::Stream::new( + lopdf::Dictionary::new(), + content.as_bytes().to_vec(), + )); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + doc.objects.insert( + pages_id, + lopdf::Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![lopdf::Object::Reference(page_id)], + "Count" => 1_i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + buf + } +} diff --git a/batdoc-core/src/pptx.rs b/batdoc-core/src/pptx.rs index 8bbf191..f09f652 100644 --- a/batdoc-core/src/pptx.rs +++ b/batdoc-core/src/pptx.rs @@ -9,12 +9,13 @@ use quick_xml::events::Event; use quick_xml::reader::Reader; use std::collections::HashMap; use std::fmt::Write as _; -use std::io::{Cursor, Read}; +use std::io::{BufRead, Cursor, Read}; use zip::ZipArchive; use crate::markup; use crate::xml_util::{self, get_attr, Rels}; use crate::ExtractOptions; +use crate::ExtractSink; /// A parsed slide: its number and extracted text runs. #[derive(Debug)] @@ -72,14 +73,26 @@ struct TextRun { /// Extract plain text from a .pptx file. pub(crate) fn extract_plain(data: &[u8], opts: ExtractOptions) -> crate::error::Result { - let (slides, _) = parse_pptx( + let mut out = String::new(); + extract_plain_to(data, opts, &mut out)?; + Ok(out) +} + +/// Stream plain text from a .pptx file into `sink`. +pub(crate) fn extract_plain_to( + data: &[u8], + opts: ExtractOptions, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + extract_to( data, ExtractOptions { images: false, ..opts }, - )?; - Ok(render_plain(&slides)) + Mode::Plain, + sink, + ) } /// Extract markdown-formatted text from a .pptx file. @@ -89,15 +102,397 @@ pub(crate) fn extract_plain(data: &[u8], opts: ExtractOptions) -> crate::error:: /// When `opts.ocr` is set, embedded images are OCR'd and rendered as a /// blockquote after each slide's images. pub(crate) fn extract_markdown(data: &[u8], opts: ExtractOptions) -> crate::error::Result { + let mut out = String::new(); + extract_markdown_to(data, opts, &mut out)?; + Ok(out) +} + +/// Stream markdown from a .pptx file into `sink`. +pub(crate) fn extract_markdown_to( + data: &[u8], + opts: ExtractOptions, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + extract_to(data, opts, Mode::Markdown, sink) +} + +#[derive(Clone, Copy)] +enum Mode { + Plain, + Markdown, +} + +fn extract_to( + data: &[u8], + opts: ExtractOptions, + mode: Mode, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + if opts.ocr { + return extract_to_buffered(data, opts, mode, sink); + } + extract_to_streaming(data, opts, mode, sink) +} + +/// Buffered path for `--ocr`: reuses the untouched `parse_pptx` → render +/// pipeline, which OCRs embedded images and renders their text. +fn extract_to_buffered( + data: &[u8], + opts: ExtractOptions, + mode: Mode, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { let (slides, image_defs) = parse_pptx(data, opts)?; - let mut md = render_markdown(&slides); - if !image_defs.is_empty() { - for def in &image_defs { - md.push_str(def); - md.push('\n'); + let out = match mode { + Mode::Plain => render_plain(&slides), + Mode::Markdown => { + let mut md = render_markdown(&slides); + if !image_defs.is_empty() { + for def in &image_defs { + md.push_str(def); + md.push('\n'); + } + } + md + } + }; + sink.write_str(&out) +} + +/// Tracks the trailing-newline state of the streaming output so the notes +/// trailer can be blank-line separated from the deck, and queues image +/// definitions during streaming for a single write at the very end. +struct StreamOut<'a, S: ExtractSink> { + sink: &'a mut S, + trailing_newlines: u8, + wrote: bool, + image_counter: usize, + image_queue: Vec<(String, String, String)>, +} + +impl<'a, S: ExtractSink> StreamOut<'a, S> { + const fn new(sink: &'a mut S) -> Self { + Self { + sink, + trailing_newlines: 0, + wrote: false, + image_counter: 0, + image_queue: Vec::new(), + } + } + + #[allow(clippy::cast_possible_truncation)] + fn write_str(&mut self, s: &str) -> crate::error::Result<()> { + if s.is_empty() { + return Ok(()); + } + self.sink.write_str(s)?; + self.wrote = true; + let bytes = s.as_bytes(); + let mut n = 0u8; + for &b in bytes.iter().rev() { + if b == b'\n' { + n = n.saturating_add(1); + } else { + break; + } + } + // `bytes.len() < 256` makes the cast safe; n is already u8 and + // bounded by the trailing-newline count of the input string. + if n == bytes.len() as u8 && bytes.len() < 256 { + self.trailing_newlines = self.trailing_newlines.saturating_add(n); + } else { + self.trailing_newlines = n; + } + Ok(()) + } + + fn ensure_trailer_blank_line(&mut self) -> crate::error::Result<()> { + if !self.wrote || self.trailing_newlines >= 2 { + return Ok(()); + } + if self.trailing_newlines == 1 { + self.write_str("\n") + } else { + self.write_str("\n\n") } } - Ok(md) +} + +fn extract_to_streaming( + data: &[u8], + opts: ExtractOptions, + mode: Mode, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + let markdown = matches!(mode, Mode::Markdown); + let cursor = Cursor::new(data); + let mut archive = ZipArchive::new(cursor)?; + + let mut slide_paths = discover_slides(&mut archive)?; + + // Slides whose parts are missing are dropped (the buffered `parse_pptx` + // `continue`s on a failed `by_name`), so the multi-slide heading decision + // must see the same slide count as the buffered renderer. + slide_paths.retain(|(_, path)| archive.by_name(path).is_ok()); + + let multiple = slide_paths.len() > 1; + let heading_offset = if multiple { 2 } else { 0 }; + + // Separate archive for reading image bytes without holding the slide + // reader's borrow of `archive`. + let mut images = ZipArchive::new(Cursor::new(data))?; + let mut out = StreamOut::new(sink); + + // Buffered speaker notes (small): emitted as a trailer after all slides. + let mut notes_entries: Vec<(usize, Vec)> = Vec::new(); + + for (slide_index, (num, path)) in slide_paths.into_iter().enumerate() { + let rels_path = xml_util::rels_path(&path); + let rels = xml_util::load_rels(&mut archive, &rels_path); + let image_rels = if opts.images { + xml_util::load_image_rels(&mut archive, &rels_path) + } else { + Rels::new() + }; + + let mut body = String::new(); + let mut pic_rids: Vec = Vec::new(); + let has_shapes = { + let mut reader = xml_util::open_xml(&mut archive, &path)?; + let mut buf = Vec::new(); + let mut first_shape = true; + emit_slide_body( + &mut reader, + &mut buf, + &rels, + markdown, + heading_offset, + &mut first_shape, + &mut body, + &mut pic_rids, + opts.images, + ) + }; + + let mut has_images = false; + if markdown && opts.images && !image_rels.is_empty() { + let base_dir = path.rsplit_once('/').map_or("ppt", |(dir, _)| dir); + for rid in &pic_rids { + if let Some(target) = image_rels.get(rid) { + if let Some(data) = xml_util::read_image_from_zip(&mut images, target, base_dir) + { + out.image_counter += 1; + let id = format!("image{}", out.image_counter); + if let Some(img_ref) = crate::markup::image_to_base64_ref(&data, &id) { + body.push_str(&img_ref.inline); + body.push_str("\n\n"); + out.image_queue + .push((id, target.clone(), base_dir.to_string())); + has_images = true; + } + } + } + } + } + + let has_content = has_shapes || has_images; + if has_content { + if markdown { + if multiple { + out.write_str(&format!("## Slide {num}\n\n"))?; + } + } else if multiple { + if slide_index > 0 { + out.write_str("\n")?; + } + out.write_str(&format!("--- Slide {num} ---\n"))?; + } + out.write_str(&body)?; + } + + // Speaker notes (buffered; small). Emitted as a trailer after the deck. + let notes = load_slide_notes(&mut archive, &path); + if !notes.is_empty() { + notes_entries.push((num, notes)); + } + } + + match mode { + Mode::Plain => append_notes_plain_streaming(¬es_entries, &mut out)?, + Mode::Markdown => append_notes_markdown_streaming(¬es_entries, &mut out)?, + } + + if opts.images { + let queue = std::mem::take(&mut out.image_queue); + write_image_defs(&mut images, &queue, &mut out)?; + } + + Ok(()) +} + +/// Write queued image definitions at the very end, in global counter order. +fn write_image_defs( + archive: &mut ZipArchive>, + queue: &[(String, String, String)], + out: &mut StreamOut<'_, S>, +) -> crate::error::Result<()> { + for (id, target, base_dir) in queue { + if let Some(data) = xml_util::read_image_from_zip(archive, target, base_dir) { + if let Some(img_ref) = crate::markup::image_to_base64_ref(&data, id) { + out.write_str(&img_ref.definition)?; + out.write_str("\n")?; + } + } + } + Ok(()) +} + +/// Stream a slide's XML: render shapes as they close and collect `` +/// rIds in document order. Returns whether the slide contained any shape +/// with a non-empty text body (mirrors the buffered `shapes.is_empty()` skip +/// condition, which counts parsed shapes rather than rendered text). +#[allow(clippy::too_many_arguments)] +fn emit_slide_body( + reader: &mut Reader, + buf: &mut Vec, + rels: &Rels, + markdown: bool, + heading_offset: u8, + first_shape: &mut bool, + body: &mut String, + pic_rids: &mut Vec, + collect_pics: bool, +) -> bool { + let mut has_shapes = false; + let mut in_pic = false; + let mut pic_depth = 0u32; + + loop { + let mut start_sp = false; + let mut start_gf = false; + let mut start_pic = false; + let mut pic_rid: Option = None; + let mut end_pic = false; + let mut done = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) => { + let name = e.local_name(); + match name.as_ref() { + b"sp" => start_sp = true, + b"graphicFrame" => start_gf = true, + b"pic" => start_pic = true, + _ => { + if in_pic { + pic_depth += 1; + if collect_pics && name.as_ref() == b"blip" { + pic_rid = get_attr(e, b"r:embed"); + } + } + } + } + } + Ok(Event::Empty(ref e)) + if in_pic && collect_pics && e.local_name().as_ref() == b"blip" => + { + pic_rid = get_attr(e, b"r:embed"); + } + Ok(Event::End(ref e)) => { + if e.local_name().as_ref() == b"sld" { + done = true; + } else if in_pic { + if e.local_name().as_ref() == b"pic" { + end_pic = true; + } else { + pic_depth -= 1; + if pic_depth == 0 { + end_pic = true; + } + } + } + } + Ok(Event::Eof) | Err(_) => done = true, + _ => {} + } + buf.clear(); + if done { + break; + } + if start_pic { + in_pic = true; + pic_depth = 1; + } else if start_sp || start_gf { + let end_tag: &[u8] = if start_sp { b"sp" } else { b"graphicFrame" }; + if let Some(shape) = parse_shape(reader, buf, rels, end_tag) { + has_shapes = true; + if !*first_shape { + body.push('\n'); + } + *first_shape = false; + if markdown { + render_shape_markdown(&shape, body, heading_offset); + } else { + render_shape_plain(&shape, body); + } + } + } else if let Some(rid) = pic_rid { + pic_rids.push(rid); + } else if end_pic { + in_pic = false; + } + } + + has_shapes +} + +/// Append the streaming notes trailer to a markdown body. +fn append_notes_markdown_streaming( + entries: &[(usize, Vec)], + out: &mut StreamOut<'_, S>, +) -> crate::error::Result<()> { + let mut any = false; + for (num, notes) in entries { + if !notes_nonempty(notes) { + continue; + } + if !any { + out.ensure_trailer_blank_line()?; + out.write_str("## Notes\n\n")?; + any = true; + } + out.write_str(&format!("### Slide {num}\n\n"))?; + for shape in notes { + let mut rendered = String::new(); + render_shape_markdown(shape, &mut rendered, 3); + out.write_str(&rendered)?; + } + } + Ok(()) +} + +/// Append the streaming notes trailer to a plain-text body. +fn append_notes_plain_streaming( + entries: &[(usize, Vec)], + out: &mut StreamOut<'_, S>, +) -> crate::error::Result<()> { + let mut any = false; + for (num, notes) in entries { + if !notes_nonempty(notes) { + continue; + } + if !any { + out.ensure_trailer_blank_line()?; + out.write_str("--- Notes ---\n")?; + any = true; + } + out.write_str(&format!("[Slide {num}]\n"))?; + for shape in notes { + let mut rendered = String::new(); + render_shape_plain(shape, &mut rendered); + out.write_str(&rendered)?; + } + } + Ok(()) } // ── Parsing ──────────────────────────────────────────────────────── @@ -195,44 +590,59 @@ fn parse_pptx( /// /// Returns the rIds in document order. fn parse_slide_pic_rids(xml: &str) -> Vec { - let mut rids = Vec::new(); let mut reader = Reader::from_str(xml); + let mut buf = Vec::new(); + parse_slide_pic_rids_reader(&mut reader, &mut buf) +} + +fn parse_slide_pic_rids_reader( + reader: &mut Reader, + buf: &mut Vec, +) -> Vec { + let mut rids = Vec::new(); let mut in_pic = false; let mut depth = 0u32; loop { - match reader.read_event() { + let mut start_rid: Option = None; + let mut ended = false; + let mut start_pic = false; + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) => { if e.local_name().as_ref() == b"pic" { - in_pic = true; - depth = 1; + start_pic = true; } else if in_pic { depth += 1; if e.local_name().as_ref() == b"blip" { - if let Some(rid) = get_attr(e, b"r:embed") { - rids.push(rid); - } + start_rid = get_attr(e, b"r:embed"); } } } Ok(Event::Empty(ref e)) if in_pic && e.local_name().as_ref() == b"blip" => { - if let Some(rid) = get_attr(e, b"r:embed") { - rids.push(rid); - } + start_rid = get_attr(e, b"r:embed"); } Ok(Event::End(ref e)) if in_pic => { if e.local_name().as_ref() == b"pic" { - in_pic = false; + ended = true; } else { depth -= 1; if depth == 0 { - in_pic = false; + ended = true; } } } Ok(Event::Eof) | Err(_) => break, _ => {} } + buf.clear(); + if start_pic { + in_pic = true; + depth = 1; + } else if let Some(rid) = start_rid { + rids.push(rid); + } else if ended { + in_pic = false; + } } rids @@ -315,44 +725,76 @@ fn discover_slides( /// Parse a single slide's XML, extracting text from all shapes. fn parse_slide_xml(xml: &str, rels: &Rels) -> Vec { let mut reader = Reader::from_str(xml); + let mut buf = Vec::new(); + parse_slide_reader(&mut reader, &mut buf, rels) +} + +fn parse_slide_reader( + reader: &mut Reader, + buf: &mut Vec, + rels: &Rels, +) -> Vec { let mut shapes = Vec::new(); loop { - match reader.read_event() { + let mut start_tag: Option> = None; + let mut done = false; + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) => { let name = e.local_name(); // = shape, = table/chart, = group if name.as_ref() == b"sp" || name.as_ref() == b"graphicFrame" { - if let Some(shape) = parse_shape(&mut reader, rels, e.local_name().as_ref()) { - if !shape.paragraphs.is_empty() { - shapes.push(shape); - } - } + start_tag = Some(name.as_ref().to_vec()); } } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if let Some(tag) = start_tag { + if let Some(shape) = parse_shape(reader, buf, rels, &tag) { + if !shape.paragraphs.is_empty() { + shapes.push(shape); + } + } + } } shapes } /// Parse a shape element (`` or ``), extracting its text body. -fn parse_shape(reader: &mut Reader<&[u8]>, rels: &Rels, end_tag: &[u8]) -> Option { +fn parse_shape( + reader: &mut Reader, + buf: &mut Vec, + rels: &Rels, + end_tag: &[u8], +) -> Option { let mut paragraphs = Vec::new(); loop { - match reader.read_event() { + let mut start_body = false; + let mut done = false; + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"txBody" => { - parse_text_body(reader, rels, &mut paragraphs); + start_body = true; } Ok(Event::End(ref e)) if e.local_name().as_ref() == end_tag => { - break; + done = true; } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_body { + parse_text_body(reader, buf, rels, &mut paragraphs); + } } if paragraphs.is_empty() { @@ -411,19 +853,29 @@ fn load_slide_notes(archive: &mut ZipArchive>, slide_path: &str) - /// so the run walker runs against an empty `Rels`. fn parse_notes_slide_xml(xml: &str) -> Vec { let mut reader = Reader::from_str(xml); + let mut buf = Vec::new(); let rels = Rels::new(); let mut shapes = Vec::new(); loop { - match reader.read_event() { + let mut start_sp = false; + let mut done = false; + match reader.read_event_into(&mut buf) { Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"sp" => { - if let Some(shape) = parse_notes_shape(&mut reader, &rels, b"sp") { - shapes.push(shape); - } + start_sp = true; } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_sp { + if let Some(shape) = parse_notes_shape(&mut reader, &mut buf, &rels, b"sp") { + shapes.push(shape); + } + } } shapes @@ -435,26 +887,40 @@ fn parse_notes_slide_xml(xml: &str) -> Vec { /// `/` before the text body), then parses `txBody` with /// the same walker used for slides. Shapes whose placeholder type is not /// `body` — including freeform shapes with no `` at all — are dropped. -fn parse_notes_shape(reader: &mut Reader<&[u8]>, rels: &Rels, end_tag: &[u8]) -> Option { +fn parse_notes_shape( + reader: &mut Reader, + buf: &mut Vec, + rels: &Rels, + end_tag: &[u8], +) -> Option { let mut ph_type: Option = None; let mut paragraphs = Vec::new(); loop { - match reader.read_event() { + let mut start_body = false; + let mut done = false; + match reader.read_event_into(buf) { Ok(Event::Empty(ref e) | Event::Start(ref e)) if e.local_name().as_ref() == b"ph" && ph_type.is_none() => { ph_type = get_attr(e, b"type"); } Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"txBody" => { - parse_text_body(reader, rels, &mut paragraphs); + start_body = true; } Ok(Event::End(ref e)) if e.local_name().as_ref() == end_tag => { - break; + done = true; } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_body { + parse_text_body(reader, buf, rels, &mut paragraphs); + } } // Only shapes anchored to a `body` placeholder carry speaker notes. @@ -468,92 +934,107 @@ fn parse_notes_shape(reader: &mut Reader<&[u8]>, rels: &Rels, end_tag: &[u8]) -> } /// Parse a `` (or ``) element. -fn parse_text_body(reader: &mut Reader<&[u8]>, rels: &Rels, paragraphs: &mut Vec) { +fn parse_text_body( + reader: &mut Reader, + buf: &mut Vec, + rels: &Rels, + paragraphs: &mut Vec, +) { loop { - match reader.read_event() { + let mut start_p = false; + let mut done = false; + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"p" => { - let para = parse_para(reader, rels); - if !para.runs.is_empty() { - paragraphs.push(para); - } + start_p = true; } Ok(Event::End(ref e)) if e.local_name().as_ref() == b"txBody" => { - break; + done = true; } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_p { + let para = parse_para(reader, buf, rels); + if !para.runs.is_empty() { + paragraphs.push(para); + } + } } } /// Parse a `` paragraph element within a text body. -fn parse_para(reader: &mut Reader<&[u8]>, rels: &Rels) -> Paragraph { +fn parse_para(reader: &mut Reader, buf: &mut Vec, rels: &Rels) -> Paragraph { let mut runs = Vec::new(); let mut max_font_size: Option = None; let mut bullet = BulletKind::None; loop { - match reader.read_event() { + let mut start_ppr = false; + let mut ppr_lvl = 0u8; + let mut start_r = false; + let mut start_fld = false; + let mut empty_br = false; + let mut done = false; + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) => { let name = e.local_name(); match name.as_ref() { b"pPr" => { - parse_para_props(reader, e, &mut bullet); - } - b"r" => { - let run = parse_text_run(reader, rels, None); - if let Some(fs) = run.font_size { - max_font_size = Some(max_font_size.map_or(fs, |prev| prev.max(fs))); - } - if !run.text.is_empty() { - runs.push(run); - } - } - b"fld" => { - // Field element (slide number, date, etc.) — extract text - let run = parse_text_run(reader, rels, Some(b"fld")); - if !run.text.is_empty() { - runs.push(run); - } - } - _ => {} - } - } - Ok(Event::Empty(ref e)) => { - let name = e.local_name(); - match name.as_ref() { - b"br" => { - runs.push(TextRun { - text: "\n".into(), - bold: false, - italic: false, - link_url: None, - font_size: None, - }); - } - b"pPr" => { - // Self-closing with no bullet children - // means default bullet for body placeholders - let lvl = get_attr(e, b"lvl") + start_ppr = true; + ppr_lvl = get_attr(e, b"lvl") .and_then(|v| v.parse::().ok()) .unwrap_or(0); - // Self-closing pPr has no child elements, so we can't tell - // bullet vs. no-bullet — leave as None (will be plain text). - // In practice, self-closing pPr without buNone in a body - // placeholder still gets a bullet from the master, but we - // can't know that without parsing the slide layout/master. - // We'll rely on the Start form for bullet info. - let _ = lvl; } + b"r" => start_r = true, + b"fld" => start_fld = true, _ => {} } } + // A self-closing has no bullet children, so + // (as with the Start form) we leave the bullet as None and rely + // on the non-empty `pPr` form for bullet info. + Ok(Event::Empty(ref e)) if e.local_name().as_ref() == b"br" => { + empty_br = true; + } Ok(Event::End(ref e)) if e.local_name().as_ref() == b"p" => { - break; + done = true; } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_ppr { + parse_para_props(reader, buf, ppr_lvl, &mut bullet); + } else if start_r { + let run = parse_text_run(reader, buf, rels, None); + if let Some(fs) = run.font_size { + max_font_size = Some(max_font_size.map_or(fs, |prev| prev.max(fs))); + } + if !run.text.is_empty() { + runs.push(run); + } + } else if start_fld { + // Field element (slide number, date, etc.) — extract text + let run = parse_text_run(reader, buf, rels, Some(b"fld")); + if !run.text.is_empty() { + runs.push(run); + } + } else if empty_br { + runs.push(TextRun { + text: "\n".into(), + bold: false, + italic: false, + link_url: None, + font_size: None, + }); + } } // Infer heading level from font size (in hundredths of a point): @@ -593,22 +1074,20 @@ fn parse_para(reader: &mut Reader<&[u8]>, rels: &Rels) -> Paragraph { /// to an unordered bullet — `PowerPoint` body placeholders inherit bullets /// from the slide layout/master, and the `lvl` attribute alone indicates /// list membership in practice. -fn parse_para_props( - reader: &mut Reader<&[u8]>, - start: &quick_xml::events::BytesStart, +fn parse_para_props( + reader: &mut Reader, + buf: &mut Vec, + lvl: u8, bullet: &mut BulletKind, ) { - let lvl = get_attr(start, b"lvl") - .and_then(|v| v.parse::().ok()) - .unwrap_or(0); - let mut found_bu_char = false; let mut found_bu_auto = false; let mut found_bu_none = false; let mut depth = 1u32; loop { - match reader.read_event() { + let mut done = false; + match reader.read_event_into(buf) { Ok(Event::Start(_)) => { depth += 1; } @@ -624,12 +1103,16 @@ fn parse_para_props( Ok(Event::End(_)) => { depth -= 1; if depth == 0 { - break; + done = true; } } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } } if found_bu_none { @@ -648,7 +1131,12 @@ fn parse_para_props( /// Parse a `` text run (or `` field) element. /// /// Extracts text, bold/italic, font size, and hyperlink URL. -fn parse_text_run(reader: &mut Reader<&[u8]>, rels: &Rels, end_tag: Option<&[u8]>) -> TextRun { +fn parse_text_run( + reader: &mut Reader, + buf: &mut Vec, + rels: &Rels, + end_tag: Option<&[u8]>, +) -> TextRun { let end_name = end_tag.unwrap_or(b"r"); let mut text = String::new(); let mut bold = false; @@ -657,22 +1145,20 @@ fn parse_text_run(reader: &mut Reader<&[u8]>, rels: &Rels, end_tag: Option<&[u8] let mut font_size: Option = None; loop { - match reader.read_event() { + let mut start_rpr = false; + let mut start_t = false; + let mut done = false; + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) => { let name = e.local_name(); match name.as_ref() { b"rPr" => { // Read attributes from the start tag read_rpr_attrs(e, &mut bold, &mut italic, &mut font_size); - // Parse children for hyperlinks - parse_run_props_children(reader, &mut link_url, rels); + start_rpr = true; } b"t" => { - if let Ok(Event::Text(t)) = reader.read_event() { - if let Ok(s) = t.unescape() { - text.push_str(&s); - } - } + start_t = true; } _ => {} } @@ -685,11 +1171,26 @@ fn parse_text_run(reader: &mut Reader<&[u8]>, rels: &Rels, end_tag: Option<&[u8] } } Ok(Event::End(ref e)) if e.local_name().as_ref() == end_name => { - break; + done = true; } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if start_rpr { + // Parse children for hyperlinks + parse_run_props_children(reader, buf, &mut link_url, rels); + } else if start_t { + if let Ok(Event::Text(t)) = reader.read_event_into(buf) { + if let Ok(s) = t.unescape() { + text.push_str(&s); + } + } + buf.clear(); + } } TextRun { @@ -723,20 +1224,23 @@ fn read_rpr_attrs( } /// Parse children of `` to find hyperlink references. -fn parse_run_props_children( - reader: &mut Reader<&[u8]>, +fn parse_run_props_children( + reader: &mut Reader, + buf: &mut Vec, link_url: &mut Option, rels: &Rels, ) { let mut depth = 1u32; loop { - match reader.read_event() { + let mut hlink: Option = None; + let mut done = false; + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) => { if e.local_name().as_ref() == b"hlinkClick" { if let Some(rid) = get_attr(e, b"r:id") { if let Some(url) = rels.get(&rid) { - *link_url = Some(url.clone()); + hlink = Some(url.clone()); } } } @@ -745,22 +1249,30 @@ fn parse_run_props_children( Ok(Event::Empty(ref e)) if e.local_name().as_ref() == b"hlinkClick" => { if let Some(rid) = get_attr(e, b"r:id") { if let Some(url) = rels.get(&rid) { - *link_url = Some(url.clone()); + hlink = Some(url.clone()); } } } Ok(Event::End(ref e)) => { if e.local_name().as_ref() == b"rPr" { - break; - } - depth -= 1; - if depth == 0 { - break; + done = true; + } else { + depth -= 1; + if depth == 0 { + done = true; + } } } - Ok(Event::Eof) | Err(_) => break, + Ok(Event::Eof) | Err(_) => done = true, _ => {} } + buf.clear(); + if done { + break; + } + if let Some(url) = hlink { + *link_url = Some(url); + } } } @@ -1838,4 +2350,217 @@ mod tests { assert!(plain.contains("[Slide 1]"), "plain: {plain:?}"); assert!(plain.contains("Speak slowly"), "plain: {plain:?}"); } + + // ── streaming vs buffered equivalence ───────────────────────── + + /// Wrap raw `` children in a full `` document. + fn slide_xml(inner: &str) -> String { + format!( + r#" + + {inner} +"# + ) + } + + /// Build an N-slide deck from fully-specified slide XML bodies. + fn minimal_pptx_multi(slides: &[String]) -> Vec { + let buf = Cursor::new(Vec::new()); + let mut z = ZipWriter::new(buf); + zip_entry( + &mut z, + "[Content_Types].xml", + r#""#, + ); + + let mut sld_ids = String::new(); + let mut rels = String::from( + r#" +"#, + ); + for i in 0..slides.len() { + sld_ids.push_str(&format!( + r#""#, + 256 + i, + i + 1 + )); + rels.push_str(&format!( + r#""#, + i + 1, + i + 1 + )); + } + rels.push_str(""); + + let pres = format!( + r#" + + {sld_ids} +"# + ); + zip_entry(&mut z, "ppt/presentation.xml", &pres); + zip_entry(&mut z, "ppt/_rels/presentation.xml.rels", &rels); + + for (i, slide) in slides.iter().enumerate() { + zip_entry(&mut z, &format!("ppt/slides/slide{}.xml", i + 1), slide); + } + + z.finish().unwrap().into_inner() + } + + /// The buffered reference output for the markdown extractor: the same + /// `parse_pptx` → `render_markdown` → image definitions pipeline that + /// `extract_markdown` used before the streaming rewrite. + fn buffered_markdown(data: &[u8], opts: crate::ExtractOptions) -> String { + let (slides, image_defs) = parse_pptx(data, opts).unwrap(); + let mut md = render_markdown(&slides); + if !image_defs.is_empty() { + for def in &image_defs { + md.push_str(def); + md.push('\n'); + } + } + md + } + + #[test] + fn extract_to_equals_buffered_on_multi_slide_deck() { + let slide1 = slide_xml( + r#" + Title + Body text + "#, + ); + let slide2 = slide_xml( + r#" + Item one + Sub item + "#, + ); + let data = minimal_pptx_multi(&[slide1, slide2]); + let opts = crate::ExtractOptions::default(); + + assert_eq!(buffered_markdown(&data, opts), { + let mut out = String::new(); + extract_markdown_to(&data, opts, &mut out).unwrap(); + out + }); + + let (slides, _) = parse_pptx( + &data, + crate::ExtractOptions { + images: false, + ..opts + }, + ) + .unwrap(); + let ref_plain = render_plain(&slides); + let mut plain = String::new(); + extract_plain_to(&data, opts, &mut plain).unwrap(); + assert_eq!(ref_plain, plain); + + let md = buffered_markdown(&data, opts); + assert!(md.contains("## Slide 1"), "md: {md:?}"); + assert!(md.contains("### **Title**"), "md: {md:?}"); + assert!(md.contains("## Slide 2"), "md: {md:?}"); + assert!(md.contains("- Item one"), "md: {md:?}"); + assert!(md.contains(" - Sub item"), "md: {md:?}"); + assert!(plain.contains("--- Slide 1 ---"), "plain: {plain:?}"); + assert!(plain.contains("--- Slide 2 ---"), "plain: {plain:?}"); + } + + /// Two-slide deck; each slide has a text shape and one PNG image (`rId5`). + fn minimal_pptx_with_images() -> Vec { + const CONTENT_TYPES: &str = r#""#; + const RELS_NS: &str = "http://schemas.openxmlformats.org/package/2006/relationships"; + const SLIDE_TYPE: &str = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide"; + const IMAGE_TYPE: &str = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/image"; + const PNG_SIGNATURE: [u8; 8] = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; + + let slide = |text: &str| -> String { + format!( + r#" + + + {text} + + +"# + ) + }; + + let buf = Cursor::new(Vec::new()); + let mut z = ZipWriter::new(buf); + zip_entry(&mut z, "[Content_Types].xml", CONTENT_TYPES); + zip_entry( + &mut z, + "ppt/presentation.xml", + r#" + + + + + +"#, + ); + zip_entry( + &mut z, + "ppt/_rels/presentation.xml.rels", + &format!( + r#" + + + +"# + ), + ); + zip_entry(&mut z, "ppt/slides/slide1.xml", &slide("One")); + zip_entry(&mut z, "ppt/slides/slide2.xml", &slide("Two")); + for n in 1..=2 { + zip_entry( + &mut z, + &format!("ppt/slides/_rels/slide{n}.xml.rels"), + &format!( + r#" + + +"# + ), + ); + } + z.start_file("ppt/media/image1.png", SimpleFileOptions::default()) + .unwrap(); + z.write_all(&PNG_SIGNATURE).unwrap(); + z.finish().unwrap().into_inner() + } + + #[test] + fn extract_markdown_to_equals_buffered_on_images() { + let data = minimal_pptx_with_images(); + let opts = crate::ExtractOptions { + images: true, + ..crate::ExtractOptions::default() + }; + + let expected = buffered_markdown(&data, opts); + let mut actual = String::new(); + extract_markdown_to(&data, opts, &mut actual).unwrap(); + + assert_eq!(expected, actual); + assert!( + actual.contains("[image1]: "), + "actual: {actual:?}" + ); + assert!( + actual.contains("[image2]: "), + "actual: {actual:?}" + ); + } } diff --git a/batdoc-core/src/sheet.rs b/batdoc-core/src/sheet.rs index a7a3089..5a3173d 100644 --- a/batdoc-core/src/sheet.rs +++ b/batdoc-core/src/sheet.rs @@ -1,353 +1,110 @@ -//! Shared spreadsheet rendering used by both `.xlsx` and `.xls` parsers. -//! -//! Provides the `Sheet` struct (a named 2D grid of cell values) and renderers -//! that produce either tab-separated plain text or markdown tables. +//! Shared spreadsheet row writers used by `.xlsx` and `.xls` parsers. -/// A parsed worksheet: a name and a 2D grid of cell values. -#[derive(Debug)] -pub(crate) struct Sheet { - pub(crate) name: String, - pub(crate) rows: Vec>, -} - -// ── Plain text rendering ────────────────────────────────────────── - -pub(crate) fn render_plain(sheets: &[Sheet]) -> String { - let mut out = String::new(); - let multiple = sheets.len() > 1; - - for (i, sheet) in sheets.iter().enumerate() { - if skip_empty_sheet(sheet) { - continue; - } +use crate::error::Result; +use crate::ExtractSink; - if multiple { - if i > 0 { - out.push('\n'); - } - out.push_str("--- "); - out.push_str(&sheet.name); - out.push_str(" ---\n"); - } +pub(crate) const MAX_COLS: usize = 512; - for row in &sheet.rows { - let line: String = row.join("\t"); - let line = line.trim_end(); - if !line.is_empty() { - out.push_str(line); - out.push('\n'); - } - } - } - - out +/// Occupancy from pass 1. +pub(crate) struct TableShape { + pub first_col: usize, + pub last_col: usize, + pub last_row: usize, // inclusive, 0-based among observed rows in order } -// ── Markdown rendering ──────────────────────────────────────────── - -pub(crate) fn render_markdown(sheets: &[Sheet]) -> String { - let mut out = String::new(); - let multiple = sheets.len() > 1; - - for sheet in sheets { - if skip_empty_sheet(sheet) { - continue; - } - - // Strip trailing empty rows - let rows = strip_trailing_empty_rows(&sheet.rows); - if rows.is_empty() { - continue; - } - - // Strip leading empty columns and trailing empty columns - let (rows, ncols) = strip_empty_cols(&rows); - if ncols == 0 { - continue; - } - - if multiple { - out.push_str("## "); - out.push_str(&sheet.name); - out.push_str("\n\n"); - } - - // First row is the header - if let Some(header) = rows.first() { - out.push_str("| "); - out.push_str( - &header - .iter() - .map(|c| escape_pipe(c)) - .collect::>() - .join(" | "), - ); - out.push_str(" |\n"); - - // Separator - out.push('|'); - for _ in 0..ncols { - out.push_str(" --- |"); - } - out.push('\n'); - - // Data rows - for row in rows.iter().skip(1) { - out.push_str("| "); - out.push_str( - &row.iter() - .map(|c| escape_pipe(c)) - .collect::>() - .join(" | "), - ); - out.push_str(" |\n"); - } - out.push('\n'); +pub(crate) fn write_plain_row( + sink: &mut impl ExtractSink, + cells: impl IntoIterator>, +) -> Result<()> { + let mut line = String::new(); + let mut first = true; + for cell in cells { + if !first { + line.push('\t'); } + first = false; + line.push_str(cell.as_ref()); } - - out -} - -/// Returns true if the sheet has no non-empty cells. -pub(crate) fn skip_empty_sheet(sheet: &Sheet) -> bool { - sheet - .rows - .iter() - .all(|row| row.iter().all(|cell| cell.trim().is_empty())) + let line = line.trim_end(); + if line.is_empty() { + return Ok(()); + } + sink.write_str(line)?; + sink.write_str("\n") } -/// Strip trailing rows that are entirely empty. -fn strip_trailing_empty_rows(rows: &[Vec]) -> Vec> { - let last_nonempty = rows - .iter() - .rposition(|row| row.iter().any(|cell| !cell.trim().is_empty())); - last_nonempty.map_or_else(Vec::new, |idx| rows[..=idx].to_vec()) +pub(crate) fn write_markdown_header>( + sink: &mut impl ExtractSink, + cells: &[C], +) -> Result<()> { + write_markdown_cells(sink, cells) } -/// Strip leading and trailing columns that are entirely empty. -/// Returns the trimmed rows and the new column count. -fn strip_empty_cols(rows: &[Vec]) -> (Vec>, usize) { - if rows.is_empty() { - return (Vec::new(), 0); - } - - let ncols = rows.iter().map(Vec::len).max().unwrap_or(0); - if ncols == 0 { - return (Vec::new(), 0); +pub(crate) fn write_markdown_separator(sink: &mut impl ExtractSink, ncols: usize) -> Result<()> { + sink.write_str("|")?; + for _ in 0..ncols { + sink.write_str(" --- |")?; } + sink.write_str("\n") +} - // Find first non-empty column - let first_col = (0..ncols) - .find(|&c| { - rows.iter() - .any(|r| r.get(c).is_some_and(|v| !v.trim().is_empty())) - }) - .unwrap_or(0); - - // Find last non-empty column - let last_col = (0..ncols) - .rfind(|&c| { - rows.iter() - .any(|r| r.get(c).is_some_and(|v| !v.trim().is_empty())) - }) - .unwrap_or(0); +pub(crate) fn write_markdown_data_row>( + sink: &mut impl ExtractSink, + cells: &[C], +) -> Result<()> { + write_markdown_cells(sink, cells) +} - if first_col > last_col { - return (Vec::new(), 0); +fn write_markdown_cells>(sink: &mut impl ExtractSink, cells: &[C]) -> Result<()> { + sink.write_str("| ")?; + for (i, cell) in cells.iter().enumerate() { + if i > 0 { + sink.write_str(" | ")?; + } + write_escaped_pipes(sink, cell.as_ref())?; } - - let trimmed: Vec> = rows - .iter() - .map(|row| { - (first_col..=last_col) - .map(|c| row.get(c).cloned().unwrap_or_default()) - .collect() - }) - .collect(); - - let new_ncols = last_col - first_col + 1; - (trimmed, new_ncols) + sink.write_str(" |\n") } -/// Escape pipe characters for markdown table cells. -fn escape_pipe(s: &str) -> String { - s.replace('|', "\\|") +fn write_escaped_pipes(sink: &mut impl ExtractSink, s: &str) -> Result<()> { + let mut rest = s; + while let Some(idx) = rest.find('|') { + sink.write_str(&rest[..idx])?; + sink.write_str("\\|")?; + rest = &rest[idx + 1..]; + } + sink.write_str(rest) } #[cfg(test)] mod tests { use super::*; - // ── render_markdown ────────────────────────────────────────── - #[test] - fn render_single_sheet_markdown() { - let sheets = vec![Sheet { - name: "Sheet1".into(), - rows: vec![ - vec!["Name".into(), "Age".into()], - vec!["Alice".into(), "30".into()], - vec!["Bob".into(), "25".into()], - ], - }]; - - let md = render_markdown(&sheets); - assert!(md.contains("| Name | Age |")); - assert!(md.contains("| --- | --- |")); - assert!(md.contains("| Alice | 30 |")); - assert!(md.contains("| Bob | 25 |")); - // Single sheet — no heading - assert!(!md.contains("## Sheet1")); + fn write_plain_row_skips_empty() { + let mut s = String::new(); + write_plain_row(&mut s, ["", " "]).unwrap(); + assert_eq!(s, ""); } #[test] - fn render_multi_sheet_markdown() { - let sheets = vec![ - Sheet { - name: "People".into(), - rows: vec![vec!["Name".into()], vec!["Alice".into()]], - }, - Sheet { - name: "Places".into(), - rows: vec![vec!["City".into()], vec!["NYC".into()]], - }, - ]; - - let md = render_markdown(&sheets); - assert!(md.contains("## People")); - assert!(md.contains("## Places")); - assert!(md.contains("| Name |")); - assert!(md.contains("| City |")); + fn write_plain_row_tabs_and_trim() { + let mut s = String::new(); + write_plain_row(&mut s, ["a", "b", ""]).unwrap(); + assert_eq!(s, "a\tb\n"); } #[test] - fn render_empty_sheet_skipped() { - let sheets = vec![ - Sheet { - name: "Empty".into(), - rows: vec![vec![String::new(), String::new()]], - }, - Sheet { - name: "Data".into(), - rows: vec![vec!["Hello".into()]], - }, - ]; - - let md = render_markdown(&sheets); - assert!(!md.contains("Empty")); - assert!(md.contains("| Hello |")); - } - - #[test] - fn render_pipe_escaped() { - let sheets = vec![Sheet { - name: "Sheet1".into(), - rows: vec![vec!["A|B".into()], vec!["C".into()]], - }]; - - let md = render_markdown(&sheets); - assert!(md.contains("A\\|B")); - } - - // ── render_plain ───────────────────────────────────────────── - - #[test] - fn render_plain_tsv() { - let sheets = vec![Sheet { - name: "Sheet1".into(), - rows: vec![ - vec!["Name".into(), "Age".into()], - vec!["Alice".into(), "30".into()], - ], - }]; - - let text = render_plain(&sheets); - assert!(text.contains("Name\tAge")); - assert!(text.contains("Alice\t30")); - } - - #[test] - fn render_plain_multi_sheet() { - let sheets = vec![ - Sheet { - name: "People".into(), - rows: vec![vec!["Alice".into()]], - }, - Sheet { - name: "Places".into(), - rows: vec![vec!["NYC".into()]], - }, - ]; - - let text = render_plain(&sheets); - assert!(text.contains("--- People ---")); - assert!(text.contains("--- Places ---")); - } - - // ── skip_empty_sheet ───────────────────────────────────────── - - #[test] - fn empty_sheet_detected() { - let sheet = Sheet { - name: "Empty".into(), - rows: vec![ - vec![String::new(), " ".into()], - vec![String::new(), String::new()], - ], - }; - assert!(skip_empty_sheet(&sheet)); - } - - #[test] - fn nonempty_sheet_not_skipped() { - let sheet = Sheet { - name: "Data".into(), - rows: vec![vec![String::new(), "Hello".into()]], - }; - assert!(!skip_empty_sheet(&sheet)); - } - - // ── strip_trailing_empty_rows ──────────────────────────────── - - #[test] - fn strips_trailing_empty() { - let rows = vec![vec!["A".into()], vec![String::new()], vec![String::new()]]; - let result = strip_trailing_empty_rows(&rows); - assert_eq!(result.len(), 1); - assert_eq!(result[0], vec!["A"]); - } - - #[test] - fn keeps_all_nonempty() { - let rows = vec![vec!["A".into()], vec!["B".into()]]; - let result = strip_trailing_empty_rows(&rows); - assert_eq!(result.len(), 2); - } - - // ── strip_empty_cols ───────────────────────────────────────── - - #[test] - fn strips_leading_trailing_empty_cols() { - let rows = vec![ - vec![String::new(), "A".into(), "B".into(), String::new()], - vec![String::new(), "C".into(), "D".into(), String::new()], - ]; - let (result, ncols) = strip_empty_cols(&rows); - assert_eq!(ncols, 2); - assert_eq!(result[0], vec!["A", "B"]); - assert_eq!(result[1], vec!["C", "D"]); - } - - // ── escape_pipe ────────────────────────────────────────────── - - #[test] - fn escape_pipe_in_text() { - assert_eq!(escape_pipe("A|B|C"), "A\\|B\\|C"); + fn write_markdown_escapes_pipe() { + let mut s = String::new(); + write_markdown_header(&mut s, &["A|B"]).unwrap(); + write_markdown_separator(&mut s, 1).unwrap(); + write_markdown_data_row(&mut s, &["C"]).unwrap(); + assert_eq!(s, "| A\\|B |\n| --- |\n| C |\n"); } #[test] - fn escape_pipe_no_pipes() { - assert_eq!(escape_pipe("hello"), "hello"); + fn max_cols_constant_is_512() { + assert_eq!(MAX_COLS, 512); } } diff --git a/batdoc-core/src/sink.rs b/batdoc-core/src/sink.rs new file mode 100644 index 0000000..56bd062 --- /dev/null +++ b/batdoc-core/src/sink.rs @@ -0,0 +1,112 @@ +use crate::error::{BatdocError, Result}; + +pub trait ExtractSink { + /// Write a string to the sink. + /// + /// # Errors + /// Returns an error if the underlying sink or output budget cannot accept + /// the write. + fn write_str(&mut self, s: &str) -> Result<()>; +} + +impl ExtractSink for String { + fn write_str(&mut self, s: &str) -> Result<()> { + self.push_str(s); + Ok(()) + } +} + +impl ExtractSink for &mut S { + fn write_str(&mut self, s: &str) -> Result<()> { + (**self).write_str(s) + } +} + +pub struct IoSink(pub W); + +impl ExtractSink for IoSink { + fn write_str(&mut self, s: &str) -> Result<()> { + self.0.write_all(s.as_bytes())?; + Ok(()) + } +} + +pub struct BudgetSink { + inner: S, + written: u64, + max: u64, +} + +impl BudgetSink { + pub const fn new(inner: S, max: u64) -> Self { + Self { + inner, + written: 0, + max, + } + } +} + +impl ExtractSink for BudgetSink { + fn write_str(&mut self, s: &str) -> Result<()> { + let add = s.len() as u64; + if self.written.saturating_add(add) > self.max { + return Err(BatdocError::Document(format!( + "output exceeded {} bytes", + self.max + ))); + } + self.inner.write_str(s)?; + self.written += add; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn string_sink_appends() { + let mut s = String::new(); + ExtractSink::write_str(&mut s, "ab").unwrap(); + ExtractSink::write_str(&mut s, "c").unwrap(); + assert_eq!(s, "abc"); + } + + #[test] + fn io_sink_writes_bytes() { + let mut buf = Vec::new(); + let mut sink = IoSink(&mut buf); + sink.write_str("hi").unwrap(); + assert_eq!(buf, b"hi"); + } + + #[test] + fn budget_sink_errors_and_stops() { + let mut s = String::new(); + let mut sink = BudgetSink::new(&mut s, 4); + sink.write_str("abc").unwrap(); + let err = sink.write_str("de").unwrap_err().to_string(); + assert_eq!(err, "output exceeded 4 bytes"); + assert_eq!(s, "abc"); + } + + #[test] + fn extract_plain_to_matches_extract_plain_on_garbage() { + let data = b"hello world, definitely not a document"; + let a = crate::to_plain(data).unwrap_err().to_string(); + let mut out = String::new(); + let b = crate::extract_plain_to( + data, + crate::Format::Doc, + crate::ExtractOptions::default(), + &mut out, + ); + // detect fails first in to_plain; here we pass an explicit format so + // we only check the dispatch exists. Use detect + extract_plain_to: + let format = crate::detect_format(data).unwrap_err().to_string(); + assert_eq!(a, format); + assert!(b.is_err() || out.is_empty()); + } +} diff --git a/batdoc-core/src/wasm.rs b/batdoc-core/src/wasm.rs new file mode 100644 index 0000000..6d22649 --- /dev/null +++ b/batdoc-core/src/wasm.rs @@ -0,0 +1,49 @@ +//! Browser-facing entry points for the `wasm32-unknown-unknown` build. +//! +//! Bytes in, text out — no file I/O, no terminal. The OCR *model download* +//! path is feature-gated off (`--no-default-features`); OCR still works when +//! model files have been seeded via `BATDOC_MODELS_DIR` (or embedded by the +//! caller before invoking these functions). +//! +//! This module is only compiled for `wasm32` targets; native builds never +//! pull in `wasm-bindgen`. + +#![cfg(target_arch = "wasm32")] + +use crate::{detect_format, extract_markdown_with, extract_plain_with, ExtractOptions}; +use wasm_bindgen::prelude::*; + +/// Detect the document format from magic bytes and return its name +/// (`DOC`, `XLS`, `DOCX`, `XLSX`, `PPTX`, `PDF`, `IMAGE`) or an error string. +#[wasm_bindgen] +pub fn detect(data: &[u8]) -> String { + detect_format(data) + .map(|f| f.to_string()) + .unwrap_or_else(|e| e.to_string()) +} + +/// Detect + extract plain text. `data` is the raw file bytes (a `Uint8Array` +/// from JS). Returns the extracted text, or throws with a descriptive message. +#[wasm_bindgen] +pub fn to_plain(data: &[u8]) -> Result { + let format = detect_format(data).map_err(|e| e.to_string())?; + extract_plain_with(data, format, ExtractOptions::default()).map_err(|e| e.to_string()) +} + +/// Detect + extract Markdown. `images` embeds DOCX/XLSX/PPTX images as +/// base64 data URIs; `ocr` OCRs DOCX/PPTX embedded images (needs seeded +/// models). Returns the Markdown, or throws with a descriptive message. +#[wasm_bindgen] +pub fn to_markdown(data: &[u8], images: bool, ocr: bool) -> Result { + let format = detect_format(data).map_err(|e| e.to_string())?; + extract_markdown_with( + data, + format, + ExtractOptions { + images, + ocr, + ..Default::default() + }, + ) + .map_err(|e| e.to_string()) +} diff --git a/batdoc-core/src/xls.rs b/batdoc-core/src/xls.rs index fe75ac8..8675c03 100644 --- a/batdoc-core/src/xls.rs +++ b/batdoc-core/src/xls.rs @@ -3,16 +3,20 @@ //! Reads the `Workbook` (or `Book`) stream from the OLE2 compound file, //! parses the BIFF8 record stream to extract the Shared String Table (SST), //! sheet metadata (`BoundSheet8`), and cell records (LABELSST, NUMBER, RK, -//! MULRK, FORMULA, LABEL, BOOLERR). Produces the same `Sheet` type used -//! by the `.xlsx` parser for rendering. +//! MULRK, FORMULA, LABEL, BOOLERR). Emits rows through the shared sheet writers. use cfb::CompoundFile; use std::io::{Cursor, Read}; +use crate::arena::StringArena; use crate::codepage; use crate::dateconv; use crate::error::BatdocError; -use crate::sheet::Sheet; +use crate::sheet::{ + write_markdown_data_row, write_markdown_header, write_markdown_separator, write_plain_row, + TableShape, MAX_COLS, +}; +use crate::ExtractSink; // ── BIFF8 record types ──────────────────────────────────────────── @@ -37,14 +41,86 @@ const REC_CODEPAGE: u16 = 0x0042; /// Extract plain text (TSV) from a BIFF8 .xls file. pub(crate) fn extract_plain(data: &[u8]) -> crate::error::Result { - let sheets = parse_xls(data)?; - Ok(crate::sheet::render_plain(&sheets)) + let mut out = String::new(); + extract_plain_to(data, &mut out)?; + Ok(out) +} + +/// Stream plain text (TSV) from a BIFF8 .xls file into `sink`. +pub(crate) fn extract_plain_to( + data: &[u8], + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + let wb = open_workbook(data)?; + let visible: Vec<&SheetEntry> = wb + .sheets + .iter() + .filter(|e| e.sheet_type == 0 && e.visibility == 0) + .collect(); + let mut emitted_any = false; + for entry in visible { + emit_sheet_plain( + &wb.buf, + entry, + &wb.sst, + &wb.xf_styles, + wb.cp, + &mut emitted_any, + sink, + )?; + } + Ok(()) } /// Extract markdown-formatted text from a BIFF8 .xls file. pub(crate) fn extract_markdown(data: &[u8]) -> crate::error::Result { - let sheets = parse_xls(data)?; - Ok(crate::sheet::render_markdown(&sheets)) + let mut out = String::new(); + extract_markdown_to(data, &mut out)?; + Ok(out) +} + +/// Stream markdown from a BIFF8 .xls file into `sink`. +pub(crate) fn extract_markdown_to( + data: &[u8], + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + let wb = open_workbook(data)?; + let visible: Vec<&SheetEntry> = wb + .sheets + .iter() + .filter(|e| e.sheet_type == 0 && e.visibility == 0) + .collect(); + let mut shapes = Vec::with_capacity(visible.len()); + for entry in &visible { + shapes.push(scan_sheet_shape( + &wb.buf, + entry.bof_offset, + &wb.sst, + &wb.xf_styles, + wb.cp, + )?); + } + let multiple = shapes.iter().filter(|s| s.is_some()).count() > 1; + for (entry, shape) in visible.iter().zip(shapes.iter()) { + let Some(shape) = shape else { + continue; + }; + if multiple { + sink.write_str("## ")?; + sink.write_str(&entry.name)?; + sink.write_str("\n\n")?; + } + emit_sheet_markdown( + &wb.buf, + entry.bof_offset, + shape, + &wb.sst, + &wb.xf_styles, + wb.cp, + sink, + )?; + } + Ok(()) } // ── Record-level types ───────────────────────────────────────────── @@ -65,21 +141,20 @@ struct SheetEntry { sheet_type: u8, } -/// A cell being placed into the grid. -#[derive(Debug)] -struct Cell { - row: u16, - col: u16, - value: String, -} - // ── Main parser ──────────────────────────────────────────────────── -fn parse_xls(data: &[u8]) -> crate::error::Result> { +struct Workbook { + buf: Vec, + sst: StringArena, + sheets: Vec, + xf_styles: XfStyles, + cp: u16, +} + +fn open_workbook(data: &[u8]) -> crate::error::Result { let cursor = Cursor::new(data); let mut cfb = CompoundFile::open(cursor)?; - // Try "Workbook" first (BIFF8), then "Book" (BIFF5 compat) let stream_name = if cfb.exists("/Workbook") { "/Workbook" } else if cfb.exists("/Book") { @@ -94,32 +169,15 @@ fn parse_xls(data: &[u8]) -> crate::error::Result> { let mut buf = Vec::new(); stream.read_to_end(&mut buf)?; - // Parse all records let records = parse_records(&buf); - - // Phase 1: Parse workbook globals (SST + sheet entries + XF styles + codepage) - // This also detects encryption (FILEPASS record) early. - let (sst, sheet_entries, xf_styles, cp) = parse_globals(&records)?; - - // Phase 2: Parse each worksheet substream - let mut sheets = Vec::new(); - for entry in &sheet_entries { - // Skip non-worksheet types (charts, macros, VB modules) - if entry.sheet_type != 0 { - continue; - } - // Skip hidden sheets - if entry.visibility != 0 { - continue; - } - let rows = parse_sheet_substream(&buf, entry.bof_offset, &sst, &xf_styles, cp); - sheets.push(Sheet { - name: entry.name.clone(), - rows, - }); - } - - Ok(sheets) + let (sst, sheets, xf_styles, cp) = parse_globals(&records)?; + Ok(Workbook { + buf, + sst, + sheets, + xf_styles, + cp, + }) } /// Maximum number of BIFF8 records to parse (defense-in-depth against @@ -186,8 +244,8 @@ impl XfStyles { /// an error before doing any further parsing. fn parse_globals( records: &[Record<'_>], -) -> crate::error::Result<(Vec, Vec, XfStyles, u16)> { - let mut sst = Vec::new(); +) -> crate::error::Result<(StringArena, Vec, XfStyles, u16)> { + let mut sst = StringArena::new(); let mut sheet_entries = Vec::new(); // Custom FORMAT records: numFmtId → format string let mut custom_formats: Vec<(u16, String)> = Vec::new(); @@ -229,7 +287,8 @@ fn parse_globals( combined.extend_from_slice(records[j].data); j += 1; } - sst = parse_sst(&combined, &continue_boundaries, cp); + sst = StringArena::new(); + parse_sst(&combined, &continue_boundaries, cp, &mut sst); i = j; continue; } @@ -279,13 +338,12 @@ fn parse_format_record(data: &[u8], cp: u16) -> Option<(u16, String)> { /// BIFF8 strings that span a CONTINUE boundary have a special encoding: /// at the CONTINUE boundary, a new "grbit" byte indicates whether the /// remaining characters are compressed (0) or uncompressed (1). -fn parse_sst(data: &[u8], continue_boundaries: &[usize], cp: u16) -> Vec { +fn parse_sst(data: &[u8], continue_boundaries: &[usize], cp: u16, arena: &mut StringArena) { if data.len() < 8 { - return Vec::new(); + return; } let unique_count = u32::from_le_bytes([data[4], data[5], data[6], data[7]]) as usize; - let mut strings = Vec::with_capacity(unique_count.min(65536)); let mut pos = 8; for _ in 0..unique_count { @@ -294,11 +352,9 @@ fn parse_sst(data: &[u8], continue_boundaries: &[usize], cp: u16) -> Vec } let (s, new_pos) = read_biff8_string(data, pos, continue_boundaries, cp); - strings.push(s); + arena.push(&s); pos = new_pos; } - - strings } /// Read a BIFF8 unicode string from a buffer, handling CONTINUE boundaries. @@ -464,194 +520,421 @@ fn parse_boundsheet(data: &[u8], cp: u16) -> Option { // ── Sheet substream parsing ──────────────────────────────────────── -/// Accumulator for collecting cells while parsing a sheet substream. -struct GridBuilder { - cells: Vec, - max_row: usize, - max_col: usize, +struct CurrentRow { + row: Option, + cells: Vec<(u16, String)>, } -impl GridBuilder { +impl CurrentRow { const fn new() -> Self { Self { + row: None, cells: Vec::new(), - max_row: 0, - max_col: 0, } } - fn push(&mut self, row: u16, col: u16, value: String) { - let r = usize::from(row); - let c = usize::from(col); - if r + 1 > self.max_row { - self.max_row = r + 1; + fn push( + &mut self, + row: u16, + col: u16, + value: String, + ) -> crate::error::Result>> { + check_col(col)?; + if self.row.is_some_and(|r| r != row) { + let flushed = std::mem::take(&mut self.cells); + self.row = Some(row); + self.cells.push((col, value)); + return Ok(Some(flushed)); } - if c + 1 > self.max_col { - self.max_col = c + 1; - } - self.cells.push(Cell { row, col, value }); + self.row = Some(row); + self.cells.push((col, value)); + Ok(None) } - fn into_grid(self) -> Vec> { - cells_to_grid(self.cells, self.max_row, self.max_col) + fn take(&mut self) -> Vec<(u16, String)> { + self.row = None; + std::mem::take(&mut self.cells) } } -/// Parse a sheet substream starting at `bof_offset` in the raw data, -/// extracting cell values into a 2D grid. -fn parse_sheet_substream( - data: &[u8], - bof_offset: u32, - sst: &[String], - xf_styles: &XfStyles, - cp: u16, -) -> Vec> { - let mut grid = GridBuilder::new(); - let mut offset = bof_offset as usize; // u32 → usize: lossless on 32+ bit - let mut pending_string_cell: Option<(u16, u16)> = None; +fn check_col(col: u16) -> crate::error::Result<()> { + if usize::from(col) >= MAX_COLS { + Err(BatdocError::Document("sheet exceeds 512 columns".into())) + } else { + Ok(()) + } +} - // Verify BOF +fn skip_sheet_bof(data: &[u8], bof_offset: u32) -> Option { + let offset = bof_offset as usize; if offset + 4 > data.len() { - return Vec::new(); + return None; } let rec_type = u16::from_le_bytes([data[offset], data[offset + 1]]); if rec_type != REC_BOF { - return Vec::new(); + return None; } - - // Skip BOF record let rec_len = usize::from(u16::from_le_bytes([data[offset + 2], data[offset + 3]])); - offset += 4 + rec_len; + Some(offset + 4 + rec_len) +} - while offset + 4 <= data.len() { - let rec_type = u16::from_le_bytes([data[offset], data[offset + 1]]); - let rec_len = usize::from(u16::from_le_bytes([data[offset + 2], data[offset + 3]])); - let rec_end = (offset + 4 + rec_len).min(data.len()); - let rec_data = &data[offset + 4..rec_end]; - - match rec_type { - REC_EOF => break, - REC_LABELSST => handle_labelsst(rec_data, sst, &mut grid), - REC_LABEL | REC_RSTRING => handle_label(rec_data, &mut grid, cp), - REC_NUMBER => handle_number(rec_data, &mut grid, xf_styles), - REC_RK => handle_rk(rec_data, &mut grid, xf_styles), - REC_MULRK => handle_mulrk(rec_data, &mut grid, xf_styles), - REC_FORMULA => { - handle_formula(rec_data, &mut grid, &mut pending_string_cell, xf_styles); - } - REC_STRING => handle_string(rec_data, &mut grid, &mut pending_string_cell, cp), - REC_BOOLERR => handle_boolerr(rec_data, &mut grid), - _ => { - // Clear pending string cell on any non-STRING record - // (STRING must immediately follow FORMULA) - if rec_type != REC_CONTINUE { - pending_string_cell = None; +struct SheetWalk<'a> { + data: &'a [u8], + offset: usize, + pending_string_cell: Option<(u16, u16)>, + pending_cells: Vec<(u16, u16, String)>, +} + +impl<'a> SheetWalk<'a> { + fn start(data: &'a [u8], bof_offset: u32) -> Option { + Some(Self { + data, + offset: skip_sheet_bof(data, bof_offset)?, + pending_string_cell: None, + pending_cells: Vec::new(), + }) + } + + #[allow(clippy::unnecessary_wraps)] + fn next_cell( + &mut self, + sst: &StringArena, + xf_styles: &XfStyles, + cp: u16, + ) -> crate::error::Result> { + if !self.pending_cells.is_empty() { + return Ok(Some(self.pending_cells.remove(0))); + } + while self.offset + 4 <= self.data.len() { + let rec_type = u16::from_le_bytes([self.data[self.offset], self.data[self.offset + 1]]); + let rec_len = usize::from(u16::from_le_bytes([ + self.data[self.offset + 2], + self.data[self.offset + 3], + ])); + let rec_end = (self.offset + 4 + rec_len).min(self.data.len()); + let rec_data = &self.data[self.offset + 4..rec_end]; + self.offset = rec_end; + + let emitted = match rec_type { + REC_EOF => return Ok(None), + REC_LABELSST => handle_labelsst(rec_data, sst), + REC_LABEL | REC_RSTRING => handle_label(rec_data, cp), + REC_NUMBER => handle_number(rec_data, xf_styles), + REC_RK => handle_rk(rec_data, xf_styles), + REC_MULRK => { + let mut cells = handle_mulrk(rec_data, xf_styles); + if cells.is_empty() { + None + } else { + let first = cells.remove(0); + self.pending_cells = cells; + Some(first) + } + } + REC_FORMULA => handle_formula(rec_data, &mut self.pending_string_cell, xf_styles), + REC_STRING => handle_string(rec_data, &mut self.pending_string_cell, cp), + REC_BOOLERR => handle_boolerr(rec_data), + _ => { + if rec_type != REC_CONTINUE { + self.pending_string_cell = None; + } + None } + }; + if emitted.is_some() { + return Ok(emitted); } } + Ok(None) + } +} + +fn densify_plain_row(cells: &[(u16, String)]) -> Vec { + let Some(max_col) = cells.iter().map(|(col, _)| *col).max() else { + return Vec::new(); + }; + let mut dense = vec![String::new(); usize::from(max_col) + 1]; + for (col, value) in cells { + let idx = usize::from(*col); + if idx < dense.len() { + dense[idx].clone_from(value); + } + } + dense +} - offset = rec_end; +fn densify_markdown_row(cells: &[(u16, String)], shape: &TableShape) -> Vec { + let ncols = shape.last_col - shape.first_col + 1; + let mut dense = vec![String::new(); ncols]; + for (col, value) in cells { + let col = usize::from(*col); + if col < shape.first_col || col > shape.last_col { + continue; + } + dense[col - shape.first_col].clone_from(value); } + dense +} + +fn row_has_content(cells: &[(u16, String)]) -> bool { + cells.iter().any(|(_, v)| !v.trim().is_empty()) +} + +fn emit_sheet_plain( + data: &[u8], + entry: &SheetEntry, + sst: &StringArena, + xf_styles: &XfStyles, + cp: u16, + emitted_any: &mut bool, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + let Some(mut walk) = SheetWalk::start(data, entry.bof_offset) else { + return Ok(()); + }; + let mut current = CurrentRow::new(); + let mut wrote_header = false; - grid.into_grid() + while let Some((row, col, value)) = walk.next_cell(sst, xf_styles, cp)? { + if let Some(prev) = current.push(row, col, value)? { + flush_plain_row(&prev, entry, &mut wrote_header, emitted_any, sink)?; + } + } + let last = current.take(); + if !last.is_empty() { + flush_plain_row(&last, entry, &mut wrote_header, emitted_any, sink)?; + } + Ok(()) +} + +fn flush_plain_row( + cells: &[(u16, String)], + entry: &SheetEntry, + wrote_header: &mut bool, + emitted_any: &mut bool, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + let dense = densify_plain_row(cells); + let mut line = String::new(); + write_plain_row(&mut line, dense.iter().map(String::as_str))?; + if line.is_empty() { + return Ok(()); + } + if !*wrote_header { + if *emitted_any { + sink.write_str("\n--- ")?; + sink.write_str(&entry.name)?; + sink.write_str(" ---\n")?; + } + *wrote_header = true; + *emitted_any = true; + } + sink.write_str(&line) +} + +fn scan_sheet_shape( + data: &[u8], + bof_offset: u32, + sst: &StringArena, + xf_styles: &XfStyles, + cp: u16, +) -> crate::error::Result> { + let Some(mut walk) = SheetWalk::start(data, bof_offset) else { + return Ok(None); + }; + let mut current = CurrentRow::new(); + let mut used = vec![false; MAX_COLS]; + let mut last_nonempty_row = None; + let mut row_idx = 0usize; + + let mut mark = |cells: &[(u16, String)]| { + if !row_has_content(cells) { + row_idx += 1; + return; + } + for (col, value) in cells { + if !value.trim().is_empty() { + used[usize::from(*col)] = true; + } + } + last_nonempty_row = Some(row_idx); + row_idx += 1; + }; + + while let Some((row, col, value)) = walk.next_cell(sst, xf_styles, cp)? { + if let Some(prev) = current.push(row, col, value)? { + mark(&prev); + } + } + let last = current.take(); + if !last.is_empty() { + mark(&last); + } + + let Some(last_row) = last_nonempty_row else { + return Ok(None); + }; + let Some(first_col) = used.iter().position(|&u| u) else { + return Ok(None); + }; + let last_col = used.iter().rposition(|&u| u).unwrap_or(first_col); + Ok(Some(TableShape { + first_col, + last_col, + last_row, + })) +} + +fn emit_sheet_markdown( + data: &[u8], + bof_offset: u32, + shape: &TableShape, + sst: &StringArena, + xf_styles: &XfStyles, + cp: u16, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + let Some(mut walk) = SheetWalk::start(data, bof_offset) else { + return Ok(()); + }; + let mut current = CurrentRow::new(); + let mut row_idx = 0usize; + let mut is_header = true; + let ncols = shape.last_col - shape.first_col + 1; + + while let Some((row, col, value)) = walk.next_cell(sst, xf_styles, cp)? { + if let Some(prev) = current.push(row, col, value)? { + emit_markdown_row(&prev, shape, ncols, &mut row_idx, &mut is_header, sink)?; + } + } + let last = current.take(); + if !last.is_empty() { + emit_markdown_row(&last, shape, ncols, &mut row_idx, &mut is_header, sink)?; + } + sink.write_str("\n") +} + +fn emit_markdown_row( + cells: &[(u16, String)], + shape: &TableShape, + ncols: usize, + row_idx: &mut usize, + is_header: &mut bool, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + if *row_idx > shape.last_row { + return Ok(()); + } + let slice = densify_markdown_row(cells, shape); + if *is_header { + write_markdown_header(sink, &slice)?; + write_markdown_separator(sink, ncols)?; + *is_header = false; + } else { + write_markdown_data_row(sink, &slice)?; + } + *row_idx += 1; + Ok(()) } // ── Cell record handlers ─────────────────────────────────────────── -fn handle_labelsst(rec_data: &[u8], sst: &[String], grid: &mut GridBuilder) { - if rec_data.len() >= 10 { - let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); - let col = u16::from_le_bytes([rec_data[2], rec_data[3]]); - let sst_idx = - u32::from_le_bytes([rec_data[6], rec_data[7], rec_data[8], rec_data[9]]) as usize; - let value = sst.get(sst_idx).cloned().unwrap_or_default(); - grid.push(row, col, value); +fn handle_labelsst(rec_data: &[u8], sst: &StringArena) -> Option<(u16, u16, String)> { + if rec_data.len() < 10 { + return None; } + let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); + let col = u16::from_le_bytes([rec_data[2], rec_data[3]]); + let sst_idx = u32::from_le_bytes([rec_data[6], rec_data[7], rec_data[8], rec_data[9]]) as usize; + let value = sst.get(sst_idx).unwrap_or("").to_string(); + Some((row, col, value)) } -fn handle_label(rec_data: &[u8], grid: &mut GridBuilder, cp: u16) { - if rec_data.len() >= 8 { - let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); - let col = u16::from_le_bytes([rec_data[2], rec_data[3]]); - let (s, _) = read_biff8_string(rec_data, 6, &[], cp); - grid.push(row, col, s); +fn handle_label(rec_data: &[u8], cp: u16) -> Option<(u16, u16, String)> { + if rec_data.len() < 8 { + return None; } + let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); + let col = u16::from_le_bytes([rec_data[2], rec_data[3]]); + let (s, _) = read_biff8_string(rec_data, 6, &[], cp); + Some((row, col, s)) } -fn handle_number(rec_data: &[u8], grid: &mut GridBuilder, xf_styles: &XfStyles) { - if rec_data.len() >= 14 { - let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); - let col = u16::from_le_bytes([rec_data[2], rec_data[3]]); - let ixfe = u16::from_le_bytes([rec_data[4], rec_data[5]]); - let val = f64::from_le_bytes([ - rec_data[6], - rec_data[7], - rec_data[8], - rec_data[9], - rec_data[10], - rec_data[11], - rec_data[12], - rec_data[13], - ]); - grid.push(row, col, format_maybe_date(val, ixfe, xf_styles)); +fn handle_number(rec_data: &[u8], xf_styles: &XfStyles) -> Option<(u16, u16, String)> { + if rec_data.len() < 14 { + return None; } + let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); + let col = u16::from_le_bytes([rec_data[2], rec_data[3]]); + let ixfe = u16::from_le_bytes([rec_data[4], rec_data[5]]); + let val = f64::from_le_bytes([ + rec_data[6], + rec_data[7], + rec_data[8], + rec_data[9], + rec_data[10], + rec_data[11], + rec_data[12], + rec_data[13], + ]); + Some((row, col, format_maybe_date(val, ixfe, xf_styles))) } -fn handle_rk(rec_data: &[u8], grid: &mut GridBuilder, xf_styles: &XfStyles) { - if rec_data.len() >= 10 { - let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); - let col = u16::from_le_bytes([rec_data[2], rec_data[3]]); - let ixfe = u16::from_le_bytes([rec_data[4], rec_data[5]]); - let rk = u32::from_le_bytes([rec_data[6], rec_data[7], rec_data[8], rec_data[9]]); - grid.push(row, col, format_maybe_date(decode_rk(rk), ixfe, xf_styles)); +fn handle_rk(rec_data: &[u8], xf_styles: &XfStyles) -> Option<(u16, u16, String)> { + if rec_data.len() < 10 { + return None; } + let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); + let col = u16::from_le_bytes([rec_data[2], rec_data[3]]); + let ixfe = u16::from_le_bytes([rec_data[4], rec_data[5]]); + let rk = u32::from_le_bytes([rec_data[6], rec_data[7], rec_data[8], rec_data[9]]); + Some((row, col, format_maybe_date(decode_rk(rk), ixfe, xf_styles))) } -fn handle_mulrk(rec_data: &[u8], grid: &mut GridBuilder, xf_styles: &XfStyles) { - if rec_data.len() >= 6 { - let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); - let first_col = u16::from_le_bytes([rec_data[2], rec_data[3]]); - let last_col = - u16::from_le_bytes([rec_data[rec_data.len() - 2], rec_data[rec_data.len() - 1]]); - let mut pos = 4; - for c in first_col..=last_col { - if pos + 6 > rec_data.len() - 2 { - break; - } - // Each MULRK entry: 2 bytes ixfe + 4 bytes RK value - let ixfe = u16::from_le_bytes([rec_data[pos], rec_data[pos + 1]]); - let rk = u32::from_le_bytes([ - rec_data[pos + 2], - rec_data[pos + 3], - rec_data[pos + 4], - rec_data[pos + 5], - ]); - grid.push(row, c, format_maybe_date(decode_rk(rk), ixfe, xf_styles)); - pos += 6; +fn handle_mulrk(rec_data: &[u8], xf_styles: &XfStyles) -> Vec<(u16, u16, String)> { + let mut cells = Vec::new(); + if rec_data.len() < 6 { + return cells; + } + let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); + let first_col = u16::from_le_bytes([rec_data[2], rec_data[3]]); + let last_col = u16::from_le_bytes([rec_data[rec_data.len() - 2], rec_data[rec_data.len() - 1]]); + let mut pos = 4; + for c in first_col..=last_col { + if pos + 6 > rec_data.len() - 2 { + break; } + let ixfe = u16::from_le_bytes([rec_data[pos], rec_data[pos + 1]]); + let rk = u32::from_le_bytes([ + rec_data[pos + 2], + rec_data[pos + 3], + rec_data[pos + 4], + rec_data[pos + 5], + ]); + cells.push((row, c, format_maybe_date(decode_rk(rk), ixfe, xf_styles))); + pos += 6; } + cells } fn handle_formula( rec_data: &[u8], - grid: &mut GridBuilder, pending_string_cell: &mut Option<(u16, u16)>, xf_styles: &XfStyles, -) { +) -> Option<(u16, u16, String)> { if rec_data.len() < 20 { - return; + return None; } let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); let col = u16::from_le_bytes([rec_data[2], rec_data[3]]); let ixfe = u16::from_le_bytes([rec_data[4], rec_data[5]]); let result_bytes = &rec_data[6..14]; - // Special type (string, bool, error, empty) indicated by 0xFFFF marker if result_bytes[6] == 0xFF && result_bytes[7] == 0xFF { match result_bytes[0] { 0 => { - // String result — follows in a STRING record *pending_string_cell = Some((row, col)); + None } 1 => { let val = if result_bytes[2] != 0 { @@ -659,11 +942,10 @@ fn handle_formula( } else { "FALSE" }; - grid.push(row, col, val.to_string()); + Some((row, col, val.to_string())) } - // 2 = error, skip. 3 = empty string. - 3 => grid.push(row, col, String::new()), - _ => {} + 3 => Some((row, col, String::new())), + _ => None, } } else { let val = f64::from_le_bytes([ @@ -676,63 +958,35 @@ fn handle_formula( result_bytes[6], result_bytes[7], ]); - grid.push(row, col, format_maybe_date(val, ixfe, xf_styles)); + Some((row, col, format_maybe_date(val, ixfe, xf_styles))) } } fn handle_string( rec_data: &[u8], - grid: &mut GridBuilder, pending_string_cell: &mut Option<(u16, u16)>, cp: u16, -) { - if let Some((row, col)) = pending_string_cell.take() { - if rec_data.len() >= 3 { - let (s, _) = read_biff8_string(rec_data, 0, &[], cp); - grid.push(row, col, s); - } - } -} - -fn handle_boolerr(rec_data: &[u8], grid: &mut GridBuilder) { - if rec_data.len() >= 8 { - let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); - let col = u16::from_le_bytes([rec_data[2], rec_data[3]]); - let is_error = rec_data[7]; - if is_error == 0 { - let val = if rec_data[6] != 0 { "TRUE" } else { "FALSE" }; - grid.push(row, col, val.to_string()); - } +) -> Option<(u16, u16, String)> { + let (row, col) = pending_string_cell.take()?; + if rec_data.len() < 3 { + return None; } + let (s, _) = read_biff8_string(rec_data, 0, &[], cp); + Some((row, col, s)) } -/// Maximum grid cells to allocate (defense-in-depth against crafted files -/// with extreme row/col indices that would cause OOM). -const MAX_GRID_CELLS: usize = 1_000_000; - -/// Convert sparse cell list into a dense 2D grid. -/// -/// Returns an empty grid if the dimensions would exceed `MAX_GRID_CELLS`. -fn cells_to_grid(cells: Vec, max_row: usize, max_col: usize) -> Vec> { - if max_row - .checked_mul(max_col) - .is_none_or(|n| n > MAX_GRID_CELLS) - { - // Dimensions too large — return empty rather than OOM - return Vec::new(); +fn handle_boolerr(rec_data: &[u8]) -> Option<(u16, u16, String)> { + if rec_data.len() < 8 { + return None; } - - let mut grid: Vec> = vec![vec![String::new(); max_col]; max_row]; - - for cell in cells { - let r = usize::from(cell.row); - let c = usize::from(cell.col); - if r < max_row && c < max_col { - grid[r][c] = cell.value; - } + let row = u16::from_le_bytes([rec_data[0], rec_data[1]]); + let col = u16::from_le_bytes([rec_data[2], rec_data[3]]); + let is_error = rec_data[7]; + if is_error != 0 { + return None; } - - grid + let val = if rec_data[6] != 0 { "TRUE" } else { "FALSE" }; + Some((row, col, val.to_string())) } // ── Date-aware number formatting ──────────────────────────────────── @@ -1018,50 +1272,6 @@ mod tests { assert!(parse_boundsheet(&data, 1252).is_none()); } - // ── cells_to_grid ───────────────────────────────────────────── - - #[test] - fn grid_basic() { - let cells = vec![ - Cell { - row: 0, - col: 0, - value: "A".into(), - }, - Cell { - row: 0, - col: 1, - value: "B".into(), - }, - Cell { - row: 1, - col: 0, - value: "C".into(), - }, - ]; - let grid = cells_to_grid(cells, 2, 2); - assert_eq!(grid.len(), 2); - assert_eq!(grid[0], vec!["A", "B"]); - assert_eq!(grid[1], vec!["C", ""]); - } - - #[test] - fn grid_sparse() { - let cells = vec![Cell { - row: 0, - col: 2, - value: "X".into(), - }]; - let grid = cells_to_grid(cells, 1, 3); - assert_eq!(grid[0], vec!["", "", "X"]); - } - - #[test] - fn grid_empty() { - let grid = cells_to_grid(Vec::new(), 0, 0); - assert!(grid.is_empty()); - } - // ── parse_sst ───────────────────────────────────────────────── #[test] @@ -1074,8 +1284,11 @@ mod tests { 0x02, 0x00, 0x00, b'H', b'i', // String 2: "Go" compressed 0x02, 0x00, 0x00, b'G', b'o', ]; - let strings = parse_sst(&data, &[data.len()], 1252); - assert_eq!(strings, vec!["Hi", "Go"]); + let mut arena = StringArena::new(); + parse_sst(&data, &[data.len()], 1252, &mut arena); + assert_eq!(arena.get(0), Some("Hi")); + assert_eq!(arena.get(1), Some("Go")); + assert_eq!(arena.len(), 2); } #[test] @@ -1086,15 +1299,18 @@ mod tests { // String: "A" in unicode 0x01, 0x00, 0x01, 0x41, 0x00, ]; - let strings = parse_sst(&data, &[data.len()], 1252); - assert_eq!(strings, vec!["A"]); + let mut arena = StringArena::new(); + parse_sst(&data, &[data.len()], 1252, &mut arena); + assert_eq!(arena.get(0), Some("A")); + assert_eq!(arena.len(), 1); } #[test] fn sst_empty() { let data = vec![0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; - let strings = parse_sst(&data, &[data.len()], 1252); - assert!(strings.is_empty()); + let mut arena = StringArena::new(); + parse_sst(&data, &[data.len()], 1252, &mut arena); + assert_eq!(arena.len(), 0); } // ── SST with CONTINUE boundary ──────────────────────────────── @@ -1114,8 +1330,9 @@ mod tests { data.push(0x00); data.extend_from_slice(b"llo"); - let strings = parse_sst(&data, &[boundary], 1252); - assert_eq!(strings, vec!["Hello"]); + let mut arena = StringArena::new(); + parse_sst(&data, &[boundary], 1252, &mut arena); + assert_eq!(arena.get(0), Some("Hello")); } #[test] @@ -1133,7 +1350,174 @@ mod tests { // "BC" in UTF-16LE data.extend_from_slice(&[0x42, 0x00, 0x43, 0x00]); - let strings = parse_sst(&data, &[boundary], 1252); - assert_eq!(strings, vec!["ABC"]); + let mut arena = StringArena::new(); + parse_sst(&data, &[boundary], 1252, &mut arena); + assert_eq!(arena.get(0), Some("ABC")); + } + + // ── streaming extract (OLE fixtures) ────────────────────────── + + use std::io::Write; + + fn rec(rec_type: u16, data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(4 + data.len()); + out.extend_from_slice(&rec_type.to_le_bytes()); + out.extend_from_slice(&(u16::try_from(data.len()).unwrap()).to_le_bytes()); + out.extend_from_slice(data); + out + } + + fn bof(dt: u16) -> Vec { + let mut data = vec![0; 16]; + data[0..2].copy_from_slice(&0x0600u16.to_le_bytes()); + data[2..4].copy_from_slice(&dt.to_le_bytes()); + rec(REC_BOF, &data) + } + + fn eof() -> Vec { + rec(REC_EOF, &[]) + } + + fn boundsheet(bof_offset: u32, name: &str) -> Vec { + let mut data = Vec::new(); + data.extend_from_slice(&bof_offset.to_le_bytes()); + data.push(0); + data.push(0); + data.push(u8::try_from(name.len()).unwrap()); + data.push(0); + data.extend_from_slice(name.as_bytes()); + rec(REC_BOUNDSHEET, &data) + } + + fn label(row: u16, col: u16, text: &str) -> Vec { + let mut data = Vec::new(); + data.extend_from_slice(&row.to_le_bytes()); + data.extend_from_slice(&col.to_le_bytes()); + data.extend_from_slice(&0u16.to_le_bytes()); + data.extend_from_slice(&(u16::try_from(text.len()).unwrap()).to_le_bytes()); + data.push(0); + data.extend_from_slice(text.as_bytes()); + rec(REC_LABEL, &data) + } + + fn ole_workbook(workbook: &[u8]) -> Vec { + let mut cfb = CompoundFile::create(Cursor::new(Vec::new())).unwrap(); + { + let mut stream = cfb.create_stream("/Workbook").unwrap(); + stream.write_all(workbook).unwrap(); + } + cfb.flush().unwrap(); + cfb.into_inner().into_inner() + } + + fn xls_with_label(row: u16, col: u16, text: &str) -> Vec { + xls_with_sheets(&[("Sheet1", vec![label(row, col, text)])]) + } + + fn xls_with_sheets(sheets: &[(&str, Vec>)]) -> Vec { + let sheet_payloads: Vec> = sheets + .iter() + .map(|(_, cells)| { + let mut sheet = bof(0x0010); + for cell in cells { + sheet.extend_from_slice(cell); + } + sheet.extend_from_slice(&eof()); + sheet + }) + .collect(); + + let boundsheet_lens: Vec = sheets.iter().map(|(name, _)| 12 + name.len()).collect(); + let globals_len: usize = 20 + boundsheet_lens.iter().sum::() + 4; + + let mut workbook = bof(0x0005); + let mut offset = globals_len; + for ((name, _), payload) in sheets.iter().zip(sheet_payloads.iter()) { + workbook.extend_from_slice(&boundsheet(u32::try_from(offset).unwrap(), name)); + offset += payload.len(); + } + workbook.extend_from_slice(&eof()); + for payload in &sheet_payloads { + workbook.extend_from_slice(payload); + } + ole_workbook(&workbook) + } + + fn two_row_xls() -> Vec { + xls_with_sheets(&[( + "Sheet1", + vec![ + label(0, 0, "Name"), + label(0, 1, "Age"), + label(1, 0, "Alice"), + label(1, 1, "30"), + ], + )]) + } + + #[test] + fn xls_plain_rejects_col_512() { + let data = xls_with_label(0, 512, "X"); + let err = crate::extract_plain(&data, crate::Format::Xls) + .unwrap_err() + .to_string(); + assert_eq!(err, "sheet exceeds 512 columns"); + } + + #[test] + fn xls_plain_accepts_col_511() { + let data = xls_with_label(0, 511, "X"); + let text = crate::extract_plain(&data, crate::Format::Xls).unwrap(); + assert!(text.contains("X")); + } + + #[test] + fn xls_plain_to_equals_extract_plain_on_current_fixtures() { + let data = two_row_xls(); + let buffered = extract_plain(&data).unwrap(); + let mut streamed = String::new(); + extract_plain_to(&data, &mut streamed).unwrap(); + assert_eq!(buffered, streamed); + assert_eq!(buffered, "Name\tAge\nAlice\t30\n"); + } + + #[test] + fn xls_markdown_two_rows_locked() { + let data = two_row_xls(); + let md = extract_markdown(&data).unwrap(); + assert_eq!(md, "| Name | Age |\n| --- | --- |\n| Alice | 30 |\n\n"); + } + + #[test] + fn xls_plain_multi_sheet_headers() { + let data = xls_with_sheets(&[ + ("People", vec![label(0, 0, "Alice")]), + ("Places", vec![label(0, 0, "NYC")]), + ]); + let text = extract_plain(&data).unwrap(); + assert_eq!(text, "Alice\n\n--- Places ---\nNYC\n"); + } + + #[test] + fn xls_markdown_multi_sheet_headings() { + let data = xls_with_sheets(&[ + ("People", vec![label(0, 0, "Alice")]), + ("Places", vec![label(0, 0, "NYC")]), + ]); + let md = extract_markdown(&data).unwrap(); + assert_eq!( + md, + "## People\n\n| Alice |\n| --- |\n\n## Places\n\n| NYC |\n| --- |\n\n" + ); + } + + #[test] + fn xls_markdown_skips_empty_sheet_and_omits_single_heading() { + let data = xls_with_sheets(&[ + ("Empty", vec![label(0, 0, " ")]), + ("Data", vec![label(0, 0, "Hello")]), + ]); + let md = extract_markdown(&data).unwrap(); + assert_eq!(md, "| Hello |\n| --- |\n\n"); } } diff --git a/batdoc-core/src/xlsx.rs b/batdoc-core/src/xlsx.rs index 35b76a9..3e64d86 100644 --- a/batdoc-core/src/xlsx.rs +++ b/batdoc-core/src/xlsx.rs @@ -7,17 +7,61 @@ use quick_xml::events::Event; use quick_xml::reader::Reader; -use std::io::{Cursor, Read}; +use std::collections::HashMap; +use std::io::{BufRead, Cursor, Read}; use zip::ZipArchive; +use crate::arena::StringArena; use crate::dateconv; -use crate::sheet::Sheet; +use crate::error::BatdocError; +use crate::sheet::{ + write_markdown_data_row, write_markdown_header, write_markdown_separator, write_plain_row, + TableShape, MAX_COLS, +}; use crate::xml_util::{self, get_attr, Rels}; +use crate::ExtractSink; /// Extract plain text (TSV) from an .xlsx file. pub(crate) fn extract_plain(data: &[u8]) -> crate::error::Result { - let sheets = parse_xlsx(data)?; - Ok(crate::sheet::render_plain(&sheets)) + let mut out = String::new(); + extract_plain_to(data, &mut out)?; + Ok(out) +} + +/// Stream plain text (TSV) from an .xlsx file into `sink`. +pub(crate) fn extract_plain_to( + data: &[u8], + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + let cursor = Cursor::new(data); + let mut archive = ZipArchive::new(cursor)?; + + let shared_strings = parse_shared_strings_arena(&mut archive); + let styles = parse_styles(&mut archive); + let sheet_info = discover_sheets(&mut archive)?; + + let mut emitted_any = false; + for (name, path) in &sheet_info { + let sheet_rels_path = xml_util::rels_path(path); + let rels = xml_util::load_rels(&mut archive, &sheet_rels_path); + let hyperlinks = if rels.is_empty() { + HashMap::new() + } else { + load_sheet_hyperlinks(&mut archive, path, &rels) + }; + emit_sheet_plain( + &mut archive, + path, + name, + &shared_strings, + &styles, + &hyperlinks, + &mut emitted_any, + sink, + )?; + } + + Ok(()) } /// Extract markdown-formatted text from an .xlsx file. @@ -25,63 +69,482 @@ pub(crate) fn extract_plain(data: &[u8]) -> crate::error::Result { /// When `images` is true, embedded images from drawings are extracted /// and appended as reference-style base64 images with definitions at the end. pub(crate) fn extract_markdown(data: &[u8], images: bool) -> crate::error::Result { - let sheets = parse_xlsx(data)?; - let mut md = crate::sheet::render_markdown(&sheets); + let mut out = String::new(); + extract_markdown_to(data, images, &mut out)?; + Ok(out) +} + +/// Stream markdown from an .xlsx file into `sink`. +pub(crate) fn extract_markdown_to( + data: &[u8], + images: bool, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + let cursor = Cursor::new(data); + let mut archive = ZipArchive::new(cursor)?; + + let shared_strings = parse_shared_strings_arena(&mut archive); + let styles = parse_styles(&mut archive); + let sheet_info = discover_sheets(&mut archive)?; + + let mut shapes = Vec::with_capacity(sheet_info.len()); + for (_, path) in &sheet_info { + shapes.push(scan_sheet_shape( + &mut archive, + path, + &shared_strings, + &styles, + )?); + } + let multiple = shapes.iter().filter(|s| s.is_some()).count() > 1; + + for ((name, path), shape) in sheet_info.iter().zip(shapes.iter()) { + let Some(shape) = shape else { + continue; + }; + let sheet_rels_path = xml_util::rels_path(path); + let rels = xml_util::load_rels(&mut archive, &sheet_rels_path); + let hyperlinks = if rels.is_empty() { + HashMap::new() + } else { + load_sheet_hyperlinks(&mut archive, path, &rels) + }; + if multiple { + sink.write_str("## ")?; + sink.write_str(name)?; + sink.write_str("\n\n")?; + } + emit_sheet_markdown( + &mut archive, + path, + shape, + &shared_strings, + &styles, + &hyperlinks, + sink, + )?; + } if images { - let cursor = Cursor::new(data); - let mut archive = ZipArchive::new(cursor)?; - let sheet_info = discover_sheets(&mut archive)?; - append_sheet_images(&mut md, &sheet_info, &mut archive); + let mut extra = String::new(); + append_sheet_images(&mut extra, &sheet_info, &mut archive); + if !extra.is_empty() { + sink.write_str(&extra)?; + } } - Ok(md) + Ok(()) } // ── Parsing ──────────────────────────────────────────────────────── -/// Parse the xlsx archive into a list of sheets. -fn parse_xlsx(data: &[u8]) -> crate::error::Result> { - let cursor = Cursor::new(data); - let mut archive = ZipArchive::new(cursor)?; +fn parse_shared_strings_arena(archive: &mut ZipArchive>) -> StringArena { + let mut arena = StringArena::new(); + let Ok(mut reader) = xml_util::open_xml(archive, "xl/sharedStrings.xml") else { + return arena; + }; + let mut buf = Vec::new(); + fill_shared_strings(&mut reader, &mut buf, &mut arena); + arena +} - // 1. Load shared strings table (optional — some files use inline strings) - let shared_strings = parse_shared_strings(&mut archive); +fn fill_shared_strings( + reader: &mut Reader, + buf: &mut Vec, + arena: &mut StringArena, +) { + let mut in_si = false; + let mut current = String::new(); - // 2. Load styles (for date format detection) - let styles = parse_styles(&mut archive); + loop { + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"si" => { + in_si = true; + current.clear(); + } + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"si" => { + arena.push(¤t); + in_si = false; + } + Ok(Event::Text(ref t)) if in_si => { + if let Ok(s) = t.unescape() { + current.push_str(&s); + } + } + Ok(Event::Eof) | Err(_) => break, + _ => {} + } + buf.clear(); + } +} - // 3. Discover sheets: name + file path - let sheet_info = discover_sheets(&mut archive)?; +fn load_sheet_hyperlinks( + archive: &mut ZipArchive>, + path: &str, + rels: &Rels, +) -> HashMap { + let mut map = HashMap::new(); + let Ok(mut reader) = xml_util::open_xml(archive, path) else { + return map; + }; + let mut buf = Vec::new(); + let mut in_hyperlinks = false; - // 4. Parse each sheet - let mut sheets = Vec::new(); - for (name, path) in &sheet_info { - let mut xml = String::new(); - match archive.by_name(path) { - Ok(mut entry) => { - entry.read_to_string(&mut xml)?; + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"hyperlinks" => { + in_hyperlinks = true; + } + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"hyperlinks" => { + break; + } + Ok(Event::Empty(ref e) | Event::Start(ref e)) + if in_hyperlinks && e.local_name().as_ref() == b"hyperlink" => + { + let cell_ref = get_attr(e, b"ref").unwrap_or_default(); + let rid = get_attr(e, b"r:id").unwrap_or_default(); + if let Some(url) = rels.get(&rid) { + map.insert(cell_ref, url.clone()); + } } - Err(_) => continue, + Ok(Event::Eof) | Err(_) => break, + _ => {} } + buf.clear(); + } - // Load hyperlink relationships for this sheet - let sheet_rels_path = xml_util::rels_path(path); - let rels = xml_util::load_rels(&mut archive, &sheet_rels_path); + map +} + +#[allow(clippy::too_many_arguments)] +fn emit_sheet_plain( + archive: &mut ZipArchive>, + path: &str, + name: &str, + shared_strings: &StringArena, + styles: &Styles, + hyperlinks: &HashMap, + emitted_any: &mut bool, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + let Ok(mut reader) = xml_util::open_xml(archive, path) else { + return Ok(()); + }; + let mut buf = Vec::new(); + let mut wrote_header = false; + + loop { + let kind = match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"row" => 1u8, + Ok(Event::Eof) | Err(_) => 2, + _ => 0, + }; + buf.clear(); + match kind { + 1 => { + let cells = parse_row_arena(&mut reader, &mut buf, shared_strings, styles)?; + let dense = densify_plain_row(&cells, hyperlinks); + let mut line = String::new(); + write_plain_row(&mut line, dense.iter().map(String::as_str))?; + if !line.is_empty() { + if !wrote_header { + if *emitted_any { + sink.write_str("\n--- ")?; + sink.write_str(name)?; + sink.write_str(" ---\n")?; + } + wrote_header = true; + *emitted_any = true; + } + sink.write_str(&line)?; + } + } + 2 => break, + _ => {} + } + } + + Ok(()) +} + +fn densify_plain_row( + cells: &[(usize, String, String)], + hyperlinks: &HashMap, +) -> Vec { + let Some(max_col) = cells.iter().map(|(col, _, _)| *col).max() else { + return Vec::new(); + }; + let mut dense = vec![String::new(); max_col + 1]; + for (col, value, cell_ref) in cells { + if *col >= dense.len() { + continue; + } + dense[*col] = apply_cell_hyperlink(value, cell_ref, hyperlinks); + } + dense +} + +fn apply_cell_hyperlink( + value: &str, + cell_ref: &str, + hyperlinks: &HashMap, +) -> String { + if value.is_empty() { + return String::new(); + } + hyperlinks + .get(cell_ref) + .map_or_else(|| value.to_string(), |url| format!("[{value}]({url})")) +} + +fn check_col(col: usize) -> crate::error::Result<()> { + if col >= MAX_COLS { + Err(BatdocError::Document("sheet exceeds 512 columns".into())) + } else { + Ok(()) + } +} + +fn scan_sheet_shape( + archive: &mut ZipArchive>, + path: &str, + shared_strings: &StringArena, + styles: &Styles, +) -> crate::error::Result> { + let Ok(mut reader) = xml_util::open_xml(archive, path) else { + return Ok(None); + }; + let mut buf = Vec::new(); + let mut used = vec![false; MAX_COLS]; + let mut last_nonempty_row = None; + let mut row_idx = 0usize; + + loop { + let kind = match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"row" => 1u8, + Ok(Event::Eof) | Err(_) => 2, + _ => 0, + }; + buf.clear(); + match kind { + 1 => { + let cells = parse_row_arena(&mut reader, &mut buf, shared_strings, styles)?; + let mut nonempty = false; + for (col, value, _) in &cells { + if !value.trim().is_empty() { + used[*col] = true; + nonempty = true; + } + } + if nonempty { + last_nonempty_row = Some(row_idx); + } + row_idx += 1; + } + 2 => break, + _ => {} + } + } + + let Some(last_row) = last_nonempty_row else { + return Ok(None); + }; + let Some(first_col) = used.iter().position(|&u| u) else { + return Ok(None); + }; + let last_col = used.iter().rposition(|&u| u).unwrap_or(first_col); + Ok(Some(TableShape { + first_col, + last_col, + last_row, + })) +} + +fn emit_sheet_markdown( + archive: &mut ZipArchive>, + path: &str, + shape: &TableShape, + shared_strings: &StringArena, + styles: &Styles, + hyperlinks: &HashMap, + sink: &mut impl ExtractSink, +) -> crate::error::Result<()> { + let Ok(mut reader) = xml_util::open_xml(archive, path) else { + return Ok(()); + }; + let mut buf = Vec::new(); + let mut row_idx = 0usize; + let mut is_header = true; + let ncols = shape.last_col - shape.first_col + 1; + + loop { + let kind = match reader.read_event_into(&mut buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"row" => 1u8, + Ok(Event::Eof) | Err(_) => 2, + _ => 0, + }; + buf.clear(); + match kind { + 1 => { + if row_idx > shape.last_row { + break; + } + let cells = parse_row_arena(&mut reader, &mut buf, shared_strings, styles)?; + let slice = densify_markdown_row(&cells, shape, hyperlinks); + if is_header { + write_markdown_header(sink, &slice)?; + write_markdown_separator(sink, ncols)?; + is_header = false; + } else { + write_markdown_data_row(sink, &slice)?; + } + row_idx += 1; + } + 2 => break, + _ => {} + } + } + + sink.write_str("\n") +} + +fn densify_markdown_row( + cells: &[(usize, String, String)], + shape: &TableShape, + hyperlinks: &HashMap, +) -> Vec { + let ncols = shape.last_col - shape.first_col + 1; + let mut dense = vec![String::new(); ncols]; + for (col, value, cell_ref) in cells { + if *col < shape.first_col || *col > shape.last_col { + continue; + } + dense[*col - shape.first_col] = apply_cell_hyperlink(value, cell_ref, hyperlinks); + } + dense +} + +fn parse_row_arena( + reader: &mut Reader, + buf: &mut Vec, + shared_strings: &StringArena, + styles: &Styles, +) -> crate::error::Result> { + let mut cells: Vec<(usize, String, String)> = Vec::new(); + buf.clear(); + + loop { + let mut start_cell = None; + let mut empty_cell = None; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) if e.local_name().as_ref() == b"c" => { + let cell_ref = get_attr(e, b"r").unwrap_or_default(); + let col_idx = if cell_ref.is_empty() { + cells.len() + } else { + col_ref_to_index(&cell_ref) + }; + check_col(col_idx)?; + let cell_type = get_attr(e, b"t").unwrap_or_default(); + let style_idx: usize = get_attr(e, b"s").and_then(|s| s.parse().ok()).unwrap_or(0); + start_cell = Some((col_idx, cell_ref, cell_type, style_idx)); + } + Ok(Event::Empty(ref e)) if e.local_name().as_ref() == b"c" => { + let cell_ref = get_attr(e, b"r").unwrap_or_default(); + let col_idx = if cell_ref.is_empty() { + cells.len() + } else { + col_ref_to_index(&cell_ref) + }; + check_col(col_idx)?; + empty_cell = Some((col_idx, cell_ref)); + } + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"row" => break, + Ok(Event::Eof) | Err(_) => break, + _ => {} + } + buf.clear(); + if let Some((col_idx, cell_ref, cell_type, style_idx)) = start_cell { + let value = + parse_cell_arena(reader, buf, &cell_type, shared_strings, style_idx, styles); + cells.push((col_idx, value, cell_ref)); + } else if let Some((col_idx, cell_ref)) = empty_cell { + cells.push((col_idx, String::new(), cell_ref)); + } + } + + Ok(cells) +} + +fn parse_cell_arena( + reader: &mut Reader, + buf: &mut Vec, + cell_type: &str, + shared_strings: &StringArena, + style_idx: usize, + styles: &Styles, +) -> String { + let mut value = String::new(); + let mut inline_text = String::new(); + buf.clear(); - let mut rows = parse_sheet_xml(&xml, &shared_strings, &styles); + loop { + let mut read_v = false; + let mut read_is = false; + match reader.read_event_into(buf) { + Ok(Event::Start(ref e)) => match e.local_name().as_ref() { + b"v" => read_v = true, + b"is" => read_is = true, + _ => {} + }, + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"c" => break, + Ok(Event::Eof) | Err(_) => break, + _ => {} + } + buf.clear(); + if read_v { + if let Ok(Event::Text(t)) = reader.read_event_into(buf) { + if let Ok(s) = t.unescape() { + value = s.into_owned(); + } + } + buf.clear(); + } else if read_is { + inline_text = parse_inline_string_into(reader, buf); + } + } + + match cell_type { + "s" => value + .parse::() + .ok() + .and_then(|idx| shared_strings.get(idx).map(str::to_string)) + .unwrap_or_default(), + "inlineStr" => inline_text, + "" | "n" => maybe_convert_date(&value, style_idx, styles), + _ => value, + } +} - // Apply hyperlinks: parse from sheet XML and - // resolve URLs from the rels map - apply_hyperlinks(&xml, &rels, &mut rows); +fn parse_inline_string_into(reader: &mut Reader, buf: &mut Vec) -> String { + let mut text = String::new(); + buf.clear(); - sheets.push(Sheet { - name: name.clone(), - rows, - }); + loop { + match reader.read_event_into(buf) { + Ok(Event::Text(ref t)) => { + if let Ok(s) = t.unescape() { + text.push_str(&s); + } + } + Ok(Event::End(ref e)) if e.local_name().as_ref() == b"is" => break, + Ok(Event::Eof) | Err(_) => break, + _ => {} + } + buf.clear(); } - Ok(sheets) + text } // ── Style / date format detection ────────────────────────────────── @@ -106,31 +569,28 @@ impl Styles { /// Reads `` elements for custom format strings and `` elements /// in `` for the numFmtId associated with each style index. fn parse_styles(archive: &mut ZipArchive>) -> Styles { - let mut xml = String::new(); - match archive.by_name("xl/styles.xml") { - Ok(mut entry) => { - if entry.read_to_string(&mut xml).is_err() { - return Styles::default(); - } - } - Err(_) => return Styles::default(), - } - - parse_styles_xml(&xml) + let Ok(mut reader) = xml_util::open_xml(archive, "xl/styles.xml") else { + return Styles::default(); + }; + let mut buf = Vec::new(); + fill_styles(&mut reader, &mut buf) } /// Parse styles XML into resolved style info (separated for testability). +#[cfg(test)] fn parse_styles_xml(xml: &str) -> Styles { let mut reader = Reader::from_str(xml); + let mut buf = Vec::new(); + fill_styles(&mut reader, &mut buf) +} - // Custom number formats: numFmtId → format string +fn fill_styles(reader: &mut Reader, buf: &mut Vec) -> Styles { let mut custom_formats: Vec<(u16, String)> = Vec::new(); - // Cell xf entries: each entry's numFmtId let mut cell_xf_fmt_ids: Vec = Vec::new(); let mut in_cell_xfs = false; loop { - match reader.read_event() { + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) => { let name = e.local_name(); match name.as_ref() { @@ -171,6 +631,7 @@ fn parse_styles_xml(xml: &str) -> Styles { Ok(Event::Eof) | Err(_) => break, _ => {} } + buf.clear(); } Styles { @@ -185,6 +646,7 @@ fn parse_styles_xml(xml: &str) -> Styles { /// Each `` maps a cell reference to /// a relationship ID. We resolve the rId to a URL and wrap the existing /// cell value as `[value](url)`. +#[cfg(test)] fn apply_hyperlinks(xml: &str, rels: &Rels, rows: &mut [Vec]) { if rels.is_empty() { return; @@ -376,6 +838,7 @@ fn normalize_dotdot(path: &str) -> String { } /// Extract the 0-based row number from a cell reference like "B3" → 2. +#[cfg(test)] fn cell_ref_to_row(cell_ref: &str) -> usize { let digits: String = cell_ref .chars() @@ -384,58 +847,19 @@ fn cell_ref_to_row(cell_ref: &str) -> usize { digits.parse::().unwrap_or(1).saturating_sub(1) } -/// Parse `xl/sharedStrings.xml` into a lookup table. -/// -/// Each `` element contributes one string at its positional index. -/// Strings may be plain `` text or rich text with multiple `` runs. -fn parse_shared_strings(archive: &mut ZipArchive>) -> Vec { - let mut xml = String::new(); - match archive.by_name("xl/sharedStrings.xml") { - Ok(mut entry) => { - if entry.read_to_string(&mut xml).is_err() { - return Vec::new(); - } - } - Err(_) => return Vec::new(), - } - - parse_shared_strings_xml(&xml) -} - /// Parse shared string table XML into a list of strings. /// /// Separated from `parse_shared_strings` for testability (avoids needing /// a ZIP archive in tests). +#[cfg(test)] fn parse_shared_strings_xml(xml: &str) -> Vec { let mut reader = Reader::from_str(xml); - let mut strings = Vec::new(); - let mut in_si = false; - let mut current = String::new(); - - loop { - match reader.read_event() { - Ok(Event::Start(ref e)) => { - let name = e.local_name(); - if name.as_ref() == b"si" { - in_si = true; - current.clear(); - } - } - Ok(Event::End(ref e)) if e.local_name().as_ref() == b"si" => { - strings.push(std::mem::take(&mut current)); - in_si = false; - } - Ok(Event::Text(ref t)) if in_si => { - if let Ok(s) = t.unescape() { - current.push_str(&s); - } - } - Ok(Event::Eof) | Err(_) => break, - _ => {} - } - } - - strings + let mut buf = Vec::new(); + let mut arena = StringArena::new(); + fill_shared_strings(&mut reader, &mut buf, &mut arena); + (0..arena.len()) + .map(|i| arena.get(i).unwrap_or("").to_string()) + .collect() } /// Discover sheet names and their file paths from workbook.xml and relationships. @@ -444,63 +868,56 @@ fn parse_shared_strings_xml(xml: &str) -> Vec { fn discover_sheets( archive: &mut ZipArchive>, ) -> crate::error::Result> { - // Parse workbook.xml for sheet name → rId mapping - let mut workbook_xml = String::new(); - archive - .by_name("xl/workbook.xml")? - .read_to_string(&mut workbook_xml)?; - - let mut sheet_entries: Vec<(String, String)> = Vec::new(); // (name, rId) - let mut reader = Reader::from_str(&workbook_xml); - - loop { - match reader.read_event() { - Ok(Event::Empty(ref e) | Event::Start(ref e)) - if e.local_name().as_ref() == b"sheet" => - { - let name = get_attr(e, b"name").unwrap_or_default(); - let rid = get_attr(e, b"r:id").unwrap_or_default(); - let state = get_attr(e, b"state").unwrap_or_default(); - // Skip hidden sheets - if state != "hidden" && !name.is_empty() && !rid.is_empty() { - sheet_entries.push((name, rid)); + let sheet_entries = { + let mut reader = xml_util::open_xml(archive, "xl/workbook.xml")?; + let mut buf = Vec::new(); + let mut sheet_entries: Vec<(String, String)> = Vec::new(); + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Empty(ref e) | Event::Start(ref e)) + if e.local_name().as_ref() == b"sheet" => + { + let name = get_attr(e, b"name").unwrap_or_default(); + let rid = get_attr(e, b"r:id").unwrap_or_default(); + let state = get_attr(e, b"state").unwrap_or_default(); + if state != "hidden" && !name.is_empty() && !rid.is_empty() { + sheet_entries.push((name, rid)); + } } + Ok(Event::Eof) | Err(_) => break, + _ => {} } - Ok(Event::Eof) | Err(_) => break, - _ => {} + buf.clear(); } - } - - // Parse workbook.xml.rels for rId → Target path mapping - let mut rels_xml = String::new(); - archive - .by_name("xl/_rels/workbook.xml.rels")? - .read_to_string(&mut rels_xml)?; - - let mut rid_to_target: Vec<(String, String)> = Vec::new(); - let mut reader = Reader::from_str(&rels_xml); - - loop { - match reader.read_event() { - Ok(Event::Empty(ref e) | Event::Start(ref e)) - if e.local_name().as_ref() == b"Relationship" => - { - let id = get_attr(e, b"Id").unwrap_or_default(); - let target = get_attr(e, b"Target").unwrap_or_default(); - if !id.is_empty() && !target.is_empty() { - rid_to_target.push((id, target)); + sheet_entries + }; + + let rid_to_target = { + let mut reader = xml_util::open_xml(archive, "xl/_rels/workbook.xml.rels")?; + let mut buf = Vec::new(); + let mut rid_to_target: Vec<(String, String)> = Vec::new(); + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Empty(ref e) | Event::Start(ref e)) + if e.local_name().as_ref() == b"Relationship" => + { + let id = get_attr(e, b"Id").unwrap_or_default(); + let target = get_attr(e, b"Target").unwrap_or_default(); + if !id.is_empty() && !target.is_empty() { + rid_to_target.push((id, target)); + } } + Ok(Event::Eof) | Err(_) => break, + _ => {} } - Ok(Event::Eof) | Err(_) => break, - _ => {} + buf.clear(); } - } + rid_to_target + }; - // Resolve: sheet name → zip path let mut result = Vec::new(); for (name, rid) in &sheet_entries { if let Some((_, target)) = rid_to_target.iter().find(|(id, _)| id == rid) { - // Target is relative to xl/, may have leading / let path = if target.starts_with('/') { target.trim_start_matches('/').to_string() } else { @@ -521,6 +938,7 @@ fn discover_sheets( /// - Otherwise: raw value from `` (numbers, dates, formulas with cached values) /// /// Numeric cells whose style maps to a date format are converted to ISO dates. +#[cfg(test)] fn parse_sheet_xml(xml: &str, shared_strings: &[String], styles: &Styles) -> Vec> { let mut reader = Reader::from_str(xml); let mut sparse_rows: Vec> = Vec::new(); @@ -558,6 +976,7 @@ fn parse_sheet_xml(xml: &str, shared_strings: &[String], styles: &Styles) -> Vec } /// Parse a `` element, returning `(column_index, value)` pairs. +#[cfg(test)] fn parse_row( reader: &mut Reader<&[u8]>, shared_strings: &[String], @@ -597,6 +1016,7 @@ fn parse_row( /// For numeric cells (no `t` attribute or `t="n"`), checks the style /// to see if the number format is a date — if so, converts the serial /// number to an ISO date string. +#[cfg(test)] fn parse_cell( reader: &mut Reader<&[u8]>, cell_type: &str, @@ -661,6 +1081,7 @@ fn maybe_convert_date(value: &str, style_idx: usize, styles: &Styles) -> String } /// Parse an `` inline string element, collecting all `` text. +#[cfg(test)] fn parse_inline_string(reader: &mut Reader<&[u8]>) -> String { let mut text = String::new(); @@ -928,4 +1349,503 @@ mod tests { apply_hyperlinks("", &rels, &mut rows); assert_eq!(rows[0][0], "Hello"); } + + // ── streaming plain (zip fixtures) ─────────────────────────── + + use std::io::Write; + use zip::write::SimpleFileOptions; + use zip::ZipWriter; + + fn zip_xlsx(parts: &[(&str, &str)]) -> Vec { + let buf = Cursor::new(Vec::new()); + let mut z = ZipWriter::new(buf); + for (name, body) in parts { + z.start_file(*name, SimpleFileOptions::default()).unwrap(); + z.write_all(body.as_bytes()).unwrap(); + } + z.finish().unwrap().into_inner() + } + + fn minimal_xlsx_two_rows() -> Vec { + zip_xlsx(&[ + ( + "[Content_Types].xml", + r#""#, + ), + ( + "xl/workbook.xml", + r#" + + + + +"#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#" + + +"#, + ), + ( + "xl/sharedStrings.xml", + r#" + + Name + Age + Alice + 30 +"#, + ), + ( + "xl/worksheets/sheet1.xml", + r#" + + + + 0 + 1 + + + 2 + 3 + + +"#, + ), + ]) + } + + #[test] + fn xlsx_plain_to_matches_buffered_and_has_no_trailing_tab_row() { + let data = minimal_xlsx_two_rows(); + let buffered = crate::extract_plain(&data, crate::Format::Xlsx).unwrap(); + let mut streamed = String::new(); + crate::extract_plain_to( + &data, + crate::Format::Xlsx, + crate::ExtractOptions::default(), + &mut streamed, + ) + .unwrap(); + assert_eq!(buffered, streamed); + assert_eq!(buffered, "Name\tAge\nAlice\t30\n"); + } + + #[test] + fn xlsx_plain_to_respects_max_output_bytes() { + let data = minimal_xlsx_two_rows(); + let mut out = String::new(); + let err = crate::extract_plain_to( + &data, + crate::Format::Xlsx, + crate::ExtractOptions { + max_output_bytes: Some(16), + ..Default::default() + }, + &mut out, + ) + .unwrap_err() + .to_string(); + assert_eq!(err, "output exceeded 16 bytes"); + assert!(out.len() <= 16); + } + + fn workbook_two_sheets() -> &'static str { + r#" + + + + + +"# + } + + fn workbook_two_sheet_rels() -> &'static str { + r#" + + + +"# + } + + fn sheet_xml(body: &str) -> String { + format!( + r#" + + {body} +"# + ) + } + + #[test] + fn xlsx_plain_skips_empty_sheet_and_omits_single_sheet_header() { + let data = zip_xlsx(&[ + ("[Content_Types].xml", r#""#), + ("xl/workbook.xml", workbook_two_sheets()), + ("xl/_rels/workbook.xml.rels", workbook_two_sheet_rels()), + ( + "xl/worksheets/sheet1.xml", + &sheet_xml(r#""#), + ), + ( + "xl/worksheets/sheet2.xml", + &sheet_xml(r#"NYC"#), + ), + ]); + let text = crate::extract_plain(&data, crate::Format::Xlsx).unwrap(); + assert_eq!(text, "NYC\n"); + } + + #[test] + fn xlsx_plain_multi_sheet_header_only_before_non_first_emitted() { + let data = zip_xlsx(&[ + ("[Content_Types].xml", r#""#), + ("xl/workbook.xml", workbook_two_sheets()), + ("xl/_rels/workbook.xml.rels", workbook_two_sheet_rels()), + ( + "xl/worksheets/sheet1.xml", + &sheet_xml(r#"Alice"#), + ), + ( + "xl/worksheets/sheet2.xml", + &sheet_xml(r#"NYC"#), + ), + ]); + let text = crate::extract_plain(&data, crate::Format::Xlsx).unwrap(); + assert_eq!(text, "Alice\n\n--- Places ---\nNYC\n"); + } + + #[test] + fn xlsx_plain_sparse_row_keeps_middle_gap() { + let data = zip_xlsx(&[ + ("[Content_Types].xml", r#""#), + ( + "xl/workbook.xml", + r#" + + +"#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#" + + +"#, + ), + ( + "xl/sharedStrings.xml", + r#"FirstThird"#, + ), + ( + "xl/worksheets/sheet1.xml", + &sheet_xml( + r#"01"#, + ), + ), + ]); + let text = crate::extract_plain(&data, crate::Format::Xlsx).unwrap(); + assert_eq!(text, "First\t\tThird\n"); + } + + #[test] + fn xlsx_plain_missing_sst_index_is_empty_cell() { + let data = zip_xlsx(&[ + ("[Content_Types].xml", r#""#), + ( + "xl/workbook.xml", + r#" + + +"#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#" + + +"#, + ), + ("xl/sharedStrings.xml", r#"Only"#), + ( + "xl/worksheets/sheet1.xml", + &sheet_xml( + r#"099"#, + ), + ), + ]); + let text = crate::extract_plain(&data, crate::Format::Xlsx).unwrap(); + assert_eq!(text, "Only\n"); + } + + #[test] + fn xlsx_plain_date_styled_numeric_is_iso() { + let data = zip_xlsx(&[ + ("[Content_Types].xml", r#""#), + ( + "xl/workbook.xml", + r#" + + +"#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#" + + +"#, + ), + ( + "xl/styles.xml", + r#" + + + + +"#, + ), + ( + "xl/worksheets/sheet1.xml", + &sheet_xml( + r#"4245292"#, + ), + ), + ]); + let text = crate::extract_plain(&data, crate::Format::Xlsx).unwrap(); + assert_eq!(text, "42\t2024-01-01\n"); + } + + #[test] + fn xlsx_plain_hyperlink_wraps_cell() { + let data = zip_xlsx(&[ + ("[Content_Types].xml", r#""#), + ( + "xl/workbook.xml", + r#" + + +"#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#" + + +"#, + ), + ( + "xl/worksheets/_rels/sheet1.xml.rels", + r#" + + +"#, + ), + ( + "xl/worksheets/sheet1.xml", + r#" + + + Click here + + + + +"#, + ), + ]); + let text = crate::extract_plain(&data, crate::Format::Xlsx).unwrap(); + assert_eq!(text, "[Click here](https://example.com)\n"); + } + + fn xlsx_with_cell_at(cell_ref: &str) -> Vec { + let body = + format!(r#"X"#); + zip_xlsx(&[ + ("[Content_Types].xml", r#""#), + ( + "xl/workbook.xml", + r#" + + +"#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#" + + +"#, + ), + ("xl/worksheets/sheet1.xml", &sheet_xml(&body)), + ]) + } + + #[test] + fn xlsx_markdown_rejects_col_513() { + let data = xlsx_with_cell_at("SS1"); + let err = crate::extract_markdown(&data, crate::Format::Xlsx, false) + .unwrap_err() + .to_string(); + assert_eq!(err, "sheet exceeds 512 columns"); + } + + #[test] + fn xlsx_markdown_accepts_col_ja() { + let data = xlsx_with_cell_at("JA1"); + let md = crate::extract_markdown(&data, crate::Format::Xlsx, false).unwrap(); + assert_eq!(md, "| X |\n| --- |\n\n"); + } + + #[test] + fn xlsx_markdown_two_rows_locked() { + let data = minimal_xlsx_two_rows(); + let md = crate::extract_markdown(&data, crate::Format::Xlsx, false).unwrap(); + assert_eq!(md, "| Name | Age |\n| --- | --- |\n| Alice | 30 |\n\n"); + } + + #[test] + fn xlsx_markdown_to_respects_max_output_bytes() { + let data = minimal_xlsx_two_rows(); + let mut out = String::new(); + let err = crate::extract_markdown_to( + &data, + crate::Format::Xlsx, + crate::ExtractOptions { + max_output_bytes: Some(16), + ..Default::default() + }, + &mut out, + ) + .unwrap_err() + .to_string(); + assert_eq!(err, "output exceeded 16 bytes"); + assert!(out.len() <= 16); + } + + #[test] + fn xlsx_markdown_skips_empty_sheet_and_omits_single_sheet_heading() { + let data = zip_xlsx(&[ + ("[Content_Types].xml", r#""#), + ("xl/workbook.xml", workbook_two_sheets()), + ("xl/_rels/workbook.xml.rels", workbook_two_sheet_rels()), + ( + "xl/worksheets/sheet1.xml", + &sheet_xml(r#""#), + ), + ( + "xl/worksheets/sheet2.xml", + &sheet_xml(r#"NYC"#), + ), + ]); + let md = crate::extract_markdown(&data, crate::Format::Xlsx, false).unwrap(); + assert_eq!(md, "| NYC |\n| --- |\n\n"); + } + + #[test] + fn xlsx_markdown_multi_sheet_headings() { + let data = zip_xlsx(&[ + ("[Content_Types].xml", r#""#), + ("xl/workbook.xml", workbook_two_sheets()), + ("xl/_rels/workbook.xml.rels", workbook_two_sheet_rels()), + ( + "xl/worksheets/sheet1.xml", + &sheet_xml(r#"Alice"#), + ), + ( + "xl/worksheets/sheet2.xml", + &sheet_xml(r#"NYC"#), + ), + ]); + let md = crate::extract_markdown(&data, crate::Format::Xlsx, false).unwrap(); + assert_eq!( + md, + "## People\n\n| Alice |\n| --- |\n\n## Places\n\n| NYC |\n| --- |\n\n" + ); + } + + #[test] + fn xlsx_markdown_strips_leading_empty_col_and_trailing_empty_row() { + let data = zip_xlsx(&[ + ("[Content_Types].xml", r#""#), + ( + "xl/workbook.xml", + r#" + + +"#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#" + + +"#, + ), + ( + "xl/worksheets/sheet1.xml", + &sheet_xml( + r#"NameAlice "#, + ), + ), + ]); + let md = crate::extract_markdown(&data, crate::Format::Xlsx, false).unwrap(); + assert_eq!(md, "| Name |\n| --- |\n| Alice |\n\n"); + } + + #[test] + fn xlsx_markdown_hyperlink_wraps_cell() { + let data = zip_xlsx(&[ + ("[Content_Types].xml", r#""#), + ( + "xl/workbook.xml", + r#" + + +"#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#" + + +"#, + ), + ( + "xl/worksheets/_rels/sheet1.xml.rels", + r#" + + +"#, + ), + ( + "xl/worksheets/sheet1.xml", + r#" + + + Click here + + + + +"#, + ), + ]); + let md = crate::extract_markdown(&data, crate::Format::Xlsx, false).unwrap(); + assert_eq!(md, "| [Click here](https://example.com) |\n| --- |\n\n"); + } } diff --git a/batdoc-core/src/xml_util.rs b/batdoc-core/src/xml_util.rs index 248fdf2..48f3b81 100644 --- a/batdoc-core/src/xml_util.rs +++ b/batdoc-core/src/xml_util.rs @@ -201,6 +201,17 @@ fn normalize_zip_path(path: &str) -> String { parts.join("/") } +pub(crate) type XmlReader<'a> = Reader>>; + +/// Open a ZIP entry as a streaming `quick_xml` reader. +pub(crate) fn open_xml<'arc>( + archive: &'arc mut ZipArchive>, + path: &str, +) -> crate::error::Result> { + let entry = archive.by_name(path)?; + Ok(Reader::from_reader(std::io::BufReader::new(entry))) +} + /// Compute the `_rels` file path for a given OOXML part path. /// /// For `xl/worksheets/sheet1.xml`, returns `xl/worksheets/_rels/sheet1.xml.rels`. @@ -370,4 +381,50 @@ mod tests { "xl/media/image1.png" ); } + + // ── open_xml ────────────────────────────────────────────────── + + #[test] + fn open_xml_reads_streamed_entry() { + use std::io::{Cursor, Write}; + let mut buf = Vec::new(); + { + let mut zw = zip::ZipWriter::new(Cursor::new(&mut buf)); + zw.start_file("hi.xml", zip::write::SimpleFileOptions::default()) + .unwrap(); + zw.write_all(br#""#).unwrap(); + zw.finish().unwrap(); + } + let mut archive = zip::ZipArchive::new(Cursor::new(buf.as_slice())).unwrap(); + let mut reader = open_xml(&mut archive, "hi.xml").unwrap(); + let mut buf = Vec::new(); + let mut found = false; + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Empty(ref e)) if e.local_name().as_ref() == b"item" => { + assert_eq!(get_attr(e, b"id").as_deref(), Some("1")); + found = true; + } + Ok(Event::Eof) | Err(_) => break, + _ => {} + } + buf.clear(); + } + assert!(found); + } + + #[test] + fn open_xml_missing_entry_is_err() { + use std::io::{Cursor, Write}; + let mut buf = Vec::new(); + { + let mut zw = zip::ZipWriter::new(Cursor::new(&mut buf)); + zw.start_file("other.xml", zip::write::SimpleFileOptions::default()) + .unwrap(); + zw.write_all(b"").unwrap(); + zw.finish().unwrap(); + } + let mut archive = zip::ZipArchive::new(Cursor::new(buf.as_slice())).unwrap(); + assert!(open_xml(&mut archive, "hi.xml").is_err()); + } } diff --git a/batdoc-core/tests/ocr_e2e.rs b/batdoc-core/tests/ocr_e2e.rs index a8d648f..7cbc95b 100644 --- a/batdoc-core/tests/ocr_e2e.rs +++ b/batdoc-core/tests/ocr_e2e.rs @@ -253,6 +253,7 @@ fn ocr_docx_embedded_image_markdown() { ExtractOptions { images: true, ocr: true, + ..Default::default() }, ) .unwrap(); @@ -267,6 +268,7 @@ fn ocr_docx_embedded_image_markdown() { ExtractOptions { images: true, ocr: false, + ..Default::default() }, ) .unwrap(); @@ -278,20 +280,36 @@ fn ocr_docx_embedded_image_markdown() { #[ignore = "requires OCR models (downloaded on first use)"] fn ocr_pdf_embedded_image_page() { let pdf = build_pdf_with_image(&render_test_image()); - // Today's behavior: no --ocr → clean error. - let err = batdoc_core::extract_plain_with(&pdf, Format::Pdf, ExtractOptions::default()) - .unwrap_err() - .to_string(); - assert!(err.contains("scanned/image-only"), "got: {err}"); - // With --ocr → OCR'd text. + // A textless PDF auto-falls back to OCR even without --ocr… + let auto = + batdoc_core::extract_plain_with(&pdf, Format::Pdf, ExtractOptions::default()).unwrap(); + assert!(auto.contains("123"), "auto OCR text missing: {auto:?}"); + // …and an explicit --ocr behaves identically. let text = batdoc_core::extract_plain_with( &pdf, Format::Pdf, ExtractOptions { ocr: true, images: false, + ..Default::default() }, ) .unwrap(); assert!(text.contains("123"), "OCR text missing: {text:?}"); } + +#[test] +#[ignore = "requires OCR models (downloaded on first use)"] +fn ocr_pdf_markdown_mode_region_merge() { + let pdf = build_pdf_with_image(&render_test_image()); + // Markdown mode on a textless PDF must also run OCR and include the + // recognized text via the region-merge path. (Note this is *not* shared + // with plain mode: `extract_plain` textless pages fall back to the + // page-end `ocr_page` append instead.) + let md = batdoc_core::extract_markdown(&pdf, Format::Pdf, false).unwrap(); + assert!(md.contains("123"), "OCR text missing in markdown: {md:?}"); + assert!( + md.contains("BATDOC"), + "OCR text missing in markdown: {md:?}" + ); +} diff --git a/batdoc-core/tests/pdf_positioned_e2e.rs b/batdoc-core/tests/pdf_positioned_e2e.rs new file mode 100644 index 0000000..7765f8d --- /dev/null +++ b/batdoc-core/tests/pdf_positioned_e2e.rs @@ -0,0 +1,53 @@ +//! End-to-end tests against real PDFs supplied out-of-tree (fixtures are +//! not committed; AGENTS.md convention). Set BATDOC_SOAK_DIR to a +//! directory containing: +//! garbled-cid.pdf — a CID-keyed PDF whose /ToUnicode is broken +//! scanned.pdf — an image-only scan +//! two-column.pdf — a real multi-column document +//! All tests are #[ignore]d: run with +//! BATDOC_SOAK_DIR=/path cargo test --release -p batdoc-core --test pdf_positioned_e2e -- --ignored + +use std::path::PathBuf; + +fn soak_pdf(name: &str) -> Option> { + let dir = std::env::var_os("BATDOC_SOAK_DIR")?; + let path = PathBuf::from(dir).join(name); + std::fs::read(path).ok() +} + +#[test] +#[ignore] +fn garbled_cid_pdf_extracts_readable_text() { + let Some(data) = soak_pdf("garbled-cid.pdf") else { + return; + }; + let text = batdoc_core::extract_plain(&data, batdoc_core::Format::Pdf).unwrap(); + // Owner: replace with a phrase known to appear in the document. + assert!(!text.trim().is_empty()); + assert!( + !text.contains('\u{FFFD}'), + "recovery left replacement chars" + ); +} + +#[test] +#[ignore] +fn scanned_pdf_ocr_text_has_no_duplication() { + let Some(data) = soak_pdf("scanned.pdf") else { + return; + }; + let md = batdoc_core::extract_markdown(&data, batdoc_core::Format::Pdf, false).unwrap(); + assert!(!md.trim().is_empty()); + // Owner: assert a known line appears exactly once. +} + +#[test] +#[ignore] +fn two_column_pdf_reads_columns_in_order() { + let Some(data) = soak_pdf("two-column.pdf") else { + return; + }; + let md = batdoc_core::extract_markdown(&data, batdoc_core::Format::Pdf, false).unwrap(); + // Owner: assert find(phrase_col1_end) < find(phrase_col2_start). + assert!(!md.trim().is_empty()); +} diff --git a/build.rs b/build.rs index 225fb88..ee58407 100644 --- a/build.rs +++ b/build.rs @@ -62,6 +62,11 @@ fn main() { (OOXML), .pdf, and raster images (.png, .jpg, .gif, .webp, \ .bmp).", ) + .paragraph( + "PDF glyph recovery depends on the vendored pdf-extract fork \ + under crates/pdf-extract; crates.io installs use upstream \ + pdf-extract and do not recover garbled fonts.", + ) .paragraph( "When stdout is a terminal, output is pretty-printed as \ syntax-highlighted markdown via bat(1) with paging. When \ diff --git a/crates/pdf-extract/Cargo.toml b/crates/pdf-extract/Cargo.toml new file mode 100644 index 0000000..b87bb70 --- /dev/null +++ b/crates/pdf-extract/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "pdf-extract" +version = "0.10.0" +edition = "2018" +authors = ["Jeff Muizelaar "] +description = "A library to extract content from pdfs (batdoc vendored fork: panic-free decode + glyph recovery)" +license = "MIT" +repository = "https://github.com/jrmuizel/pdf-extract" +readme = "README.md" + +[lib] +name = "pdf_extract" +path = "src/lib.rs" + +[dependencies] +adobe-cmap-parser = "0.4.1" +cff-parser = "0.1.0" +encoding_rs = "0.8.34" +euclid = "0.20.5" +log = "0.4.22" +lopdf = { version = "0.38", default-features = false } +postscript = "0.14" +ttf-parser = "0.25" +type1-encoding-parser = "0.1.0" +unicode-normalization = "0.1.19" + +[dev-dependencies] +lopdf = { version = "0.38", default-features = false } diff --git a/crates/pdf-extract/README.md b/crates/pdf-extract/README.md new file mode 100644 index 0000000..7b23be8 --- /dev/null +++ b/crates/pdf-extract/README.md @@ -0,0 +1,26 @@ +## pdf-extract + +[![Build Status](https://github.com/jrmuizel/pdf-extract/actions/workflows/rust.yml/badge.svg)](https://github.com/jrmuizel/pdf-extract/actions) +[![crates.io](https://img.shields.io/crates/v/pdf-extract.svg)](https://crates.io/crates/pdf-extract) +[![Documentation](https://docs.rs/pdf-extract/badge.svg)](https://docs.rs/pdf-extract) + +A rust library to extract content from PDF files. + +```rust +let bytes = std::fs::read("tests/docs/simple.pdf").unwrap(); +let out = pdf_extract::extract_text_from_mem(&bytes).unwrap(); +assert!(out.contains("This is a small demonstration")); +``` + +## See also + +- +- / +- +- +- used by [grobid](https://github.com/kermitt2/grobid/) +- (uses PyMuPDF and pdfminer.six) + +### Not PDF specific + +- diff --git a/crates/pdf-extract/src/core_fonts.rs b/crates/pdf-extract/src/core_fonts.rs new file mode 100644 index 0000000..54e94b3 --- /dev/null +++ b/crates/pdf-extract/src/core_fonts.rs @@ -0,0 +1,4267 @@ +// Generated by afm/example/metrics.rs +pub fn metrics() -> Box< + [( + &'static str, + &'static str, + &'static [(i64, f64, &'static str)], + )], +> { + Box::new([ + ( + "Courier-Bold", + "AdobeStandardEncoding", + &[ + (32, 600.0, "space"), + (33, 600.0, "exclam"), + (34, 600.0, "quotedbl"), + (35, 600.0, "numbersign"), + (36, 600.0, "dollar"), + (37, 600.0, "percent"), + (38, 600.0, "ampersand"), + (39, 600.0, "quoteright"), + (40, 600.0, "parenleft"), + (41, 600.0, "parenright"), + (42, 600.0, "asterisk"), + (43, 600.0, "plus"), + (44, 600.0, "comma"), + (45, 600.0, "hyphen"), + (46, 600.0, "period"), + (47, 600.0, "slash"), + (48, 600.0, "zero"), + (49, 600.0, "one"), + (50, 600.0, "two"), + (51, 600.0, "three"), + (52, 600.0, "four"), + (53, 600.0, "five"), + (54, 600.0, "six"), + (55, 600.0, "seven"), + (56, 600.0, "eight"), + (57, 600.0, "nine"), + (58, 600.0, "colon"), + (59, 600.0, "semicolon"), + (60, 600.0, "less"), + (61, 600.0, "equal"), + (62, 600.0, "greater"), + (63, 600.0, "question"), + (64, 600.0, "at"), + (65, 600.0, "A"), + (66, 600.0, "B"), + (67, 600.0, "C"), + (68, 600.0, "D"), + (69, 600.0, "E"), + (70, 600.0, "F"), + (71, 600.0, "G"), + (72, 600.0, "H"), + (73, 600.0, "I"), + (74, 600.0, "J"), + (75, 600.0, "K"), + (76, 600.0, "L"), + (77, 600.0, "M"), + (78, 600.0, "N"), + (79, 600.0, "O"), + (80, 600.0, "P"), + (81, 600.0, "Q"), + (82, 600.0, "R"), + (83, 600.0, "S"), + (84, 600.0, "T"), + (85, 600.0, "U"), + (86, 600.0, "V"), + (87, 600.0, "W"), + (88, 600.0, "X"), + (89, 600.0, "Y"), + (90, 600.0, "Z"), + (91, 600.0, "bracketleft"), + (92, 600.0, "backslash"), + (93, 600.0, "bracketright"), + (94, 600.0, "asciicircum"), + (95, 600.0, "underscore"), + (96, 600.0, "quoteleft"), + (97, 600.0, "a"), + (98, 600.0, "b"), + (99, 600.0, "c"), + (100, 600.0, "d"), + (101, 600.0, "e"), + (102, 600.0, "f"), + (103, 600.0, "g"), + (104, 600.0, "h"), + (105, 600.0, "i"), + (106, 600.0, "j"), + (107, 600.0, "k"), + (108, 600.0, "l"), + (109, 600.0, "m"), + (110, 600.0, "n"), + (111, 600.0, "o"), + (112, 600.0, "p"), + (113, 600.0, "q"), + (114, 600.0, "r"), + (115, 600.0, "s"), + (116, 600.0, "t"), + (117, 600.0, "u"), + (118, 600.0, "v"), + (119, 600.0, "w"), + (120, 600.0, "x"), + (121, 600.0, "y"), + (122, 600.0, "z"), + (123, 600.0, "braceleft"), + (124, 600.0, "bar"), + (125, 600.0, "braceright"), + (126, 600.0, "asciitilde"), + (161, 600.0, "exclamdown"), + (162, 600.0, "cent"), + (163, 600.0, "sterling"), + (164, 600.0, "fraction"), + (165, 600.0, "yen"), + (166, 600.0, "florin"), + (167, 600.0, "section"), + (168, 600.0, "currency"), + (169, 600.0, "quotesingle"), + (170, 600.0, "quotedblleft"), + (171, 600.0, "guillemotleft"), + (172, 600.0, "guilsinglleft"), + (173, 600.0, "guilsinglright"), + (174, 600.0, "fi"), + (175, 600.0, "fl"), + (177, 600.0, "endash"), + (178, 600.0, "dagger"), + (179, 600.0, "daggerdbl"), + (180, 600.0, "periodcentered"), + (182, 600.0, "paragraph"), + (183, 600.0, "bullet"), + (184, 600.0, "quotesinglbase"), + (185, 600.0, "quotedblbase"), + (186, 600.0, "quotedblright"), + (187, 600.0, "guillemotright"), + (188, 600.0, "ellipsis"), + (189, 600.0, "perthousand"), + (191, 600.0, "questiondown"), + (193, 600.0, "grave"), + (194, 600.0, "acute"), + (195, 600.0, "circumflex"), + (196, 600.0, "tilde"), + (197, 600.0, "macron"), + (198, 600.0, "breve"), + (199, 600.0, "dotaccent"), + (200, 600.0, "dieresis"), + (202, 600.0, "ring"), + (203, 600.0, "cedilla"), + (205, 600.0, "hungarumlaut"), + (206, 600.0, "ogonek"), + (207, 600.0, "caron"), + (208, 600.0, "emdash"), + (225, 600.0, "AE"), + (227, 600.0, "ordfeminine"), + (232, 600.0, "Lslash"), + (233, 600.0, "Oslash"), + (234, 600.0, "OE"), + (235, 600.0, "ordmasculine"), + (241, 600.0, "ae"), + (245, 600.0, "dotlessi"), + (248, 600.0, "lslash"), + (249, 600.0, "oslash"), + (250, 600.0, "oe"), + (251, 600.0, "germandbls"), + (-1, 600.0, "Idieresis"), + (-1, 600.0, "eacute"), + (-1, 600.0, "abreve"), + (-1, 600.0, "uhungarumlaut"), + (-1, 600.0, "ecaron"), + (-1, 600.0, "Ydieresis"), + (-1, 600.0, "divide"), + (-1, 600.0, "Yacute"), + (-1, 600.0, "Acircumflex"), + (-1, 600.0, "aacute"), + (-1, 600.0, "Ucircumflex"), + (-1, 600.0, "yacute"), + (-1, 600.0, "scommaaccent"), + (-1, 600.0, "ecircumflex"), + (-1, 600.0, "Uring"), + (-1, 600.0, "Udieresis"), + (-1, 600.0, "aogonek"), + (-1, 600.0, "Uacute"), + (-1, 600.0, "uogonek"), + (-1, 600.0, "Edieresis"), + (-1, 600.0, "Dcroat"), + (-1, 600.0, "commaaccent"), + (-1, 600.0, "copyright"), + (-1, 600.0, "Emacron"), + (-1, 600.0, "ccaron"), + (-1, 600.0, "aring"), + (-1, 600.0, "Ncommaaccent"), + (-1, 600.0, "lacute"), + (-1, 600.0, "agrave"), + (-1, 600.0, "Tcommaaccent"), + (-1, 600.0, "Cacute"), + (-1, 600.0, "atilde"), + (-1, 600.0, "Edotaccent"), + (-1, 600.0, "scaron"), + (-1, 600.0, "scedilla"), + (-1, 600.0, "iacute"), + (-1, 600.0, "lozenge"), + (-1, 600.0, "Rcaron"), + (-1, 600.0, "Gcommaaccent"), + (-1, 600.0, "ucircumflex"), + (-1, 600.0, "acircumflex"), + (-1, 600.0, "Amacron"), + (-1, 600.0, "rcaron"), + (-1, 600.0, "ccedilla"), + (-1, 600.0, "Zdotaccent"), + (-1, 600.0, "Thorn"), + (-1, 600.0, "Omacron"), + (-1, 600.0, "Racute"), + (-1, 600.0, "Sacute"), + (-1, 600.0, "dcaron"), + (-1, 600.0, "Umacron"), + (-1, 600.0, "uring"), + (-1, 600.0, "threesuperior"), + (-1, 600.0, "Ograve"), + (-1, 600.0, "Agrave"), + (-1, 600.0, "Abreve"), + (-1, 600.0, "multiply"), + (-1, 600.0, "uacute"), + (-1, 600.0, "Tcaron"), + (-1, 600.0, "partialdiff"), + (-1, 600.0, "ydieresis"), + (-1, 600.0, "Nacute"), + (-1, 600.0, "icircumflex"), + (-1, 600.0, "Ecircumflex"), + (-1, 600.0, "adieresis"), + (-1, 600.0, "edieresis"), + (-1, 600.0, "cacute"), + (-1, 600.0, "nacute"), + (-1, 600.0, "umacron"), + (-1, 600.0, "Ncaron"), + (-1, 600.0, "Iacute"), + (-1, 600.0, "plusminus"), + (-1, 600.0, "brokenbar"), + (-1, 600.0, "registered"), + (-1, 600.0, "Gbreve"), + (-1, 600.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 600.0, "Egrave"), + (-1, 600.0, "racute"), + (-1, 600.0, "omacron"), + (-1, 600.0, "Zacute"), + (-1, 600.0, "Zcaron"), + (-1, 600.0, "greaterequal"), + (-1, 600.0, "Eth"), + (-1, 600.0, "Ccedilla"), + (-1, 600.0, "lcommaaccent"), + (-1, 600.0, "tcaron"), + (-1, 600.0, "eogonek"), + (-1, 600.0, "Uogonek"), + (-1, 600.0, "Aacute"), + (-1, 600.0, "Adieresis"), + (-1, 600.0, "egrave"), + (-1, 600.0, "zacute"), + (-1, 600.0, "iogonek"), + (-1, 600.0, "Oacute"), + (-1, 600.0, "oacute"), + (-1, 600.0, "amacron"), + (-1, 600.0, "sacute"), + (-1, 600.0, "idieresis"), + (-1, 600.0, "Ocircumflex"), + (-1, 600.0, "Ugrave"), + (-1, 600.0, "Delta"), + (-1, 600.0, "thorn"), + (-1, 600.0, "twosuperior"), + (-1, 600.0, "Odieresis"), + (-1, 600.0, "mu"), + (-1, 600.0, "igrave"), + (-1, 600.0, "ohungarumlaut"), + (-1, 600.0, "Eogonek"), + (-1, 600.0, "dcroat"), + (-1, 600.0, "threequarters"), + (-1, 600.0, "Scedilla"), + (-1, 600.0, "lcaron"), + (-1, 600.0, "Kcommaaccent"), + (-1, 600.0, "Lacute"), + (-1, 600.0, "trademark"), + (-1, 600.0, "edotaccent"), + (-1, 600.0, "Igrave"), + (-1, 600.0, "Imacron"), + (-1, 600.0, "Lcaron"), + (-1, 600.0, "onehalf"), + (-1, 600.0, "lessequal"), + (-1, 600.0, "ocircumflex"), + (-1, 600.0, "ntilde"), + (-1, 600.0, "Uhungarumlaut"), + (-1, 600.0, "Eacute"), + (-1, 600.0, "emacron"), + (-1, 600.0, "gbreve"), + (-1, 600.0, "onequarter"), + (-1, 600.0, "Scaron"), + (-1, 600.0, "Scommaaccent"), + (-1, 600.0, "Ohungarumlaut"), + (-1, 600.0, "degree"), + (-1, 600.0, "ograve"), + (-1, 600.0, "Ccaron"), + (-1, 600.0, "ugrave"), + (-1, 600.0, "radical"), + (-1, 600.0, "Dcaron"), + (-1, 600.0, "rcommaaccent"), + (-1, 600.0, "Ntilde"), + (-1, 600.0, "otilde"), + (-1, 600.0, "Rcommaaccent"), + (-1, 600.0, "Lcommaaccent"), + (-1, 600.0, "Atilde"), + (-1, 600.0, "Aogonek"), + (-1, 600.0, "Aring"), + (-1, 600.0, "Otilde"), + (-1, 600.0, "zdotaccent"), + (-1, 600.0, "Ecaron"), + (-1, 600.0, "Iogonek"), + (-1, 600.0, "kcommaaccent"), + (-1, 600.0, "minus"), + (-1, 600.0, "Icircumflex"), + (-1, 600.0, "ncaron"), + (-1, 600.0, "tcommaaccent"), + (-1, 600.0, "logicalnot"), + (-1, 600.0, "odieresis"), + (-1, 600.0, "udieresis"), + (-1, 600.0, "notequal"), + (-1, 600.0, "gcommaaccent"), + (-1, 600.0, "eth"), + (-1, 600.0, "zcaron"), + (-1, 600.0, "ncommaaccent"), + (-1, 600.0, "onesuperior"), + (-1, 600.0, "imacron"), + (-1, 600.0, "Euro"), + ][..], + ), + ( + "Courier-BoldOblique", + "AdobeStandardEncoding", + &[ + (32, 600.0, "space"), + (33, 600.0, "exclam"), + (34, 600.0, "quotedbl"), + (35, 600.0, "numbersign"), + (36, 600.0, "dollar"), + (37, 600.0, "percent"), + (38, 600.0, "ampersand"), + (39, 600.0, "quoteright"), + (40, 600.0, "parenleft"), + (41, 600.0, "parenright"), + (42, 600.0, "asterisk"), + (43, 600.0, "plus"), + (44, 600.0, "comma"), + (45, 600.0, "hyphen"), + (46, 600.0, "period"), + (47, 600.0, "slash"), + (48, 600.0, "zero"), + (49, 600.0, "one"), + (50, 600.0, "two"), + (51, 600.0, "three"), + (52, 600.0, "four"), + (53, 600.0, "five"), + (54, 600.0, "six"), + (55, 600.0, "seven"), + (56, 600.0, "eight"), + (57, 600.0, "nine"), + (58, 600.0, "colon"), + (59, 600.0, "semicolon"), + (60, 600.0, "less"), + (61, 600.0, "equal"), + (62, 600.0, "greater"), + (63, 600.0, "question"), + (64, 600.0, "at"), + (65, 600.0, "A"), + (66, 600.0, "B"), + (67, 600.0, "C"), + (68, 600.0, "D"), + (69, 600.0, "E"), + (70, 600.0, "F"), + (71, 600.0, "G"), + (72, 600.0, "H"), + (73, 600.0, "I"), + (74, 600.0, "J"), + (75, 600.0, "K"), + (76, 600.0, "L"), + (77, 600.0, "M"), + (78, 600.0, "N"), + (79, 600.0, "O"), + (80, 600.0, "P"), + (81, 600.0, "Q"), + (82, 600.0, "R"), + (83, 600.0, "S"), + (84, 600.0, "T"), + (85, 600.0, "U"), + (86, 600.0, "V"), + (87, 600.0, "W"), + (88, 600.0, "X"), + (89, 600.0, "Y"), + (90, 600.0, "Z"), + (91, 600.0, "bracketleft"), + (92, 600.0, "backslash"), + (93, 600.0, "bracketright"), + (94, 600.0, "asciicircum"), + (95, 600.0, "underscore"), + (96, 600.0, "quoteleft"), + (97, 600.0, "a"), + (98, 600.0, "b"), + (99, 600.0, "c"), + (100, 600.0, "d"), + (101, 600.0, "e"), + (102, 600.0, "f"), + (103, 600.0, "g"), + (104, 600.0, "h"), + (105, 600.0, "i"), + (106, 600.0, "j"), + (107, 600.0, "k"), + (108, 600.0, "l"), + (109, 600.0, "m"), + (110, 600.0, "n"), + (111, 600.0, "o"), + (112, 600.0, "p"), + (113, 600.0, "q"), + (114, 600.0, "r"), + (115, 600.0, "s"), + (116, 600.0, "t"), + (117, 600.0, "u"), + (118, 600.0, "v"), + (119, 600.0, "w"), + (120, 600.0, "x"), + (121, 600.0, "y"), + (122, 600.0, "z"), + (123, 600.0, "braceleft"), + (124, 600.0, "bar"), + (125, 600.0, "braceright"), + (126, 600.0, "asciitilde"), + (161, 600.0, "exclamdown"), + (162, 600.0, "cent"), + (163, 600.0, "sterling"), + (164, 600.0, "fraction"), + (165, 600.0, "yen"), + (166, 600.0, "florin"), + (167, 600.0, "section"), + (168, 600.0, "currency"), + (169, 600.0, "quotesingle"), + (170, 600.0, "quotedblleft"), + (171, 600.0, "guillemotleft"), + (172, 600.0, "guilsinglleft"), + (173, 600.0, "guilsinglright"), + (174, 600.0, "fi"), + (175, 600.0, "fl"), + (177, 600.0, "endash"), + (178, 600.0, "dagger"), + (179, 600.0, "daggerdbl"), + (180, 600.0, "periodcentered"), + (182, 600.0, "paragraph"), + (183, 600.0, "bullet"), + (184, 600.0, "quotesinglbase"), + (185, 600.0, "quotedblbase"), + (186, 600.0, "quotedblright"), + (187, 600.0, "guillemotright"), + (188, 600.0, "ellipsis"), + (189, 600.0, "perthousand"), + (191, 600.0, "questiondown"), + (193, 600.0, "grave"), + (194, 600.0, "acute"), + (195, 600.0, "circumflex"), + (196, 600.0, "tilde"), + (197, 600.0, "macron"), + (198, 600.0, "breve"), + (199, 600.0, "dotaccent"), + (200, 600.0, "dieresis"), + (202, 600.0, "ring"), + (203, 600.0, "cedilla"), + (205, 600.0, "hungarumlaut"), + (206, 600.0, "ogonek"), + (207, 600.0, "caron"), + (208, 600.0, "emdash"), + (225, 600.0, "AE"), + (227, 600.0, "ordfeminine"), + (232, 600.0, "Lslash"), + (233, 600.0, "Oslash"), + (234, 600.0, "OE"), + (235, 600.0, "ordmasculine"), + (241, 600.0, "ae"), + (245, 600.0, "dotlessi"), + (248, 600.0, "lslash"), + (249, 600.0, "oslash"), + (250, 600.0, "oe"), + (251, 600.0, "germandbls"), + (-1, 600.0, "Idieresis"), + (-1, 600.0, "eacute"), + (-1, 600.0, "abreve"), + (-1, 600.0, "uhungarumlaut"), + (-1, 600.0, "ecaron"), + (-1, 600.0, "Ydieresis"), + (-1, 600.0, "divide"), + (-1, 600.0, "Yacute"), + (-1, 600.0, "Acircumflex"), + (-1, 600.0, "aacute"), + (-1, 600.0, "Ucircumflex"), + (-1, 600.0, "yacute"), + (-1, 600.0, "scommaaccent"), + (-1, 600.0, "ecircumflex"), + (-1, 600.0, "Uring"), + (-1, 600.0, "Udieresis"), + (-1, 600.0, "aogonek"), + (-1, 600.0, "Uacute"), + (-1, 600.0, "uogonek"), + (-1, 600.0, "Edieresis"), + (-1, 600.0, "Dcroat"), + (-1, 600.0, "commaaccent"), + (-1, 600.0, "copyright"), + (-1, 600.0, "Emacron"), + (-1, 600.0, "ccaron"), + (-1, 600.0, "aring"), + (-1, 600.0, "Ncommaaccent"), + (-1, 600.0, "lacute"), + (-1, 600.0, "agrave"), + (-1, 600.0, "Tcommaaccent"), + (-1, 600.0, "Cacute"), + (-1, 600.0, "atilde"), + (-1, 600.0, "Edotaccent"), + (-1, 600.0, "scaron"), + (-1, 600.0, "scedilla"), + (-1, 600.0, "iacute"), + (-1, 600.0, "lozenge"), + (-1, 600.0, "Rcaron"), + (-1, 600.0, "Gcommaaccent"), + (-1, 600.0, "ucircumflex"), + (-1, 600.0, "acircumflex"), + (-1, 600.0, "Amacron"), + (-1, 600.0, "rcaron"), + (-1, 600.0, "ccedilla"), + (-1, 600.0, "Zdotaccent"), + (-1, 600.0, "Thorn"), + (-1, 600.0, "Omacron"), + (-1, 600.0, "Racute"), + (-1, 600.0, "Sacute"), + (-1, 600.0, "dcaron"), + (-1, 600.0, "Umacron"), + (-1, 600.0, "uring"), + (-1, 600.0, "threesuperior"), + (-1, 600.0, "Ograve"), + (-1, 600.0, "Agrave"), + (-1, 600.0, "Abreve"), + (-1, 600.0, "multiply"), + (-1, 600.0, "uacute"), + (-1, 600.0, "Tcaron"), + (-1, 600.0, "partialdiff"), + (-1, 600.0, "ydieresis"), + (-1, 600.0, "Nacute"), + (-1, 600.0, "icircumflex"), + (-1, 600.0, "Ecircumflex"), + (-1, 600.0, "adieresis"), + (-1, 600.0, "edieresis"), + (-1, 600.0, "cacute"), + (-1, 600.0, "nacute"), + (-1, 600.0, "umacron"), + (-1, 600.0, "Ncaron"), + (-1, 600.0, "Iacute"), + (-1, 600.0, "plusminus"), + (-1, 600.0, "brokenbar"), + (-1, 600.0, "registered"), + (-1, 600.0, "Gbreve"), + (-1, 600.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 600.0, "Egrave"), + (-1, 600.0, "racute"), + (-1, 600.0, "omacron"), + (-1, 600.0, "Zacute"), + (-1, 600.0, "Zcaron"), + (-1, 600.0, "greaterequal"), + (-1, 600.0, "Eth"), + (-1, 600.0, "Ccedilla"), + (-1, 600.0, "lcommaaccent"), + (-1, 600.0, "tcaron"), + (-1, 600.0, "eogonek"), + (-1, 600.0, "Uogonek"), + (-1, 600.0, "Aacute"), + (-1, 600.0, "Adieresis"), + (-1, 600.0, "egrave"), + (-1, 600.0, "zacute"), + (-1, 600.0, "iogonek"), + (-1, 600.0, "Oacute"), + (-1, 600.0, "oacute"), + (-1, 600.0, "amacron"), + (-1, 600.0, "sacute"), + (-1, 600.0, "idieresis"), + (-1, 600.0, "Ocircumflex"), + (-1, 600.0, "Ugrave"), + (-1, 600.0, "Delta"), + (-1, 600.0, "thorn"), + (-1, 600.0, "twosuperior"), + (-1, 600.0, "Odieresis"), + (-1, 600.0, "mu"), + (-1, 600.0, "igrave"), + (-1, 600.0, "ohungarumlaut"), + (-1, 600.0, "Eogonek"), + (-1, 600.0, "dcroat"), + (-1, 600.0, "threequarters"), + (-1, 600.0, "Scedilla"), + (-1, 600.0, "lcaron"), + (-1, 600.0, "Kcommaaccent"), + (-1, 600.0, "Lacute"), + (-1, 600.0, "trademark"), + (-1, 600.0, "edotaccent"), + (-1, 600.0, "Igrave"), + (-1, 600.0, "Imacron"), + (-1, 600.0, "Lcaron"), + (-1, 600.0, "onehalf"), + (-1, 600.0, "lessequal"), + (-1, 600.0, "ocircumflex"), + (-1, 600.0, "ntilde"), + (-1, 600.0, "Uhungarumlaut"), + (-1, 600.0, "Eacute"), + (-1, 600.0, "emacron"), + (-1, 600.0, "gbreve"), + (-1, 600.0, "onequarter"), + (-1, 600.0, "Scaron"), + (-1, 600.0, "Scommaaccent"), + (-1, 600.0, "Ohungarumlaut"), + (-1, 600.0, "degree"), + (-1, 600.0, "ograve"), + (-1, 600.0, "Ccaron"), + (-1, 600.0, "ugrave"), + (-1, 600.0, "radical"), + (-1, 600.0, "Dcaron"), + (-1, 600.0, "rcommaaccent"), + (-1, 600.0, "Ntilde"), + (-1, 600.0, "otilde"), + (-1, 600.0, "Rcommaaccent"), + (-1, 600.0, "Lcommaaccent"), + (-1, 600.0, "Atilde"), + (-1, 600.0, "Aogonek"), + (-1, 600.0, "Aring"), + (-1, 600.0, "Otilde"), + (-1, 600.0, "zdotaccent"), + (-1, 600.0, "Ecaron"), + (-1, 600.0, "Iogonek"), + (-1, 600.0, "kcommaaccent"), + (-1, 600.0, "minus"), + (-1, 600.0, "Icircumflex"), + (-1, 600.0, "ncaron"), + (-1, 600.0, "tcommaaccent"), + (-1, 600.0, "logicalnot"), + (-1, 600.0, "odieresis"), + (-1, 600.0, "udieresis"), + (-1, 600.0, "notequal"), + (-1, 600.0, "gcommaaccent"), + (-1, 600.0, "eth"), + (-1, 600.0, "zcaron"), + (-1, 600.0, "ncommaaccent"), + (-1, 600.0, "onesuperior"), + (-1, 600.0, "imacron"), + (-1, 600.0, "Euro"), + ][..], + ), + ( + "Courier-Oblique", + "AdobeStandardEncoding", + &[ + (32, 600.0, "space"), + (33, 600.0, "exclam"), + (34, 600.0, "quotedbl"), + (35, 600.0, "numbersign"), + (36, 600.0, "dollar"), + (37, 600.0, "percent"), + (38, 600.0, "ampersand"), + (39, 600.0, "quoteright"), + (40, 600.0, "parenleft"), + (41, 600.0, "parenright"), + (42, 600.0, "asterisk"), + (43, 600.0, "plus"), + (44, 600.0, "comma"), + (45, 600.0, "hyphen"), + (46, 600.0, "period"), + (47, 600.0, "slash"), + (48, 600.0, "zero"), + (49, 600.0, "one"), + (50, 600.0, "two"), + (51, 600.0, "three"), + (52, 600.0, "four"), + (53, 600.0, "five"), + (54, 600.0, "six"), + (55, 600.0, "seven"), + (56, 600.0, "eight"), + (57, 600.0, "nine"), + (58, 600.0, "colon"), + (59, 600.0, "semicolon"), + (60, 600.0, "less"), + (61, 600.0, "equal"), + (62, 600.0, "greater"), + (63, 600.0, "question"), + (64, 600.0, "at"), + (65, 600.0, "A"), + (66, 600.0, "B"), + (67, 600.0, "C"), + (68, 600.0, "D"), + (69, 600.0, "E"), + (70, 600.0, "F"), + (71, 600.0, "G"), + (72, 600.0, "H"), + (73, 600.0, "I"), + (74, 600.0, "J"), + (75, 600.0, "K"), + (76, 600.0, "L"), + (77, 600.0, "M"), + (78, 600.0, "N"), + (79, 600.0, "O"), + (80, 600.0, "P"), + (81, 600.0, "Q"), + (82, 600.0, "R"), + (83, 600.0, "S"), + (84, 600.0, "T"), + (85, 600.0, "U"), + (86, 600.0, "V"), + (87, 600.0, "W"), + (88, 600.0, "X"), + (89, 600.0, "Y"), + (90, 600.0, "Z"), + (91, 600.0, "bracketleft"), + (92, 600.0, "backslash"), + (93, 600.0, "bracketright"), + (94, 600.0, "asciicircum"), + (95, 600.0, "underscore"), + (96, 600.0, "quoteleft"), + (97, 600.0, "a"), + (98, 600.0, "b"), + (99, 600.0, "c"), + (100, 600.0, "d"), + (101, 600.0, "e"), + (102, 600.0, "f"), + (103, 600.0, "g"), + (104, 600.0, "h"), + (105, 600.0, "i"), + (106, 600.0, "j"), + (107, 600.0, "k"), + (108, 600.0, "l"), + (109, 600.0, "m"), + (110, 600.0, "n"), + (111, 600.0, "o"), + (112, 600.0, "p"), + (113, 600.0, "q"), + (114, 600.0, "r"), + (115, 600.0, "s"), + (116, 600.0, "t"), + (117, 600.0, "u"), + (118, 600.0, "v"), + (119, 600.0, "w"), + (120, 600.0, "x"), + (121, 600.0, "y"), + (122, 600.0, "z"), + (123, 600.0, "braceleft"), + (124, 600.0, "bar"), + (125, 600.0, "braceright"), + (126, 600.0, "asciitilde"), + (161, 600.0, "exclamdown"), + (162, 600.0, "cent"), + (163, 600.0, "sterling"), + (164, 600.0, "fraction"), + (165, 600.0, "yen"), + (166, 600.0, "florin"), + (167, 600.0, "section"), + (168, 600.0, "currency"), + (169, 600.0, "quotesingle"), + (170, 600.0, "quotedblleft"), + (171, 600.0, "guillemotleft"), + (172, 600.0, "guilsinglleft"), + (173, 600.0, "guilsinglright"), + (174, 600.0, "fi"), + (175, 600.0, "fl"), + (177, 600.0, "endash"), + (178, 600.0, "dagger"), + (179, 600.0, "daggerdbl"), + (180, 600.0, "periodcentered"), + (182, 600.0, "paragraph"), + (183, 600.0, "bullet"), + (184, 600.0, "quotesinglbase"), + (185, 600.0, "quotedblbase"), + (186, 600.0, "quotedblright"), + (187, 600.0, "guillemotright"), + (188, 600.0, "ellipsis"), + (189, 600.0, "perthousand"), + (191, 600.0, "questiondown"), + (193, 600.0, "grave"), + (194, 600.0, "acute"), + (195, 600.0, "circumflex"), + (196, 600.0, "tilde"), + (197, 600.0, "macron"), + (198, 600.0, "breve"), + (199, 600.0, "dotaccent"), + (200, 600.0, "dieresis"), + (202, 600.0, "ring"), + (203, 600.0, "cedilla"), + (205, 600.0, "hungarumlaut"), + (206, 600.0, "ogonek"), + (207, 600.0, "caron"), + (208, 600.0, "emdash"), + (225, 600.0, "AE"), + (227, 600.0, "ordfeminine"), + (232, 600.0, "Lslash"), + (233, 600.0, "Oslash"), + (234, 600.0, "OE"), + (235, 600.0, "ordmasculine"), + (241, 600.0, "ae"), + (245, 600.0, "dotlessi"), + (248, 600.0, "lslash"), + (249, 600.0, "oslash"), + (250, 600.0, "oe"), + (251, 600.0, "germandbls"), + (-1, 600.0, "Idieresis"), + (-1, 600.0, "eacute"), + (-1, 600.0, "abreve"), + (-1, 600.0, "uhungarumlaut"), + (-1, 600.0, "ecaron"), + (-1, 600.0, "Ydieresis"), + (-1, 600.0, "divide"), + (-1, 600.0, "Yacute"), + (-1, 600.0, "Acircumflex"), + (-1, 600.0, "aacute"), + (-1, 600.0, "Ucircumflex"), + (-1, 600.0, "yacute"), + (-1, 600.0, "scommaaccent"), + (-1, 600.0, "ecircumflex"), + (-1, 600.0, "Uring"), + (-1, 600.0, "Udieresis"), + (-1, 600.0, "aogonek"), + (-1, 600.0, "Uacute"), + (-1, 600.0, "uogonek"), + (-1, 600.0, "Edieresis"), + (-1, 600.0, "Dcroat"), + (-1, 600.0, "commaaccent"), + (-1, 600.0, "copyright"), + (-1, 600.0, "Emacron"), + (-1, 600.0, "ccaron"), + (-1, 600.0, "aring"), + (-1, 600.0, "Ncommaaccent"), + (-1, 600.0, "lacute"), + (-1, 600.0, "agrave"), + (-1, 600.0, "Tcommaaccent"), + (-1, 600.0, "Cacute"), + (-1, 600.0, "atilde"), + (-1, 600.0, "Edotaccent"), + (-1, 600.0, "scaron"), + (-1, 600.0, "scedilla"), + (-1, 600.0, "iacute"), + (-1, 600.0, "lozenge"), + (-1, 600.0, "Rcaron"), + (-1, 600.0, "Gcommaaccent"), + (-1, 600.0, "ucircumflex"), + (-1, 600.0, "acircumflex"), + (-1, 600.0, "Amacron"), + (-1, 600.0, "rcaron"), + (-1, 600.0, "ccedilla"), + (-1, 600.0, "Zdotaccent"), + (-1, 600.0, "Thorn"), + (-1, 600.0, "Omacron"), + (-1, 600.0, "Racute"), + (-1, 600.0, "Sacute"), + (-1, 600.0, "dcaron"), + (-1, 600.0, "Umacron"), + (-1, 600.0, "uring"), + (-1, 600.0, "threesuperior"), + (-1, 600.0, "Ograve"), + (-1, 600.0, "Agrave"), + (-1, 600.0, "Abreve"), + (-1, 600.0, "multiply"), + (-1, 600.0, "uacute"), + (-1, 600.0, "Tcaron"), + (-1, 600.0, "partialdiff"), + (-1, 600.0, "ydieresis"), + (-1, 600.0, "Nacute"), + (-1, 600.0, "icircumflex"), + (-1, 600.0, "Ecircumflex"), + (-1, 600.0, "adieresis"), + (-1, 600.0, "edieresis"), + (-1, 600.0, "cacute"), + (-1, 600.0, "nacute"), + (-1, 600.0, "umacron"), + (-1, 600.0, "Ncaron"), + (-1, 600.0, "Iacute"), + (-1, 600.0, "plusminus"), + (-1, 600.0, "brokenbar"), + (-1, 600.0, "registered"), + (-1, 600.0, "Gbreve"), + (-1, 600.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 600.0, "Egrave"), + (-1, 600.0, "racute"), + (-1, 600.0, "omacron"), + (-1, 600.0, "Zacute"), + (-1, 600.0, "Zcaron"), + (-1, 600.0, "greaterequal"), + (-1, 600.0, "Eth"), + (-1, 600.0, "Ccedilla"), + (-1, 600.0, "lcommaaccent"), + (-1, 600.0, "tcaron"), + (-1, 600.0, "eogonek"), + (-1, 600.0, "Uogonek"), + (-1, 600.0, "Aacute"), + (-1, 600.0, "Adieresis"), + (-1, 600.0, "egrave"), + (-1, 600.0, "zacute"), + (-1, 600.0, "iogonek"), + (-1, 600.0, "Oacute"), + (-1, 600.0, "oacute"), + (-1, 600.0, "amacron"), + (-1, 600.0, "sacute"), + (-1, 600.0, "idieresis"), + (-1, 600.0, "Ocircumflex"), + (-1, 600.0, "Ugrave"), + (-1, 600.0, "Delta"), + (-1, 600.0, "thorn"), + (-1, 600.0, "twosuperior"), + (-1, 600.0, "Odieresis"), + (-1, 600.0, "mu"), + (-1, 600.0, "igrave"), + (-1, 600.0, "ohungarumlaut"), + (-1, 600.0, "Eogonek"), + (-1, 600.0, "dcroat"), + (-1, 600.0, "threequarters"), + (-1, 600.0, "Scedilla"), + (-1, 600.0, "lcaron"), + (-1, 600.0, "Kcommaaccent"), + (-1, 600.0, "Lacute"), + (-1, 600.0, "trademark"), + (-1, 600.0, "edotaccent"), + (-1, 600.0, "Igrave"), + (-1, 600.0, "Imacron"), + (-1, 600.0, "Lcaron"), + (-1, 600.0, "onehalf"), + (-1, 600.0, "lessequal"), + (-1, 600.0, "ocircumflex"), + (-1, 600.0, "ntilde"), + (-1, 600.0, "Uhungarumlaut"), + (-1, 600.0, "Eacute"), + (-1, 600.0, "emacron"), + (-1, 600.0, "gbreve"), + (-1, 600.0, "onequarter"), + (-1, 600.0, "Scaron"), + (-1, 600.0, "Scommaaccent"), + (-1, 600.0, "Ohungarumlaut"), + (-1, 600.0, "degree"), + (-1, 600.0, "ograve"), + (-1, 600.0, "Ccaron"), + (-1, 600.0, "ugrave"), + (-1, 600.0, "radical"), + (-1, 600.0, "Dcaron"), + (-1, 600.0, "rcommaaccent"), + (-1, 600.0, "Ntilde"), + (-1, 600.0, "otilde"), + (-1, 600.0, "Rcommaaccent"), + (-1, 600.0, "Lcommaaccent"), + (-1, 600.0, "Atilde"), + (-1, 600.0, "Aogonek"), + (-1, 600.0, "Aring"), + (-1, 600.0, "Otilde"), + (-1, 600.0, "zdotaccent"), + (-1, 600.0, "Ecaron"), + (-1, 600.0, "Iogonek"), + (-1, 600.0, "kcommaaccent"), + (-1, 600.0, "minus"), + (-1, 600.0, "Icircumflex"), + (-1, 600.0, "ncaron"), + (-1, 600.0, "tcommaaccent"), + (-1, 600.0, "logicalnot"), + (-1, 600.0, "odieresis"), + (-1, 600.0, "udieresis"), + (-1, 600.0, "notequal"), + (-1, 600.0, "gcommaaccent"), + (-1, 600.0, "eth"), + (-1, 600.0, "zcaron"), + (-1, 600.0, "ncommaaccent"), + (-1, 600.0, "onesuperior"), + (-1, 600.0, "imacron"), + (-1, 600.0, "Euro"), + ][..], + ), + ( + "Courier", + "AdobeStandardEncoding", + &[ + (32, 600.0, "space"), + (33, 600.0, "exclam"), + (34, 600.0, "quotedbl"), + (35, 600.0, "numbersign"), + (36, 600.0, "dollar"), + (37, 600.0, "percent"), + (38, 600.0, "ampersand"), + (39, 600.0, "quoteright"), + (40, 600.0, "parenleft"), + (41, 600.0, "parenright"), + (42, 600.0, "asterisk"), + (43, 600.0, "plus"), + (44, 600.0, "comma"), + (45, 600.0, "hyphen"), + (46, 600.0, "period"), + (47, 600.0, "slash"), + (48, 600.0, "zero"), + (49, 600.0, "one"), + (50, 600.0, "two"), + (51, 600.0, "three"), + (52, 600.0, "four"), + (53, 600.0, "five"), + (54, 600.0, "six"), + (55, 600.0, "seven"), + (56, 600.0, "eight"), + (57, 600.0, "nine"), + (58, 600.0, "colon"), + (59, 600.0, "semicolon"), + (60, 600.0, "less"), + (61, 600.0, "equal"), + (62, 600.0, "greater"), + (63, 600.0, "question"), + (64, 600.0, "at"), + (65, 600.0, "A"), + (66, 600.0, "B"), + (67, 600.0, "C"), + (68, 600.0, "D"), + (69, 600.0, "E"), + (70, 600.0, "F"), + (71, 600.0, "G"), + (72, 600.0, "H"), + (73, 600.0, "I"), + (74, 600.0, "J"), + (75, 600.0, "K"), + (76, 600.0, "L"), + (77, 600.0, "M"), + (78, 600.0, "N"), + (79, 600.0, "O"), + (80, 600.0, "P"), + (81, 600.0, "Q"), + (82, 600.0, "R"), + (83, 600.0, "S"), + (84, 600.0, "T"), + (85, 600.0, "U"), + (86, 600.0, "V"), + (87, 600.0, "W"), + (88, 600.0, "X"), + (89, 600.0, "Y"), + (90, 600.0, "Z"), + (91, 600.0, "bracketleft"), + (92, 600.0, "backslash"), + (93, 600.0, "bracketright"), + (94, 600.0, "asciicircum"), + (95, 600.0, "underscore"), + (96, 600.0, "quoteleft"), + (97, 600.0, "a"), + (98, 600.0, "b"), + (99, 600.0, "c"), + (100, 600.0, "d"), + (101, 600.0, "e"), + (102, 600.0, "f"), + (103, 600.0, "g"), + (104, 600.0, "h"), + (105, 600.0, "i"), + (106, 600.0, "j"), + (107, 600.0, "k"), + (108, 600.0, "l"), + (109, 600.0, "m"), + (110, 600.0, "n"), + (111, 600.0, "o"), + (112, 600.0, "p"), + (113, 600.0, "q"), + (114, 600.0, "r"), + (115, 600.0, "s"), + (116, 600.0, "t"), + (117, 600.0, "u"), + (118, 600.0, "v"), + (119, 600.0, "w"), + (120, 600.0, "x"), + (121, 600.0, "y"), + (122, 600.0, "z"), + (123, 600.0, "braceleft"), + (124, 600.0, "bar"), + (125, 600.0, "braceright"), + (126, 600.0, "asciitilde"), + (161, 600.0, "exclamdown"), + (162, 600.0, "cent"), + (163, 600.0, "sterling"), + (164, 600.0, "fraction"), + (165, 600.0, "yen"), + (166, 600.0, "florin"), + (167, 600.0, "section"), + (168, 600.0, "currency"), + (169, 600.0, "quotesingle"), + (170, 600.0, "quotedblleft"), + (171, 600.0, "guillemotleft"), + (172, 600.0, "guilsinglleft"), + (173, 600.0, "guilsinglright"), + (174, 600.0, "fi"), + (175, 600.0, "fl"), + (177, 600.0, "endash"), + (178, 600.0, "dagger"), + (179, 600.0, "daggerdbl"), + (180, 600.0, "periodcentered"), + (182, 600.0, "paragraph"), + (183, 600.0, "bullet"), + (184, 600.0, "quotesinglbase"), + (185, 600.0, "quotedblbase"), + (186, 600.0, "quotedblright"), + (187, 600.0, "guillemotright"), + (188, 600.0, "ellipsis"), + (189, 600.0, "perthousand"), + (191, 600.0, "questiondown"), + (193, 600.0, "grave"), + (194, 600.0, "acute"), + (195, 600.0, "circumflex"), + (196, 600.0, "tilde"), + (197, 600.0, "macron"), + (198, 600.0, "breve"), + (199, 600.0, "dotaccent"), + (200, 600.0, "dieresis"), + (202, 600.0, "ring"), + (203, 600.0, "cedilla"), + (205, 600.0, "hungarumlaut"), + (206, 600.0, "ogonek"), + (207, 600.0, "caron"), + (208, 600.0, "emdash"), + (225, 600.0, "AE"), + (227, 600.0, "ordfeminine"), + (232, 600.0, "Lslash"), + (233, 600.0, "Oslash"), + (234, 600.0, "OE"), + (235, 600.0, "ordmasculine"), + (241, 600.0, "ae"), + (245, 600.0, "dotlessi"), + (248, 600.0, "lslash"), + (249, 600.0, "oslash"), + (250, 600.0, "oe"), + (251, 600.0, "germandbls"), + (-1, 600.0, "Idieresis"), + (-1, 600.0, "eacute"), + (-1, 600.0, "abreve"), + (-1, 600.0, "uhungarumlaut"), + (-1, 600.0, "ecaron"), + (-1, 600.0, "Ydieresis"), + (-1, 600.0, "divide"), + (-1, 600.0, "Yacute"), + (-1, 600.0, "Acircumflex"), + (-1, 600.0, "aacute"), + (-1, 600.0, "Ucircumflex"), + (-1, 600.0, "yacute"), + (-1, 600.0, "scommaaccent"), + (-1, 600.0, "ecircumflex"), + (-1, 600.0, "Uring"), + (-1, 600.0, "Udieresis"), + (-1, 600.0, "aogonek"), + (-1, 600.0, "Uacute"), + (-1, 600.0, "uogonek"), + (-1, 600.0, "Edieresis"), + (-1, 600.0, "Dcroat"), + (-1, 600.0, "commaaccent"), + (-1, 600.0, "copyright"), + (-1, 600.0, "Emacron"), + (-1, 600.0, "ccaron"), + (-1, 600.0, "aring"), + (-1, 600.0, "Ncommaaccent"), + (-1, 600.0, "lacute"), + (-1, 600.0, "agrave"), + (-1, 600.0, "Tcommaaccent"), + (-1, 600.0, "Cacute"), + (-1, 600.0, "atilde"), + (-1, 600.0, "Edotaccent"), + (-1, 600.0, "scaron"), + (-1, 600.0, "scedilla"), + (-1, 600.0, "iacute"), + (-1, 600.0, "lozenge"), + (-1, 600.0, "Rcaron"), + (-1, 600.0, "Gcommaaccent"), + (-1, 600.0, "ucircumflex"), + (-1, 600.0, "acircumflex"), + (-1, 600.0, "Amacron"), + (-1, 600.0, "rcaron"), + (-1, 600.0, "ccedilla"), + (-1, 600.0, "Zdotaccent"), + (-1, 600.0, "Thorn"), + (-1, 600.0, "Omacron"), + (-1, 600.0, "Racute"), + (-1, 600.0, "Sacute"), + (-1, 600.0, "dcaron"), + (-1, 600.0, "Umacron"), + (-1, 600.0, "uring"), + (-1, 600.0, "threesuperior"), + (-1, 600.0, "Ograve"), + (-1, 600.0, "Agrave"), + (-1, 600.0, "Abreve"), + (-1, 600.0, "multiply"), + (-1, 600.0, "uacute"), + (-1, 600.0, "Tcaron"), + (-1, 600.0, "partialdiff"), + (-1, 600.0, "ydieresis"), + (-1, 600.0, "Nacute"), + (-1, 600.0, "icircumflex"), + (-1, 600.0, "Ecircumflex"), + (-1, 600.0, "adieresis"), + (-1, 600.0, "edieresis"), + (-1, 600.0, "cacute"), + (-1, 600.0, "nacute"), + (-1, 600.0, "umacron"), + (-1, 600.0, "Ncaron"), + (-1, 600.0, "Iacute"), + (-1, 600.0, "plusminus"), + (-1, 600.0, "brokenbar"), + (-1, 600.0, "registered"), + (-1, 600.0, "Gbreve"), + (-1, 600.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 600.0, "Egrave"), + (-1, 600.0, "racute"), + (-1, 600.0, "omacron"), + (-1, 600.0, "Zacute"), + (-1, 600.0, "Zcaron"), + (-1, 600.0, "greaterequal"), + (-1, 600.0, "Eth"), + (-1, 600.0, "Ccedilla"), + (-1, 600.0, "lcommaaccent"), + (-1, 600.0, "tcaron"), + (-1, 600.0, "eogonek"), + (-1, 600.0, "Uogonek"), + (-1, 600.0, "Aacute"), + (-1, 600.0, "Adieresis"), + (-1, 600.0, "egrave"), + (-1, 600.0, "zacute"), + (-1, 600.0, "iogonek"), + (-1, 600.0, "Oacute"), + (-1, 600.0, "oacute"), + (-1, 600.0, "amacron"), + (-1, 600.0, "sacute"), + (-1, 600.0, "idieresis"), + (-1, 600.0, "Ocircumflex"), + (-1, 600.0, "Ugrave"), + (-1, 600.0, "Delta"), + (-1, 600.0, "thorn"), + (-1, 600.0, "twosuperior"), + (-1, 600.0, "Odieresis"), + (-1, 600.0, "mu"), + (-1, 600.0, "igrave"), + (-1, 600.0, "ohungarumlaut"), + (-1, 600.0, "Eogonek"), + (-1, 600.0, "dcroat"), + (-1, 600.0, "threequarters"), + (-1, 600.0, "Scedilla"), + (-1, 600.0, "lcaron"), + (-1, 600.0, "Kcommaaccent"), + (-1, 600.0, "Lacute"), + (-1, 600.0, "trademark"), + (-1, 600.0, "edotaccent"), + (-1, 600.0, "Igrave"), + (-1, 600.0, "Imacron"), + (-1, 600.0, "Lcaron"), + (-1, 600.0, "onehalf"), + (-1, 600.0, "lessequal"), + (-1, 600.0, "ocircumflex"), + (-1, 600.0, "ntilde"), + (-1, 600.0, "Uhungarumlaut"), + (-1, 600.0, "Eacute"), + (-1, 600.0, "emacron"), + (-1, 600.0, "gbreve"), + (-1, 600.0, "onequarter"), + (-1, 600.0, "Scaron"), + (-1, 600.0, "Scommaaccent"), + (-1, 600.0, "Ohungarumlaut"), + (-1, 600.0, "degree"), + (-1, 600.0, "ograve"), + (-1, 600.0, "Ccaron"), + (-1, 600.0, "ugrave"), + (-1, 600.0, "radical"), + (-1, 600.0, "Dcaron"), + (-1, 600.0, "rcommaaccent"), + (-1, 600.0, "Ntilde"), + (-1, 600.0, "otilde"), + (-1, 600.0, "Rcommaaccent"), + (-1, 600.0, "Lcommaaccent"), + (-1, 600.0, "Atilde"), + (-1, 600.0, "Aogonek"), + (-1, 600.0, "Aring"), + (-1, 600.0, "Otilde"), + (-1, 600.0, "zdotaccent"), + (-1, 600.0, "Ecaron"), + (-1, 600.0, "Iogonek"), + (-1, 600.0, "kcommaaccent"), + (-1, 600.0, "minus"), + (-1, 600.0, "Icircumflex"), + (-1, 600.0, "ncaron"), + (-1, 600.0, "tcommaaccent"), + (-1, 600.0, "logicalnot"), + (-1, 600.0, "odieresis"), + (-1, 600.0, "udieresis"), + (-1, 600.0, "notequal"), + (-1, 600.0, "gcommaaccent"), + (-1, 600.0, "eth"), + (-1, 600.0, "zcaron"), + (-1, 600.0, "ncommaaccent"), + (-1, 600.0, "onesuperior"), + (-1, 600.0, "imacron"), + (-1, 600.0, "Euro"), + ][..], + ), + ( + "Helvetica-Bold", + "AdobeStandardEncoding", + &[ + (32, 278.0, "space"), + (33, 333.0, "exclam"), + (34, 474.0, "quotedbl"), + (35, 556.0, "numbersign"), + (36, 556.0, "dollar"), + (37, 889.0, "percent"), + (38, 722.0, "ampersand"), + (39, 278.0, "quoteright"), + (40, 333.0, "parenleft"), + (41, 333.0, "parenright"), + (42, 389.0, "asterisk"), + (43, 584.0, "plus"), + (44, 278.0, "comma"), + (45, 333.0, "hyphen"), + (46, 278.0, "period"), + (47, 278.0, "slash"), + (48, 556.0, "zero"), + (49, 556.0, "one"), + (50, 556.0, "two"), + (51, 556.0, "three"), + (52, 556.0, "four"), + (53, 556.0, "five"), + (54, 556.0, "six"), + (55, 556.0, "seven"), + (56, 556.0, "eight"), + (57, 556.0, "nine"), + (58, 333.0, "colon"), + (59, 333.0, "semicolon"), + (60, 584.0, "less"), + (61, 584.0, "equal"), + (62, 584.0, "greater"), + (63, 611.0, "question"), + (64, 975.0, "at"), + (65, 722.0, "A"), + (66, 722.0, "B"), + (67, 722.0, "C"), + (68, 722.0, "D"), + (69, 667.0, "E"), + (70, 611.0, "F"), + (71, 778.0, "G"), + (72, 722.0, "H"), + (73, 278.0, "I"), + (74, 556.0, "J"), + (75, 722.0, "K"), + (76, 611.0, "L"), + (77, 833.0, "M"), + (78, 722.0, "N"), + (79, 778.0, "O"), + (80, 667.0, "P"), + (81, 778.0, "Q"), + (82, 722.0, "R"), + (83, 667.0, "S"), + (84, 611.0, "T"), + (85, 722.0, "U"), + (86, 667.0, "V"), + (87, 944.0, "W"), + (88, 667.0, "X"), + (89, 667.0, "Y"), + (90, 611.0, "Z"), + (91, 333.0, "bracketleft"), + (92, 278.0, "backslash"), + (93, 333.0, "bracketright"), + (94, 584.0, "asciicircum"), + (95, 556.0, "underscore"), + (96, 278.0, "quoteleft"), + (97, 556.0, "a"), + (98, 611.0, "b"), + (99, 556.0, "c"), + (100, 611.0, "d"), + (101, 556.0, "e"), + (102, 333.0, "f"), + (103, 611.0, "g"), + (104, 611.0, "h"), + (105, 278.0, "i"), + (106, 278.0, "j"), + (107, 556.0, "k"), + (108, 278.0, "l"), + (109, 889.0, "m"), + (110, 611.0, "n"), + (111, 611.0, "o"), + (112, 611.0, "p"), + (113, 611.0, "q"), + (114, 389.0, "r"), + (115, 556.0, "s"), + (116, 333.0, "t"), + (117, 611.0, "u"), + (118, 556.0, "v"), + (119, 778.0, "w"), + (120, 556.0, "x"), + (121, 556.0, "y"), + (122, 500.0, "z"), + (123, 389.0, "braceleft"), + (124, 280.0, "bar"), + (125, 389.0, "braceright"), + (126, 584.0, "asciitilde"), + (161, 333.0, "exclamdown"), + (162, 556.0, "cent"), + (163, 556.0, "sterling"), + (164, 167.0, "fraction"), + (165, 556.0, "yen"), + (166, 556.0, "florin"), + (167, 556.0, "section"), + (168, 556.0, "currency"), + (169, 238.0, "quotesingle"), + (170, 500.0, "quotedblleft"), + (171, 556.0, "guillemotleft"), + (172, 333.0, "guilsinglleft"), + (173, 333.0, "guilsinglright"), + (174, 611.0, "fi"), + (175, 611.0, "fl"), + (177, 556.0, "endash"), + (178, 556.0, "dagger"), + (179, 556.0, "daggerdbl"), + (180, 278.0, "periodcentered"), + (182, 556.0, "paragraph"), + (183, 350.0, "bullet"), + (184, 278.0, "quotesinglbase"), + (185, 500.0, "quotedblbase"), + (186, 500.0, "quotedblright"), + (187, 556.0, "guillemotright"), + (188, 1000.0, "ellipsis"), + (189, 1000.0, "perthousand"), + (191, 611.0, "questiondown"), + (193, 333.0, "grave"), + (194, 333.0, "acute"), + (195, 333.0, "circumflex"), + (196, 333.0, "tilde"), + (197, 333.0, "macron"), + (198, 333.0, "breve"), + (199, 333.0, "dotaccent"), + (200, 333.0, "dieresis"), + (202, 333.0, "ring"), + (203, 333.0, "cedilla"), + (205, 333.0, "hungarumlaut"), + (206, 333.0, "ogonek"), + (207, 333.0, "caron"), + (208, 1000.0, "emdash"), + (225, 1000.0, "AE"), + (227, 370.0, "ordfeminine"), + (232, 611.0, "Lslash"), + (233, 778.0, "Oslash"), + (234, 1000.0, "OE"), + (235, 365.0, "ordmasculine"), + (241, 889.0, "ae"), + (245, 278.0, "dotlessi"), + (248, 278.0, "lslash"), + (249, 611.0, "oslash"), + (250, 944.0, "oe"), + (251, 611.0, "germandbls"), + (-1, 278.0, "Idieresis"), + (-1, 556.0, "eacute"), + (-1, 556.0, "abreve"), + (-1, 611.0, "uhungarumlaut"), + (-1, 556.0, "ecaron"), + (-1, 667.0, "Ydieresis"), + (-1, 584.0, "divide"), + (-1, 667.0, "Yacute"), + (-1, 722.0, "Acircumflex"), + (-1, 556.0, "aacute"), + (-1, 722.0, "Ucircumflex"), + (-1, 556.0, "yacute"), + (-1, 556.0, "scommaaccent"), + (-1, 556.0, "ecircumflex"), + (-1, 722.0, "Uring"), + (-1, 722.0, "Udieresis"), + (-1, 556.0, "aogonek"), + (-1, 722.0, "Uacute"), + (-1, 611.0, "uogonek"), + (-1, 667.0, "Edieresis"), + (-1, 722.0, "Dcroat"), + (-1, 250.0, "commaaccent"), + (-1, 737.0, "copyright"), + (-1, 667.0, "Emacron"), + (-1, 556.0, "ccaron"), + (-1, 556.0, "aring"), + (-1, 722.0, "Ncommaaccent"), + (-1, 278.0, "lacute"), + (-1, 556.0, "agrave"), + (-1, 611.0, "Tcommaaccent"), + (-1, 722.0, "Cacute"), + (-1, 556.0, "atilde"), + (-1, 667.0, "Edotaccent"), + (-1, 556.0, "scaron"), + (-1, 556.0, "scedilla"), + (-1, 278.0, "iacute"), + (-1, 494.0, "lozenge"), + (-1, 722.0, "Rcaron"), + (-1, 778.0, "Gcommaaccent"), + (-1, 611.0, "ucircumflex"), + (-1, 556.0, "acircumflex"), + (-1, 722.0, "Amacron"), + (-1, 389.0, "rcaron"), + (-1, 556.0, "ccedilla"), + (-1, 611.0, "Zdotaccent"), + (-1, 667.0, "Thorn"), + (-1, 778.0, "Omacron"), + (-1, 722.0, "Racute"), + (-1, 667.0, "Sacute"), + (-1, 743.0, "dcaron"), + (-1, 722.0, "Umacron"), + (-1, 611.0, "uring"), + (-1, 333.0, "threesuperior"), + (-1, 778.0, "Ograve"), + (-1, 722.0, "Agrave"), + (-1, 722.0, "Abreve"), + (-1, 584.0, "multiply"), + (-1, 611.0, "uacute"), + (-1, 611.0, "Tcaron"), + (-1, 494.0, "partialdiff"), + (-1, 556.0, "ydieresis"), + (-1, 722.0, "Nacute"), + (-1, 278.0, "icircumflex"), + (-1, 667.0, "Ecircumflex"), + (-1, 556.0, "adieresis"), + (-1, 556.0, "edieresis"), + (-1, 556.0, "cacute"), + (-1, 611.0, "nacute"), + (-1, 611.0, "umacron"), + (-1, 722.0, "Ncaron"), + (-1, 278.0, "Iacute"), + (-1, 584.0, "plusminus"), + (-1, 280.0, "brokenbar"), + (-1, 737.0, "registered"), + (-1, 778.0, "Gbreve"), + (-1, 278.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 667.0, "Egrave"), + (-1, 389.0, "racute"), + (-1, 611.0, "omacron"), + (-1, 611.0, "Zacute"), + (-1, 611.0, "Zcaron"), + (-1, 549.0, "greaterequal"), + (-1, 722.0, "Eth"), + (-1, 722.0, "Ccedilla"), + (-1, 278.0, "lcommaaccent"), + (-1, 389.0, "tcaron"), + (-1, 556.0, "eogonek"), + (-1, 722.0, "Uogonek"), + (-1, 722.0, "Aacute"), + (-1, 722.0, "Adieresis"), + (-1, 556.0, "egrave"), + (-1, 500.0, "zacute"), + (-1, 278.0, "iogonek"), + (-1, 778.0, "Oacute"), + (-1, 611.0, "oacute"), + (-1, 556.0, "amacron"), + (-1, 556.0, "sacute"), + (-1, 278.0, "idieresis"), + (-1, 778.0, "Ocircumflex"), + (-1, 722.0, "Ugrave"), + (-1, 612.0, "Delta"), + (-1, 611.0, "thorn"), + (-1, 333.0, "twosuperior"), + (-1, 778.0, "Odieresis"), + (-1, 611.0, "mu"), + (-1, 278.0, "igrave"), + (-1, 611.0, "ohungarumlaut"), + (-1, 667.0, "Eogonek"), + (-1, 611.0, "dcroat"), + (-1, 834.0, "threequarters"), + (-1, 667.0, "Scedilla"), + (-1, 400.0, "lcaron"), + (-1, 722.0, "Kcommaaccent"), + (-1, 611.0, "Lacute"), + (-1, 1000.0, "trademark"), + (-1, 556.0, "edotaccent"), + (-1, 278.0, "Igrave"), + (-1, 278.0, "Imacron"), + (-1, 611.0, "Lcaron"), + (-1, 834.0, "onehalf"), + (-1, 549.0, "lessequal"), + (-1, 611.0, "ocircumflex"), + (-1, 611.0, "ntilde"), + (-1, 722.0, "Uhungarumlaut"), + (-1, 667.0, "Eacute"), + (-1, 556.0, "emacron"), + (-1, 611.0, "gbreve"), + (-1, 834.0, "onequarter"), + (-1, 667.0, "Scaron"), + (-1, 667.0, "Scommaaccent"), + (-1, 778.0, "Ohungarumlaut"), + (-1, 400.0, "degree"), + (-1, 611.0, "ograve"), + (-1, 722.0, "Ccaron"), + (-1, 611.0, "ugrave"), + (-1, 549.0, "radical"), + (-1, 722.0, "Dcaron"), + (-1, 389.0, "rcommaaccent"), + (-1, 722.0, "Ntilde"), + (-1, 611.0, "otilde"), + (-1, 722.0, "Rcommaaccent"), + (-1, 611.0, "Lcommaaccent"), + (-1, 722.0, "Atilde"), + (-1, 722.0, "Aogonek"), + (-1, 722.0, "Aring"), + (-1, 778.0, "Otilde"), + (-1, 500.0, "zdotaccent"), + (-1, 667.0, "Ecaron"), + (-1, 278.0, "Iogonek"), + (-1, 556.0, "kcommaaccent"), + (-1, 584.0, "minus"), + (-1, 278.0, "Icircumflex"), + (-1, 611.0, "ncaron"), + (-1, 333.0, "tcommaaccent"), + (-1, 584.0, "logicalnot"), + (-1, 611.0, "odieresis"), + (-1, 611.0, "udieresis"), + (-1, 549.0, "notequal"), + (-1, 611.0, "gcommaaccent"), + (-1, 611.0, "eth"), + (-1, 500.0, "zcaron"), + (-1, 611.0, "ncommaaccent"), + (-1, 333.0, "onesuperior"), + (-1, 278.0, "imacron"), + (-1, 556.0, "Euro"), + ][..], + ), + ( + "Helvetica-BoldOblique", + "AdobeStandardEncoding", + &[ + (32, 278.0, "space"), + (33, 333.0, "exclam"), + (34, 474.0, "quotedbl"), + (35, 556.0, "numbersign"), + (36, 556.0, "dollar"), + (37, 889.0, "percent"), + (38, 722.0, "ampersand"), + (39, 278.0, "quoteright"), + (40, 333.0, "parenleft"), + (41, 333.0, "parenright"), + (42, 389.0, "asterisk"), + (43, 584.0, "plus"), + (44, 278.0, "comma"), + (45, 333.0, "hyphen"), + (46, 278.0, "period"), + (47, 278.0, "slash"), + (48, 556.0, "zero"), + (49, 556.0, "one"), + (50, 556.0, "two"), + (51, 556.0, "three"), + (52, 556.0, "four"), + (53, 556.0, "five"), + (54, 556.0, "six"), + (55, 556.0, "seven"), + (56, 556.0, "eight"), + (57, 556.0, "nine"), + (58, 333.0, "colon"), + (59, 333.0, "semicolon"), + (60, 584.0, "less"), + (61, 584.0, "equal"), + (62, 584.0, "greater"), + (63, 611.0, "question"), + (64, 975.0, "at"), + (65, 722.0, "A"), + (66, 722.0, "B"), + (67, 722.0, "C"), + (68, 722.0, "D"), + (69, 667.0, "E"), + (70, 611.0, "F"), + (71, 778.0, "G"), + (72, 722.0, "H"), + (73, 278.0, "I"), + (74, 556.0, "J"), + (75, 722.0, "K"), + (76, 611.0, "L"), + (77, 833.0, "M"), + (78, 722.0, "N"), + (79, 778.0, "O"), + (80, 667.0, "P"), + (81, 778.0, "Q"), + (82, 722.0, "R"), + (83, 667.0, "S"), + (84, 611.0, "T"), + (85, 722.0, "U"), + (86, 667.0, "V"), + (87, 944.0, "W"), + (88, 667.0, "X"), + (89, 667.0, "Y"), + (90, 611.0, "Z"), + (91, 333.0, "bracketleft"), + (92, 278.0, "backslash"), + (93, 333.0, "bracketright"), + (94, 584.0, "asciicircum"), + (95, 556.0, "underscore"), + (96, 278.0, "quoteleft"), + (97, 556.0, "a"), + (98, 611.0, "b"), + (99, 556.0, "c"), + (100, 611.0, "d"), + (101, 556.0, "e"), + (102, 333.0, "f"), + (103, 611.0, "g"), + (104, 611.0, "h"), + (105, 278.0, "i"), + (106, 278.0, "j"), + (107, 556.0, "k"), + (108, 278.0, "l"), + (109, 889.0, "m"), + (110, 611.0, "n"), + (111, 611.0, "o"), + (112, 611.0, "p"), + (113, 611.0, "q"), + (114, 389.0, "r"), + (115, 556.0, "s"), + (116, 333.0, "t"), + (117, 611.0, "u"), + (118, 556.0, "v"), + (119, 778.0, "w"), + (120, 556.0, "x"), + (121, 556.0, "y"), + (122, 500.0, "z"), + (123, 389.0, "braceleft"), + (124, 280.0, "bar"), + (125, 389.0, "braceright"), + (126, 584.0, "asciitilde"), + (161, 333.0, "exclamdown"), + (162, 556.0, "cent"), + (163, 556.0, "sterling"), + (164, 167.0, "fraction"), + (165, 556.0, "yen"), + (166, 556.0, "florin"), + (167, 556.0, "section"), + (168, 556.0, "currency"), + (169, 238.0, "quotesingle"), + (170, 500.0, "quotedblleft"), + (171, 556.0, "guillemotleft"), + (172, 333.0, "guilsinglleft"), + (173, 333.0, "guilsinglright"), + (174, 611.0, "fi"), + (175, 611.0, "fl"), + (177, 556.0, "endash"), + (178, 556.0, "dagger"), + (179, 556.0, "daggerdbl"), + (180, 278.0, "periodcentered"), + (182, 556.0, "paragraph"), + (183, 350.0, "bullet"), + (184, 278.0, "quotesinglbase"), + (185, 500.0, "quotedblbase"), + (186, 500.0, "quotedblright"), + (187, 556.0, "guillemotright"), + (188, 1000.0, "ellipsis"), + (189, 1000.0, "perthousand"), + (191, 611.0, "questiondown"), + (193, 333.0, "grave"), + (194, 333.0, "acute"), + (195, 333.0, "circumflex"), + (196, 333.0, "tilde"), + (197, 333.0, "macron"), + (198, 333.0, "breve"), + (199, 333.0, "dotaccent"), + (200, 333.0, "dieresis"), + (202, 333.0, "ring"), + (203, 333.0, "cedilla"), + (205, 333.0, "hungarumlaut"), + (206, 333.0, "ogonek"), + (207, 333.0, "caron"), + (208, 1000.0, "emdash"), + (225, 1000.0, "AE"), + (227, 370.0, "ordfeminine"), + (232, 611.0, "Lslash"), + (233, 778.0, "Oslash"), + (234, 1000.0, "OE"), + (235, 365.0, "ordmasculine"), + (241, 889.0, "ae"), + (245, 278.0, "dotlessi"), + (248, 278.0, "lslash"), + (249, 611.0, "oslash"), + (250, 944.0, "oe"), + (251, 611.0, "germandbls"), + (-1, 278.0, "Idieresis"), + (-1, 556.0, "eacute"), + (-1, 556.0, "abreve"), + (-1, 611.0, "uhungarumlaut"), + (-1, 556.0, "ecaron"), + (-1, 667.0, "Ydieresis"), + (-1, 584.0, "divide"), + (-1, 667.0, "Yacute"), + (-1, 722.0, "Acircumflex"), + (-1, 556.0, "aacute"), + (-1, 722.0, "Ucircumflex"), + (-1, 556.0, "yacute"), + (-1, 556.0, "scommaaccent"), + (-1, 556.0, "ecircumflex"), + (-1, 722.0, "Uring"), + (-1, 722.0, "Udieresis"), + (-1, 556.0, "aogonek"), + (-1, 722.0, "Uacute"), + (-1, 611.0, "uogonek"), + (-1, 667.0, "Edieresis"), + (-1, 722.0, "Dcroat"), + (-1, 250.0, "commaaccent"), + (-1, 737.0, "copyright"), + (-1, 667.0, "Emacron"), + (-1, 556.0, "ccaron"), + (-1, 556.0, "aring"), + (-1, 722.0, "Ncommaaccent"), + (-1, 278.0, "lacute"), + (-1, 556.0, "agrave"), + (-1, 611.0, "Tcommaaccent"), + (-1, 722.0, "Cacute"), + (-1, 556.0, "atilde"), + (-1, 667.0, "Edotaccent"), + (-1, 556.0, "scaron"), + (-1, 556.0, "scedilla"), + (-1, 278.0, "iacute"), + (-1, 494.0, "lozenge"), + (-1, 722.0, "Rcaron"), + (-1, 778.0, "Gcommaaccent"), + (-1, 611.0, "ucircumflex"), + (-1, 556.0, "acircumflex"), + (-1, 722.0, "Amacron"), + (-1, 389.0, "rcaron"), + (-1, 556.0, "ccedilla"), + (-1, 611.0, "Zdotaccent"), + (-1, 667.0, "Thorn"), + (-1, 778.0, "Omacron"), + (-1, 722.0, "Racute"), + (-1, 667.0, "Sacute"), + (-1, 743.0, "dcaron"), + (-1, 722.0, "Umacron"), + (-1, 611.0, "uring"), + (-1, 333.0, "threesuperior"), + (-1, 778.0, "Ograve"), + (-1, 722.0, "Agrave"), + (-1, 722.0, "Abreve"), + (-1, 584.0, "multiply"), + (-1, 611.0, "uacute"), + (-1, 611.0, "Tcaron"), + (-1, 494.0, "partialdiff"), + (-1, 556.0, "ydieresis"), + (-1, 722.0, "Nacute"), + (-1, 278.0, "icircumflex"), + (-1, 667.0, "Ecircumflex"), + (-1, 556.0, "adieresis"), + (-1, 556.0, "edieresis"), + (-1, 556.0, "cacute"), + (-1, 611.0, "nacute"), + (-1, 611.0, "umacron"), + (-1, 722.0, "Ncaron"), + (-1, 278.0, "Iacute"), + (-1, 584.0, "plusminus"), + (-1, 280.0, "brokenbar"), + (-1, 737.0, "registered"), + (-1, 778.0, "Gbreve"), + (-1, 278.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 667.0, "Egrave"), + (-1, 389.0, "racute"), + (-1, 611.0, "omacron"), + (-1, 611.0, "Zacute"), + (-1, 611.0, "Zcaron"), + (-1, 549.0, "greaterequal"), + (-1, 722.0, "Eth"), + (-1, 722.0, "Ccedilla"), + (-1, 278.0, "lcommaaccent"), + (-1, 389.0, "tcaron"), + (-1, 556.0, "eogonek"), + (-1, 722.0, "Uogonek"), + (-1, 722.0, "Aacute"), + (-1, 722.0, "Adieresis"), + (-1, 556.0, "egrave"), + (-1, 500.0, "zacute"), + (-1, 278.0, "iogonek"), + (-1, 778.0, "Oacute"), + (-1, 611.0, "oacute"), + (-1, 556.0, "amacron"), + (-1, 556.0, "sacute"), + (-1, 278.0, "idieresis"), + (-1, 778.0, "Ocircumflex"), + (-1, 722.0, "Ugrave"), + (-1, 612.0, "Delta"), + (-1, 611.0, "thorn"), + (-1, 333.0, "twosuperior"), + (-1, 778.0, "Odieresis"), + (-1, 611.0, "mu"), + (-1, 278.0, "igrave"), + (-1, 611.0, "ohungarumlaut"), + (-1, 667.0, "Eogonek"), + (-1, 611.0, "dcroat"), + (-1, 834.0, "threequarters"), + (-1, 667.0, "Scedilla"), + (-1, 400.0, "lcaron"), + (-1, 722.0, "Kcommaaccent"), + (-1, 611.0, "Lacute"), + (-1, 1000.0, "trademark"), + (-1, 556.0, "edotaccent"), + (-1, 278.0, "Igrave"), + (-1, 278.0, "Imacron"), + (-1, 611.0, "Lcaron"), + (-1, 834.0, "onehalf"), + (-1, 549.0, "lessequal"), + (-1, 611.0, "ocircumflex"), + (-1, 611.0, "ntilde"), + (-1, 722.0, "Uhungarumlaut"), + (-1, 667.0, "Eacute"), + (-1, 556.0, "emacron"), + (-1, 611.0, "gbreve"), + (-1, 834.0, "onequarter"), + (-1, 667.0, "Scaron"), + (-1, 667.0, "Scommaaccent"), + (-1, 778.0, "Ohungarumlaut"), + (-1, 400.0, "degree"), + (-1, 611.0, "ograve"), + (-1, 722.0, "Ccaron"), + (-1, 611.0, "ugrave"), + (-1, 549.0, "radical"), + (-1, 722.0, "Dcaron"), + (-1, 389.0, "rcommaaccent"), + (-1, 722.0, "Ntilde"), + (-1, 611.0, "otilde"), + (-1, 722.0, "Rcommaaccent"), + (-1, 611.0, "Lcommaaccent"), + (-1, 722.0, "Atilde"), + (-1, 722.0, "Aogonek"), + (-1, 722.0, "Aring"), + (-1, 778.0, "Otilde"), + (-1, 500.0, "zdotaccent"), + (-1, 667.0, "Ecaron"), + (-1, 278.0, "Iogonek"), + (-1, 556.0, "kcommaaccent"), + (-1, 584.0, "minus"), + (-1, 278.0, "Icircumflex"), + (-1, 611.0, "ncaron"), + (-1, 333.0, "tcommaaccent"), + (-1, 584.0, "logicalnot"), + (-1, 611.0, "odieresis"), + (-1, 611.0, "udieresis"), + (-1, 549.0, "notequal"), + (-1, 611.0, "gcommaaccent"), + (-1, 611.0, "eth"), + (-1, 500.0, "zcaron"), + (-1, 611.0, "ncommaaccent"), + (-1, 333.0, "onesuperior"), + (-1, 278.0, "imacron"), + (-1, 556.0, "Euro"), + ][..], + ), + ( + "Helvetica-Oblique", + "AdobeStandardEncoding", + &[ + (32, 278.0, "space"), + (33, 278.0, "exclam"), + (34, 355.0, "quotedbl"), + (35, 556.0, "numbersign"), + (36, 556.0, "dollar"), + (37, 889.0, "percent"), + (38, 667.0, "ampersand"), + (39, 222.0, "quoteright"), + (40, 333.0, "parenleft"), + (41, 333.0, "parenright"), + (42, 389.0, "asterisk"), + (43, 584.0, "plus"), + (44, 278.0, "comma"), + (45, 333.0, "hyphen"), + (46, 278.0, "period"), + (47, 278.0, "slash"), + (48, 556.0, "zero"), + (49, 556.0, "one"), + (50, 556.0, "two"), + (51, 556.0, "three"), + (52, 556.0, "four"), + (53, 556.0, "five"), + (54, 556.0, "six"), + (55, 556.0, "seven"), + (56, 556.0, "eight"), + (57, 556.0, "nine"), + (58, 278.0, "colon"), + (59, 278.0, "semicolon"), + (60, 584.0, "less"), + (61, 584.0, "equal"), + (62, 584.0, "greater"), + (63, 556.0, "question"), + (64, 1015.0, "at"), + (65, 667.0, "A"), + (66, 667.0, "B"), + (67, 722.0, "C"), + (68, 722.0, "D"), + (69, 667.0, "E"), + (70, 611.0, "F"), + (71, 778.0, "G"), + (72, 722.0, "H"), + (73, 278.0, "I"), + (74, 500.0, "J"), + (75, 667.0, "K"), + (76, 556.0, "L"), + (77, 833.0, "M"), + (78, 722.0, "N"), + (79, 778.0, "O"), + (80, 667.0, "P"), + (81, 778.0, "Q"), + (82, 722.0, "R"), + (83, 667.0, "S"), + (84, 611.0, "T"), + (85, 722.0, "U"), + (86, 667.0, "V"), + (87, 944.0, "W"), + (88, 667.0, "X"), + (89, 667.0, "Y"), + (90, 611.0, "Z"), + (91, 278.0, "bracketleft"), + (92, 278.0, "backslash"), + (93, 278.0, "bracketright"), + (94, 469.0, "asciicircum"), + (95, 556.0, "underscore"), + (96, 222.0, "quoteleft"), + (97, 556.0, "a"), + (98, 556.0, "b"), + (99, 500.0, "c"), + (100, 556.0, "d"), + (101, 556.0, "e"), + (102, 278.0, "f"), + (103, 556.0, "g"), + (104, 556.0, "h"), + (105, 222.0, "i"), + (106, 222.0, "j"), + (107, 500.0, "k"), + (108, 222.0, "l"), + (109, 833.0, "m"), + (110, 556.0, "n"), + (111, 556.0, "o"), + (112, 556.0, "p"), + (113, 556.0, "q"), + (114, 333.0, "r"), + (115, 500.0, "s"), + (116, 278.0, "t"), + (117, 556.0, "u"), + (118, 500.0, "v"), + (119, 722.0, "w"), + (120, 500.0, "x"), + (121, 500.0, "y"), + (122, 500.0, "z"), + (123, 334.0, "braceleft"), + (124, 260.0, "bar"), + (125, 334.0, "braceright"), + (126, 584.0, "asciitilde"), + (161, 333.0, "exclamdown"), + (162, 556.0, "cent"), + (163, 556.0, "sterling"), + (164, 167.0, "fraction"), + (165, 556.0, "yen"), + (166, 556.0, "florin"), + (167, 556.0, "section"), + (168, 556.0, "currency"), + (169, 191.0, "quotesingle"), + (170, 333.0, "quotedblleft"), + (171, 556.0, "guillemotleft"), + (172, 333.0, "guilsinglleft"), + (173, 333.0, "guilsinglright"), + (174, 500.0, "fi"), + (175, 500.0, "fl"), + (177, 556.0, "endash"), + (178, 556.0, "dagger"), + (179, 556.0, "daggerdbl"), + (180, 278.0, "periodcentered"), + (182, 537.0, "paragraph"), + (183, 350.0, "bullet"), + (184, 222.0, "quotesinglbase"), + (185, 333.0, "quotedblbase"), + (186, 333.0, "quotedblright"), + (187, 556.0, "guillemotright"), + (188, 1000.0, "ellipsis"), + (189, 1000.0, "perthousand"), + (191, 611.0, "questiondown"), + (193, 333.0, "grave"), + (194, 333.0, "acute"), + (195, 333.0, "circumflex"), + (196, 333.0, "tilde"), + (197, 333.0, "macron"), + (198, 333.0, "breve"), + (199, 333.0, "dotaccent"), + (200, 333.0, "dieresis"), + (202, 333.0, "ring"), + (203, 333.0, "cedilla"), + (205, 333.0, "hungarumlaut"), + (206, 333.0, "ogonek"), + (207, 333.0, "caron"), + (208, 1000.0, "emdash"), + (225, 1000.0, "AE"), + (227, 370.0, "ordfeminine"), + (232, 556.0, "Lslash"), + (233, 778.0, "Oslash"), + (234, 1000.0, "OE"), + (235, 365.0, "ordmasculine"), + (241, 889.0, "ae"), + (245, 278.0, "dotlessi"), + (248, 222.0, "lslash"), + (249, 611.0, "oslash"), + (250, 944.0, "oe"), + (251, 611.0, "germandbls"), + (-1, 278.0, "Idieresis"), + (-1, 556.0, "eacute"), + (-1, 556.0, "abreve"), + (-1, 556.0, "uhungarumlaut"), + (-1, 556.0, "ecaron"), + (-1, 667.0, "Ydieresis"), + (-1, 584.0, "divide"), + (-1, 667.0, "Yacute"), + (-1, 667.0, "Acircumflex"), + (-1, 556.0, "aacute"), + (-1, 722.0, "Ucircumflex"), + (-1, 500.0, "yacute"), + (-1, 500.0, "scommaaccent"), + (-1, 556.0, "ecircumflex"), + (-1, 722.0, "Uring"), + (-1, 722.0, "Udieresis"), + (-1, 556.0, "aogonek"), + (-1, 722.0, "Uacute"), + (-1, 556.0, "uogonek"), + (-1, 667.0, "Edieresis"), + (-1, 722.0, "Dcroat"), + (-1, 250.0, "commaaccent"), + (-1, 737.0, "copyright"), + (-1, 667.0, "Emacron"), + (-1, 500.0, "ccaron"), + (-1, 556.0, "aring"), + (-1, 722.0, "Ncommaaccent"), + (-1, 222.0, "lacute"), + (-1, 556.0, "agrave"), + (-1, 611.0, "Tcommaaccent"), + (-1, 722.0, "Cacute"), + (-1, 556.0, "atilde"), + (-1, 667.0, "Edotaccent"), + (-1, 500.0, "scaron"), + (-1, 500.0, "scedilla"), + (-1, 278.0, "iacute"), + (-1, 471.0, "lozenge"), + (-1, 722.0, "Rcaron"), + (-1, 778.0, "Gcommaaccent"), + (-1, 556.0, "ucircumflex"), + (-1, 556.0, "acircumflex"), + (-1, 667.0, "Amacron"), + (-1, 333.0, "rcaron"), + (-1, 500.0, "ccedilla"), + (-1, 611.0, "Zdotaccent"), + (-1, 667.0, "Thorn"), + (-1, 778.0, "Omacron"), + (-1, 722.0, "Racute"), + (-1, 667.0, "Sacute"), + (-1, 643.0, "dcaron"), + (-1, 722.0, "Umacron"), + (-1, 556.0, "uring"), + (-1, 333.0, "threesuperior"), + (-1, 778.0, "Ograve"), + (-1, 667.0, "Agrave"), + (-1, 667.0, "Abreve"), + (-1, 584.0, "multiply"), + (-1, 556.0, "uacute"), + (-1, 611.0, "Tcaron"), + (-1, 476.0, "partialdiff"), + (-1, 500.0, "ydieresis"), + (-1, 722.0, "Nacute"), + (-1, 278.0, "icircumflex"), + (-1, 667.0, "Ecircumflex"), + (-1, 556.0, "adieresis"), + (-1, 556.0, "edieresis"), + (-1, 500.0, "cacute"), + (-1, 556.0, "nacute"), + (-1, 556.0, "umacron"), + (-1, 722.0, "Ncaron"), + (-1, 278.0, "Iacute"), + (-1, 584.0, "plusminus"), + (-1, 260.0, "brokenbar"), + (-1, 737.0, "registered"), + (-1, 778.0, "Gbreve"), + (-1, 278.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 667.0, "Egrave"), + (-1, 333.0, "racute"), + (-1, 556.0, "omacron"), + (-1, 611.0, "Zacute"), + (-1, 611.0, "Zcaron"), + (-1, 549.0, "greaterequal"), + (-1, 722.0, "Eth"), + (-1, 722.0, "Ccedilla"), + (-1, 222.0, "lcommaaccent"), + (-1, 317.0, "tcaron"), + (-1, 556.0, "eogonek"), + (-1, 722.0, "Uogonek"), + (-1, 667.0, "Aacute"), + (-1, 667.0, "Adieresis"), + (-1, 556.0, "egrave"), + (-1, 500.0, "zacute"), + (-1, 222.0, "iogonek"), + (-1, 778.0, "Oacute"), + (-1, 556.0, "oacute"), + (-1, 556.0, "amacron"), + (-1, 500.0, "sacute"), + (-1, 278.0, "idieresis"), + (-1, 778.0, "Ocircumflex"), + (-1, 722.0, "Ugrave"), + (-1, 612.0, "Delta"), + (-1, 556.0, "thorn"), + (-1, 333.0, "twosuperior"), + (-1, 778.0, "Odieresis"), + (-1, 556.0, "mu"), + (-1, 278.0, "igrave"), + (-1, 556.0, "ohungarumlaut"), + (-1, 667.0, "Eogonek"), + (-1, 556.0, "dcroat"), + (-1, 834.0, "threequarters"), + (-1, 667.0, "Scedilla"), + (-1, 299.0, "lcaron"), + (-1, 667.0, "Kcommaaccent"), + (-1, 556.0, "Lacute"), + (-1, 1000.0, "trademark"), + (-1, 556.0, "edotaccent"), + (-1, 278.0, "Igrave"), + (-1, 278.0, "Imacron"), + (-1, 556.0, "Lcaron"), + (-1, 834.0, "onehalf"), + (-1, 549.0, "lessequal"), + (-1, 556.0, "ocircumflex"), + (-1, 556.0, "ntilde"), + (-1, 722.0, "Uhungarumlaut"), + (-1, 667.0, "Eacute"), + (-1, 556.0, "emacron"), + (-1, 556.0, "gbreve"), + (-1, 834.0, "onequarter"), + (-1, 667.0, "Scaron"), + (-1, 667.0, "Scommaaccent"), + (-1, 778.0, "Ohungarumlaut"), + (-1, 400.0, "degree"), + (-1, 556.0, "ograve"), + (-1, 722.0, "Ccaron"), + (-1, 556.0, "ugrave"), + (-1, 453.0, "radical"), + (-1, 722.0, "Dcaron"), + (-1, 333.0, "rcommaaccent"), + (-1, 722.0, "Ntilde"), + (-1, 556.0, "otilde"), + (-1, 722.0, "Rcommaaccent"), + (-1, 556.0, "Lcommaaccent"), + (-1, 667.0, "Atilde"), + (-1, 667.0, "Aogonek"), + (-1, 667.0, "Aring"), + (-1, 778.0, "Otilde"), + (-1, 500.0, "zdotaccent"), + (-1, 667.0, "Ecaron"), + (-1, 278.0, "Iogonek"), + (-1, 500.0, "kcommaaccent"), + (-1, 584.0, "minus"), + (-1, 278.0, "Icircumflex"), + (-1, 556.0, "ncaron"), + (-1, 278.0, "tcommaaccent"), + (-1, 584.0, "logicalnot"), + (-1, 556.0, "odieresis"), + (-1, 556.0, "udieresis"), + (-1, 549.0, "notequal"), + (-1, 556.0, "gcommaaccent"), + (-1, 556.0, "eth"), + (-1, 500.0, "zcaron"), + (-1, 556.0, "ncommaaccent"), + (-1, 333.0, "onesuperior"), + (-1, 278.0, "imacron"), + (-1, 556.0, "Euro"), + ][..], + ), + ( + "Helvetica", + "AdobeStandardEncoding", + &[ + (32, 278.0, "space"), + (33, 278.0, "exclam"), + (34, 355.0, "quotedbl"), + (35, 556.0, "numbersign"), + (36, 556.0, "dollar"), + (37, 889.0, "percent"), + (38, 667.0, "ampersand"), + (39, 222.0, "quoteright"), + (40, 333.0, "parenleft"), + (41, 333.0, "parenright"), + (42, 389.0, "asterisk"), + (43, 584.0, "plus"), + (44, 278.0, "comma"), + (45, 333.0, "hyphen"), + (46, 278.0, "period"), + (47, 278.0, "slash"), + (48, 556.0, "zero"), + (49, 556.0, "one"), + (50, 556.0, "two"), + (51, 556.0, "three"), + (52, 556.0, "four"), + (53, 556.0, "five"), + (54, 556.0, "six"), + (55, 556.0, "seven"), + (56, 556.0, "eight"), + (57, 556.0, "nine"), + (58, 278.0, "colon"), + (59, 278.0, "semicolon"), + (60, 584.0, "less"), + (61, 584.0, "equal"), + (62, 584.0, "greater"), + (63, 556.0, "question"), + (64, 1015.0, "at"), + (65, 667.0, "A"), + (66, 667.0, "B"), + (67, 722.0, "C"), + (68, 722.0, "D"), + (69, 667.0, "E"), + (70, 611.0, "F"), + (71, 778.0, "G"), + (72, 722.0, "H"), + (73, 278.0, "I"), + (74, 500.0, "J"), + (75, 667.0, "K"), + (76, 556.0, "L"), + (77, 833.0, "M"), + (78, 722.0, "N"), + (79, 778.0, "O"), + (80, 667.0, "P"), + (81, 778.0, "Q"), + (82, 722.0, "R"), + (83, 667.0, "S"), + (84, 611.0, "T"), + (85, 722.0, "U"), + (86, 667.0, "V"), + (87, 944.0, "W"), + (88, 667.0, "X"), + (89, 667.0, "Y"), + (90, 611.0, "Z"), + (91, 278.0, "bracketleft"), + (92, 278.0, "backslash"), + (93, 278.0, "bracketright"), + (94, 469.0, "asciicircum"), + (95, 556.0, "underscore"), + (96, 222.0, "quoteleft"), + (97, 556.0, "a"), + (98, 556.0, "b"), + (99, 500.0, "c"), + (100, 556.0, "d"), + (101, 556.0, "e"), + (102, 278.0, "f"), + (103, 556.0, "g"), + (104, 556.0, "h"), + (105, 222.0, "i"), + (106, 222.0, "j"), + (107, 500.0, "k"), + (108, 222.0, "l"), + (109, 833.0, "m"), + (110, 556.0, "n"), + (111, 556.0, "o"), + (112, 556.0, "p"), + (113, 556.0, "q"), + (114, 333.0, "r"), + (115, 500.0, "s"), + (116, 278.0, "t"), + (117, 556.0, "u"), + (118, 500.0, "v"), + (119, 722.0, "w"), + (120, 500.0, "x"), + (121, 500.0, "y"), + (122, 500.0, "z"), + (123, 334.0, "braceleft"), + (124, 260.0, "bar"), + (125, 334.0, "braceright"), + (126, 584.0, "asciitilde"), + (161, 333.0, "exclamdown"), + (162, 556.0, "cent"), + (163, 556.0, "sterling"), + (164, 167.0, "fraction"), + (165, 556.0, "yen"), + (166, 556.0, "florin"), + (167, 556.0, "section"), + (168, 556.0, "currency"), + (169, 191.0, "quotesingle"), + (170, 333.0, "quotedblleft"), + (171, 556.0, "guillemotleft"), + (172, 333.0, "guilsinglleft"), + (173, 333.0, "guilsinglright"), + (174, 500.0, "fi"), + (175, 500.0, "fl"), + (177, 556.0, "endash"), + (178, 556.0, "dagger"), + (179, 556.0, "daggerdbl"), + (180, 278.0, "periodcentered"), + (182, 537.0, "paragraph"), + (183, 350.0, "bullet"), + (184, 222.0, "quotesinglbase"), + (185, 333.0, "quotedblbase"), + (186, 333.0, "quotedblright"), + (187, 556.0, "guillemotright"), + (188, 1000.0, "ellipsis"), + (189, 1000.0, "perthousand"), + (191, 611.0, "questiondown"), + (193, 333.0, "grave"), + (194, 333.0, "acute"), + (195, 333.0, "circumflex"), + (196, 333.0, "tilde"), + (197, 333.0, "macron"), + (198, 333.0, "breve"), + (199, 333.0, "dotaccent"), + (200, 333.0, "dieresis"), + (202, 333.0, "ring"), + (203, 333.0, "cedilla"), + (205, 333.0, "hungarumlaut"), + (206, 333.0, "ogonek"), + (207, 333.0, "caron"), + (208, 1000.0, "emdash"), + (225, 1000.0, "AE"), + (227, 370.0, "ordfeminine"), + (232, 556.0, "Lslash"), + (233, 778.0, "Oslash"), + (234, 1000.0, "OE"), + (235, 365.0, "ordmasculine"), + (241, 889.0, "ae"), + (245, 278.0, "dotlessi"), + (248, 222.0, "lslash"), + (249, 611.0, "oslash"), + (250, 944.0, "oe"), + (251, 611.0, "germandbls"), + (-1, 278.0, "Idieresis"), + (-1, 556.0, "eacute"), + (-1, 556.0, "abreve"), + (-1, 556.0, "uhungarumlaut"), + (-1, 556.0, "ecaron"), + (-1, 667.0, "Ydieresis"), + (-1, 584.0, "divide"), + (-1, 667.0, "Yacute"), + (-1, 667.0, "Acircumflex"), + (-1, 556.0, "aacute"), + (-1, 722.0, "Ucircumflex"), + (-1, 500.0, "yacute"), + (-1, 500.0, "scommaaccent"), + (-1, 556.0, "ecircumflex"), + (-1, 722.0, "Uring"), + (-1, 722.0, "Udieresis"), + (-1, 556.0, "aogonek"), + (-1, 722.0, "Uacute"), + (-1, 556.0, "uogonek"), + (-1, 667.0, "Edieresis"), + (-1, 722.0, "Dcroat"), + (-1, 250.0, "commaaccent"), + (-1, 737.0, "copyright"), + (-1, 667.0, "Emacron"), + (-1, 500.0, "ccaron"), + (-1, 556.0, "aring"), + (-1, 722.0, "Ncommaaccent"), + (-1, 222.0, "lacute"), + (-1, 556.0, "agrave"), + (-1, 611.0, "Tcommaaccent"), + (-1, 722.0, "Cacute"), + (-1, 556.0, "atilde"), + (-1, 667.0, "Edotaccent"), + (-1, 500.0, "scaron"), + (-1, 500.0, "scedilla"), + (-1, 278.0, "iacute"), + (-1, 471.0, "lozenge"), + (-1, 722.0, "Rcaron"), + (-1, 778.0, "Gcommaaccent"), + (-1, 556.0, "ucircumflex"), + (-1, 556.0, "acircumflex"), + (-1, 667.0, "Amacron"), + (-1, 333.0, "rcaron"), + (-1, 500.0, "ccedilla"), + (-1, 611.0, "Zdotaccent"), + (-1, 667.0, "Thorn"), + (-1, 778.0, "Omacron"), + (-1, 722.0, "Racute"), + (-1, 667.0, "Sacute"), + (-1, 643.0, "dcaron"), + (-1, 722.0, "Umacron"), + (-1, 556.0, "uring"), + (-1, 333.0, "threesuperior"), + (-1, 778.0, "Ograve"), + (-1, 667.0, "Agrave"), + (-1, 667.0, "Abreve"), + (-1, 584.0, "multiply"), + (-1, 556.0, "uacute"), + (-1, 611.0, "Tcaron"), + (-1, 476.0, "partialdiff"), + (-1, 500.0, "ydieresis"), + (-1, 722.0, "Nacute"), + (-1, 278.0, "icircumflex"), + (-1, 667.0, "Ecircumflex"), + (-1, 556.0, "adieresis"), + (-1, 556.0, "edieresis"), + (-1, 500.0, "cacute"), + (-1, 556.0, "nacute"), + (-1, 556.0, "umacron"), + (-1, 722.0, "Ncaron"), + (-1, 278.0, "Iacute"), + (-1, 584.0, "plusminus"), + (-1, 260.0, "brokenbar"), + (-1, 737.0, "registered"), + (-1, 778.0, "Gbreve"), + (-1, 278.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 667.0, "Egrave"), + (-1, 333.0, "racute"), + (-1, 556.0, "omacron"), + (-1, 611.0, "Zacute"), + (-1, 611.0, "Zcaron"), + (-1, 549.0, "greaterequal"), + (-1, 722.0, "Eth"), + (-1, 722.0, "Ccedilla"), + (-1, 222.0, "lcommaaccent"), + (-1, 317.0, "tcaron"), + (-1, 556.0, "eogonek"), + (-1, 722.0, "Uogonek"), + (-1, 667.0, "Aacute"), + (-1, 667.0, "Adieresis"), + (-1, 556.0, "egrave"), + (-1, 500.0, "zacute"), + (-1, 222.0, "iogonek"), + (-1, 778.0, "Oacute"), + (-1, 556.0, "oacute"), + (-1, 556.0, "amacron"), + (-1, 500.0, "sacute"), + (-1, 278.0, "idieresis"), + (-1, 778.0, "Ocircumflex"), + (-1, 722.0, "Ugrave"), + (-1, 612.0, "Delta"), + (-1, 556.0, "thorn"), + (-1, 333.0, "twosuperior"), + (-1, 778.0, "Odieresis"), + (-1, 556.0, "mu"), + (-1, 278.0, "igrave"), + (-1, 556.0, "ohungarumlaut"), + (-1, 667.0, "Eogonek"), + (-1, 556.0, "dcroat"), + (-1, 834.0, "threequarters"), + (-1, 667.0, "Scedilla"), + (-1, 299.0, "lcaron"), + (-1, 667.0, "Kcommaaccent"), + (-1, 556.0, "Lacute"), + (-1, 1000.0, "trademark"), + (-1, 556.0, "edotaccent"), + (-1, 278.0, "Igrave"), + (-1, 278.0, "Imacron"), + (-1, 556.0, "Lcaron"), + (-1, 834.0, "onehalf"), + (-1, 549.0, "lessequal"), + (-1, 556.0, "ocircumflex"), + (-1, 556.0, "ntilde"), + (-1, 722.0, "Uhungarumlaut"), + (-1, 667.0, "Eacute"), + (-1, 556.0, "emacron"), + (-1, 556.0, "gbreve"), + (-1, 834.0, "onequarter"), + (-1, 667.0, "Scaron"), + (-1, 667.0, "Scommaaccent"), + (-1, 778.0, "Ohungarumlaut"), + (-1, 400.0, "degree"), + (-1, 556.0, "ograve"), + (-1, 722.0, "Ccaron"), + (-1, 556.0, "ugrave"), + (-1, 453.0, "radical"), + (-1, 722.0, "Dcaron"), + (-1, 333.0, "rcommaaccent"), + (-1, 722.0, "Ntilde"), + (-1, 556.0, "otilde"), + (-1, 722.0, "Rcommaaccent"), + (-1, 556.0, "Lcommaaccent"), + (-1, 667.0, "Atilde"), + (-1, 667.0, "Aogonek"), + (-1, 667.0, "Aring"), + (-1, 778.0, "Otilde"), + (-1, 500.0, "zdotaccent"), + (-1, 667.0, "Ecaron"), + (-1, 278.0, "Iogonek"), + (-1, 500.0, "kcommaaccent"), + (-1, 584.0, "minus"), + (-1, 278.0, "Icircumflex"), + (-1, 556.0, "ncaron"), + (-1, 278.0, "tcommaaccent"), + (-1, 584.0, "logicalnot"), + (-1, 556.0, "odieresis"), + (-1, 556.0, "udieresis"), + (-1, 549.0, "notequal"), + (-1, 556.0, "gcommaaccent"), + (-1, 556.0, "eth"), + (-1, 500.0, "zcaron"), + (-1, 556.0, "ncommaaccent"), + (-1, 333.0, "onesuperior"), + (-1, 278.0, "imacron"), + (-1, 556.0, "Euro"), + ][..], + ), + ( + "Symbol", + "FontSpecific", + &[ + (32, 250.0, "space"), + (33, 333.0, "exclam"), + (34, 713.0, "universal"), + (35, 500.0, "numbersign"), + (36, 549.0, "existential"), + (37, 833.0, "percent"), + (38, 778.0, "ampersand"), + (39, 439.0, "suchthat"), + (40, 333.0, "parenleft"), + (41, 333.0, "parenright"), + (42, 500.0, "asteriskmath"), + (43, 549.0, "plus"), + (44, 250.0, "comma"), + (45, 549.0, "minus"), + (46, 250.0, "period"), + (47, 278.0, "slash"), + (48, 500.0, "zero"), + (49, 500.0, "one"), + (50, 500.0, "two"), + (51, 500.0, "three"), + (52, 500.0, "four"), + (53, 500.0, "five"), + (54, 500.0, "six"), + (55, 500.0, "seven"), + (56, 500.0, "eight"), + (57, 500.0, "nine"), + (58, 278.0, "colon"), + (59, 278.0, "semicolon"), + (60, 549.0, "less"), + (61, 549.0, "equal"), + (62, 549.0, "greater"), + (63, 444.0, "question"), + (64, 549.0, "congruent"), + (65, 722.0, "Alpha"), + (66, 667.0, "Beta"), + (67, 722.0, "Chi"), + (68, 612.0, "Delta"), + (69, 611.0, "Epsilon"), + (70, 763.0, "Phi"), + (71, 603.0, "Gamma"), + (72, 722.0, "Eta"), + (73, 333.0, "Iota"), + (74, 631.0, "theta1"), + (75, 722.0, "Kappa"), + (76, 686.0, "Lambda"), + (77, 889.0, "Mu"), + (78, 722.0, "Nu"), + (79, 722.0, "Omicron"), + (80, 768.0, "Pi"), + (81, 741.0, "Theta"), + (82, 556.0, "Rho"), + (83, 592.0, "Sigma"), + (84, 611.0, "Tau"), + (85, 690.0, "Upsilon"), + (86, 439.0, "sigma1"), + (87, 768.0, "Omega"), + (88, 645.0, "Xi"), + (89, 795.0, "Psi"), + (90, 611.0, "Zeta"), + (91, 333.0, "bracketleft"), + (92, 863.0, "therefore"), + (93, 333.0, "bracketright"), + (94, 658.0, "perpendicular"), + (95, 500.0, "underscore"), + (96, 500.0, "radicalex"), + (97, 631.0, "alpha"), + (98, 549.0, "beta"), + (99, 549.0, "chi"), + (100, 494.0, "delta"), + (101, 439.0, "epsilon"), + (102, 521.0, "phi"), + (103, 411.0, "gamma"), + (104, 603.0, "eta"), + (105, 329.0, "iota"), + (106, 603.0, "phi1"), + (107, 549.0, "kappa"), + (108, 549.0, "lambda"), + (109, 576.0, "mu"), + (110, 521.0, "nu"), + (111, 549.0, "omicron"), + (112, 549.0, "pi"), + (113, 521.0, "theta"), + (114, 549.0, "rho"), + (115, 603.0, "sigma"), + (116, 439.0, "tau"), + (117, 576.0, "upsilon"), + (118, 713.0, "omega1"), + (119, 686.0, "omega"), + (120, 493.0, "xi"), + (121, 686.0, "psi"), + (122, 494.0, "zeta"), + (123, 480.0, "braceleft"), + (124, 200.0, "bar"), + (125, 480.0, "braceright"), + (126, 549.0, "similar"), + (160, 750.0, "Euro"), + (161, 620.0, "Upsilon1"), + (162, 247.0, "minute"), + (163, 549.0, "lessequal"), + (164, 167.0, "fraction"), + (165, 713.0, "infinity"), + (166, 500.0, "florin"), + (167, 753.0, "club"), + (168, 753.0, "diamond"), + (169, 753.0, "heart"), + (170, 753.0, "spade"), + (171, 1042.0, "arrowboth"), + (172, 987.0, "arrowleft"), + (173, 603.0, "arrowup"), + (174, 987.0, "arrowright"), + (175, 603.0, "arrowdown"), + (176, 400.0, "degree"), + (177, 549.0, "plusminus"), + (178, 411.0, "second"), + (179, 549.0, "greaterequal"), + (180, 549.0, "multiply"), + (181, 713.0, "proportional"), + (182, 494.0, "partialdiff"), + (183, 460.0, "bullet"), + (184, 549.0, "divide"), + (185, 549.0, "notequal"), + (186, 549.0, "equivalence"), + (187, 549.0, "approxequal"), + (188, 1000.0, "ellipsis"), + (189, 603.0, "arrowvertex"), + (190, 1000.0, "arrowhorizex"), + (191, 658.0, "carriagereturn"), + (192, 823.0, "aleph"), + (193, 686.0, "Ifraktur"), + (194, 795.0, "Rfraktur"), + (195, 987.0, "weierstrass"), + (196, 768.0, "circlemultiply"), + (197, 768.0, "circleplus"), + (198, 823.0, "emptyset"), + (199, 768.0, "intersection"), + (200, 768.0, "union"), + (201, 713.0, "propersuperset"), + (202, 713.0, "reflexsuperset"), + (203, 713.0, "notsubset"), + (204, 713.0, "propersubset"), + (205, 713.0, "reflexsubset"), + (206, 713.0, "element"), + (207, 713.0, "notelement"), + (208, 768.0, "angle"), + (209, 713.0, "gradient"), + (210, 790.0, "registerserif"), + (211, 790.0, "copyrightserif"), + (212, 890.0, "trademarkserif"), + (213, 823.0, "product"), + (214, 549.0, "radical"), + (215, 250.0, "dotmath"), + (216, 713.0, "logicalnot"), + (217, 603.0, "logicaland"), + (218, 603.0, "logicalor"), + (219, 1042.0, "arrowdblboth"), + (220, 987.0, "arrowdblleft"), + (221, 603.0, "arrowdblup"), + (222, 987.0, "arrowdblright"), + (223, 603.0, "arrowdbldown"), + (224, 494.0, "lozenge"), + (225, 329.0, "angleleft"), + (226, 790.0, "registersans"), + (227, 790.0, "copyrightsans"), + (228, 786.0, "trademarksans"), + (229, 713.0, "summation"), + (230, 384.0, "parenlefttp"), + (231, 384.0, "parenleftex"), + (232, 384.0, "parenleftbt"), + (233, 384.0, "bracketlefttp"), + (234, 384.0, "bracketleftex"), + (235, 384.0, "bracketleftbt"), + (236, 494.0, "bracelefttp"), + (237, 494.0, "braceleftmid"), + (238, 494.0, "braceleftbt"), + (239, 494.0, "braceex"), + (241, 329.0, "angleright"), + (242, 274.0, "integral"), + (243, 686.0, "integraltp"), + (244, 686.0, "integralex"), + (245, 686.0, "integralbt"), + (246, 384.0, "parenrighttp"), + (247, 384.0, "parenrightex"), + (248, 384.0, "parenrightbt"), + (249, 384.0, "bracketrighttp"), + (250, 384.0, "bracketrightex"), + (251, 384.0, "bracketrightbt"), + (252, 494.0, "bracerighttp"), + (253, 494.0, "bracerightmid"), + (254, 494.0, "bracerightbt"), + (-1, 790.0, "apple"), + ][..], + ), + ( + "Times-Bold", + "AdobeStandardEncoding", + &[ + (32, 250.0, "space"), + (33, 333.0, "exclam"), + (34, 555.0, "quotedbl"), + (35, 500.0, "numbersign"), + (36, 500.0, "dollar"), + (37, 1000.0, "percent"), + (38, 833.0, "ampersand"), + (39, 333.0, "quoteright"), + (40, 333.0, "parenleft"), + (41, 333.0, "parenright"), + (42, 500.0, "asterisk"), + (43, 570.0, "plus"), + (44, 250.0, "comma"), + (45, 333.0, "hyphen"), + (46, 250.0, "period"), + (47, 278.0, "slash"), + (48, 500.0, "zero"), + (49, 500.0, "one"), + (50, 500.0, "two"), + (51, 500.0, "three"), + (52, 500.0, "four"), + (53, 500.0, "five"), + (54, 500.0, "six"), + (55, 500.0, "seven"), + (56, 500.0, "eight"), + (57, 500.0, "nine"), + (58, 333.0, "colon"), + (59, 333.0, "semicolon"), + (60, 570.0, "less"), + (61, 570.0, "equal"), + (62, 570.0, "greater"), + (63, 500.0, "question"), + (64, 930.0, "at"), + (65, 722.0, "A"), + (66, 667.0, "B"), + (67, 722.0, "C"), + (68, 722.0, "D"), + (69, 667.0, "E"), + (70, 611.0, "F"), + (71, 778.0, "G"), + (72, 778.0, "H"), + (73, 389.0, "I"), + (74, 500.0, "J"), + (75, 778.0, "K"), + (76, 667.0, "L"), + (77, 944.0, "M"), + (78, 722.0, "N"), + (79, 778.0, "O"), + (80, 611.0, "P"), + (81, 778.0, "Q"), + (82, 722.0, "R"), + (83, 556.0, "S"), + (84, 667.0, "T"), + (85, 722.0, "U"), + (86, 722.0, "V"), + (87, 1000.0, "W"), + (88, 722.0, "X"), + (89, 722.0, "Y"), + (90, 667.0, "Z"), + (91, 333.0, "bracketleft"), + (92, 278.0, "backslash"), + (93, 333.0, "bracketright"), + (94, 581.0, "asciicircum"), + (95, 500.0, "underscore"), + (96, 333.0, "quoteleft"), + (97, 500.0, "a"), + (98, 556.0, "b"), + (99, 444.0, "c"), + (100, 556.0, "d"), + (101, 444.0, "e"), + (102, 333.0, "f"), + (103, 500.0, "g"), + (104, 556.0, "h"), + (105, 278.0, "i"), + (106, 333.0, "j"), + (107, 556.0, "k"), + (108, 278.0, "l"), + (109, 833.0, "m"), + (110, 556.0, "n"), + (111, 500.0, "o"), + (112, 556.0, "p"), + (113, 556.0, "q"), + (114, 444.0, "r"), + (115, 389.0, "s"), + (116, 333.0, "t"), + (117, 556.0, "u"), + (118, 500.0, "v"), + (119, 722.0, "w"), + (120, 500.0, "x"), + (121, 500.0, "y"), + (122, 444.0, "z"), + (123, 394.0, "braceleft"), + (124, 220.0, "bar"), + (125, 394.0, "braceright"), + (126, 520.0, "asciitilde"), + (161, 333.0, "exclamdown"), + (162, 500.0, "cent"), + (163, 500.0, "sterling"), + (164, 167.0, "fraction"), + (165, 500.0, "yen"), + (166, 500.0, "florin"), + (167, 500.0, "section"), + (168, 500.0, "currency"), + (169, 278.0, "quotesingle"), + (170, 500.0, "quotedblleft"), + (171, 500.0, "guillemotleft"), + (172, 333.0, "guilsinglleft"), + (173, 333.0, "guilsinglright"), + (174, 556.0, "fi"), + (175, 556.0, "fl"), + (177, 500.0, "endash"), + (178, 500.0, "dagger"), + (179, 500.0, "daggerdbl"), + (180, 250.0, "periodcentered"), + (182, 540.0, "paragraph"), + (183, 350.0, "bullet"), + (184, 333.0, "quotesinglbase"), + (185, 500.0, "quotedblbase"), + (186, 500.0, "quotedblright"), + (187, 500.0, "guillemotright"), + (188, 1000.0, "ellipsis"), + (189, 1000.0, "perthousand"), + (191, 500.0, "questiondown"), + (193, 333.0, "grave"), + (194, 333.0, "acute"), + (195, 333.0, "circumflex"), + (196, 333.0, "tilde"), + (197, 333.0, "macron"), + (198, 333.0, "breve"), + (199, 333.0, "dotaccent"), + (200, 333.0, "dieresis"), + (202, 333.0, "ring"), + (203, 333.0, "cedilla"), + (205, 333.0, "hungarumlaut"), + (206, 333.0, "ogonek"), + (207, 333.0, "caron"), + (208, 1000.0, "emdash"), + (225, 1000.0, "AE"), + (227, 300.0, "ordfeminine"), + (232, 667.0, "Lslash"), + (233, 778.0, "Oslash"), + (234, 1000.0, "OE"), + (235, 330.0, "ordmasculine"), + (241, 722.0, "ae"), + (245, 278.0, "dotlessi"), + (248, 278.0, "lslash"), + (249, 500.0, "oslash"), + (250, 722.0, "oe"), + (251, 556.0, "germandbls"), + (-1, 389.0, "Idieresis"), + (-1, 444.0, "eacute"), + (-1, 500.0, "abreve"), + (-1, 556.0, "uhungarumlaut"), + (-1, 444.0, "ecaron"), + (-1, 722.0, "Ydieresis"), + (-1, 570.0, "divide"), + (-1, 722.0, "Yacute"), + (-1, 722.0, "Acircumflex"), + (-1, 500.0, "aacute"), + (-1, 722.0, "Ucircumflex"), + (-1, 500.0, "yacute"), + (-1, 389.0, "scommaaccent"), + (-1, 444.0, "ecircumflex"), + (-1, 722.0, "Uring"), + (-1, 722.0, "Udieresis"), + (-1, 500.0, "aogonek"), + (-1, 722.0, "Uacute"), + (-1, 556.0, "uogonek"), + (-1, 667.0, "Edieresis"), + (-1, 722.0, "Dcroat"), + (-1, 250.0, "commaaccent"), + (-1, 747.0, "copyright"), + (-1, 667.0, "Emacron"), + (-1, 444.0, "ccaron"), + (-1, 500.0, "aring"), + (-1, 722.0, "Ncommaaccent"), + (-1, 278.0, "lacute"), + (-1, 500.0, "agrave"), + (-1, 667.0, "Tcommaaccent"), + (-1, 722.0, "Cacute"), + (-1, 500.0, "atilde"), + (-1, 667.0, "Edotaccent"), + (-1, 389.0, "scaron"), + (-1, 389.0, "scedilla"), + (-1, 278.0, "iacute"), + (-1, 494.0, "lozenge"), + (-1, 722.0, "Rcaron"), + (-1, 778.0, "Gcommaaccent"), + (-1, 556.0, "ucircumflex"), + (-1, 500.0, "acircumflex"), + (-1, 722.0, "Amacron"), + (-1, 444.0, "rcaron"), + (-1, 444.0, "ccedilla"), + (-1, 667.0, "Zdotaccent"), + (-1, 611.0, "Thorn"), + (-1, 778.0, "Omacron"), + (-1, 722.0, "Racute"), + (-1, 556.0, "Sacute"), + (-1, 672.0, "dcaron"), + (-1, 722.0, "Umacron"), + (-1, 556.0, "uring"), + (-1, 300.0, "threesuperior"), + (-1, 778.0, "Ograve"), + (-1, 722.0, "Agrave"), + (-1, 722.0, "Abreve"), + (-1, 570.0, "multiply"), + (-1, 556.0, "uacute"), + (-1, 667.0, "Tcaron"), + (-1, 494.0, "partialdiff"), + (-1, 500.0, "ydieresis"), + (-1, 722.0, "Nacute"), + (-1, 278.0, "icircumflex"), + (-1, 667.0, "Ecircumflex"), + (-1, 500.0, "adieresis"), + (-1, 444.0, "edieresis"), + (-1, 444.0, "cacute"), + (-1, 556.0, "nacute"), + (-1, 556.0, "umacron"), + (-1, 722.0, "Ncaron"), + (-1, 389.0, "Iacute"), + (-1, 570.0, "plusminus"), + (-1, 220.0, "brokenbar"), + (-1, 747.0, "registered"), + (-1, 778.0, "Gbreve"), + (-1, 389.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 667.0, "Egrave"), + (-1, 444.0, "racute"), + (-1, 500.0, "omacron"), + (-1, 667.0, "Zacute"), + (-1, 667.0, "Zcaron"), + (-1, 549.0, "greaterequal"), + (-1, 722.0, "Eth"), + (-1, 722.0, "Ccedilla"), + (-1, 278.0, "lcommaaccent"), + (-1, 416.0, "tcaron"), + (-1, 444.0, "eogonek"), + (-1, 722.0, "Uogonek"), + (-1, 722.0, "Aacute"), + (-1, 722.0, "Adieresis"), + (-1, 444.0, "egrave"), + (-1, 444.0, "zacute"), + (-1, 278.0, "iogonek"), + (-1, 778.0, "Oacute"), + (-1, 500.0, "oacute"), + (-1, 500.0, "amacron"), + (-1, 389.0, "sacute"), + (-1, 278.0, "idieresis"), + (-1, 778.0, "Ocircumflex"), + (-1, 722.0, "Ugrave"), + (-1, 612.0, "Delta"), + (-1, 556.0, "thorn"), + (-1, 300.0, "twosuperior"), + (-1, 778.0, "Odieresis"), + (-1, 556.0, "mu"), + (-1, 278.0, "igrave"), + (-1, 500.0, "ohungarumlaut"), + (-1, 667.0, "Eogonek"), + (-1, 556.0, "dcroat"), + (-1, 750.0, "threequarters"), + (-1, 556.0, "Scedilla"), + (-1, 394.0, "lcaron"), + (-1, 778.0, "Kcommaaccent"), + (-1, 667.0, "Lacute"), + (-1, 1000.0, "trademark"), + (-1, 444.0, "edotaccent"), + (-1, 389.0, "Igrave"), + (-1, 389.0, "Imacron"), + (-1, 667.0, "Lcaron"), + (-1, 750.0, "onehalf"), + (-1, 549.0, "lessequal"), + (-1, 500.0, "ocircumflex"), + (-1, 556.0, "ntilde"), + (-1, 722.0, "Uhungarumlaut"), + (-1, 667.0, "Eacute"), + (-1, 444.0, "emacron"), + (-1, 500.0, "gbreve"), + (-1, 750.0, "onequarter"), + (-1, 556.0, "Scaron"), + (-1, 556.0, "Scommaaccent"), + (-1, 778.0, "Ohungarumlaut"), + (-1, 400.0, "degree"), + (-1, 500.0, "ograve"), + (-1, 722.0, "Ccaron"), + (-1, 556.0, "ugrave"), + (-1, 549.0, "radical"), + (-1, 722.0, "Dcaron"), + (-1, 444.0, "rcommaaccent"), + (-1, 722.0, "Ntilde"), + (-1, 500.0, "otilde"), + (-1, 722.0, "Rcommaaccent"), + (-1, 667.0, "Lcommaaccent"), + (-1, 722.0, "Atilde"), + (-1, 722.0, "Aogonek"), + (-1, 722.0, "Aring"), + (-1, 778.0, "Otilde"), + (-1, 444.0, "zdotaccent"), + (-1, 667.0, "Ecaron"), + (-1, 389.0, "Iogonek"), + (-1, 556.0, "kcommaaccent"), + (-1, 570.0, "minus"), + (-1, 389.0, "Icircumflex"), + (-1, 556.0, "ncaron"), + (-1, 333.0, "tcommaaccent"), + (-1, 570.0, "logicalnot"), + (-1, 500.0, "odieresis"), + (-1, 556.0, "udieresis"), + (-1, 549.0, "notequal"), + (-1, 500.0, "gcommaaccent"), + (-1, 500.0, "eth"), + (-1, 444.0, "zcaron"), + (-1, 556.0, "ncommaaccent"), + (-1, 300.0, "onesuperior"), + (-1, 278.0, "imacron"), + (-1, 500.0, "Euro"), + ][..], + ), + ( + "Times-BoldItalic", + "AdobeStandardEncoding", + &[ + (32, 250.0, "space"), + (33, 389.0, "exclam"), + (34, 555.0, "quotedbl"), + (35, 500.0, "numbersign"), + (36, 500.0, "dollar"), + (37, 833.0, "percent"), + (38, 778.0, "ampersand"), + (39, 333.0, "quoteright"), + (40, 333.0, "parenleft"), + (41, 333.0, "parenright"), + (42, 500.0, "asterisk"), + (43, 570.0, "plus"), + (44, 250.0, "comma"), + (45, 333.0, "hyphen"), + (46, 250.0, "period"), + (47, 278.0, "slash"), + (48, 500.0, "zero"), + (49, 500.0, "one"), + (50, 500.0, "two"), + (51, 500.0, "three"), + (52, 500.0, "four"), + (53, 500.0, "five"), + (54, 500.0, "six"), + (55, 500.0, "seven"), + (56, 500.0, "eight"), + (57, 500.0, "nine"), + (58, 333.0, "colon"), + (59, 333.0, "semicolon"), + (60, 570.0, "less"), + (61, 570.0, "equal"), + (62, 570.0, "greater"), + (63, 500.0, "question"), + (64, 832.0, "at"), + (65, 667.0, "A"), + (66, 667.0, "B"), + (67, 667.0, "C"), + (68, 722.0, "D"), + (69, 667.0, "E"), + (70, 667.0, "F"), + (71, 722.0, "G"), + (72, 778.0, "H"), + (73, 389.0, "I"), + (74, 500.0, "J"), + (75, 667.0, "K"), + (76, 611.0, "L"), + (77, 889.0, "M"), + (78, 722.0, "N"), + (79, 722.0, "O"), + (80, 611.0, "P"), + (81, 722.0, "Q"), + (82, 667.0, "R"), + (83, 556.0, "S"), + (84, 611.0, "T"), + (85, 722.0, "U"), + (86, 667.0, "V"), + (87, 889.0, "W"), + (88, 667.0, "X"), + (89, 611.0, "Y"), + (90, 611.0, "Z"), + (91, 333.0, "bracketleft"), + (92, 278.0, "backslash"), + (93, 333.0, "bracketright"), + (94, 570.0, "asciicircum"), + (95, 500.0, "underscore"), + (96, 333.0, "quoteleft"), + (97, 500.0, "a"), + (98, 500.0, "b"), + (99, 444.0, "c"), + (100, 500.0, "d"), + (101, 444.0, "e"), + (102, 333.0, "f"), + (103, 500.0, "g"), + (104, 556.0, "h"), + (105, 278.0, "i"), + (106, 278.0, "j"), + (107, 500.0, "k"), + (108, 278.0, "l"), + (109, 778.0, "m"), + (110, 556.0, "n"), + (111, 500.0, "o"), + (112, 500.0, "p"), + (113, 500.0, "q"), + (114, 389.0, "r"), + (115, 389.0, "s"), + (116, 278.0, "t"), + (117, 556.0, "u"), + (118, 444.0, "v"), + (119, 667.0, "w"), + (120, 500.0, "x"), + (121, 444.0, "y"), + (122, 389.0, "z"), + (123, 348.0, "braceleft"), + (124, 220.0, "bar"), + (125, 348.0, "braceright"), + (126, 570.0, "asciitilde"), + (161, 389.0, "exclamdown"), + (162, 500.0, "cent"), + (163, 500.0, "sterling"), + (164, 167.0, "fraction"), + (165, 500.0, "yen"), + (166, 500.0, "florin"), + (167, 500.0, "section"), + (168, 500.0, "currency"), + (169, 278.0, "quotesingle"), + (170, 500.0, "quotedblleft"), + (171, 500.0, "guillemotleft"), + (172, 333.0, "guilsinglleft"), + (173, 333.0, "guilsinglright"), + (174, 556.0, "fi"), + (175, 556.0, "fl"), + (177, 500.0, "endash"), + (178, 500.0, "dagger"), + (179, 500.0, "daggerdbl"), + (180, 250.0, "periodcentered"), + (182, 500.0, "paragraph"), + (183, 350.0, "bullet"), + (184, 333.0, "quotesinglbase"), + (185, 500.0, "quotedblbase"), + (186, 500.0, "quotedblright"), + (187, 500.0, "guillemotright"), + (188, 1000.0, "ellipsis"), + (189, 1000.0, "perthousand"), + (191, 500.0, "questiondown"), + (193, 333.0, "grave"), + (194, 333.0, "acute"), + (195, 333.0, "circumflex"), + (196, 333.0, "tilde"), + (197, 333.0, "macron"), + (198, 333.0, "breve"), + (199, 333.0, "dotaccent"), + (200, 333.0, "dieresis"), + (202, 333.0, "ring"), + (203, 333.0, "cedilla"), + (205, 333.0, "hungarumlaut"), + (206, 333.0, "ogonek"), + (207, 333.0, "caron"), + (208, 1000.0, "emdash"), + (225, 944.0, "AE"), + (227, 266.0, "ordfeminine"), + (232, 611.0, "Lslash"), + (233, 722.0, "Oslash"), + (234, 944.0, "OE"), + (235, 300.0, "ordmasculine"), + (241, 722.0, "ae"), + (245, 278.0, "dotlessi"), + (248, 278.0, "lslash"), + (249, 500.0, "oslash"), + (250, 722.0, "oe"), + (251, 500.0, "germandbls"), + (-1, 389.0, "Idieresis"), + (-1, 444.0, "eacute"), + (-1, 500.0, "abreve"), + (-1, 556.0, "uhungarumlaut"), + (-1, 444.0, "ecaron"), + (-1, 611.0, "Ydieresis"), + (-1, 570.0, "divide"), + (-1, 611.0, "Yacute"), + (-1, 667.0, "Acircumflex"), + (-1, 500.0, "aacute"), + (-1, 722.0, "Ucircumflex"), + (-1, 444.0, "yacute"), + (-1, 389.0, "scommaaccent"), + (-1, 444.0, "ecircumflex"), + (-1, 722.0, "Uring"), + (-1, 722.0, "Udieresis"), + (-1, 500.0, "aogonek"), + (-1, 722.0, "Uacute"), + (-1, 556.0, "uogonek"), + (-1, 667.0, "Edieresis"), + (-1, 722.0, "Dcroat"), + (-1, 250.0, "commaaccent"), + (-1, 747.0, "copyright"), + (-1, 667.0, "Emacron"), + (-1, 444.0, "ccaron"), + (-1, 500.0, "aring"), + (-1, 722.0, "Ncommaaccent"), + (-1, 278.0, "lacute"), + (-1, 500.0, "agrave"), + (-1, 611.0, "Tcommaaccent"), + (-1, 667.0, "Cacute"), + (-1, 500.0, "atilde"), + (-1, 667.0, "Edotaccent"), + (-1, 389.0, "scaron"), + (-1, 389.0, "scedilla"), + (-1, 278.0, "iacute"), + (-1, 494.0, "lozenge"), + (-1, 667.0, "Rcaron"), + (-1, 722.0, "Gcommaaccent"), + (-1, 556.0, "ucircumflex"), + (-1, 500.0, "acircumflex"), + (-1, 667.0, "Amacron"), + (-1, 389.0, "rcaron"), + (-1, 444.0, "ccedilla"), + (-1, 611.0, "Zdotaccent"), + (-1, 611.0, "Thorn"), + (-1, 722.0, "Omacron"), + (-1, 667.0, "Racute"), + (-1, 556.0, "Sacute"), + (-1, 608.0, "dcaron"), + (-1, 722.0, "Umacron"), + (-1, 556.0, "uring"), + (-1, 300.0, "threesuperior"), + (-1, 722.0, "Ograve"), + (-1, 667.0, "Agrave"), + (-1, 667.0, "Abreve"), + (-1, 570.0, "multiply"), + (-1, 556.0, "uacute"), + (-1, 611.0, "Tcaron"), + (-1, 494.0, "partialdiff"), + (-1, 444.0, "ydieresis"), + (-1, 722.0, "Nacute"), + (-1, 278.0, "icircumflex"), + (-1, 667.0, "Ecircumflex"), + (-1, 500.0, "adieresis"), + (-1, 444.0, "edieresis"), + (-1, 444.0, "cacute"), + (-1, 556.0, "nacute"), + (-1, 556.0, "umacron"), + (-1, 722.0, "Ncaron"), + (-1, 389.0, "Iacute"), + (-1, 570.0, "plusminus"), + (-1, 220.0, "brokenbar"), + (-1, 747.0, "registered"), + (-1, 722.0, "Gbreve"), + (-1, 389.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 667.0, "Egrave"), + (-1, 389.0, "racute"), + (-1, 500.0, "omacron"), + (-1, 611.0, "Zacute"), + (-1, 611.0, "Zcaron"), + (-1, 549.0, "greaterequal"), + (-1, 722.0, "Eth"), + (-1, 667.0, "Ccedilla"), + (-1, 278.0, "lcommaaccent"), + (-1, 366.0, "tcaron"), + (-1, 444.0, "eogonek"), + (-1, 722.0, "Uogonek"), + (-1, 667.0, "Aacute"), + (-1, 667.0, "Adieresis"), + (-1, 444.0, "egrave"), + (-1, 389.0, "zacute"), + (-1, 278.0, "iogonek"), + (-1, 722.0, "Oacute"), + (-1, 500.0, "oacute"), + (-1, 500.0, "amacron"), + (-1, 389.0, "sacute"), + (-1, 278.0, "idieresis"), + (-1, 722.0, "Ocircumflex"), + (-1, 722.0, "Ugrave"), + (-1, 612.0, "Delta"), + (-1, 500.0, "thorn"), + (-1, 300.0, "twosuperior"), + (-1, 722.0, "Odieresis"), + (-1, 576.0, "mu"), + (-1, 278.0, "igrave"), + (-1, 500.0, "ohungarumlaut"), + (-1, 667.0, "Eogonek"), + (-1, 500.0, "dcroat"), + (-1, 750.0, "threequarters"), + (-1, 556.0, "Scedilla"), + (-1, 382.0, "lcaron"), + (-1, 667.0, "Kcommaaccent"), + (-1, 611.0, "Lacute"), + (-1, 1000.0, "trademark"), + (-1, 444.0, "edotaccent"), + (-1, 389.0, "Igrave"), + (-1, 389.0, "Imacron"), + (-1, 611.0, "Lcaron"), + (-1, 750.0, "onehalf"), + (-1, 549.0, "lessequal"), + (-1, 500.0, "ocircumflex"), + (-1, 556.0, "ntilde"), + (-1, 722.0, "Uhungarumlaut"), + (-1, 667.0, "Eacute"), + (-1, 444.0, "emacron"), + (-1, 500.0, "gbreve"), + (-1, 750.0, "onequarter"), + (-1, 556.0, "Scaron"), + (-1, 556.0, "Scommaaccent"), + (-1, 722.0, "Ohungarumlaut"), + (-1, 400.0, "degree"), + (-1, 500.0, "ograve"), + (-1, 667.0, "Ccaron"), + (-1, 556.0, "ugrave"), + (-1, 549.0, "radical"), + (-1, 722.0, "Dcaron"), + (-1, 389.0, "rcommaaccent"), + (-1, 722.0, "Ntilde"), + (-1, 500.0, "otilde"), + (-1, 667.0, "Rcommaaccent"), + (-1, 611.0, "Lcommaaccent"), + (-1, 667.0, "Atilde"), + (-1, 667.0, "Aogonek"), + (-1, 667.0, "Aring"), + (-1, 722.0, "Otilde"), + (-1, 389.0, "zdotaccent"), + (-1, 667.0, "Ecaron"), + (-1, 389.0, "Iogonek"), + (-1, 500.0, "kcommaaccent"), + (-1, 606.0, "minus"), + (-1, 389.0, "Icircumflex"), + (-1, 556.0, "ncaron"), + (-1, 278.0, "tcommaaccent"), + (-1, 606.0, "logicalnot"), + (-1, 500.0, "odieresis"), + (-1, 556.0, "udieresis"), + (-1, 549.0, "notequal"), + (-1, 500.0, "gcommaaccent"), + (-1, 500.0, "eth"), + (-1, 389.0, "zcaron"), + (-1, 556.0, "ncommaaccent"), + (-1, 300.0, "onesuperior"), + (-1, 278.0, "imacron"), + (-1, 500.0, "Euro"), + ][..], + ), + ( + "Times-Italic", + "AdobeStandardEncoding", + &[ + (32, 250.0, "space"), + (33, 333.0, "exclam"), + (34, 420.0, "quotedbl"), + (35, 500.0, "numbersign"), + (36, 500.0, "dollar"), + (37, 833.0, "percent"), + (38, 778.0, "ampersand"), + (39, 333.0, "quoteright"), + (40, 333.0, "parenleft"), + (41, 333.0, "parenright"), + (42, 500.0, "asterisk"), + (43, 675.0, "plus"), + (44, 250.0, "comma"), + (45, 333.0, "hyphen"), + (46, 250.0, "period"), + (47, 278.0, "slash"), + (48, 500.0, "zero"), + (49, 500.0, "one"), + (50, 500.0, "two"), + (51, 500.0, "three"), + (52, 500.0, "four"), + (53, 500.0, "five"), + (54, 500.0, "six"), + (55, 500.0, "seven"), + (56, 500.0, "eight"), + (57, 500.0, "nine"), + (58, 333.0, "colon"), + (59, 333.0, "semicolon"), + (60, 675.0, "less"), + (61, 675.0, "equal"), + (62, 675.0, "greater"), + (63, 500.0, "question"), + (64, 920.0, "at"), + (65, 611.0, "A"), + (66, 611.0, "B"), + (67, 667.0, "C"), + (68, 722.0, "D"), + (69, 611.0, "E"), + (70, 611.0, "F"), + (71, 722.0, "G"), + (72, 722.0, "H"), + (73, 333.0, "I"), + (74, 444.0, "J"), + (75, 667.0, "K"), + (76, 556.0, "L"), + (77, 833.0, "M"), + (78, 667.0, "N"), + (79, 722.0, "O"), + (80, 611.0, "P"), + (81, 722.0, "Q"), + (82, 611.0, "R"), + (83, 500.0, "S"), + (84, 556.0, "T"), + (85, 722.0, "U"), + (86, 611.0, "V"), + (87, 833.0, "W"), + (88, 611.0, "X"), + (89, 556.0, "Y"), + (90, 556.0, "Z"), + (91, 389.0, "bracketleft"), + (92, 278.0, "backslash"), + (93, 389.0, "bracketright"), + (94, 422.0, "asciicircum"), + (95, 500.0, "underscore"), + (96, 333.0, "quoteleft"), + (97, 500.0, "a"), + (98, 500.0, "b"), + (99, 444.0, "c"), + (100, 500.0, "d"), + (101, 444.0, "e"), + (102, 278.0, "f"), + (103, 500.0, "g"), + (104, 500.0, "h"), + (105, 278.0, "i"), + (106, 278.0, "j"), + (107, 444.0, "k"), + (108, 278.0, "l"), + (109, 722.0, "m"), + (110, 500.0, "n"), + (111, 500.0, "o"), + (112, 500.0, "p"), + (113, 500.0, "q"), + (114, 389.0, "r"), + (115, 389.0, "s"), + (116, 278.0, "t"), + (117, 500.0, "u"), + (118, 444.0, "v"), + (119, 667.0, "w"), + (120, 444.0, "x"), + (121, 444.0, "y"), + (122, 389.0, "z"), + (123, 400.0, "braceleft"), + (124, 275.0, "bar"), + (125, 400.0, "braceright"), + (126, 541.0, "asciitilde"), + (161, 389.0, "exclamdown"), + (162, 500.0, "cent"), + (163, 500.0, "sterling"), + (164, 167.0, "fraction"), + (165, 500.0, "yen"), + (166, 500.0, "florin"), + (167, 500.0, "section"), + (168, 500.0, "currency"), + (169, 214.0, "quotesingle"), + (170, 556.0, "quotedblleft"), + (171, 500.0, "guillemotleft"), + (172, 333.0, "guilsinglleft"), + (173, 333.0, "guilsinglright"), + (174, 500.0, "fi"), + (175, 500.0, "fl"), + (177, 500.0, "endash"), + (178, 500.0, "dagger"), + (179, 500.0, "daggerdbl"), + (180, 250.0, "periodcentered"), + (182, 523.0, "paragraph"), + (183, 350.0, "bullet"), + (184, 333.0, "quotesinglbase"), + (185, 556.0, "quotedblbase"), + (186, 556.0, "quotedblright"), + (187, 500.0, "guillemotright"), + (188, 889.0, "ellipsis"), + (189, 1000.0, "perthousand"), + (191, 500.0, "questiondown"), + (193, 333.0, "grave"), + (194, 333.0, "acute"), + (195, 333.0, "circumflex"), + (196, 333.0, "tilde"), + (197, 333.0, "macron"), + (198, 333.0, "breve"), + (199, 333.0, "dotaccent"), + (200, 333.0, "dieresis"), + (202, 333.0, "ring"), + (203, 333.0, "cedilla"), + (205, 333.0, "hungarumlaut"), + (206, 333.0, "ogonek"), + (207, 333.0, "caron"), + (208, 889.0, "emdash"), + (225, 889.0, "AE"), + (227, 276.0, "ordfeminine"), + (232, 556.0, "Lslash"), + (233, 722.0, "Oslash"), + (234, 944.0, "OE"), + (235, 310.0, "ordmasculine"), + (241, 667.0, "ae"), + (245, 278.0, "dotlessi"), + (248, 278.0, "lslash"), + (249, 500.0, "oslash"), + (250, 667.0, "oe"), + (251, 500.0, "germandbls"), + (-1, 333.0, "Idieresis"), + (-1, 444.0, "eacute"), + (-1, 500.0, "abreve"), + (-1, 500.0, "uhungarumlaut"), + (-1, 444.0, "ecaron"), + (-1, 556.0, "Ydieresis"), + (-1, 675.0, "divide"), + (-1, 556.0, "Yacute"), + (-1, 611.0, "Acircumflex"), + (-1, 500.0, "aacute"), + (-1, 722.0, "Ucircumflex"), + (-1, 444.0, "yacute"), + (-1, 389.0, "scommaaccent"), + (-1, 444.0, "ecircumflex"), + (-1, 722.0, "Uring"), + (-1, 722.0, "Udieresis"), + (-1, 500.0, "aogonek"), + (-1, 722.0, "Uacute"), + (-1, 500.0, "uogonek"), + (-1, 611.0, "Edieresis"), + (-1, 722.0, "Dcroat"), + (-1, 250.0, "commaaccent"), + (-1, 760.0, "copyright"), + (-1, 611.0, "Emacron"), + (-1, 444.0, "ccaron"), + (-1, 500.0, "aring"), + (-1, 667.0, "Ncommaaccent"), + (-1, 278.0, "lacute"), + (-1, 500.0, "agrave"), + (-1, 556.0, "Tcommaaccent"), + (-1, 667.0, "Cacute"), + (-1, 500.0, "atilde"), + (-1, 611.0, "Edotaccent"), + (-1, 389.0, "scaron"), + (-1, 389.0, "scedilla"), + (-1, 278.0, "iacute"), + (-1, 471.0, "lozenge"), + (-1, 611.0, "Rcaron"), + (-1, 722.0, "Gcommaaccent"), + (-1, 500.0, "ucircumflex"), + (-1, 500.0, "acircumflex"), + (-1, 611.0, "Amacron"), + (-1, 389.0, "rcaron"), + (-1, 444.0, "ccedilla"), + (-1, 556.0, "Zdotaccent"), + (-1, 611.0, "Thorn"), + (-1, 722.0, "Omacron"), + (-1, 611.0, "Racute"), + (-1, 500.0, "Sacute"), + (-1, 544.0, "dcaron"), + (-1, 722.0, "Umacron"), + (-1, 500.0, "uring"), + (-1, 300.0, "threesuperior"), + (-1, 722.0, "Ograve"), + (-1, 611.0, "Agrave"), + (-1, 611.0, "Abreve"), + (-1, 675.0, "multiply"), + (-1, 500.0, "uacute"), + (-1, 556.0, "Tcaron"), + (-1, 476.0, "partialdiff"), + (-1, 444.0, "ydieresis"), + (-1, 667.0, "Nacute"), + (-1, 278.0, "icircumflex"), + (-1, 611.0, "Ecircumflex"), + (-1, 500.0, "adieresis"), + (-1, 444.0, "edieresis"), + (-1, 444.0, "cacute"), + (-1, 500.0, "nacute"), + (-1, 500.0, "umacron"), + (-1, 667.0, "Ncaron"), + (-1, 333.0, "Iacute"), + (-1, 675.0, "plusminus"), + (-1, 275.0, "brokenbar"), + (-1, 760.0, "registered"), + (-1, 722.0, "Gbreve"), + (-1, 333.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 611.0, "Egrave"), + (-1, 389.0, "racute"), + (-1, 500.0, "omacron"), + (-1, 556.0, "Zacute"), + (-1, 556.0, "Zcaron"), + (-1, 549.0, "greaterequal"), + (-1, 722.0, "Eth"), + (-1, 667.0, "Ccedilla"), + (-1, 278.0, "lcommaaccent"), + (-1, 300.0, "tcaron"), + (-1, 444.0, "eogonek"), + (-1, 722.0, "Uogonek"), + (-1, 611.0, "Aacute"), + (-1, 611.0, "Adieresis"), + (-1, 444.0, "egrave"), + (-1, 389.0, "zacute"), + (-1, 278.0, "iogonek"), + (-1, 722.0, "Oacute"), + (-1, 500.0, "oacute"), + (-1, 500.0, "amacron"), + (-1, 389.0, "sacute"), + (-1, 278.0, "idieresis"), + (-1, 722.0, "Ocircumflex"), + (-1, 722.0, "Ugrave"), + (-1, 612.0, "Delta"), + (-1, 500.0, "thorn"), + (-1, 300.0, "twosuperior"), + (-1, 722.0, "Odieresis"), + (-1, 500.0, "mu"), + (-1, 278.0, "igrave"), + (-1, 500.0, "ohungarumlaut"), + (-1, 611.0, "Eogonek"), + (-1, 500.0, "dcroat"), + (-1, 750.0, "threequarters"), + (-1, 500.0, "Scedilla"), + (-1, 300.0, "lcaron"), + (-1, 667.0, "Kcommaaccent"), + (-1, 556.0, "Lacute"), + (-1, 980.0, "trademark"), + (-1, 444.0, "edotaccent"), + (-1, 333.0, "Igrave"), + (-1, 333.0, "Imacron"), + (-1, 611.0, "Lcaron"), + (-1, 750.0, "onehalf"), + (-1, 549.0, "lessequal"), + (-1, 500.0, "ocircumflex"), + (-1, 500.0, "ntilde"), + (-1, 722.0, "Uhungarumlaut"), + (-1, 611.0, "Eacute"), + (-1, 444.0, "emacron"), + (-1, 500.0, "gbreve"), + (-1, 750.0, "onequarter"), + (-1, 500.0, "Scaron"), + (-1, 500.0, "Scommaaccent"), + (-1, 722.0, "Ohungarumlaut"), + (-1, 400.0, "degree"), + (-1, 500.0, "ograve"), + (-1, 667.0, "Ccaron"), + (-1, 500.0, "ugrave"), + (-1, 453.0, "radical"), + (-1, 722.0, "Dcaron"), + (-1, 389.0, "rcommaaccent"), + (-1, 667.0, "Ntilde"), + (-1, 500.0, "otilde"), + (-1, 611.0, "Rcommaaccent"), + (-1, 556.0, "Lcommaaccent"), + (-1, 611.0, "Atilde"), + (-1, 611.0, "Aogonek"), + (-1, 611.0, "Aring"), + (-1, 722.0, "Otilde"), + (-1, 389.0, "zdotaccent"), + (-1, 611.0, "Ecaron"), + (-1, 333.0, "Iogonek"), + (-1, 444.0, "kcommaaccent"), + (-1, 675.0, "minus"), + (-1, 333.0, "Icircumflex"), + (-1, 500.0, "ncaron"), + (-1, 278.0, "tcommaaccent"), + (-1, 675.0, "logicalnot"), + (-1, 500.0, "odieresis"), + (-1, 500.0, "udieresis"), + (-1, 549.0, "notequal"), + (-1, 500.0, "gcommaaccent"), + (-1, 500.0, "eth"), + (-1, 389.0, "zcaron"), + (-1, 500.0, "ncommaaccent"), + (-1, 300.0, "onesuperior"), + (-1, 278.0, "imacron"), + (-1, 500.0, "Euro"), + ][..], + ), + ( + "Times-Roman", + "AdobeStandardEncoding", + &[ + (32, 250.0, "space"), + (33, 333.0, "exclam"), + (34, 408.0, "quotedbl"), + (35, 500.0, "numbersign"), + (36, 500.0, "dollar"), + (37, 833.0, "percent"), + (38, 778.0, "ampersand"), + (39, 333.0, "quoteright"), + (40, 333.0, "parenleft"), + (41, 333.0, "parenright"), + (42, 500.0, "asterisk"), + (43, 564.0, "plus"), + (44, 250.0, "comma"), + (45, 333.0, "hyphen"), + (46, 250.0, "period"), + (47, 278.0, "slash"), + (48, 500.0, "zero"), + (49, 500.0, "one"), + (50, 500.0, "two"), + (51, 500.0, "three"), + (52, 500.0, "four"), + (53, 500.0, "five"), + (54, 500.0, "six"), + (55, 500.0, "seven"), + (56, 500.0, "eight"), + (57, 500.0, "nine"), + (58, 278.0, "colon"), + (59, 278.0, "semicolon"), + (60, 564.0, "less"), + (61, 564.0, "equal"), + (62, 564.0, "greater"), + (63, 444.0, "question"), + (64, 921.0, "at"), + (65, 722.0, "A"), + (66, 667.0, "B"), + (67, 667.0, "C"), + (68, 722.0, "D"), + (69, 611.0, "E"), + (70, 556.0, "F"), + (71, 722.0, "G"), + (72, 722.0, "H"), + (73, 333.0, "I"), + (74, 389.0, "J"), + (75, 722.0, "K"), + (76, 611.0, "L"), + (77, 889.0, "M"), + (78, 722.0, "N"), + (79, 722.0, "O"), + (80, 556.0, "P"), + (81, 722.0, "Q"), + (82, 667.0, "R"), + (83, 556.0, "S"), + (84, 611.0, "T"), + (85, 722.0, "U"), + (86, 722.0, "V"), + (87, 944.0, "W"), + (88, 722.0, "X"), + (89, 722.0, "Y"), + (90, 611.0, "Z"), + (91, 333.0, "bracketleft"), + (92, 278.0, "backslash"), + (93, 333.0, "bracketright"), + (94, 469.0, "asciicircum"), + (95, 500.0, "underscore"), + (96, 333.0, "quoteleft"), + (97, 444.0, "a"), + (98, 500.0, "b"), + (99, 444.0, "c"), + (100, 500.0, "d"), + (101, 444.0, "e"), + (102, 333.0, "f"), + (103, 500.0, "g"), + (104, 500.0, "h"), + (105, 278.0, "i"), + (106, 278.0, "j"), + (107, 500.0, "k"), + (108, 278.0, "l"), + (109, 778.0, "m"), + (110, 500.0, "n"), + (111, 500.0, "o"), + (112, 500.0, "p"), + (113, 500.0, "q"), + (114, 333.0, "r"), + (115, 389.0, "s"), + (116, 278.0, "t"), + (117, 500.0, "u"), + (118, 500.0, "v"), + (119, 722.0, "w"), + (120, 500.0, "x"), + (121, 500.0, "y"), + (122, 444.0, "z"), + (123, 480.0, "braceleft"), + (124, 200.0, "bar"), + (125, 480.0, "braceright"), + (126, 541.0, "asciitilde"), + (161, 333.0, "exclamdown"), + (162, 500.0, "cent"), + (163, 500.0, "sterling"), + (164, 167.0, "fraction"), + (165, 500.0, "yen"), + (166, 500.0, "florin"), + (167, 500.0, "section"), + (168, 500.0, "currency"), + (169, 180.0, "quotesingle"), + (170, 444.0, "quotedblleft"), + (171, 500.0, "guillemotleft"), + (172, 333.0, "guilsinglleft"), + (173, 333.0, "guilsinglright"), + (174, 556.0, "fi"), + (175, 556.0, "fl"), + (177, 500.0, "endash"), + (178, 500.0, "dagger"), + (179, 500.0, "daggerdbl"), + (180, 250.0, "periodcentered"), + (182, 453.0, "paragraph"), + (183, 350.0, "bullet"), + (184, 333.0, "quotesinglbase"), + (185, 444.0, "quotedblbase"), + (186, 444.0, "quotedblright"), + (187, 500.0, "guillemotright"), + (188, 1000.0, "ellipsis"), + (189, 1000.0, "perthousand"), + (191, 444.0, "questiondown"), + (193, 333.0, "grave"), + (194, 333.0, "acute"), + (195, 333.0, "circumflex"), + (196, 333.0, "tilde"), + (197, 333.0, "macron"), + (198, 333.0, "breve"), + (199, 333.0, "dotaccent"), + (200, 333.0, "dieresis"), + (202, 333.0, "ring"), + (203, 333.0, "cedilla"), + (205, 333.0, "hungarumlaut"), + (206, 333.0, "ogonek"), + (207, 333.0, "caron"), + (208, 1000.0, "emdash"), + (225, 889.0, "AE"), + (227, 276.0, "ordfeminine"), + (232, 611.0, "Lslash"), + (233, 722.0, "Oslash"), + (234, 889.0, "OE"), + (235, 310.0, "ordmasculine"), + (241, 667.0, "ae"), + (245, 278.0, "dotlessi"), + (248, 278.0, "lslash"), + (249, 500.0, "oslash"), + (250, 722.0, "oe"), + (251, 500.0, "germandbls"), + (-1, 333.0, "Idieresis"), + (-1, 444.0, "eacute"), + (-1, 444.0, "abreve"), + (-1, 500.0, "uhungarumlaut"), + (-1, 444.0, "ecaron"), + (-1, 722.0, "Ydieresis"), + (-1, 564.0, "divide"), + (-1, 722.0, "Yacute"), + (-1, 722.0, "Acircumflex"), + (-1, 444.0, "aacute"), + (-1, 722.0, "Ucircumflex"), + (-1, 500.0, "yacute"), + (-1, 389.0, "scommaaccent"), + (-1, 444.0, "ecircumflex"), + (-1, 722.0, "Uring"), + (-1, 722.0, "Udieresis"), + (-1, 444.0, "aogonek"), + (-1, 722.0, "Uacute"), + (-1, 500.0, "uogonek"), + (-1, 611.0, "Edieresis"), + (-1, 722.0, "Dcroat"), + (-1, 250.0, "commaaccent"), + (-1, 760.0, "copyright"), + (-1, 611.0, "Emacron"), + (-1, 444.0, "ccaron"), + (-1, 444.0, "aring"), + (-1, 722.0, "Ncommaaccent"), + (-1, 278.0, "lacute"), + (-1, 444.0, "agrave"), + (-1, 611.0, "Tcommaaccent"), + (-1, 667.0, "Cacute"), + (-1, 444.0, "atilde"), + (-1, 611.0, "Edotaccent"), + (-1, 389.0, "scaron"), + (-1, 389.0, "scedilla"), + (-1, 278.0, "iacute"), + (-1, 471.0, "lozenge"), + (-1, 667.0, "Rcaron"), + (-1, 722.0, "Gcommaaccent"), + (-1, 500.0, "ucircumflex"), + (-1, 444.0, "acircumflex"), + (-1, 722.0, "Amacron"), + (-1, 333.0, "rcaron"), + (-1, 444.0, "ccedilla"), + (-1, 611.0, "Zdotaccent"), + (-1, 556.0, "Thorn"), + (-1, 722.0, "Omacron"), + (-1, 667.0, "Racute"), + (-1, 556.0, "Sacute"), + (-1, 588.0, "dcaron"), + (-1, 722.0, "Umacron"), + (-1, 500.0, "uring"), + (-1, 300.0, "threesuperior"), + (-1, 722.0, "Ograve"), + (-1, 722.0, "Agrave"), + (-1, 722.0, "Abreve"), + (-1, 564.0, "multiply"), + (-1, 500.0, "uacute"), + (-1, 611.0, "Tcaron"), + (-1, 476.0, "partialdiff"), + (-1, 500.0, "ydieresis"), + (-1, 722.0, "Nacute"), + (-1, 278.0, "icircumflex"), + (-1, 611.0, "Ecircumflex"), + (-1, 444.0, "adieresis"), + (-1, 444.0, "edieresis"), + (-1, 444.0, "cacute"), + (-1, 500.0, "nacute"), + (-1, 500.0, "umacron"), + (-1, 722.0, "Ncaron"), + (-1, 333.0, "Iacute"), + (-1, 564.0, "plusminus"), + (-1, 200.0, "brokenbar"), + (-1, 760.0, "registered"), + (-1, 722.0, "Gbreve"), + (-1, 333.0, "Idotaccent"), + (-1, 600.0, "summation"), + (-1, 611.0, "Egrave"), + (-1, 333.0, "racute"), + (-1, 500.0, "omacron"), + (-1, 611.0, "Zacute"), + (-1, 611.0, "Zcaron"), + (-1, 549.0, "greaterequal"), + (-1, 722.0, "Eth"), + (-1, 667.0, "Ccedilla"), + (-1, 278.0, "lcommaaccent"), + (-1, 326.0, "tcaron"), + (-1, 444.0, "eogonek"), + (-1, 722.0, "Uogonek"), + (-1, 722.0, "Aacute"), + (-1, 722.0, "Adieresis"), + (-1, 444.0, "egrave"), + (-1, 444.0, "zacute"), + (-1, 278.0, "iogonek"), + (-1, 722.0, "Oacute"), + (-1, 500.0, "oacute"), + (-1, 444.0, "amacron"), + (-1, 389.0, "sacute"), + (-1, 278.0, "idieresis"), + (-1, 722.0, "Ocircumflex"), + (-1, 722.0, "Ugrave"), + (-1, 612.0, "Delta"), + (-1, 500.0, "thorn"), + (-1, 300.0, "twosuperior"), + (-1, 722.0, "Odieresis"), + (-1, 500.0, "mu"), + (-1, 278.0, "igrave"), + (-1, 500.0, "ohungarumlaut"), + (-1, 611.0, "Eogonek"), + (-1, 500.0, "dcroat"), + (-1, 750.0, "threequarters"), + (-1, 556.0, "Scedilla"), + (-1, 344.0, "lcaron"), + (-1, 722.0, "Kcommaaccent"), + (-1, 611.0, "Lacute"), + (-1, 980.0, "trademark"), + (-1, 444.0, "edotaccent"), + (-1, 333.0, "Igrave"), + (-1, 333.0, "Imacron"), + (-1, 611.0, "Lcaron"), + (-1, 750.0, "onehalf"), + (-1, 549.0, "lessequal"), + (-1, 500.0, "ocircumflex"), + (-1, 500.0, "ntilde"), + (-1, 722.0, "Uhungarumlaut"), + (-1, 611.0, "Eacute"), + (-1, 444.0, "emacron"), + (-1, 500.0, "gbreve"), + (-1, 750.0, "onequarter"), + (-1, 556.0, "Scaron"), + (-1, 556.0, "Scommaaccent"), + (-1, 722.0, "Ohungarumlaut"), + (-1, 400.0, "degree"), + (-1, 500.0, "ograve"), + (-1, 667.0, "Ccaron"), + (-1, 500.0, "ugrave"), + (-1, 453.0, "radical"), + (-1, 722.0, "Dcaron"), + (-1, 333.0, "rcommaaccent"), + (-1, 722.0, "Ntilde"), + (-1, 500.0, "otilde"), + (-1, 667.0, "Rcommaaccent"), + (-1, 611.0, "Lcommaaccent"), + (-1, 722.0, "Atilde"), + (-1, 722.0, "Aogonek"), + (-1, 722.0, "Aring"), + (-1, 722.0, "Otilde"), + (-1, 444.0, "zdotaccent"), + (-1, 611.0, "Ecaron"), + (-1, 333.0, "Iogonek"), + (-1, 500.0, "kcommaaccent"), + (-1, 564.0, "minus"), + (-1, 333.0, "Icircumflex"), + (-1, 500.0, "ncaron"), + (-1, 278.0, "tcommaaccent"), + (-1, 564.0, "logicalnot"), + (-1, 500.0, "odieresis"), + (-1, 500.0, "udieresis"), + (-1, 549.0, "notequal"), + (-1, 500.0, "gcommaaccent"), + (-1, 500.0, "eth"), + (-1, 444.0, "zcaron"), + (-1, 500.0, "ncommaaccent"), + (-1, 300.0, "onesuperior"), + (-1, 278.0, "imacron"), + (-1, 500.0, "Euro"), + ][..], + ), + ( + "ZapfDingbats", + "FontSpecific", + &[ + (32, 278.0, "space"), + (33, 974.0, "a1"), + (34, 961.0, "a2"), + (35, 974.0, "a202"), + (36, 980.0, "a3"), + (37, 719.0, "a4"), + (38, 789.0, "a5"), + (39, 790.0, "a119"), + (40, 791.0, "a118"), + (41, 690.0, "a117"), + (42, 960.0, "a11"), + (43, 939.0, "a12"), + (44, 549.0, "a13"), + (45, 855.0, "a14"), + (46, 911.0, "a15"), + (47, 933.0, "a16"), + (48, 911.0, "a105"), + (49, 945.0, "a17"), + (50, 974.0, "a18"), + (51, 755.0, "a19"), + (52, 846.0, "a20"), + (53, 762.0, "a21"), + (54, 761.0, "a22"), + (55, 571.0, "a23"), + (56, 677.0, "a24"), + (57, 763.0, "a25"), + (58, 760.0, "a26"), + (59, 759.0, "a27"), + (60, 754.0, "a28"), + (61, 494.0, "a6"), + (62, 552.0, "a7"), + (63, 537.0, "a8"), + (64, 577.0, "a9"), + (65, 692.0, "a10"), + (66, 786.0, "a29"), + (67, 788.0, "a30"), + (68, 788.0, "a31"), + (69, 790.0, "a32"), + (70, 793.0, "a33"), + (71, 794.0, "a34"), + (72, 816.0, "a35"), + (73, 823.0, "a36"), + (74, 789.0, "a37"), + (75, 841.0, "a38"), + (76, 823.0, "a39"), + (77, 833.0, "a40"), + (78, 816.0, "a41"), + (79, 831.0, "a42"), + (80, 923.0, "a43"), + (81, 744.0, "a44"), + (82, 723.0, "a45"), + (83, 749.0, "a46"), + (84, 790.0, "a47"), + (85, 792.0, "a48"), + (86, 695.0, "a49"), + (87, 776.0, "a50"), + (88, 768.0, "a51"), + (89, 792.0, "a52"), + (90, 759.0, "a53"), + (91, 707.0, "a54"), + (92, 708.0, "a55"), + (93, 682.0, "a56"), + (94, 701.0, "a57"), + (95, 826.0, "a58"), + (96, 815.0, "a59"), + (97, 789.0, "a60"), + (98, 789.0, "a61"), + (99, 707.0, "a62"), + (100, 687.0, "a63"), + (101, 696.0, "a64"), + (102, 689.0, "a65"), + (103, 786.0, "a66"), + (104, 787.0, "a67"), + (105, 713.0, "a68"), + (106, 791.0, "a69"), + (107, 785.0, "a70"), + (108, 791.0, "a71"), + (109, 873.0, "a72"), + (110, 761.0, "a73"), + (111, 762.0, "a74"), + (112, 762.0, "a203"), + (113, 759.0, "a75"), + (114, 759.0, "a204"), + (115, 892.0, "a76"), + (116, 892.0, "a77"), + (117, 788.0, "a78"), + (118, 784.0, "a79"), + (119, 438.0, "a81"), + (120, 138.0, "a82"), + (121, 277.0, "a83"), + (122, 415.0, "a84"), + (123, 392.0, "a97"), + (124, 392.0, "a98"), + (125, 668.0, "a99"), + (126, 668.0, "a100"), + (128, 390.0, "a89"), + (129, 390.0, "a90"), + (130, 317.0, "a93"), + (131, 317.0, "a94"), + (132, 276.0, "a91"), + (133, 276.0, "a92"), + (134, 509.0, "a205"), + (135, 509.0, "a85"), + (136, 410.0, "a206"), + (137, 410.0, "a86"), + (138, 234.0, "a87"), + (139, 234.0, "a88"), + (140, 334.0, "a95"), + (141, 334.0, "a96"), + (161, 732.0, "a101"), + (162, 544.0, "a102"), + (163, 544.0, "a103"), + (164, 910.0, "a104"), + (165, 667.0, "a106"), + (166, 760.0, "a107"), + (167, 760.0, "a108"), + (168, 776.0, "a112"), + (169, 595.0, "a111"), + (170, 694.0, "a110"), + (171, 626.0, "a109"), + (172, 788.0, "a120"), + (173, 788.0, "a121"), + (174, 788.0, "a122"), + (175, 788.0, "a123"), + (176, 788.0, "a124"), + (177, 788.0, "a125"), + (178, 788.0, "a126"), + (179, 788.0, "a127"), + (180, 788.0, "a128"), + (181, 788.0, "a129"), + (182, 788.0, "a130"), + (183, 788.0, "a131"), + (184, 788.0, "a132"), + (185, 788.0, "a133"), + (186, 788.0, "a134"), + (187, 788.0, "a135"), + (188, 788.0, "a136"), + (189, 788.0, "a137"), + (190, 788.0, "a138"), + (191, 788.0, "a139"), + (192, 788.0, "a140"), + (193, 788.0, "a141"), + (194, 788.0, "a142"), + (195, 788.0, "a143"), + (196, 788.0, "a144"), + (197, 788.0, "a145"), + (198, 788.0, "a146"), + (199, 788.0, "a147"), + (200, 788.0, "a148"), + (201, 788.0, "a149"), + (202, 788.0, "a150"), + (203, 788.0, "a151"), + (204, 788.0, "a152"), + (205, 788.0, "a153"), + (206, 788.0, "a154"), + (207, 788.0, "a155"), + (208, 788.0, "a156"), + (209, 788.0, "a157"), + (210, 788.0, "a158"), + (211, 788.0, "a159"), + (212, 894.0, "a160"), + (213, 838.0, "a161"), + (214, 1016.0, "a163"), + (215, 458.0, "a164"), + (216, 748.0, "a196"), + (217, 924.0, "a165"), + (218, 748.0, "a192"), + (219, 918.0, "a166"), + (220, 927.0, "a167"), + (221, 928.0, "a168"), + (222, 928.0, "a169"), + (223, 834.0, "a170"), + (224, 873.0, "a171"), + (225, 828.0, "a172"), + (226, 924.0, "a173"), + (227, 924.0, "a162"), + (228, 917.0, "a174"), + (229, 930.0, "a175"), + (230, 931.0, "a176"), + (231, 463.0, "a177"), + (232, 883.0, "a178"), + (233, 836.0, "a179"), + (234, 836.0, "a193"), + (235, 867.0, "a180"), + (236, 867.0, "a199"), + (237, 696.0, "a181"), + (238, 696.0, "a200"), + (239, 874.0, "a182"), + (241, 874.0, "a201"), + (242, 760.0, "a183"), + (243, 946.0, "a184"), + (244, 771.0, "a197"), + (245, 865.0, "a185"), + (246, 771.0, "a194"), + (247, 888.0, "a198"), + (248, 967.0, "a186"), + (249, 888.0, "a195"), + (250, 831.0, "a187"), + (251, 873.0, "a188"), + (252, 927.0, "a189"), + (253, 970.0, "a190"), + (254, 918.0, "a191"), + ][..], + ), + ]) +} diff --git a/crates/pdf-extract/src/encodings.rs b/crates/pdf-extract/src/encodings.rs new file mode 100644 index 0000000..a93b8f3 --- /dev/null +++ b/crates/pdf-extract/src/encodings.rs @@ -0,0 +1,1816 @@ +pub const MAC_ROMAN_ENCODING: [Option<&str>; 256] = [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("space"), + Some("exclam"), + Some("quotedbl"), + Some("numbersign"), + Some("dollar"), + Some("percent"), + Some("ampersand"), + Some("quotesingle"), + Some("parenleft"), + Some("parenright"), + Some("asterisk"), + Some("plus"), + Some("comma"), + Some("hyphen"), + Some("period"), + Some("slash"), + Some("zero"), + Some("one"), + Some("two"), + Some("three"), + Some("four"), + Some("five"), + Some("six"), + Some("seven"), + Some("eight"), + Some("nine"), + Some("colon"), + Some("semicolon"), + Some("less"), + Some("equal"), + Some("greater"), + Some("question"), + Some("at"), + Some("A"), + Some("B"), + Some("C"), + Some("D"), + Some("E"), + Some("F"), + Some("G"), + Some("H"), + Some("I"), + Some("J"), + Some("K"), + Some("L"), + Some("M"), + Some("N"), + Some("O"), + Some("P"), + Some("Q"), + Some("R"), + Some("S"), + Some("T"), + Some("U"), + Some("V"), + Some("W"), + Some("X"), + Some("Y"), + Some("Z"), + Some("bracketleft"), + Some("backslash"), + Some("bracketright"), + Some("asciicircum"), + Some("underscore"), + Some("grave"), + Some("a"), + Some("b"), + Some("c"), + Some("d"), + Some("e"), + Some("f"), + Some("g"), + Some("h"), + Some("i"), + Some("j"), + Some("k"), + Some("l"), + Some("m"), + Some("n"), + Some("o"), + Some("p"), + Some("q"), + Some("r"), + Some("s"), + Some("t"), + Some("u"), + Some("v"), + Some("w"), + Some("x"), + Some("y"), + Some("z"), + Some("braceleft"), + Some("bar"), + Some("braceright"), + Some("asciitilde"), + None, + Some("Adieresis"), + Some("Aring"), + Some("Ccedilla"), + Some("Eacute"), + Some("Ntilde"), + Some("Odieresis"), + Some("Udieresis"), + Some("aacute"), + Some("agrave"), + Some("acircumflex"), + Some("adieresis"), + Some("atilde"), + Some("aring"), + Some("ccedilla"), + Some("eacute"), + Some("egrave"), + Some("ecircumflex"), + Some("edieresis"), + Some("iacute"), + Some("igrave"), + Some("icircumflex"), + Some("idieresis"), + Some("ntilde"), + Some("oacute"), + Some("ograve"), + Some("ocircumflex"), + Some("odieresis"), + Some("otilde"), + Some("uacute"), + Some("ugrave"), + Some("ucircumflex"), + Some("udieresis"), + Some("dagger"), + Some("degree"), + Some("cent"), + Some("sterling"), + Some("section"), + Some("bullet"), + Some("paragraph"), + Some("germandbls"), + Some("registered"), + Some("copyright"), + Some("trademark"), + Some("acute"), + Some("dieresis"), + Some("notequal"), + Some("AE"), + Some("Oslash"), + Some("infinity"), + Some("plusminus"), + Some("lessequal"), + Some("greaterequal"), + Some("yen"), + Some("mu"), + Some("partialdiff"), + Some("summation"), + Some("product"), + Some("pi"), + Some("integral"), + Some("ordfeminine"), + Some("ordmasculine"), + Some("Omega"), + Some("ae"), + Some("oslash"), + Some("questiondown"), + Some("exclamdown"), + Some("logicalnot"), + Some("radical"), + Some("florin"), + Some("approxequal"), + Some("Delta"), + Some("guillemotleft"), + Some("guillemotright"), + Some("ellipsis"), + Some("space"), + Some("Agrave"), + Some("Atilde"), + Some("Otilde"), + Some("OE"), + Some("oe"), + Some("endash"), + Some("emdash"), + Some("quotedblleft"), + Some("quotedblright"), + Some("quoteleft"), + Some("quoteright"), + Some("divide"), + Some("lozenge"), + Some("ydieresis"), + Some("Ydieresis"), + Some("fraction"), + Some("currency"), + Some("guilsinglleft"), + Some("guilsinglright"), + Some("fi"), + Some("fl"), + Some("daggerdbl"), + Some("periodcentered"), + Some("quotesinglbase"), + Some("quotedblbase"), + Some("perthousand"), + Some("Acircumflex"), + Some("Ecircumflex"), + Some("Aacute"), + Some("Edieresis"), + Some("Egrave"), + Some("Iacute"), + Some("Icircumflex"), + Some("Idieresis"), + Some("Igrave"), + Some("Oacute"), + Some("Ocircumflex"), + Some("apple"), + Some("Ograve"), + Some("Uacute"), + Some("Ucircumflex"), + Some("Ugrave"), + Some("dotlessi"), + Some("circumflex"), + Some("tilde"), + Some("macron"), + Some("breve"), + Some("dotaccent"), + Some("ring"), + Some("cedilla"), + Some("hungarumlaut"), + Some("ogonek"), + Some("caron"), +]; + +pub const MAC_EXPERT_ENCODING: [Option<&str>; 256] = [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("space"), + Some("exclamsmall"), + Some("Hungarumlautsmall"), + Some("centoldstyle"), + Some("dollaroldstyle"), + Some("dollarsuperior"), + Some("ampersandsmall"), + Some("Acutesmall"), + Some("parenleftsuperior"), + Some("parenrightsuperior"), + Some("twodotenleader"), + Some("onedotenleader"), + Some("comma"), + Some("hyphen"), + Some("period"), + Some("fraction"), + Some("zerooldstyle"), + Some("oneoldstyle"), + Some("twooldstyle"), + Some("threeoldstyle"), + Some("fouroldstyle"), + Some("fiveoldstyle"), + Some("sixoldstyle"), + Some("sevenoldstyle"), + Some("eightoldstyle"), + Some("nineoldstyle"), + Some("colon"), + Some("semicolon"), + None, + Some("threequartersemdash"), + None, + Some("questionsmall"), + None, + None, + None, + None, + Some("Ethsmall"), + None, + None, + Some("onequarter"), + Some("onehalf"), + Some("threequarters"), + Some("oneeighth"), + Some("threeeighths"), + Some("fiveeighths"), + Some("seveneighths"), + Some("onethird"), + Some("twothirds"), + None, + None, + None, + None, + None, + None, + Some("ff"), + Some("fi"), + Some("fl"), + Some("ffi"), + Some("ffl"), + Some("parenleftinferior"), + None, + Some("parenrightinferior"), + Some("Circumflexsmall"), + Some("hypheninferior"), + Some("Gravesmall"), + Some("Asmall"), + Some("Bsmall"), + Some("Csmall"), + Some("Dsmall"), + Some("Esmall"), + Some("Fsmall"), + Some("Gsmall"), + Some("Hsmall"), + Some("Ismall"), + Some("Jsmall"), + Some("Ksmall"), + Some("Lsmall"), + Some("Msmall"), + Some("Nsmall"), + Some("Osmall"), + Some("Psmall"), + Some("Qsmall"), + Some("Rsmall"), + Some("Ssmall"), + Some("Tsmall"), + Some("Usmall"), + Some("Vsmall"), + Some("Wsmall"), + Some("Xsmall"), + Some("Ysmall"), + Some("Zsmall"), + Some("colonmonetary"), + Some("onefitted"), + Some("rupiah"), + Some("Tildesmall"), + None, + None, + Some("asuperior"), + Some("centsuperior"), + None, + None, + None, + None, + Some("Aacutesmall"), + Some("Agravesmall"), + Some("Acircumflexsmall"), + Some("Adieresissmall"), + Some("Atildesmall"), + Some("Aringsmall"), + Some("Ccedillasmall"), + Some("Eacutesmall"), + Some("Egravesmall"), + Some("Ecircumflexsmall"), + Some("Edieresissmall"), + Some("Iacutesmall"), + Some("Igravesmall"), + Some("Icircumflexsmall"), + Some("Idieresissmall"), + Some("Ntildesmall"), + Some("Oacutesmall"), + Some("Ogravesmall"), + Some("Ocircumflexsmall"), + Some("Odieresissmall"), + Some("Otildesmall"), + Some("Uacutesmall"), + Some("Ugravesmall"), + Some("Ucircumflexsmall"), + Some("Udieresissmall"), + None, + Some("eightsuperior"), + Some("fourinferior"), + Some("threeinferior"), + Some("sixinferior"), + Some("eightinferior"), + Some("seveninferior"), + Some("Scaronsmall"), + None, + Some("centinferior"), + Some("twoinferior"), + None, + Some("Dieresissmall"), + None, + Some("Caronsmall"), + Some("osuperior"), + Some("fiveinferior"), + None, + Some("commainferior"), + Some("periodinferior"), + Some("Yacutesmall"), + None, + Some("dollarinferior"), + None, + None, + Some("Thornsmall"), + None, + Some("nineinferior"), + Some("zeroinferior"), + Some("Zcaronsmall"), + Some("AEsmall"), + Some("Oslashsmall"), + Some("questiondownsmall"), + Some("oneinferior"), + Some("Lslashsmall"), + None, + None, + None, + None, + None, + None, + Some("Cedillasmall"), + None, + None, + None, + None, + None, + Some("OEsmall"), + Some("figuredash"), + Some("hyphensuperior"), + None, + None, + None, + None, + Some("exclamdownsmall"), + None, + Some("Ydieresissmall"), + None, + Some("onesuperior"), + Some("twosuperior"), + Some("threesuperior"), + Some("foursuperior"), + Some("fivesuperior"), + Some("sixsuperior"), + Some("sevensuperior"), + Some("ninesuperior"), + Some("zerosuperior"), + None, + Some("esuperior"), + Some("rsuperior"), + Some("tsuperior"), + None, + None, + Some("isuperior"), + Some("ssuperior"), + Some("dsuperior"), + None, + None, + None, + None, + None, + Some("lsuperior"), + Some("Ogoneksmall"), + Some("Brevesmall"), + Some("Macronsmall"), + Some("bsuperior"), + Some("nsuperior"), + Some("msuperior"), + Some("commasuperior"), + Some("periodsuperior"), + Some("Dotaccentsmall"), + Some("Ringsmall"), + None, + None, + None, + None, +]; + +pub const WIN_ANSI_ENCODING: [Option<&str>; 256] = [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("space"), + Some("exclam"), + Some("quotedbl"), + Some("numbersign"), + Some("dollar"), + Some("percent"), + Some("ampersand"), + Some("quotesingle"), + Some("parenleft"), + Some("parenright"), + Some("asterisk"), + Some("plus"), + Some("comma"), + Some("hyphen"), + Some("period"), + Some("slash"), + Some("zero"), + Some("one"), + Some("two"), + Some("three"), + Some("four"), + Some("five"), + Some("six"), + Some("seven"), + Some("eight"), + Some("nine"), + Some("colon"), + Some("semicolon"), + Some("less"), + Some("equal"), + Some("greater"), + Some("question"), + Some("at"), + Some("A"), + Some("B"), + Some("C"), + Some("D"), + Some("E"), + Some("F"), + Some("G"), + Some("H"), + Some("I"), + Some("J"), + Some("K"), + Some("L"), + Some("M"), + Some("N"), + Some("O"), + Some("P"), + Some("Q"), + Some("R"), + Some("S"), + Some("T"), + Some("U"), + Some("V"), + Some("W"), + Some("X"), + Some("Y"), + Some("Z"), + Some("bracketleft"), + Some("backslash"), + Some("bracketright"), + Some("asciicircum"), + Some("underscore"), + Some("grave"), + Some("a"), + Some("b"), + Some("c"), + Some("d"), + Some("e"), + Some("f"), + Some("g"), + Some("h"), + Some("i"), + Some("j"), + Some("k"), + Some("l"), + Some("m"), + Some("n"), + Some("o"), + Some("p"), + Some("q"), + Some("r"), + Some("s"), + Some("t"), + Some("u"), + Some("v"), + Some("w"), + Some("x"), + Some("y"), + Some("z"), + Some("braceleft"), + Some("bar"), + Some("braceright"), + Some("asciitilde"), + Some("bullet"), + Some("Euro"), + Some("bullet"), + Some("quotesinglbase"), + Some("florin"), + Some("quotedblbase"), + Some("ellipsis"), + Some("dagger"), + Some("daggerdbl"), + Some("circumflex"), + Some("perthousand"), + Some("Scaron"), + Some("guilsinglleft"), + Some("OE"), + Some("bullet"), + Some("Zcaron"), + Some("bullet"), + Some("bullet"), + Some("quoteleft"), + Some("quoteright"), + Some("quotedblleft"), + Some("quotedblright"), + Some("bullet"), + Some("endash"), + Some("emdash"), + Some("tilde"), + Some("trademark"), + Some("scaron"), + Some("guilsinglright"), + Some("oe"), + Some("bullet"), + Some("zcaron"), + Some("Ydieresis"), + Some("space"), + Some("exclamdown"), + Some("cent"), + Some("sterling"), + Some("currency"), + Some("yen"), + Some("brokenbar"), + Some("section"), + Some("dieresis"), + Some("copyright"), + Some("ordfeminine"), + Some("guillemotleft"), + Some("logicalnot"), + Some("hyphen"), + Some("registered"), + Some("macron"), + Some("degree"), + Some("plusminus"), + Some("twosuperior"), + Some("threesuperior"), + Some("acute"), + Some("mu"), + Some("paragraph"), + Some("periodcentered"), + Some("cedilla"), + Some("onesuperior"), + Some("ordmasculine"), + Some("guillemotright"), + Some("onequarter"), + Some("onehalf"), + Some("threequarters"), + Some("questiondown"), + Some("Agrave"), + Some("Aacute"), + Some("Acircumflex"), + Some("Atilde"), + Some("Adieresis"), + Some("Aring"), + Some("AE"), + Some("Ccedilla"), + Some("Egrave"), + Some("Eacute"), + Some("Ecircumflex"), + Some("Edieresis"), + Some("Igrave"), + Some("Iacute"), + Some("Icircumflex"), + Some("Idieresis"), + Some("Eth"), + Some("Ntilde"), + Some("Ograve"), + Some("Oacute"), + Some("Ocircumflex"), + Some("Otilde"), + Some("Odieresis"), + Some("multiply"), + Some("Oslash"), + Some("Ugrave"), + Some("Uacute"), + Some("Ucircumflex"), + Some("Udieresis"), + Some("Yacute"), + Some("Thorn"), + Some("germandbls"), + Some("agrave"), + Some("aacute"), + Some("acircumflex"), + Some("atilde"), + Some("adieresis"), + Some("aring"), + Some("ae"), + Some("ccedilla"), + Some("egrave"), + Some("eacute"), + Some("ecircumflex"), + Some("edieresis"), + Some("igrave"), + Some("iacute"), + Some("icircumflex"), + Some("idieresis"), + Some("eth"), + Some("ntilde"), + Some("ograve"), + Some("oacute"), + Some("ocircumflex"), + Some("otilde"), + Some("odieresis"), + Some("divide"), + Some("oslash"), + Some("ugrave"), + Some("uacute"), + Some("ucircumflex"), + Some("udieresis"), + Some("yacute"), + Some("thorn"), + Some("ydieresis"), +]; + +#[allow(dead_code)] +pub const STANDARD_ENCODING: [Option<&str>; 256] = [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("space"), + Some("exclam"), + Some("quotedbl"), + Some("numbersign"), + Some("dollar"), + Some("percent"), + Some("ampersand"), + Some("quoteright"), + Some("parenleft"), + Some("parenright"), + Some("asterisk"), + Some("plus"), + Some("comma"), + Some("hyphen"), + Some("period"), + Some("slash"), + Some("zero"), + Some("one"), + Some("two"), + Some("three"), + Some("four"), + Some("five"), + Some("six"), + Some("seven"), + Some("eight"), + Some("nine"), + Some("colon"), + Some("semicolon"), + Some("less"), + Some("equal"), + Some("greater"), + Some("question"), + Some("at"), + Some("A"), + Some("B"), + Some("C"), + Some("D"), + Some("E"), + Some("F"), + Some("G"), + Some("H"), + Some("I"), + Some("J"), + Some("K"), + Some("L"), + Some("M"), + Some("N"), + Some("O"), + Some("P"), + Some("Q"), + Some("R"), + Some("S"), + Some("T"), + Some("U"), + Some("V"), + Some("W"), + Some("X"), + Some("Y"), + Some("Z"), + Some("bracketleft"), + Some("backslash"), + Some("bracketright"), + Some("asciicircum"), + Some("underscore"), + Some("quoteleft"), + Some("a"), + Some("b"), + Some("c"), + Some("d"), + Some("e"), + Some("f"), + Some("g"), + Some("h"), + Some("i"), + Some("j"), + Some("k"), + Some("l"), + Some("m"), + Some("n"), + Some("o"), + Some("p"), + Some("q"), + Some("r"), + Some("s"), + Some("t"), + Some("u"), + Some("v"), + Some("w"), + Some("x"), + Some("y"), + Some("z"), + Some("braceleft"), + Some("bar"), + Some("braceright"), + Some("asciitilde"), + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("exclamdown"), + Some("cent"), + Some("sterling"), + Some("fraction"), + Some("yen"), + Some("florin"), + Some("section"), + Some("currency"), + Some("quotesingle"), + Some("quotedblleft"), + Some("guillemotleft"), + Some("guilsinglleft"), + Some("guilsinglright"), + Some("fi"), + Some("fl"), + None, + Some("endash"), + Some("dagger"), + Some("daggerdbl"), + Some("periodcentered"), + None, + Some("paragraph"), + Some("bullet"), + Some("quotesinglbase"), + Some("quotedblbase"), + Some("quotedblright"), + Some("guillemotright"), + Some("ellipsis"), + Some("perthousand"), + None, + Some("questiondown"), + None, + Some("grave"), + Some("acute"), + Some("circumflex"), + Some("tilde"), + Some("macron"), + Some("breve"), + Some("dotaccent"), + Some("dieresis"), + None, + Some("ring"), + Some("cedilla"), + None, + Some("hungarumlaut"), + Some("ogonek"), + Some("caron"), + Some("emdash"), + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("AE"), + None, + Some("ordfeminine"), + None, + None, + None, + None, + Some("Lslash"), + Some("Oslash"), + Some("OE"), + Some("ordmasculine"), + None, + None, + None, + None, + None, + Some("ae"), + None, + None, + None, + Some("dotlessi"), + None, + None, + Some("lslash"), + Some("oslash"), + Some("oe"), + Some("germandbls"), + None, + None, + None, + None, +]; + +#[allow(dead_code)] +pub const EXPERT_ENCODING: [Option<&str>; 256] = [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("space"), + Some("exclamsmall"), + Some("Hungarumlautsmall"), + None, + Some("dollaroldstyle"), + Some("dollarsuperior"), + Some("ampersandsmall"), + Some("Acutesmall"), + Some("parenleftsuperior"), + Some("parenrightsuperior"), + Some("twodotenleader"), + Some("onedotenleader"), + Some("comma"), + Some("hyphen"), + Some("period"), + Some("fraction"), + Some("zerooldstyle"), + Some("oneoldstyle"), + Some("twooldstyle"), + Some("threeoldstyle"), + Some("fouroldstyle"), + Some("fiveoldstyle"), + Some("sixoldstyle"), + Some("sevenoldstyle"), + Some("eightoldstyle"), + Some("nineoldstyle"), + Some("colon"), + Some("semicolon"), + Some("commasuperior"), + Some("threequartersemdash"), + Some("periodsuperior"), + Some("questionsmall"), + None, + Some("asuperior"), + Some("bsuperior"), + Some("centsuperior"), + Some("dsuperior"), + Some("esuperior"), + None, + None, + None, + Some("isuperior"), + None, + None, + Some("lsuperior"), + Some("msuperior"), + Some("nsuperior"), + Some("osuperior"), + None, + None, + Some("rsuperior"), + Some("ssuperior"), + Some("tsuperior"), + None, + Some("ff"), + Some("fi"), + Some("fl"), + Some("ffi"), + Some("ffl"), + Some("parenleftinferior"), + None, + Some("parenrightinferior"), + Some("Circumflexsmall"), + Some("hyphensuperior"), + Some("Gravesmall"), + Some("Asmall"), + Some("Bsmall"), + Some("Csmall"), + Some("Dsmall"), + Some("Esmall"), + Some("Fsmall"), + Some("Gsmall"), + Some("Hsmall"), + Some("Ismall"), + Some("Jsmall"), + Some("Ksmall"), + Some("Lsmall"), + Some("Msmall"), + Some("Nsmall"), + Some("Osmall"), + Some("Psmall"), + Some("Qsmall"), + Some("Rsmall"), + Some("Ssmall"), + Some("Tsmall"), + Some("Usmall"), + Some("Vsmall"), + Some("Wsmall"), + Some("Xsmall"), + Some("Ysmall"), + Some("Zsmall"), + Some("colonmonetary"), + Some("onefitted"), + Some("rupiah"), + Some("Tildesmall"), + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("exclamdownsmall"), + Some("centoldstyle"), + Some("Lslashsmall"), + None, + None, + Some("Scaronsmall"), + Some("Zcaronsmall"), + Some("Dieresissmall"), + Some("Brevesmall"), + Some("Caronsmall"), + None, + Some("Dotaccentsmall"), + None, + None, + Some("Macronsmall"), + None, + None, + Some("figuredash"), + Some("hypheninferior"), + None, + None, + Some("Ogoneksmall"), + Some("Ringsmall"), + Some("Cedillasmall"), + None, + None, + None, + Some("onequarter"), + Some("onehalf"), + Some("threequarters"), + Some("questiondownsmall"), + Some("oneeighth"), + Some("threeeighths"), + Some("fiveeighths"), + Some("seveneighths"), + Some("onethird"), + Some("twothirds"), + None, + None, + Some("zerosuperior"), + Some("onesuperior"), + Some("twosuperior"), + Some("threesuperior"), + Some("foursuperior"), + Some("fivesuperior"), + Some("sixsuperior"), + Some("sevensuperior"), + Some("eightsuperior"), + Some("ninesuperior"), + Some("zeroinferior"), + Some("oneinferior"), + Some("twoinferior"), + Some("threeinferior"), + Some("fourinferior"), + Some("fiveinferior"), + Some("sixinferior"), + Some("seveninferior"), + Some("eightinferior"), + Some("nineinferior"), + Some("centinferior"), + Some("dollarinferior"), + Some("periodinferior"), + Some("commainferior"), + Some("Agravesmall"), + Some("Aacutesmall"), + Some("Acircumflexsmall"), + Some("Atildesmall"), + Some("Adieresissmall"), + Some("Aringsmall"), + Some("AEsmall"), + Some("Ccedillasmall"), + Some("Egravesmall"), + Some("Eacutesmall"), + Some("Ecircumflexsmall"), + Some("Edieresissmall"), + Some("Igravesmall"), + Some("Iacutesmall"), + Some("Icircumflexsmall"), + Some("Idieresissmall"), + Some("Ethsmall"), + Some("Ntildesmall"), + Some("Ogravesmall"), + Some("Oacutesmall"), + Some("Ocircumflexsmall"), + Some("Otildesmall"), + Some("Odieresissmall"), + Some("OEsmall"), + Some("Oslashsmall"), + Some("Ugravesmall"), + Some("Uacutesmall"), + Some("Ucircumflexsmall"), + Some("Udieresissmall"), + Some("Yacutesmall"), + Some("Thornsmall"), + Some("Ydieresissmall"), +]; + +#[allow(dead_code)] +pub const SYMBOL_ENCODING: [Option<&str>; 256] = [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("space"), + Some("exclam"), + Some("universal"), + Some("numbersign"), + Some("existential"), + Some("percent"), + Some("ampersand"), + Some("suchthat"), + Some("parenleft"), + Some("parenright"), + Some("asteriskmath"), + Some("plus"), + Some("comma"), + Some("minus"), + Some("period"), + Some("slash"), + Some("zero"), + Some("one"), + Some("two"), + Some("three"), + Some("four"), + Some("five"), + Some("six"), + Some("seven"), + Some("eight"), + Some("nine"), + Some("colon"), + Some("semicolon"), + Some("less"), + Some("equal"), + Some("greater"), + Some("question"), + Some("congruent"), + Some("Alpha"), + Some("Beta"), + Some("Chi"), + Some("Delta"), + Some("Epsilon"), + Some("Phi"), + Some("Gamma"), + Some("Eta"), + Some("Iota"), + Some("theta1"), + Some("Kappa"), + Some("Lambda"), + Some("Mu"), + Some("Nu"), + Some("Omicron"), + Some("Pi"), + Some("Theta"), + Some("Rho"), + Some("Sigma"), + Some("Tau"), + Some("Upsilon"), + Some("sigma1"), + Some("Omega"), + Some("Xi"), + Some("Psi"), + Some("Zeta"), + Some("bracketleft"), + Some("therefore"), + Some("bracketright"), + Some("perpendicular"), + Some("underscore"), + Some("radicalex"), + Some("alpha"), + Some("beta"), + Some("chi"), + Some("delta"), + Some("epsilon"), + Some("phi"), + Some("gamma"), + Some("eta"), + Some("iota"), + Some("phi1"), + Some("kappa"), + Some("lambda"), + Some("mu"), + Some("nu"), + Some("omicron"), + Some("pi"), + Some("theta"), + Some("rho"), + Some("sigma"), + Some("tau"), + Some("upsilon"), + Some("omega1"), + Some("omega"), + Some("xi"), + Some("psi"), + Some("zeta"), + Some("braceleft"), + Some("bar"), + Some("braceright"), + Some("similar"), + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("Upsilon1"), + Some("minute"), + Some("lessequal"), + Some("fraction"), + Some("infinity"), + Some("florin"), + Some("club"), + Some("diamond"), + Some("heart"), + Some("spade"), + Some("arrowboth"), + Some("arrowleft"), + Some("arrowup"), + Some("arrowright"), + Some("arrowdown"), + Some("degree"), + Some("plusminus"), + Some("second"), + Some("greaterequal"), + Some("multiply"), + Some("proportional"), + Some("partialdiff"), + Some("bullet"), + Some("divide"), + Some("notequal"), + Some("equivalence"), + Some("approxequal"), + Some("ellipsis"), + Some("arrowvertex"), + Some("arrowhorizex"), + Some("carriagereturn"), + Some("aleph"), + Some("Ifraktur"), + Some("Rfraktur"), + Some("weierstrass"), + Some("circlemultiply"), + Some("circleplus"), + Some("emptyset"), + Some("intersection"), + Some("union"), + Some("propersuperset"), + Some("reflexsuperset"), + Some("notsubset"), + Some("propersubset"), + Some("reflexsubset"), + Some("element"), + Some("notelement"), + Some("angle"), + Some("gradient"), + Some("registerserif"), + Some("copyrightserif"), + Some("trademarkserif"), + Some("product"), + Some("radical"), + Some("dotmath"), + Some("logicalnot"), + Some("logicaland"), + Some("logicalor"), + Some("arrowdblboth"), + Some("arrowdblleft"), + Some("arrowdblup"), + Some("arrowdblright"), + Some("arrowdbldown"), + Some("lozenge"), + Some("angleleft"), + Some("registersans"), + Some("copyrightsans"), + Some("trademarksans"), + Some("summation"), + Some("parenlefttp"), + Some("parenleftex"), + Some("parenleftbt"), + Some("bracketlefttp"), + Some("bracketleftex"), + Some("bracketleftbt"), + Some("bracelefttp"), + Some("braceleftmid"), + Some("braceleftbt"), + Some("braceex"), + None, + Some("angleright"), + Some("integral"), + Some("integraltp"), + Some("integralex"), + Some("integralbt"), + Some("parenrighttp"), + Some("parenrightex"), + Some("parenrightbt"), + Some("bracketrighttp"), + Some("bracketrightex"), + Some("bracketrightbt"), + Some("bracerighttp"), + Some("bracerightmid"), + Some("bracerightbt"), + None, +]; + +#[allow(dead_code)] +pub const ZAPFDINGBATS_ENCODING: [Option<&str>; 256] = [ + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("space"), + Some("a1"), + Some("a2"), + Some("a202"), + Some("a3"), + Some("a4"), + Some("a5"), + Some("a119"), + Some("a118"), + Some("a117"), + Some("a11"), + Some("a12"), + Some("a13"), + Some("a14"), + Some("a15"), + Some("a16"), + Some("a105"), + Some("a17"), + Some("a18"), + Some("a19"), + Some("a20"), + Some("a21"), + Some("a22"), + Some("a23"), + Some("a24"), + Some("a25"), + Some("a26"), + Some("a27"), + Some("a28"), + Some("a6"), + Some("a7"), + Some("a8"), + Some("a9"), + Some("a10"), + Some("a29"), + Some("a30"), + Some("a31"), + Some("a32"), + Some("a33"), + Some("a34"), + Some("a35"), + Some("a36"), + Some("a37"), + Some("a38"), + Some("a39"), + Some("a40"), + Some("a41"), + Some("a42"), + Some("a43"), + Some("a44"), + Some("a45"), + Some("a46"), + Some("a47"), + Some("a48"), + Some("a49"), + Some("a50"), + Some("a51"), + Some("a52"), + Some("a53"), + Some("a54"), + Some("a55"), + Some("a56"), + Some("a57"), + Some("a58"), + Some("a59"), + Some("a60"), + Some("a61"), + Some("a62"), + Some("a63"), + Some("a64"), + Some("a65"), + Some("a66"), + Some("a67"), + Some("a68"), + Some("a69"), + Some("a70"), + Some("a71"), + Some("a72"), + Some("a73"), + Some("a74"), + Some("a203"), + Some("a75"), + Some("a204"), + Some("a76"), + Some("a77"), + Some("a78"), + Some("a79"), + Some("a81"), + Some("a82"), + Some("a83"), + Some("a84"), + Some("a97"), + Some("a98"), + Some("a99"), + Some("a100"), + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + Some("a101"), + Some("a102"), + Some("a103"), + Some("a104"), + Some("a106"), + Some("a107"), + Some("a108"), + Some("a112"), + Some("a111"), + Some("a110"), + Some("a109"), + Some("a120"), + Some("a121"), + Some("a122"), + Some("a123"), + Some("a124"), + Some("a125"), + Some("a126"), + Some("a127"), + Some("a128"), + Some("a129"), + Some("a130"), + Some("a131"), + Some("a132"), + Some("a133"), + Some("a134"), + Some("a135"), + Some("a136"), + Some("a137"), + Some("a138"), + Some("a139"), + Some("a140"), + Some("a141"), + Some("a142"), + Some("a143"), + Some("a144"), + Some("a145"), + Some("a146"), + Some("a147"), + Some("a148"), + Some("a149"), + Some("a150"), + Some("a151"), + Some("a152"), + Some("a153"), + Some("a154"), + Some("a155"), + Some("a156"), + Some("a157"), + Some("a158"), + Some("a159"), + Some("a160"), + Some("a161"), + Some("a163"), + Some("a164"), + Some("a196"), + Some("a165"), + Some("a192"), + Some("a166"), + Some("a167"), + Some("a168"), + Some("a169"), + Some("a170"), + Some("a171"), + Some("a172"), + Some("a173"), + Some("a162"), + Some("a174"), + Some("a175"), + Some("a176"), + Some("a177"), + Some("a178"), + Some("a179"), + Some("a193"), + Some("a180"), + Some("a199"), + Some("a181"), + Some("a200"), + Some("a182"), + None, + Some("a201"), + Some("a183"), + Some("a184"), + Some("a197"), + Some("a185"), + Some("a194"), + Some("a198"), + Some("a186"), + Some("a195"), + Some("a187"), + Some("a188"), + Some("a189"), + Some("a190"), + Some("a191"), + None, +]; diff --git a/crates/pdf-extract/src/font_cmap.rs b/crates/pdf-extract/src/font_cmap.rs new file mode 100644 index 0000000..8e25bac --- /dev/null +++ b/crates/pdf-extract/src/font_cmap.rs @@ -0,0 +1,192 @@ +//! Reverse cmap: glyph-id -> unicode, parsed from an embedded font +//! program's `cmap` table via ttf-parser. Used to recover text from +//! CID-keyed fonts whose /ToUnicode is missing or garbage. +//! +//! Algorithm derived from run-llama/liteparse `font_cmap.rs` +//! (Apache-2.0); reimplemented — no code copied. +//! +//! Preference order (liteparse): UCS-4 subtables (format 12, platform 0/4 +//! or 3/10) over BMP (format 4, platform 0/3 or 3/1); later mappings for a +//! glyph do not overwrite earlier ones; Private Use Area codepoints +//! (U+E000..=U+F8FF) are rejected as they are never real text. + +use std::collections::HashMap; + +#[derive(Clone)] +pub(crate) struct ReverseCmap { + map: HashMap, // glyph id -> unicode +} + +impl ReverseCmap { + /// Parse `data` as an sfnt and invert its unicode cmap subtables. + /// `None` if the data isn't a parseable font or has no usable cmap. + pub(crate) fn from_font_program(data: &[u8]) -> Option { + let face = ttf_parser::Face::parse(data, 0).ok()?; + let cmap = face.tables().cmap?; + // Unicode subtables, best-first: format 12 (`SegmentedCoverage`, + // full UCS-4) before BMP formats (4/0/6). + let mut subs: Vec<_> = cmap + .subtables + .into_iter() + .filter(|s| s.is_unicode()) + .collect(); + subs.sort_by_key(|s| { + u8::from(!matches!( + s.format, + ttf_parser::cmap::Format::SegmentedCoverage(_) + )) + }); + let mut map = HashMap::new(); + for sub in subs { + sub.codepoints(|cp| { + // PUA codepoints are never real text; skip. + if (0xE000..=0xF8FF).contains(&cp) { + return; + } + let Some(ch) = char::from_u32(cp) else { return }; + if let Some(glyph) = sub.glyph_index(cp) { + // First mapping for a glyph wins (UCS-4 subtable ran + // first); glyph 0 already surfaces as None. + map.entry(u32::from(glyph.0)).or_insert(ch); + } + }); + } + if map.is_empty() { + None + } else { + Some(ReverseCmap { map }) + } + } + + pub(crate) fn lookup(&self, glyph_id: u32) -> Option { + self.map.get(&glyph_id).copied() + } + + /// Build a ReverseCmap directly from glyph→char pairs. Test-only: + /// lib.rs's `decode_char_with` tests need a known mapping without + /// plumbing a synthetic sfnt through the font-decode tests too. + #[cfg(test)] + pub(crate) fn from_pairs_for_test(pairs: &[(u32, char)]) -> ReverseCmap { + ReverseCmap { + map: pairs.iter().copied().collect(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a minimal synthetic sfnt: a `cmap` table with one format-4 + /// subtable mapping A(0x41)->glyph 1, B(0x42)->glyph 2 via an idDelta + /// segment [0x41..=0x42], delta = -0x40, plus the required 0xFFFF + /// terminator segment. ttf-parser's `Face::parse` hard-requires + /// `head`/`hhea`/`maxp`, so those are included with minimal valid + /// contents. Kept synthetic: no checked-in binary fixtures. + fn tiny_sfnt() -> Vec { + // cmap format 4 layout: https://learn.microsoft.com/en-us/typography/opentype/spec/cmap#format-4 + let mut sub = Vec::new(); + let seg_count = 2_u16; // [0x41..0x42], [0xFFFF] + let seg_x2 = seg_count * 2; + sub.extend_from_slice(&4_u16.to_be_bytes()); // format + sub.extend_from_slice(&0_u16.to_be_bytes()); // length (patched below) + sub.extend_from_slice(&0_u16.to_be_bytes()); // language + sub.extend_from_slice(&(seg_x2).to_be_bytes()); + // searchRange = 2 * 2^floor(log2(segCount)) = 4 + sub.extend_from_slice(&4_u16.to_be_bytes()); + // entrySelector = floor(log2(segCount)) = 1 + sub.extend_from_slice(&1_u16.to_be_bytes()); + // rangeShift = 2*segCount - searchRange = 0 + sub.extend_from_slice(&0_u16.to_be_bytes()); + sub.extend_from_slice(&0x42_u16.to_be_bytes()); // endCode[0] + sub.extend_from_slice(&0xFFFF_u16.to_be_bytes()); // endCode[1] + sub.extend_from_slice(&0_u16.to_be_bytes()); // reservedPad + sub.extend_from_slice(&0x41_u16.to_be_bytes()); // startCode[0] + sub.extend_from_slice(&0xFFFF_u16.to_be_bytes()); // startCode[1] + sub.extend_from_slice(&(-0x40_i16).to_be_bytes()); // idDelta[0]: 0x41->1 + sub.extend_from_slice(&1_i16.to_be_bytes()); // idDelta[1]: 0xFFFF->0 + sub.extend_from_slice(&0_u16.to_be_bytes()); // idRangeOffset[0] + sub.extend_from_slice(&0_u16.to_be_bytes()); // idRangeOffset[1] + let sub_len = sub.len() as u16; + sub[2..4].copy_from_slice(&sub_len.to_be_bytes()); + + let mut cmap = Vec::new(); + cmap.extend_from_slice(&0_u16.to_be_bytes()); // version + cmap.extend_from_slice(&1_u16.to_be_bytes()); // numTables + cmap.extend_from_slice(&3_u16.to_be_bytes()); // platform: Windows + cmap.extend_from_slice(&1_u16.to_be_bytes()); // encoding: Unicode BMP + cmap.extend_from_slice(&12_u32.to_be_bytes()); // subtable offset + cmap.extend_from_slice(&sub); + + // head: 54 bytes; only unitsPerEm (offset 18) is validated + // (16..=16384). + let mut head = Vec::new(); + head.extend_from_slice(&0x0001_0000_u32.to_be_bytes()); // version + head.extend_from_slice(&0_u32.to_be_bytes()); // fontRevision + head.extend_from_slice(&0_u32.to_be_bytes()); // checkSumAdjustment + head.extend_from_slice(&0x5F0F_3CF5_u32.to_be_bytes()); // magicNumber + head.extend_from_slice(&0_u16.to_be_bytes()); // flags + head.extend_from_slice(&1000_u16.to_be_bytes()); // unitsPerEm + head.extend_from_slice(&[0; 16]); // created + modified + head.extend_from_slice(&[0; 8]); // xMin..yMax + head.extend_from_slice(&[0; 10]); // macStyle..glyphDataFormat + assert_eq!(head.len(), 54); + + // hhea: 36 bytes; numberOfMetrics (u16) at offset 34. + let mut hhea = Vec::new(); + hhea.extend_from_slice(&0x0001_0000_u32.to_be_bytes()); // version + hhea.extend_from_slice(&[0; 30]); + hhea.extend_from_slice(&3_u16.to_be_bytes()); // numberOfMetrics + assert_eq!(hhea.len(), 36); + + // maxp: version 1.0; numGlyphs (u16, nonzero) at offset 4. + let mut maxp = Vec::new(); + maxp.extend_from_slice(&0x0001_0000_u32.to_be_bytes()); // version + maxp.extend_from_slice(&3_u16.to_be_bytes()); // numGlyphs + maxp.extend_from_slice(&[0; 26]); // remaining v1.0 fields + assert_eq!(maxp.len(), 32); + + // Table directory records must be sorted by tag (ttf-parser + // binary-searches them): cmap < head < hhea < maxp. + let tables: [(&[u8; 4], &[u8]); 4] = [ + (b"cmap", &cmap), + (b"head", &head), + (b"hhea", &hhea), + (b"maxp", &maxp), + ]; + let num_tables = tables.len() as u16; + let dir_len = 12 + tables.len() * 16; + let mut sfnt = Vec::new(); + sfnt.extend_from_slice(&0x0001_0000_u32.to_be_bytes()); // sfnt version + sfnt.extend_from_slice(&num_tables.to_be_bytes()); + sfnt.extend_from_slice(&64_u16.to_be_bytes()); // searchRange = 16*2^2 + sfnt.extend_from_slice(&2_u16.to_be_bytes()); // entrySelector + sfnt.extend_from_slice(&0_u16.to_be_bytes()); // rangeShift + let mut offset = dir_len as u32; + for (tag, data) in &tables { + sfnt.extend_from_slice(tag.as_slice()); + sfnt.extend_from_slice(&0_u32.to_be_bytes()); // checksum (unused) + sfnt.extend_from_slice(&offset.to_be_bytes()); + sfnt.extend_from_slice(&(data.len() as u32).to_be_bytes()); + offset += data.len() as u32; + } + for (_, data) in &tables { + sfnt.extend_from_slice(data); + } + sfnt + } + + #[test] + fn reverse_cmap_inverts_format4() { + let rc = ReverseCmap::from_font_program(&tiny_sfnt()).unwrap(); + assert_eq!(rc.lookup(1), Some('A')); + assert_eq!(rc.lookup(2), Some('B')); + assert_eq!(rc.lookup(3), None); + } + + #[test] + fn reverse_cmap_rejects_garbage() { + assert!(ReverseCmap::from_font_program(b"not a font").is_none()); + assert!(ReverseCmap::from_font_program(&[]).is_none()); + } +} diff --git a/crates/pdf-extract/src/glyphlist-export.py b/crates/pdf-extract/src/glyphlist-export.py new file mode 100644 index 0000000..de50c0a --- /dev/null +++ b/crates/pdf-extract/src/glyphlist-export.py @@ -0,0 +1,41 @@ +glyphlist = [] +glyphs_seen = {} +def read_glyphs(name): + f = open(name) + lines = f.readlines() + import re + for l in lines: + if l[0] == '#' or l[0] == '\n': + continue + split = re.split('[; ,]+', l) + name = split[0] + val = int(split[1], 16) + if val > 0xffff: + val = int(split[-1], 16) + if val == 0xf766 and name != "Fsmall": + continue + if name in glyphs_seen: + continue + glyphs_seen[name] = True + glyphlist.append((name,val)) +read_glyphs("glyphlist-extended.txt") +read_glyphs("texglyphlist.txt") +read_glyphs("additional.txt") +# there are some conflicts between these files +# e.g. tildewide=0x02dc, vs tildewide=0x0303 +# for now we just ignore the subsequent ones +glyphlist.append(('mapsto', 0x21A6)) +glyphlist = list(set(glyphlist)) +glyphlist.sort() +print "/* Autogenerated from:" +print " https://github.com/michal-h21/htfgen/commits/master/glyphlist-extended.txt" +print " https://github.com/kohler/lcdf-typetools/blob/master/texglyphlist.txt" +print " https://github.com/apache/pdfbox/blob/trunk/pdfbox/src/main/resources/org/apache/pdfbox/resources/glyphlist/additional.txt" +print " */" +print "pub fn name_to_unicode(name: &str) -> Option {" +print " const names: [(&'static str, u16); %d] = [" % len(glyphlist) +print ",\n".join('(\"%s\", 0x%04x)' % (g[0], g[1]) for g in glyphlist) +print " ];" +print " let result = names.binary_search_by_key(&name, |&(name,_code)| &name);" +print " result.ok().map(|indx| names[indx].1)" +print "}" diff --git a/crates/pdf-extract/src/glyphlist-extended.txt b/crates/pdf-extract/src/glyphlist-extended.txt new file mode 100644 index 0000000..ca21f43 --- /dev/null +++ b/crates/pdf-extract/src/glyphlist-extended.txt @@ -0,0 +1,4553 @@ +# lcdf-typetools glyphlist.txt, Version 2.33 +# Contents: The Adobe Glyph List, followed by extensions for TeX fonts and +# encodings. We also extend the second field so that it can contain +# multiple Unicode scalar values, separated by semicolons, analogous to +# otftotfm's "% UNICODING" comments. +############################################################################### +# Name: Adobe Glyph List +# Table version: 2.0 +# Date: September 20, 2002 +# +# See http://partners.adobe.com/asn/developer/typeforum/unicodegn.html +# +# Format: Semicolon-delimited fields: +# (1) glyph name +# (2) Unicode scalar value +A;0041 +AE;00C6 +AEacute;01FC +AEmacron;01E2 +AEsmall;F7E6 +Aacute;00C1 +Aacutesmall;F7E1 +Abreve;0102 +Abreveacute;1EAE +Abrevecyrillic;04D0 +Abrevedotbelow;1EB6 +Abrevegrave;1EB0 +Abrevehookabove;1EB2 +Abrevetilde;1EB4 +Acaron;01CD +Acircle;24B6 +Acircumflex;00C2 +Acircumflexacute;1EA4 +Acircumflexdotbelow;1EAC +Acircumflexgrave;1EA6 +Acircumflexhookabove;1EA8 +Acircumflexsmall;F7E2 +Acircumflextilde;1EAA +Acute;F6C9 +Acutesmall;F7B4 +Acyrillic;0410 +Adblgrave;0200 +Adieresis;00C4 +Adieresiscyrillic;04D2 +Adieresismacron;01DE +Adieresissmall;F7E4 +Adotbelow;1EA0 +Adotmacron;01E0 +Agrave;00C0 +Agravesmall;F7E0 +Ahookabove;1EA2 +Aiecyrillic;04D4 +Ainvertedbreve;0202 +Alpha;0391 +Alphatonos;0386 +Amacron;0100 +Amonospace;FF21 +Aogonek;0104 +Aring;00C5 +Aringacute;01FA +Aringbelow;1E00 +Aringsmall;F7E5 +Asmall;F761 +Atilde;00C3 +Atildesmall;F7E3 +Aybarmenian;0531 +B;0042 +Bcircle;24B7 +Bdotaccent;1E02 +Bdotbelow;1E04 +Becyrillic;0411 +Benarmenian;0532 +Beta;0392 +Bhook;0181 +Blinebelow;1E06 +Bmonospace;FF22 +Brevesmall;F6F4 +Bsmall;F762 +Btopbar;0182 +C;0043 +Caarmenian;053E +Cacute;0106 +Caron;F6CA +Caronsmall;F6F5 +Ccaron;010C +Ccedilla;00C7 +Ccedillaacute;1E08 +Ccedillasmall;F7E7 +Ccircle;24B8 +Ccircumflex;0108 +Cdot;010A +Cdotaccent;010A +Cedillasmall;F7B8 +Chaarmenian;0549 +Cheabkhasiancyrillic;04BC +Checyrillic;0427 +Chedescenderabkhasiancyrillic;04BE +Chedescendercyrillic;04B6 +Chedieresiscyrillic;04F4 +Cheharmenian;0543 +Chekhakassiancyrillic;04CB +Cheverticalstrokecyrillic;04B8 +Chi;03A7 +Chook;0187 +Circumflexsmall;F6F6 +Cmonospace;FF23 +Coarmenian;0551 +Csmall;F763 +D;0044 +DZ;01F1 +DZcaron;01C4 +Daarmenian;0534 +Dafrican;0189 +Dcaron;010E +Dcedilla;1E10 +Dcircle;24B9 +Dcircumflexbelow;1E12 +Dcroat;0110 +Ddotaccent;1E0A +Ddotbelow;1E0C +Decyrillic;0414 +Deicoptic;03EE +Delta;2206 +Deltagreek;0394 +Dhook;018A +Dieresis;F6CB +DieresisAcute;F6CC +DieresisGrave;F6CD +Dieresissmall;F7A8 +Digammagreek;03DC +Djecyrillic;0402 +Dlinebelow;1E0E +Dmonospace;FF24 +Dotaccentsmall;F6F7 +Dslash;0110 +Dsmall;F764 +Dtopbar;018B +Dz;01F2 +Dzcaron;01C5 +Dzeabkhasiancyrillic;04E0 +Dzecyrillic;0405 +Dzhecyrillic;040F +E;0045 +Eacute;00C9 +Eacutesmall;F7E9 +Ebreve;0114 +Ecaron;011A +Ecedillabreve;1E1C +Echarmenian;0535 +Ecircle;24BA +Ecircumflex;00CA +Ecircumflexacute;1EBE +Ecircumflexbelow;1E18 +Ecircumflexdotbelow;1EC6 +Ecircumflexgrave;1EC0 +Ecircumflexhookabove;1EC2 +Ecircumflexsmall;F7EA +Ecircumflextilde;1EC4 +Ecyrillic;0404 +Edblgrave;0204 +Edieresis;00CB +Edieresissmall;F7EB +Edot;0116 +Edotaccent;0116 +Edotbelow;1EB8 +Efcyrillic;0424 +Egrave;00C8 +Egravesmall;F7E8 +Eharmenian;0537 +Ehookabove;1EBA +Eightroman;2167 +Einvertedbreve;0206 +Eiotifiedcyrillic;0464 +Elcyrillic;041B +Elevenroman;216A +Emacron;0112 +Emacronacute;1E16 +Emacrongrave;1E14 +Emcyrillic;041C +Emonospace;FF25 +Encyrillic;041D +Endescendercyrillic;04A2 +Eng;014A +Enghecyrillic;04A4 +Enhookcyrillic;04C7 +Eogonek;0118 +Eopen;0190 +Epsilon;0395 +Epsilontonos;0388 +Ercyrillic;0420 +Ereversed;018E +Ereversedcyrillic;042D +Escyrillic;0421 +Esdescendercyrillic;04AA +Esh;01A9 +Esmall;F765 +Eta;0397 +Etarmenian;0538 +Etatonos;0389 +Eth;00D0 +Ethsmall;F7F0 +Etilde;1EBC +Etildebelow;1E1A +Euro;20AC +Ezh;01B7 +Ezhcaron;01EE +Ezhreversed;01B8 +F;0046 +Fcircle;24BB +Fdotaccent;1E1E +Feharmenian;0556 +Feicoptic;03E4 +Fhook;0191 +Fitacyrillic;0472 +Fiveroman;2164 +Fmonospace;FF26 +Fourroman;2163 +Fsmall;F766 +G;0047 +GBsquare;3387 +Gacute;01F4 +Gamma;0393 +Gammaafrican;0194 +Gangiacoptic;03EA +Gbreve;011E +Gcaron;01E6 +Gcedilla;0122 +Gcircle;24BC +Gcircumflex;011C +Gcommaaccent;0122 +Gdot;0120 +Gdotaccent;0120 +Gecyrillic;0413 +Ghadarmenian;0542 +Ghemiddlehookcyrillic;0494 +Ghestrokecyrillic;0492 +Gheupturncyrillic;0490 +Ghook;0193 +Gimarmenian;0533 +Gjecyrillic;0403 +Gmacron;1E20 +Gmonospace;FF27 +Grave;F6CE +Gravesmall;F760 +Gsmall;F767 +Gsmallhook;029B +Gstroke;01E4 +H;0048 +H18533;25CF +H18543;25AA +H18551;25AB +H22073;25A1 +HPsquare;33CB +Haabkhasiancyrillic;04A8 +Hadescendercyrillic;04B2 +Hardsigncyrillic;042A +Hbar;0126 +Hbrevebelow;1E2A +Hcedilla;1E28 +Hcircle;24BD +Hcircumflex;0124 +Hdieresis;1E26 +Hdotaccent;1E22 +Hdotbelow;1E24 +Hmonospace;FF28 +Hoarmenian;0540 +Horicoptic;03E8 +Hsmall;F768 +Hungarumlaut;F6CF +Hungarumlautsmall;F6F8 +Hzsquare;3390 +I;0049 +IAcyrillic;042F +IJ;0132 +IUcyrillic;042E +Iacute;00CD +Iacutesmall;F7ED +Ibreve;012C +Icaron;01CF +Icircle;24BE +Icircumflex;00CE +Icircumflexsmall;F7EE +Icyrillic;0406 +Idblgrave;0208 +Idieresis;00CF +Idieresisacute;1E2E +Idieresiscyrillic;04E4 +Idieresissmall;F7EF +Idot;0130 +Idotaccent;0130 +Idotbelow;1ECA +Iebrevecyrillic;04D6 +Iecyrillic;0415 +Ifraktur;2111 +Igrave;00CC +Igravesmall;F7EC +Ihookabove;1EC8 +Iicyrillic;0418 +Iinvertedbreve;020A +Iishortcyrillic;0419 +Imacron;012A +Imacroncyrillic;04E2 +Imonospace;FF29 +Iniarmenian;053B +Iocyrillic;0401 +Iogonek;012E +Iota;0399 +Iotaafrican;0196 +Iotadieresis;03AA +Iotatonos;038A +Ismall;F769 +Istroke;0197 +Itilde;0128 +Itildebelow;1E2C +Izhitsacyrillic;0474 +Izhitsadblgravecyrillic;0476 +J;004A +Jaarmenian;0541 +Jcircle;24BF +Jcircumflex;0134 +Jecyrillic;0408 +Jheharmenian;054B +Jmonospace;FF2A +Jsmall;F76A +K;004B +KBsquare;3385 +KKsquare;33CD +Kabashkircyrillic;04A0 +Kacute;1E30 +Kacyrillic;041A +Kadescendercyrillic;049A +Kahookcyrillic;04C3 +Kappa;039A +Kastrokecyrillic;049E +Kaverticalstrokecyrillic;049C +Kcaron;01E8 +Kcedilla;0136 +Kcircle;24C0 +Kcommaaccent;0136 +Kdotbelow;1E32 +Keharmenian;0554 +Kenarmenian;053F +Khacyrillic;0425 +Kheicoptic;03E6 +Khook;0198 +Kjecyrillic;040C +Klinebelow;1E34 +Kmonospace;FF2B +Koppacyrillic;0480 +Koppagreek;03DE +Ksicyrillic;046E +Ksmall;F76B +L;004C +LJ;01C7 +LL;F6BF +Lacute;0139 +Lambda;039B +Lcaron;013D +Lcedilla;013B +Lcircle;24C1 +Lcircumflexbelow;1E3C +Lcommaaccent;013B +Ldot;013F +Ldotaccent;013F +Ldotbelow;1E36 +Ldotbelowmacron;1E38 +Liwnarmenian;053C +Lj;01C8 +Ljecyrillic;0409 +Llinebelow;1E3A +Lmonospace;FF2C +Lslash;0141 +Lslashsmall;F6F9 +Lsmall;F76C +M;004D +MBsquare;3386 +Macron;F6D0 +Macronsmall;F7AF +Macute;1E3E +Mcircle;24C2 +Mdotaccent;1E40 +Mdotbelow;1E42 +Menarmenian;0544 +Mmonospace;FF2D +Msmall;F76D +Mturned;019C +Mu;039C +N;004E +NJ;01CA +Nacute;0143 +Ncaron;0147 +Ncedilla;0145 +Ncircle;24C3 +Ncircumflexbelow;1E4A +Ncommaaccent;0145 +Ndotaccent;1E44 +Ndotbelow;1E46 +Nhookleft;019D +Nineroman;2168 +Nj;01CB +Njecyrillic;040A +Nlinebelow;1E48 +Nmonospace;FF2E +Nowarmenian;0546 +Nsmall;F76E +Ntilde;00D1 +Ntildesmall;F7F1 +Nu;039D +O;004F +OE;0152 +OEsmall;F6FA +Oacute;00D3 +Oacutesmall;F7F3 +Obarredcyrillic;04E8 +Obarreddieresiscyrillic;04EA +Obreve;014E +Ocaron;01D1 +Ocenteredtilde;019F +Ocircle;24C4 +Ocircumflex;00D4 +Ocircumflexacute;1ED0 +Ocircumflexdotbelow;1ED8 +Ocircumflexgrave;1ED2 +Ocircumflexhookabove;1ED4 +Ocircumflexsmall;F7F4 +Ocircumflextilde;1ED6 +Ocyrillic;041E +Odblacute;0150 +Odblgrave;020C +Odieresis;00D6 +Odieresiscyrillic;04E6 +Odieresissmall;F7F6 +Odotbelow;1ECC +Ogoneksmall;F6FB +Ograve;00D2 +Ogravesmall;F7F2 +Oharmenian;0555 +Ohm;2126 +Ohookabove;1ECE +Ohorn;01A0 +Ohornacute;1EDA +Ohorndotbelow;1EE2 +Ohorngrave;1EDC +Ohornhookabove;1EDE +Ohorntilde;1EE0 +Ohungarumlaut;0150 +Oi;01A2 +Oinvertedbreve;020E +Omacron;014C +Omacronacute;1E52 +Omacrongrave;1E50 +Omega;2126 +Omegacyrillic;0460 +Omegagreek;03A9 +Omegaroundcyrillic;047A +Omegatitlocyrillic;047C +Omegatonos;038F +Omicron;039F +Omicrontonos;038C +Omonospace;FF2F +Oneroman;2160 +Oogonek;01EA +Oogonekmacron;01EC +Oopen;0186 +Oslash;00D8 +Oslashacute;01FE +Oslashsmall;F7F8 +Osmall;F76F +Ostrokeacute;01FE +Otcyrillic;047E +Otilde;00D5 +Otildeacute;1E4C +Otildedieresis;1E4E +Otildesmall;F7F5 +P;0050 +Pacute;1E54 +Pcircle;24C5 +Pdotaccent;1E56 +Pecyrillic;041F +Peharmenian;054A +Pemiddlehookcyrillic;04A6 +Phi;03A6 +Phook;01A4 +Pi;03A0 +Piwrarmenian;0553 +Pmonospace;FF30 +Psi;03A8 +Psicyrillic;0470 +Psmall;F770 +Q;0051 +Qcircle;24C6 +Qmonospace;FF31 +Qsmall;F771 +R;0052 +Raarmenian;054C +Racute;0154 +Rcaron;0158 +Rcedilla;0156 +Rcircle;24C7 +Rcommaaccent;0156 +Rdblgrave;0210 +Rdotaccent;1E58 +Rdotbelow;1E5A +Rdotbelowmacron;1E5C +Reharmenian;0550 +Rfraktur;211C +Rho;03A1 +Ringsmall;F6FC +Rinvertedbreve;0212 +Rlinebelow;1E5E +Rmonospace;FF32 +Rsmall;F772 +Rsmallinverted;0281 +Rsmallinvertedsuperior;02B6 +S;0053 +SF010000;250C +SF020000;2514 +SF030000;2510 +SF040000;2518 +SF050000;253C +SF060000;252C +SF070000;2534 +SF080000;251C +SF090000;2524 +SF100000;2500 +SF110000;2502 +SF190000;2561 +SF200000;2562 +SF210000;2556 +SF220000;2555 +SF230000;2563 +SF240000;2551 +SF250000;2557 +SF260000;255D +SF270000;255C +SF280000;255B +SF360000;255E +SF370000;255F +SF380000;255A +SF390000;2554 +SF400000;2569 +SF410000;2566 +SF420000;2560 +SF430000;2550 +SF440000;256C +SF450000;2567 +SF460000;2568 +SF470000;2564 +SF480000;2565 +SF490000;2559 +SF500000;2558 +SF510000;2552 +SF520000;2553 +SF530000;256B +SF540000;256A +Sacute;015A +Sacutedotaccent;1E64 +Sampigreek;03E0 +Scaron;0160 +Scarondotaccent;1E66 +Scaronsmall;F6FD +Scedilla;015E +Schwa;018F +Schwacyrillic;04D8 +Schwadieresiscyrillic;04DA +Scircle;24C8 +Scircumflex;015C +Scommaaccent;0218 +Sdotaccent;1E60 +Sdotbelow;1E62 +Sdotbelowdotaccent;1E68 +Seharmenian;054D +Sevenroman;2166 +Shaarmenian;0547 +Shacyrillic;0428 +Shchacyrillic;0429 +Sheicoptic;03E2 +Shhacyrillic;04BA +Shimacoptic;03EC +Sigma;03A3 +Sixroman;2165 +Smonospace;FF33 +Softsigncyrillic;042C +Ssmall;F773 +Stigmagreek;03DA +T;0054 +Tau;03A4 +Tbar;0166 +Tcaron;0164 +Tcedilla;0162 +Tcircle;24C9 +Tcircumflexbelow;1E70 +Tcommaaccent;0162 +Tdotaccent;1E6A +Tdotbelow;1E6C +Tecyrillic;0422 +Tedescendercyrillic;04AC +Tenroman;2169 +Tetsecyrillic;04B4 +Theta;0398 +Thook;01AC +Thorn;00DE +Thornsmall;F7FE +Threeroman;2162 +Tildesmall;F6FE +Tiwnarmenian;054F +Tlinebelow;1E6E +Tmonospace;FF34 +Toarmenian;0539 +Tonefive;01BC +Tonesix;0184 +Tonetwo;01A7 +Tretroflexhook;01AE +Tsecyrillic;0426 +Tshecyrillic;040B +Tsmall;F774 +Twelveroman;216B +Tworoman;2161 +U;0055 +Uacute;00DA +Uacutesmall;F7FA +Ubreve;016C +Ucaron;01D3 +Ucircle;24CA +Ucircumflex;00DB +Ucircumflexbelow;1E76 +Ucircumflexsmall;F7FB +Ucyrillic;0423 +Udblacute;0170 +Udblgrave;0214 +Udieresis;00DC +Udieresisacute;01D7 +Udieresisbelow;1E72 +Udieresiscaron;01D9 +Udieresiscyrillic;04F0 +Udieresisgrave;01DB +Udieresismacron;01D5 +Udieresissmall;F7FC +Udotbelow;1EE4 +Ugrave;00D9 +Ugravesmall;F7F9 +Uhookabove;1EE6 +Uhorn;01AF +Uhornacute;1EE8 +Uhorndotbelow;1EF0 +Uhorngrave;1EEA +Uhornhookabove;1EEC +Uhorntilde;1EEE +Uhungarumlaut;0170 +Uhungarumlautcyrillic;04F2 +Uinvertedbreve;0216 +Ukcyrillic;0478 +Umacron;016A +Umacroncyrillic;04EE +Umacrondieresis;1E7A +Umonospace;FF35 +Uogonek;0172 +Upsilon;03A5 +Upsilon1;03D2 +Upsilonacutehooksymbolgreek;03D3 +Upsilonafrican;01B1 +Upsilondieresis;03AB +Upsilondieresishooksymbolgreek;03D4 +Upsilonhooksymbol;03D2 +Upsilontonos;038E +Uring;016E +Ushortcyrillic;040E +Usmall;F775 +Ustraightcyrillic;04AE +Ustraightstrokecyrillic;04B0 +Utilde;0168 +Utildeacute;1E78 +Utildebelow;1E74 +V;0056 +Vcircle;24CB +Vdotbelow;1E7E +Vecyrillic;0412 +Vewarmenian;054E +Vhook;01B2 +Vmonospace;FF36 +Voarmenian;0548 +Vsmall;F776 +Vtilde;1E7C +W;0057 +Wacute;1E82 +Wcircle;24CC +Wcircumflex;0174 +Wdieresis;1E84 +Wdotaccent;1E86 +Wdotbelow;1E88 +Wgrave;1E80 +Wmonospace;FF37 +Wsmall;F777 +X;0058 +Xcircle;24CD +Xdieresis;1E8C +Xdotaccent;1E8A +Xeharmenian;053D +Xi;039E +Xmonospace;FF38 +Xsmall;F778 +Y;0059 +Yacute;00DD +Yacutesmall;F7FD +Yatcyrillic;0462 +Ycircle;24CE +Ycircumflex;0176 +Ydieresis;0178 +Ydieresissmall;F7FF +Ydotaccent;1E8E +Ydotbelow;1EF4 +Yericyrillic;042B +Yerudieresiscyrillic;04F8 +Ygrave;1EF2 +Yhook;01B3 +Yhookabove;1EF6 +Yiarmenian;0545 +Yicyrillic;0407 +Yiwnarmenian;0552 +Ymonospace;FF39 +Ysmall;F779 +Ytilde;1EF8 +Yusbigcyrillic;046A +Yusbigiotifiedcyrillic;046C +Yuslittlecyrillic;0466 +Yuslittleiotifiedcyrillic;0468 +Z;005A +Zaarmenian;0536 +Zacute;0179 +Zcaron;017D +Zcaronsmall;F6FF +Zcircle;24CF +Zcircumflex;1E90 +Zdot;017B +Zdotaccent;017B +Zdotbelow;1E92 +Zecyrillic;0417 +Zedescendercyrillic;0498 +Zedieresiscyrillic;04DE +Zeta;0396 +Zhearmenian;053A +Zhebrevecyrillic;04C1 +Zhecyrillic;0416 +Zhedescendercyrillic;0496 +Zhedieresiscyrillic;04DC +Zlinebelow;1E94 +Zmonospace;FF3A +Zsmall;F77A +Zstroke;01B5 +a;0061 +aabengali;0986 +aacute;00E1 +aadeva;0906 +aagujarati;0A86 +aagurmukhi;0A06 +aamatragurmukhi;0A3E +aarusquare;3303 +aavowelsignbengali;09BE +aavowelsigndeva;093E +aavowelsigngujarati;0ABE +abbreviationmarkarmenian;055F +abbreviationsigndeva;0970 +abengali;0985 +abopomofo;311A +abreve;0103 +abreveacute;1EAF +abrevecyrillic;04D1 +abrevedotbelow;1EB7 +abrevegrave;1EB1 +abrevehookabove;1EB3 +abrevetilde;1EB5 +acaron;01CE +acircle;24D0 +acircumflex;00E2 +acircumflexacute;1EA5 +acircumflexdotbelow;1EAD +acircumflexgrave;1EA7 +acircumflexhookabove;1EA9 +acircumflextilde;1EAB +acute;00B4 +acutebelowcmb;0317 +acutecmb;0301 +acutecomb;0301 +acutedeva;0954 +acutelowmod;02CF +acutetonecmb;0341 +acyrillic;0430 +adblgrave;0201 +addakgurmukhi;0A71 +adeva;0905 +adieresis;00E4 +adieresiscyrillic;04D3 +adieresismacron;01DF +adotbelow;1EA1 +adotmacron;01E1 +ae;00E6 +aeacute;01FD +aekorean;3150 +aemacron;01E3 +afii00208;2015 +afii08941;20A4 +afii10017;0410 +afii10018;0411 +afii10019;0412 +afii10020;0413 +afii10021;0414 +afii10022;0415 +afii10023;0401 +afii10024;0416 +afii10025;0417 +afii10026;0418 +afii10027;0419 +afii10028;041A +afii10029;041B +afii10030;041C +afii10031;041D +afii10032;041E +afii10033;041F +afii10034;0420 +afii10035;0421 +afii10036;0422 +afii10037;0423 +afii10038;0424 +afii10039;0425 +afii10040;0426 +afii10041;0427 +afii10042;0428 +afii10043;0429 +afii10044;042A +afii10045;042B +afii10046;042C +afii10047;042D +afii10048;042E +afii10049;042F +afii10050;0490 +afii10051;0402 +afii10052;0403 +afii10053;0404 +afii10054;0405 +afii10055;0406 +afii10056;0407 +afii10057;0408 +afii10058;0409 +afii10059;040A +afii10060;040B +afii10061;040C +afii10062;040E +afii10063;F6C4 +afii10064;F6C5 +afii10065;0430 +afii10066;0431 +afii10067;0432 +afii10068;0433 +afii10069;0434 +afii10070;0435 +afii10071;0451 +afii10072;0436 +afii10073;0437 +afii10074;0438 +afii10075;0439 +afii10076;043A +afii10077;043B +afii10078;043C +afii10079;043D +afii10080;043E +afii10081;043F +afii10082;0440 +afii10083;0441 +afii10084;0442 +afii10085;0443 +afii10086;0444 +afii10087;0445 +afii10088;0446 +afii10089;0447 +afii10090;0448 +afii10091;0449 +afii10092;044A +afii10093;044B +afii10094;044C +afii10095;044D +afii10096;044E +afii10097;044F +afii10098;0491 +afii10099;0452 +afii10100;0453 +afii10101;0454 +afii10102;0455 +afii10103;0456 +afii10104;0457 +afii10105;0458 +afii10106;0459 +afii10107;045A +afii10108;045B +afii10109;045C +afii10110;045E +afii10145;040F +afii10146;0462 +afii10147;0472 +afii10148;0474 +afii10192;F6C6 +afii10193;045F +afii10194;0463 +afii10195;0473 +afii10196;0475 +afii10831;F6C7 +afii10832;F6C8 +afii10846;04D9 +afii299;200E +afii300;200F +afii301;200D +afii57381;066A +afii57388;060C +afii57392;0660 +afii57393;0661 +afii57394;0662 +afii57395;0663 +afii57396;0664 +afii57397;0665 +afii57398;0666 +afii57399;0667 +afii57400;0668 +afii57401;0669 +afii57403;061B +afii57407;061F +afii57409;0621 +afii57410;0622 +afii57411;0623 +afii57412;0624 +afii57413;0625 +afii57414;0626 +afii57415;0627 +afii57416;0628 +afii57417;0629 +afii57418;062A +afii57419;062B +afii57420;062C +afii57421;062D +afii57422;062E +afii57423;062F +afii57424;0630 +afii57425;0631 +afii57426;0632 +afii57427;0633 +afii57428;0634 +afii57429;0635 +afii57430;0636 +afii57431;0637 +afii57432;0638 +afii57433;0639 +afii57434;063A +afii57440;0640 +afii57441;0641 +afii57442;0642 +afii57443;0643 +afii57444;0644 +afii57445;0645 +afii57446;0646 +afii57448;0648 +afii57449;0649 +afii57450;064A +afii57451;064B +afii57452;064C +afii57453;064D +afii57454;064E +afii57455;064F +afii57456;0650 +afii57457;0651 +afii57458;0652 +afii57470;0647 +afii57505;06A4 +afii57506;067E +afii57507;0686 +afii57508;0698 +afii57509;06AF +afii57511;0679 +afii57512;0688 +afii57513;0691 +afii57514;06BA +afii57519;06D2 +afii57534;06D5 +afii57636;20AA +afii57645;05BE +afii57658;05C3 +afii57664;05D0 +afii57665;05D1 +afii57666;05D2 +afii57667;05D3 +afii57668;05D4 +afii57669;05D5 +afii57670;05D6 +afii57671;05D7 +afii57672;05D8 +afii57673;05D9 +afii57674;05DA +afii57675;05DB +afii57676;05DC +afii57677;05DD +afii57678;05DE +afii57679;05DF +afii57680;05E0 +afii57681;05E1 +afii57682;05E2 +afii57683;05E3 +afii57684;05E4 +afii57685;05E5 +afii57686;05E6 +afii57687;05E7 +afii57688;05E8 +afii57689;05E9 +afii57690;05EA +afii57694;FB2A +afii57695;FB2B +afii57700;FB4B +afii57705;FB1F +afii57716;05F0 +afii57717;05F1 +afii57718;05F2 +afii57723;FB35 +afii57793;05B4 +afii57794;05B5 +afii57795;05B6 +afii57796;05BB +afii57797;05B8 +afii57798;05B7 +afii57799;05B0 +afii57800;05B2 +afii57801;05B1 +afii57802;05B3 +afii57803;05C2 +afii57804;05C1 +afii57806;05B9 +afii57807;05BC +afii57839;05BD +afii57841;05BF +afii57842;05C0 +afii57929;02BC +afii61248;2105 +afii61289;2113 +afii61352;2116 +afii61573;202C +afii61574;202D +afii61575;202E +afii61664;200C +afii63167;066D +afii64937;02BD +agrave;00E0 +agujarati;0A85 +agurmukhi;0A05 +ahiragana;3042 +ahookabove;1EA3 +aibengali;0990 +aibopomofo;311E +aideva;0910 +aiecyrillic;04D5 +aigujarati;0A90 +aigurmukhi;0A10 +aimatragurmukhi;0A48 +ainarabic;0639 +ainfinalarabic;FECA +aininitialarabic;FECB +ainmedialarabic;FECC +ainvertedbreve;0203 +aivowelsignbengali;09C8 +aivowelsigndeva;0948 +aivowelsigngujarati;0AC8 +akatakana;30A2 +akatakanahalfwidth;FF71 +akorean;314F +alef;05D0 +alefarabic;0627 +alefdageshhebrew;FB30 +aleffinalarabic;FE8E +alefhamzaabovearabic;0623 +alefhamzaabovefinalarabic;FE84 +alefhamzabelowarabic;0625 +alefhamzabelowfinalarabic;FE88 +alefhebrew;05D0 +aleflamedhebrew;FB4F +alefmaddaabovearabic;0622 +alefmaddaabovefinalarabic;FE82 +alefmaksuraarabic;0649 +alefmaksurafinalarabic;FEF0 +alefmaksurainitialarabic;FEF3 +alefmaksuramedialarabic;FEF4 +alefpatahhebrew;FB2E +alefqamatshebrew;FB2F +aleph;2135 +allequal;224C +alpha;03B1 +alphatonos;03AC +amacron;0101 +amonospace;FF41 +ampersand;0026 +ampersandmonospace;FF06 +ampersandsmall;F726 +amsquare;33C2 +anbopomofo;3122 +angbopomofo;3124 +angkhankhuthai;0E5A +angle;2220 +anglebracketleft;3008 +anglebracketleftvertical;FE3F +anglebracketright;3009 +anglebracketrightvertical;FE40 +angleleft;2329 +angleright;232A +angstrom;212B +anoteleia;0387 +anudattadeva;0952 +anusvarabengali;0982 +anusvaradeva;0902 +anusvaragujarati;0A82 +aogonek;0105 +apaatosquare;3300 +aparen;249C +apostrophearmenian;055A +apostrophemod;02BC +apple;F8FF +approaches;2250 +approxequal;2248 +approxequalorimage;2252 +approximatelyequal;2245 +araeaekorean;318E +araeakorean;318D +arc;2312 +arighthalfring;1E9A +aring;00E5 +aringacute;01FB +aringbelow;1E01 +arrowboth;2194 +arrowdashdown;21E3 +arrowdashleft;21E0 +arrowdashright;21E2 +arrowdashup;21E1 +arrowdblboth;21D4 +arrowdbldown;21D3 +arrowdblleft;21D0 +arrowdblright;21D2 +arrowdblup;21D1 +arrowdown;2193 +arrowdownleft;2199 +arrowdownright;2198 +arrowdownwhite;21E9 +arrowheaddownmod;02C5 +arrowheadleftmod;02C2 +arrowheadrightmod;02C3 +arrowheadupmod;02C4 +arrowhorizex;F8E7 +arrowleft;2190 +arrowleftdbl;21D0 +arrowleftdblstroke;21CD +arrowleftoverright;21C6 +arrowleftwhite;21E6 +arrowright;2192 +arrowrightdblstroke;21CF +arrowrightheavy;279E +arrowrightoverleft;21C4 +arrowrightwhite;21E8 +arrowtableft;21E4 +arrowtabright;21E5 +arrowup;2191 +arrowupdn;2195 +arrowupdnbse;21A8 +arrowupdownbase;21A8 +arrowupleft;2196 +arrowupleftofdown;21C5 +arrowupright;2197 +arrowupwhite;21E7 +arrowvertex;F8E6 +asciicircum;005E +asciicircummonospace;FF3E +asciitilde;007E +asciitildemonospace;FF5E +ascript;0251 +ascriptturned;0252 +asmallhiragana;3041 +asmallkatakana;30A1 +asmallkatakanahalfwidth;FF67 +asterisk;002A +asteriskaltonearabic;066D +asteriskarabic;066D +asteriskmath;2217 +asteriskmonospace;FF0A +asterisksmall;FE61 +asterism;2042 +asuperior;F6E9 +asymptoticallyequal;2243 +at;0040 +atilde;00E3 +atmonospace;FF20 +atsmall;FE6B +aturned;0250 +aubengali;0994 +aubopomofo;3120 +audeva;0914 +augujarati;0A94 +augurmukhi;0A14 +aulengthmarkbengali;09D7 +aumatragurmukhi;0A4C +auvowelsignbengali;09CC +auvowelsigndeva;094C +auvowelsigngujarati;0ACC +avagrahadeva;093D +aybarmenian;0561 +ayin;05E2 +ayinaltonehebrew;FB20 +ayinhebrew;05E2 +b;0062 +babengali;09AC +backslash;005C +backslashmonospace;FF3C +badeva;092C +bagujarati;0AAC +bagurmukhi;0A2C +bahiragana;3070 +bahtthai;0E3F +bakatakana;30D0 +bar;007C +barmonospace;FF5C +bbopomofo;3105 +bcircle;24D1 +bdotaccent;1E03 +bdotbelow;1E05 +beamedsixteenthnotes;266C +because;2235 +becyrillic;0431 +beharabic;0628 +behfinalarabic;FE90 +behinitialarabic;FE91 +behiragana;3079 +behmedialarabic;FE92 +behmeeminitialarabic;FC9F +behmeemisolatedarabic;FC08 +behnoonfinalarabic;FC6D +bekatakana;30D9 +benarmenian;0562 +bet;05D1 +beta;03B2 +betasymbolgreek;03D0 +betdagesh;FB31 +betdageshhebrew;FB31 +bethebrew;05D1 +betrafehebrew;FB4C +bhabengali;09AD +bhadeva;092D +bhagujarati;0AAD +bhagurmukhi;0A2D +bhook;0253 +bihiragana;3073 +bikatakana;30D3 +bilabialclick;0298 +bindigurmukhi;0A02 +birusquare;3331 +blackcircle;25CF +blackdiamond;25C6 +blackdownpointingtriangle;25BC +blackleftpointingpointer;25C4 +blackleftpointingtriangle;25C0 +blacklenticularbracketleft;3010 +blacklenticularbracketleftvertical;FE3B +blacklenticularbracketright;3011 +blacklenticularbracketrightvertical;FE3C +blacklowerlefttriangle;25E3 +blacklowerrighttriangle;25E2 +blackrectangle;25AC +blackrightpointingpointer;25BA +blackrightpointingtriangle;25B6 +blacksmallsquare;25AA +blacksmilingface;263B +blacksquare;25A0 +blackstar;2605 +blackupperlefttriangle;25E4 +blackupperrighttriangle;25E5 +blackuppointingsmalltriangle;25B4 +blackuppointingtriangle;25B2 +blank;2423 +blinebelow;1E07 +block;2588 +bmonospace;FF42 +bobaimaithai;0E1A +bohiragana;307C +bokatakana;30DC +bparen;249D +bqsquare;33C3 +braceex;F8F4 +braceleft;007B +braceleftbt;F8F3 +braceleftmid;F8F2 +braceleftmonospace;FF5B +braceleftsmall;FE5B +bracelefttp;F8F1 +braceleftvertical;FE37 +braceright;007D +bracerightbt;F8FE +bracerightmid;F8FD +bracerightmonospace;FF5D +bracerightsmall;FE5C +bracerighttp;F8FC +bracerightvertical;FE38 +bracketleft;005B +bracketleftbt;F8F0 +bracketleftex;F8EF +bracketleftmonospace;FF3B +bracketlefttp;F8EE +bracketright;005D +bracketrightbt;F8FB +bracketrightex;F8FA +bracketrightmonospace;FF3D +bracketrighttp;F8F9 +breve;02D8 +brevebelowcmb;032E +brevecmb;0306 +breveinvertedbelowcmb;032F +breveinvertedcmb;0311 +breveinverteddoublecmb;0361 +bridgebelowcmb;032A +bridgeinvertedbelowcmb;033A +brokenbar;00A6 +bstroke;0180 +bsuperior;F6EA +btopbar;0183 +buhiragana;3076 +bukatakana;30D6 +bullet;2022 +bulletinverse;25D8 +bulletoperator;2219 +bullseye;25CE +c;0063 +caarmenian;056E +cabengali;099A +cacute;0107 +cadeva;091A +cagujarati;0A9A +cagurmukhi;0A1A +calsquare;3388 +candrabindubengali;0981 +candrabinducmb;0310 +candrabindudeva;0901 +candrabindugujarati;0A81 +capslock;21EA +careof;2105 +caron;02C7 +caronbelowcmb;032C +caroncmb;030C +carriagereturn;21B5 +cbopomofo;3118 +ccaron;010D +ccedilla;00E7 +ccedillaacute;1E09 +ccircle;24D2 +ccircumflex;0109 +ccurl;0255 +cdot;010B +cdotaccent;010B +cdsquare;33C5 +cedilla;00B8 +cedillacmb;0327 +cent;00A2 +centigrade;2103 +centinferior;F6DF +centmonospace;FFE0 +centoldstyle;F7A2 +centsuperior;F6E0 +chaarmenian;0579 +chabengali;099B +chadeva;091B +chagujarati;0A9B +chagurmukhi;0A1B +chbopomofo;3114 +cheabkhasiancyrillic;04BD +checkmark;2713 +checyrillic;0447 +chedescenderabkhasiancyrillic;04BF +chedescendercyrillic;04B7 +chedieresiscyrillic;04F5 +cheharmenian;0573 +chekhakassiancyrillic;04CC +cheverticalstrokecyrillic;04B9 +chi;03C7 +chieuchacirclekorean;3277 +chieuchaparenkorean;3217 +chieuchcirclekorean;3269 +chieuchkorean;314A +chieuchparenkorean;3209 +chochangthai;0E0A +chochanthai;0E08 +chochingthai;0E09 +chochoethai;0E0C +chook;0188 +cieucacirclekorean;3276 +cieucaparenkorean;3216 +cieuccirclekorean;3268 +cieuckorean;3148 +cieucparenkorean;3208 +cieucuparenkorean;321C +circle;25CB +circlemultiply;2297 +circleot;2299 +circleplus;2295 +circlepostalmark;3036 +circlewithlefthalfblack;25D0 +circlewithrighthalfblack;25D1 +circumflex;02C6 +circumflexbelowcmb;032D +circumflexcmb;0302 +clear;2327 +clickalveolar;01C2 +clickdental;01C0 +clicklateral;01C1 +clickretroflex;01C3 +club;2663 +clubsuitblack;2663 +clubsuitwhite;2667 +cmcubedsquare;33A4 +cmonospace;FF43 +cmsquaredsquare;33A0 +coarmenian;0581 +colon;003A +colonmonetary;20A1 +colonmonospace;FF1A +colonsign;20A1 +colonsmall;FE55 +colontriangularhalfmod;02D1 +colontriangularmod;02D0 +comma;002C +commaabovecmb;0313 +commaaboverightcmb;0315 +commaaccent;F6C3 +commaarabic;060C +commaarmenian;055D +commainferior;F6E1 +commamonospace;FF0C +commareversedabovecmb;0314 +commareversedmod;02BD +commasmall;FE50 +commasuperior;F6E2 +commaturnedabovecmb;0312 +commaturnedmod;02BB +compass;263C +congruent;2245 +contourintegral;222E +control;2303 +controlACK;0006 +controlBEL;0007 +controlBS;0008 +controlCAN;0018 +controlCR;000D +controlDC1;0011 +controlDC2;0012 +controlDC3;0013 +controlDC4;0014 +controlDEL;007F +controlDLE;0010 +controlEM;0019 +controlENQ;0005 +controlEOT;0004 +controlESC;001B +controlETB;0017 +controlETX;0003 +controlFF;000C +controlFS;001C +controlGS;001D +controlHT;0009 +controlLF;000A +controlNAK;0015 +controlRS;001E +controlSI;000F +controlSO;000E +controlSOT;0002 +controlSTX;0001 +controlSUB;001A +controlSYN;0016 +controlUS;001F +controlVT;000B +copyright;00A9 +copyrightsans;F8E9 +copyrightserif;F6D9 +cornerbracketleft;300C +cornerbracketlefthalfwidth;FF62 +cornerbracketleftvertical;FE41 +cornerbracketright;300D +cornerbracketrighthalfwidth;FF63 +cornerbracketrightvertical;FE42 +corporationsquare;337F +cosquare;33C7 +coverkgsquare;33C6 +cparen;249E +cruzeiro;20A2 +cstretched;0297 +curlyand;22CF +curlyor;22CE +currency;00A4 +cyrBreve;F6D1 +cyrFlex;F6D2 +cyrbreve;F6D4 +cyrflex;F6D5 +d;0064 +daarmenian;0564 +dabengali;09A6 +dadarabic;0636 +dadeva;0926 +dadfinalarabic;FEBE +dadinitialarabic;FEBF +dadmedialarabic;FEC0 +dagesh;05BC +dageshhebrew;05BC +dagger;2020 +daggerdbl;2021 +dagujarati;0AA6 +dagurmukhi;0A26 +dahiragana;3060 +dakatakana;30C0 +dalarabic;062F +dalet;05D3 +daletdagesh;FB33 +daletdageshhebrew;FB33 +dalethatafpatah;05D3 05B2 +dalethatafpatahhebrew;05D3 05B2 +dalethatafsegol;05D3 05B1 +dalethatafsegolhebrew;05D3 05B1 +dalethebrew;05D3 +dalethiriq;05D3 05B4 +dalethiriqhebrew;05D3 05B4 +daletholam;05D3 05B9 +daletholamhebrew;05D3 05B9 +daletpatah;05D3 05B7 +daletpatahhebrew;05D3 05B7 +daletqamats;05D3 05B8 +daletqamatshebrew;05D3 05B8 +daletqubuts;05D3 05BB +daletqubutshebrew;05D3 05BB +daletsegol;05D3 05B6 +daletsegolhebrew;05D3 05B6 +daletsheva;05D3 05B0 +daletshevahebrew;05D3 05B0 +dalettsere;05D3 05B5 +dalettserehebrew;05D3 05B5 +dalfinalarabic;FEAA +dammaarabic;064F +dammalowarabic;064F +dammatanaltonearabic;064C +dammatanarabic;064C +danda;0964 +dargahebrew;05A7 +dargalefthebrew;05A7 +dasiapneumatacyrilliccmb;0485 +dblGrave;F6D3 +dblanglebracketleft;300A +dblanglebracketleftvertical;FE3D +dblanglebracketright;300B +dblanglebracketrightvertical;FE3E +dblarchinvertedbelowcmb;032B +dblarrowleft;21D4 +dblarrowright;21D2 +dbldanda;0965 +dblgrave;F6D6 +dblgravecmb;030F +dblintegral;222C +dbllowline;2017 +dbllowlinecmb;0333 +dbloverlinecmb;033F +dblprimemod;02BA +dblverticalbar;2016 +dblverticallineabovecmb;030E +dbopomofo;3109 +dbsquare;33C8 +dcaron;010F +dcedilla;1E11 +dcircle;24D3 +dcircumflexbelow;1E13 +dcroat;0111 +ddabengali;09A1 +ddadeva;0921 +ddagujarati;0AA1 +ddagurmukhi;0A21 +ddalarabic;0688 +ddalfinalarabic;FB89 +dddhadeva;095C +ddhabengali;09A2 +ddhadeva;0922 +ddhagujarati;0AA2 +ddhagurmukhi;0A22 +ddotaccent;1E0B +ddotbelow;1E0D +decimalseparatorarabic;066B +decimalseparatorpersian;066B +decyrillic;0434 +degree;00B0 +dehihebrew;05AD +dehiragana;3067 +deicoptic;03EF +dekatakana;30C7 +deleteleft;232B +deleteright;2326 +delta;03B4 +deltaturned;018D +denominatorminusonenumeratorbengali;09F8 +dezh;02A4 +dhabengali;09A7 +dhadeva;0927 +dhagujarati;0AA7 +dhagurmukhi;0A27 +dhook;0257 +dialytikatonos;0385 +dialytikatonoscmb;0344 +diamond;2666 +diamondsuitwhite;2662 +dieresis;00A8 +dieresisacute;F6D7 +dieresisbelowcmb;0324 +dieresiscmb;0308 +dieresisgrave;F6D8 +dieresistonos;0385 +dihiragana;3062 +dikatakana;30C2 +dittomark;3003 +divide;00F7 +divides;2223 +divisionslash;2215 +djecyrillic;0452 +dkshade;2593 +dlinebelow;1E0F +dlsquare;3397 +dmacron;0111 +dmonospace;FF44 +dnblock;2584 +dochadathai;0E0E +dodekthai;0E14 +dohiragana;3069 +dokatakana;30C9 +dollar;0024 +dollarinferior;F6E3 +dollarmonospace;FF04 +dollaroldstyle;F724 +dollarsmall;FE69 +dollarsuperior;F6E4 +dong;20AB +dorusquare;3326 +dotaccent;02D9 +dotaccentcmb;0307 +dotbelowcmb;0323 +dotbelowcomb;0323 +dotkatakana;30FB +dotlessi;0131 +dotlessj;F6BE +dotlessjstrokehook;0284 +dotmath;22C5 +dottedcircle;25CC +doubleyodpatah;FB1F +doubleyodpatahhebrew;FB1F +downtackbelowcmb;031E +downtackmod;02D5 +dparen;249F +dsuperior;F6EB +dtail;0256 +dtopbar;018C +duhiragana;3065 +dukatakana;30C5 +dz;01F3 +dzaltone;02A3 +dzcaron;01C6 +dzcurl;02A5 +dzeabkhasiancyrillic;04E1 +dzecyrillic;0455 +dzhecyrillic;045F +e;0065 +eacute;00E9 +earth;2641 +ebengali;098F +ebopomofo;311C +ebreve;0115 +ecandradeva;090D +ecandragujarati;0A8D +ecandravowelsigndeva;0945 +ecandravowelsigngujarati;0AC5 +ecaron;011B +ecedillabreve;1E1D +echarmenian;0565 +echyiwnarmenian;0587 +ecircle;24D4 +ecircumflex;00EA +ecircumflexacute;1EBF +ecircumflexbelow;1E19 +ecircumflexdotbelow;1EC7 +ecircumflexgrave;1EC1 +ecircumflexhookabove;1EC3 +ecircumflextilde;1EC5 +ecyrillic;0454 +edblgrave;0205 +edeva;090F +edieresis;00EB +edot;0117 +edotaccent;0117 +edotbelow;1EB9 +eegurmukhi;0A0F +eematragurmukhi;0A47 +efcyrillic;0444 +egrave;00E8 +egujarati;0A8F +eharmenian;0567 +ehbopomofo;311D +ehiragana;3048 +ehookabove;1EBB +eibopomofo;311F +eight;0038 +eightarabic;0668 +eightbengali;09EE +eightcircle;2467 +eightcircleinversesansserif;2791 +eightdeva;096E +eighteencircle;2471 +eighteenparen;2485 +eighteenperiod;2499 +eightgujarati;0AEE +eightgurmukhi;0A6E +eighthackarabic;0668 +eighthangzhou;3028 +eighthnotebeamed;266B +eightideographicparen;3227 +eightinferior;2088 +eightmonospace;FF18 +eightoldstyle;F738 +eightparen;247B +eightperiod;248F +eightpersian;06F8 +eightroman;2177 +eightsuperior;2078 +eightthai;0E58 +einvertedbreve;0207 +eiotifiedcyrillic;0465 +ekatakana;30A8 +ekatakanahalfwidth;FF74 +ekonkargurmukhi;0A74 +ekorean;3154 +elcyrillic;043B +element;2208 +elevencircle;246A +elevenparen;247E +elevenperiod;2492 +elevenroman;217A +ellipsis;2026 +ellipsisvertical;22EE +emacron;0113 +emacronacute;1E17 +emacrongrave;1E15 +emcyrillic;043C +emdash;2014 +emdashvertical;FE31 +emonospace;FF45 +emphasismarkarmenian;055B +emptyset;2205 +enbopomofo;3123 +encyrillic;043D +endash;2013 +endashvertical;FE32 +endescendercyrillic;04A3 +eng;014B +engbopomofo;3125 +enghecyrillic;04A5 +enhookcyrillic;04C8 +enspace;2002 +eogonek;0119 +eokorean;3153 +eopen;025B +eopenclosed;029A +eopenreversed;025C +eopenreversedclosed;025E +eopenreversedhook;025D +eparen;24A0 +epsilon;03B5 +epsilontonos;03AD +equal;003D +equalmonospace;FF1D +equalsmall;FE66 +equalsuperior;207C +equivalence;2261 +erbopomofo;3126 +ercyrillic;0440 +ereversed;0258 +ereversedcyrillic;044D +escyrillic;0441 +esdescendercyrillic;04AB +esh;0283 +eshcurl;0286 +eshortdeva;090E +eshortvowelsigndeva;0946 +eshreversedloop;01AA +eshsquatreversed;0285 +esmallhiragana;3047 +esmallkatakana;30A7 +esmallkatakanahalfwidth;FF6A +estimated;212E +esuperior;F6EC +eta;03B7 +etarmenian;0568 +etatonos;03AE +eth;00F0 +etilde;1EBD +etildebelow;1E1B +etnahtafoukhhebrew;0591 +etnahtafoukhlefthebrew;0591 +etnahtahebrew;0591 +etnahtalefthebrew;0591 +eturned;01DD +eukorean;3161 +euro;20AC +evowelsignbengali;09C7 +evowelsigndeva;0947 +evowelsigngujarati;0AC7 +exclam;0021 +exclamarmenian;055C +exclamdbl;203C +exclamdown;00A1 +exclamdownsmall;F7A1 +exclammonospace;FF01 +exclamsmall;F721 +existential;2203 +ezh;0292 +ezhcaron;01EF +ezhcurl;0293 +ezhreversed;01B9 +ezhtail;01BA +f;0066 +fadeva;095E +fagurmukhi;0A5E +fahrenheit;2109 +fathaarabic;064E +fathalowarabic;064E +fathatanarabic;064B +fbopomofo;3108 +fcircle;24D5 +fdotaccent;1E1F +feharabic;0641 +feharmenian;0586 +fehfinalarabic;FED2 +fehinitialarabic;FED3 +fehmedialarabic;FED4 +feicoptic;03E5 +female;2640 +ff;FB00 +f_f;FB00 +ffi;FB03 +f_f_i;FB03 +ffl;FB04 +f_f_l;FB04 +fi;FB01 +f_i;FB01 +fifteencircle;246E +fifteenparen;2482 +fifteenperiod;2496 +figuredash;2012 +filledbox;25A0 +filledrect;25AC +finalkaf;05DA +finalkafdagesh;FB3A +finalkafdageshhebrew;FB3A +finalkafhebrew;05DA +finalkafqamats;05DA 05B8 +finalkafqamatshebrew;05DA 05B8 +finalkafsheva;05DA 05B0 +finalkafshevahebrew;05DA 05B0 +finalmem;05DD +finalmemhebrew;05DD +finalnun;05DF +finalnunhebrew;05DF +finalpe;05E3 +finalpehebrew;05E3 +finaltsadi;05E5 +finaltsadihebrew;05E5 +firsttonechinese;02C9 +fisheye;25C9 +fitacyrillic;0473 +five;0035 +fivearabic;0665 +fivebengali;09EB +fivecircle;2464 +fivecircleinversesansserif;278E +fivedeva;096B +fiveeighths;215D +fivegujarati;0AEB +fivegurmukhi;0A6B +fivehackarabic;0665 +fivehangzhou;3025 +fiveideographicparen;3224 +fiveinferior;2085 +fivemonospace;FF15 +fiveoldstyle;F735 +fiveparen;2478 +fiveperiod;248C +fivepersian;06F5 +fiveroman;2174 +fivesuperior;2075 +fivethai;0E55 +fl;FB02 +f_l;FB02 +florin;0192 +fmonospace;FF46 +fmsquare;3399 +fofanthai;0E1F +fofathai;0E1D +fongmanthai;0E4F +forall;2200 +four;0034 +fourarabic;0664 +fourbengali;09EA +fourcircle;2463 +fourcircleinversesansserif;278D +fourdeva;096A +fourgujarati;0AEA +fourgurmukhi;0A6A +fourhackarabic;0664 +fourhangzhou;3024 +fourideographicparen;3223 +fourinferior;2084 +fourmonospace;FF14 +fournumeratorbengali;09F7 +fouroldstyle;F734 +fourparen;2477 +fourperiod;248B +fourpersian;06F4 +fourroman;2173 +foursuperior;2074 +fourteencircle;246D +fourteenparen;2481 +fourteenperiod;2495 +fourthai;0E54 +fourthtonechinese;02CB +fparen;24A1 +fraction;2044 +franc;20A3 +g;0067 +gabengali;0997 +gacute;01F5 +gadeva;0917 +gafarabic;06AF +gaffinalarabic;FB93 +gafinitialarabic;FB94 +gafmedialarabic;FB95 +gagujarati;0A97 +gagurmukhi;0A17 +gahiragana;304C +gakatakana;30AC +gamma;03B3 +gammalatinsmall;0263 +gammasuperior;02E0 +gangiacoptic;03EB +gbopomofo;310D +gbreve;011F +gcaron;01E7 +gcedilla;0123 +gcircle;24D6 +gcircumflex;011D +gcommaaccent;0123 +gdot;0121 +gdotaccent;0121 +gecyrillic;0433 +gehiragana;3052 +gekatakana;30B2 +geometricallyequal;2251 +gereshaccenthebrew;059C +gereshhebrew;05F3 +gereshmuqdamhebrew;059D +germandbls;00DF +gershayimaccenthebrew;059E +gershayimhebrew;05F4 +getamark;3013 +ghabengali;0998 +ghadarmenian;0572 +ghadeva;0918 +ghagujarati;0A98 +ghagurmukhi;0A18 +ghainarabic;063A +ghainfinalarabic;FECE +ghaininitialarabic;FECF +ghainmedialarabic;FED0 +ghemiddlehookcyrillic;0495 +ghestrokecyrillic;0493 +gheupturncyrillic;0491 +ghhadeva;095A +ghhagurmukhi;0A5A +ghook;0260 +ghzsquare;3393 +gihiragana;304E +gikatakana;30AE +gimarmenian;0563 +gimel;05D2 +gimeldagesh;FB32 +gimeldageshhebrew;FB32 +gimelhebrew;05D2 +gjecyrillic;0453 +glottalinvertedstroke;01BE +glottalstop;0294 +glottalstopinverted;0296 +glottalstopmod;02C0 +glottalstopreversed;0295 +glottalstopreversedmod;02C1 +glottalstopreversedsuperior;02E4 +glottalstopstroke;02A1 +glottalstopstrokereversed;02A2 +gmacron;1E21 +gmonospace;FF47 +gohiragana;3054 +gokatakana;30B4 +gparen;24A2 +gpasquare;33AC +gradient;2207 +grave;0060 +gravebelowcmb;0316 +gravecmb;0300 +gravecomb;0300 +gravedeva;0953 +gravelowmod;02CE +gravemonospace;FF40 +gravetonecmb;0340 +greater;003E +greaterequal;2265 +greaterequalorless;22DB +greatermonospace;FF1E +greaterorequivalent;2273 +greaterorless;2277 +greateroverequal;2267 +greatersmall;FE65 +gscript;0261 +gstroke;01E5 +guhiragana;3050 +guillemotleft;00AB +guillemotright;00BB +guilsinglleft;2039 +guilsinglright;203A +gukatakana;30B0 +guramusquare;3318 +gysquare;33C9 +h;0068 +haabkhasiancyrillic;04A9 +haaltonearabic;06C1 +habengali;09B9 +hadescendercyrillic;04B3 +hadeva;0939 +hagujarati;0AB9 +hagurmukhi;0A39 +haharabic;062D +hahfinalarabic;FEA2 +hahinitialarabic;FEA3 +hahiragana;306F +hahmedialarabic;FEA4 +haitusquare;332A +hakatakana;30CF +hakatakanahalfwidth;FF8A +halantgurmukhi;0A4D +hamzaarabic;0621 +hamzadammaarabic;0621 064F +hamzadammatanarabic;0621 064C +hamzafathaarabic;0621 064E +hamzafathatanarabic;0621 064B +hamzalowarabic;0621 +hamzalowkasraarabic;0621 0650 +hamzalowkasratanarabic;0621 064D +hamzasukunarabic;0621 0652 +hangulfiller;3164 +hardsigncyrillic;044A +harpoonleftbarbup;21BC +harpoonrightbarbup;21C0 +hasquare;33CA +hatafpatah;05B2 +hatafpatah16;05B2 +hatafpatah23;05B2 +hatafpatah2f;05B2 +hatafpatahhebrew;05B2 +hatafpatahnarrowhebrew;05B2 +hatafpatahquarterhebrew;05B2 +hatafpatahwidehebrew;05B2 +hatafqamats;05B3 +hatafqamats1b;05B3 +hatafqamats28;05B3 +hatafqamats34;05B3 +hatafqamatshebrew;05B3 +hatafqamatsnarrowhebrew;05B3 +hatafqamatsquarterhebrew;05B3 +hatafqamatswidehebrew;05B3 +hatafsegol;05B1 +hatafsegol17;05B1 +hatafsegol24;05B1 +hatafsegol30;05B1 +hatafsegolhebrew;05B1 +hatafsegolnarrowhebrew;05B1 +hatafsegolquarterhebrew;05B1 +hatafsegolwidehebrew;05B1 +hbar;0127 +hbopomofo;310F +hbrevebelow;1E2B +hcedilla;1E29 +hcircle;24D7 +hcircumflex;0125 +hdieresis;1E27 +hdotaccent;1E23 +hdotbelow;1E25 +he;05D4 +heart;2665 +heartsuitblack;2665 +heartsuitwhite;2661 +hedagesh;FB34 +hedageshhebrew;FB34 +hehaltonearabic;06C1 +heharabic;0647 +hehebrew;05D4 +hehfinalaltonearabic;FBA7 +hehfinalalttwoarabic;FEEA +hehfinalarabic;FEEA +hehhamzaabovefinalarabic;FBA5 +hehhamzaaboveisolatedarabic;FBA4 +hehinitialaltonearabic;FBA8 +hehinitialarabic;FEEB +hehiragana;3078 +hehmedialaltonearabic;FBA9 +hehmedialarabic;FEEC +heiseierasquare;337B +hekatakana;30D8 +hekatakanahalfwidth;FF8D +hekutaarusquare;3336 +henghook;0267 +herutusquare;3339 +het;05D7 +hethebrew;05D7 +hhook;0266 +hhooksuperior;02B1 +hieuhacirclekorean;327B +hieuhaparenkorean;321B +hieuhcirclekorean;326D +hieuhkorean;314E +hieuhparenkorean;320D +hihiragana;3072 +hikatakana;30D2 +hikatakanahalfwidth;FF8B +hiriq;05B4 +hiriq14;05B4 +hiriq21;05B4 +hiriq2d;05B4 +hiriqhebrew;05B4 +hiriqnarrowhebrew;05B4 +hiriqquarterhebrew;05B4 +hiriqwidehebrew;05B4 +hlinebelow;1E96 +hmonospace;FF48 +hoarmenian;0570 +hohipthai;0E2B +hohiragana;307B +hokatakana;30DB +hokatakanahalfwidth;FF8E +holam;05B9 +holam19;05B9 +holam26;05B9 +holam32;05B9 +holamhebrew;05B9 +holamnarrowhebrew;05B9 +holamquarterhebrew;05B9 +holamwidehebrew;05B9 +honokhukthai;0E2E +hookabovecomb;0309 +hookcmb;0309 +hookpalatalizedbelowcmb;0321 +hookretroflexbelowcmb;0322 +hoonsquare;3342 +horicoptic;03E9 +horizontalbar;2015 +horncmb;031B +hotsprings;2668 +house;2302 +hparen;24A3 +hsuperior;02B0 +hturned;0265 +huhiragana;3075 +huiitosquare;3333 +hukatakana;30D5 +hukatakanahalfwidth;FF8C +hungarumlaut;02DD +hungarumlautcmb;030B +hv;0195 +hyphen;002D +hypheninferior;F6E5 +hyphenmonospace;FF0D +hyphensmall;FE63 +hyphensuperior;F6E6 +hyphentwo;2010 +i;0069 +iacute;00ED +iacyrillic;044F +ibengali;0987 +ibopomofo;3127 +ibreve;012D +icaron;01D0 +icircle;24D8 +icircumflex;00EE +icyrillic;0456 +idblgrave;0209 +ideographearthcircle;328F +ideographfirecircle;328B +ideographicallianceparen;323F +ideographiccallparen;323A +ideographiccentrecircle;32A5 +ideographicclose;3006 +ideographiccomma;3001 +ideographiccommaleft;FF64 +ideographiccongratulationparen;3237 +ideographiccorrectcircle;32A3 +ideographicearthparen;322F +ideographicenterpriseparen;323D +ideographicexcellentcircle;329D +ideographicfestivalparen;3240 +ideographicfinancialcircle;3296 +ideographicfinancialparen;3236 +ideographicfireparen;322B +ideographichaveparen;3232 +ideographichighcircle;32A4 +ideographiciterationmark;3005 +ideographiclaborcircle;3298 +ideographiclaborparen;3238 +ideographicleftcircle;32A7 +ideographiclowcircle;32A6 +ideographicmedicinecircle;32A9 +ideographicmetalparen;322E +ideographicmoonparen;322A +ideographicnameparen;3234 +ideographicperiod;3002 +ideographicprintcircle;329E +ideographicreachparen;3243 +ideographicrepresentparen;3239 +ideographicresourceparen;323E +ideographicrightcircle;32A8 +ideographicsecretcircle;3299 +ideographicselfparen;3242 +ideographicsocietyparen;3233 +ideographicspace;3000 +ideographicspecialparen;3235 +ideographicstockparen;3231 +ideographicstudyparen;323B +ideographicsunparen;3230 +ideographicsuperviseparen;323C +ideographicwaterparen;322C +ideographicwoodparen;322D +ideographiczero;3007 +ideographmetalcircle;328E +ideographmooncircle;328A +ideographnamecircle;3294 +ideographsuncircle;3290 +ideographwatercircle;328C +ideographwoodcircle;328D +ideva;0907 +idieresis;00EF +idieresisacute;1E2F +idieresiscyrillic;04E5 +idotbelow;1ECB +iebrevecyrillic;04D7 +iecyrillic;0435 +ieungacirclekorean;3275 +ieungaparenkorean;3215 +ieungcirclekorean;3267 +ieungkorean;3147 +ieungparenkorean;3207 +igrave;00EC +igujarati;0A87 +igurmukhi;0A07 +ihiragana;3044 +ihookabove;1EC9 +iibengali;0988 +iicyrillic;0438 +iideva;0908 +iigujarati;0A88 +iigurmukhi;0A08 +iimatragurmukhi;0A40 +iinvertedbreve;020B +iishortcyrillic;0439 +iivowelsignbengali;09C0 +iivowelsigndeva;0940 +iivowelsigngujarati;0AC0 +ij;0133 +ikatakana;30A4 +ikatakanahalfwidth;FF72 +ikorean;3163 +ilde;02DC +iluyhebrew;05AC +imacron;012B +imacroncyrillic;04E3 +imageorapproximatelyequal;2253 +imatragurmukhi;0A3F +imonospace;FF49 +increment;2206 +infinity;221E +iniarmenian;056B +integral;222B +integralbottom;2321 +integralbt;2321 +integralex;F8F5 +integraltop;2320 +integraltp;2320 +intersection;2229 +intisquare;3305 +invbullet;25D8 +invcircle;25D9 +invsmileface;263B +iocyrillic;0451 +iogonek;012F +iota;03B9 +iotadieresis;03CA +iotadieresistonos;0390 +iotalatin;0269 +iotatonos;03AF +iparen;24A4 +irigurmukhi;0A72 +ismallhiragana;3043 +ismallkatakana;30A3 +ismallkatakanahalfwidth;FF68 +issharbengali;09FA +istroke;0268 +isuperior;F6ED +iterationhiragana;309D +iterationkatakana;30FD +itilde;0129 +itildebelow;1E2D +iubopomofo;3129 +iucyrillic;044E +ivowelsignbengali;09BF +ivowelsigndeva;093F +ivowelsigngujarati;0ABF +izhitsacyrillic;0475 +izhitsadblgravecyrillic;0477 +j;006A +jaarmenian;0571 +jabengali;099C +jadeva;091C +jagujarati;0A9C +jagurmukhi;0A1C +jbopomofo;3110 +jcaron;01F0 +jcircle;24D9 +jcircumflex;0135 +jcrossedtail;029D +jdotlessstroke;025F +jecyrillic;0458 +jeemarabic;062C +jeemfinalarabic;FE9E +jeeminitialarabic;FE9F +jeemmedialarabic;FEA0 +jeharabic;0698 +jehfinalarabic;FB8B +jhabengali;099D +jhadeva;091D +jhagujarati;0A9D +jhagurmukhi;0A1D +jheharmenian;057B +jis;3004 +jmonospace;FF4A +jparen;24A5 +jsuperior;02B2 +k;006B +kabashkircyrillic;04A1 +kabengali;0995 +kacute;1E31 +kacyrillic;043A +kadescendercyrillic;049B +kadeva;0915 +kaf;05DB +kafarabic;0643 +kafdagesh;FB3B +kafdageshhebrew;FB3B +kaffinalarabic;FEDA +kafhebrew;05DB +kafinitialarabic;FEDB +kafmedialarabic;FEDC +kafrafehebrew;FB4D +kagujarati;0A95 +kagurmukhi;0A15 +kahiragana;304B +kahookcyrillic;04C4 +kakatakana;30AB +kakatakanahalfwidth;FF76 +kappa;03BA +kappasymbolgreek;03F0 +kapyeounmieumkorean;3171 +kapyeounphieuphkorean;3184 +kapyeounpieupkorean;3178 +kapyeounssangpieupkorean;3179 +karoriisquare;330D +kashidaautoarabic;0640 +kashidaautonosidebearingarabic;0640 +kasmallkatakana;30F5 +kasquare;3384 +kasraarabic;0650 +kasratanarabic;064D +kastrokecyrillic;049F +katahiraprolongmarkhalfwidth;FF70 +kaverticalstrokecyrillic;049D +kbopomofo;310E +kcalsquare;3389 +kcaron;01E9 +kcedilla;0137 +kcircle;24DA +kcommaaccent;0137 +kdotbelow;1E33 +keharmenian;0584 +kehiragana;3051 +kekatakana;30B1 +kekatakanahalfwidth;FF79 +kenarmenian;056F +kesmallkatakana;30F6 +kgreenlandic;0138 +khabengali;0996 +khacyrillic;0445 +khadeva;0916 +khagujarati;0A96 +khagurmukhi;0A16 +khaharabic;062E +khahfinalarabic;FEA6 +khahinitialarabic;FEA7 +khahmedialarabic;FEA8 +kheicoptic;03E7 +khhadeva;0959 +khhagurmukhi;0A59 +khieukhacirclekorean;3278 +khieukhaparenkorean;3218 +khieukhcirclekorean;326A +khieukhkorean;314B +khieukhparenkorean;320A +khokhaithai;0E02 +khokhonthai;0E05 +khokhuatthai;0E03 +khokhwaithai;0E04 +khomutthai;0E5B +khook;0199 +khorakhangthai;0E06 +khzsquare;3391 +kihiragana;304D +kikatakana;30AD +kikatakanahalfwidth;FF77 +kiroguramusquare;3315 +kiromeetorusquare;3316 +kirosquare;3314 +kiyeokacirclekorean;326E +kiyeokaparenkorean;320E +kiyeokcirclekorean;3260 +kiyeokkorean;3131 +kiyeokparenkorean;3200 +kiyeoksioskorean;3133 +kjecyrillic;045C +klinebelow;1E35 +klsquare;3398 +kmcubedsquare;33A6 +kmonospace;FF4B +kmsquaredsquare;33A2 +kohiragana;3053 +kohmsquare;33C0 +kokaithai;0E01 +kokatakana;30B3 +kokatakanahalfwidth;FF7A +kooposquare;331E +koppacyrillic;0481 +koreanstandardsymbol;327F +koroniscmb;0343 +kparen;24A6 +kpasquare;33AA +ksicyrillic;046F +ktsquare;33CF +kturned;029E +kuhiragana;304F +kukatakana;30AF +kukatakanahalfwidth;FF78 +kvsquare;33B8 +kwsquare;33BE +l;006C +labengali;09B2 +lacute;013A +ladeva;0932 +lagujarati;0AB2 +lagurmukhi;0A32 +lakkhangyaothai;0E45 +lamaleffinalarabic;FEFC +lamalefhamzaabovefinalarabic;FEF8 +lamalefhamzaaboveisolatedarabic;FEF7 +lamalefhamzabelowfinalarabic;FEFA +lamalefhamzabelowisolatedarabic;FEF9 +lamalefisolatedarabic;FEFB +lamalefmaddaabovefinalarabic;FEF6 +lamalefmaddaaboveisolatedarabic;FEF5 +lamarabic;0644 +lambda;03BB +lambdastroke;019B +lamed;05DC +lameddagesh;FB3C +lameddageshhebrew;FB3C +lamedhebrew;05DC +lamedholam;05DC 05B9 +lamedholamdagesh;05DC 05B9 05BC +lamedholamdageshhebrew;05DC 05B9 05BC +lamedholamhebrew;05DC 05B9 +lamfinalarabic;FEDE +lamhahinitialarabic;FCCA +laminitialarabic;FEDF +lamjeeminitialarabic;FCC9 +lamkhahinitialarabic;FCCB +lamlamhehisolatedarabic;FDF2 +lammedialarabic;FEE0 +lammeemhahinitialarabic;FD88 +lammeeminitialarabic;FCCC +lammeemjeeminitialarabic;FEDF FEE4 FEA0 +lammeemkhahinitialarabic;FEDF FEE4 FEA8 +largecircle;25EF +lbar;019A +lbelt;026C +lbopomofo;310C +lcaron;013E +lcedilla;013C +lcircle;24DB +lcircumflexbelow;1E3D +lcommaaccent;013C +ldot;0140 +ldotaccent;0140 +ldotbelow;1E37 +ldotbelowmacron;1E39 +leftangleabovecmb;031A +lefttackbelowcmb;0318 +less;003C +lessequal;2264 +lessequalorgreater;22DA +lessmonospace;FF1C +lessorequivalent;2272 +lessorgreater;2276 +lessoverequal;2266 +lesssmall;FE64 +lezh;026E +lfblock;258C +lhookretroflex;026D +lira;20A4 +liwnarmenian;056C +lj;01C9 +ljecyrillic;0459 +ll;F6C0 +lladeva;0933 +llagujarati;0AB3 +llinebelow;1E3B +llladeva;0934 +llvocalicbengali;09E1 +llvocalicdeva;0961 +llvocalicvowelsignbengali;09E3 +llvocalicvowelsigndeva;0963 +lmiddletilde;026B +lmonospace;FF4C +lmsquare;33D0 +lochulathai;0E2C +logicaland;2227 +logicalnot;00AC +logicalnotreversed;2310 +logicalor;2228 +lolingthai;0E25 +longs;017F +lowlinecenterline;FE4E +lowlinecmb;0332 +lowlinedashed;FE4D +lozenge;25CA +lparen;24A7 +lslash;0142 +lsquare;2113 +lsuperior;F6EE +ltshade;2591 +luthai;0E26 +lvocalicbengali;098C +lvocalicdeva;090C +lvocalicvowelsignbengali;09E2 +lvocalicvowelsigndeva;0962 +lxsquare;33D3 +m;006D +mabengali;09AE +macron;00AF +macronbelowcmb;0331 +macroncmb;0304 +macronlowmod;02CD +macronmonospace;FFE3 +macute;1E3F +madeva;092E +magujarati;0AAE +magurmukhi;0A2E +mahapakhhebrew;05A4 +mahapakhlefthebrew;05A4 +mahiragana;307E +maichattawalowleftthai;F895 +maichattawalowrightthai;F894 +maichattawathai;0E4B +maichattawaupperleftthai;F893 +maieklowleftthai;F88C +maieklowrightthai;F88B +maiekthai;0E48 +maiekupperleftthai;F88A +maihanakatleftthai;F884 +maihanakatthai;0E31 +maitaikhuleftthai;F889 +maitaikhuthai;0E47 +maitholowleftthai;F88F +maitholowrightthai;F88E +maithothai;0E49 +maithoupperleftthai;F88D +maitrilowleftthai;F892 +maitrilowrightthai;F891 +maitrithai;0E4A +maitriupperleftthai;F890 +maiyamokthai;0E46 +makatakana;30DE +makatakanahalfwidth;FF8F +male;2642 +mansyonsquare;3347 +maqafhebrew;05BE +mars;2642 +masoracirclehebrew;05AF +masquare;3383 +mbopomofo;3107 +mbsquare;33D4 +mcircle;24DC +mcubedsquare;33A5 +mdotaccent;1E41 +mdotbelow;1E43 +meemarabic;0645 +meemfinalarabic;FEE2 +meeminitialarabic;FEE3 +meemmedialarabic;FEE4 +meemmeeminitialarabic;FCD1 +meemmeemisolatedarabic;FC48 +meetorusquare;334D +mehiragana;3081 +meizierasquare;337E +mekatakana;30E1 +mekatakanahalfwidth;FF92 +mem;05DE +memdagesh;FB3E +memdageshhebrew;FB3E +memhebrew;05DE +menarmenian;0574 +merkhahebrew;05A5 +merkhakefulahebrew;05A6 +merkhakefulalefthebrew;05A6 +merkhalefthebrew;05A5 +mhook;0271 +mhzsquare;3392 +middledotkatakanahalfwidth;FF65 +middot;00B7 +mieumacirclekorean;3272 +mieumaparenkorean;3212 +mieumcirclekorean;3264 +mieumkorean;3141 +mieumpansioskorean;3170 +mieumparenkorean;3204 +mieumpieupkorean;316E +mieumsioskorean;316F +mihiragana;307F +mikatakana;30DF +mikatakanahalfwidth;FF90 +minus;2212 +minusbelowcmb;0320 +minuscircle;2296 +minusmod;02D7 +minusplus;2213 +minute;2032 +miribaarusquare;334A +mirisquare;3349 +mlonglegturned;0270 +mlsquare;3396 +mmcubedsquare;33A3 +mmonospace;FF4D +mmsquaredsquare;339F +mohiragana;3082 +mohmsquare;33C1 +mokatakana;30E2 +mokatakanahalfwidth;FF93 +molsquare;33D6 +momathai;0E21 +moverssquare;33A7 +moverssquaredsquare;33A8 +mparen;24A8 +mpasquare;33AB +mssquare;33B3 +msuperior;F6EF +mturned;026F +mu;00B5 +mu1;00B5 +muasquare;3382 +muchgreater;226B +muchless;226A +mufsquare;338C +mugreek;03BC +mugsquare;338D +muhiragana;3080 +mukatakana;30E0 +mukatakanahalfwidth;FF91 +mulsquare;3395 +multiply;00D7 +mumsquare;339B +munahhebrew;05A3 +munahlefthebrew;05A3 +musicalnote;266A +musicalnotedbl;266B +musicflatsign;266D +musicsharpsign;266F +mussquare;33B2 +muvsquare;33B6 +muwsquare;33BC +mvmegasquare;33B9 +mvsquare;33B7 +mwmegasquare;33BF +mwsquare;33BD +n;006E +nabengali;09A8 +nabla;2207 +nacute;0144 +nadeva;0928 +nagujarati;0AA8 +nagurmukhi;0A28 +nahiragana;306A +nakatakana;30CA +nakatakanahalfwidth;FF85 +napostrophe;0149 +nasquare;3381 +nbopomofo;310B +nbspace;00A0 +ncaron;0148 +ncedilla;0146 +ncircle;24DD +ncircumflexbelow;1E4B +ncommaaccent;0146 +ndotaccent;1E45 +ndotbelow;1E47 +nehiragana;306D +nekatakana;30CD +nekatakanahalfwidth;FF88 +newsheqelsign;20AA +nfsquare;338B +ngabengali;0999 +ngadeva;0919 +ngagujarati;0A99 +ngagurmukhi;0A19 +ngonguthai;0E07 +nhiragana;3093 +nhookleft;0272 +nhookretroflex;0273 +nieunacirclekorean;326F +nieunaparenkorean;320F +nieuncieuckorean;3135 +nieuncirclekorean;3261 +nieunhieuhkorean;3136 +nieunkorean;3134 +nieunpansioskorean;3168 +nieunparenkorean;3201 +nieunsioskorean;3167 +nieuntikeutkorean;3166 +nihiragana;306B +nikatakana;30CB +nikatakanahalfwidth;FF86 +nikhahitleftthai;F899 +nikhahitthai;0E4D +nine;0039 +ninearabic;0669 +ninebengali;09EF +ninecircle;2468 +ninecircleinversesansserif;2792 +ninedeva;096F +ninegujarati;0AEF +ninegurmukhi;0A6F +ninehackarabic;0669 +ninehangzhou;3029 +nineideographicparen;3228 +nineinferior;2089 +ninemonospace;FF19 +nineoldstyle;F739 +nineparen;247C +nineperiod;2490 +ninepersian;06F9 +nineroman;2178 +ninesuperior;2079 +nineteencircle;2472 +nineteenparen;2486 +nineteenperiod;249A +ninethai;0E59 +nj;01CC +njecyrillic;045A +nkatakana;30F3 +nkatakanahalfwidth;FF9D +nlegrightlong;019E +nlinebelow;1E49 +nmonospace;FF4E +nmsquare;339A +nnabengali;09A3 +nnadeva;0923 +nnagujarati;0AA3 +nnagurmukhi;0A23 +nnnadeva;0929 +nohiragana;306E +nokatakana;30CE +nokatakanahalfwidth;FF89 +nonbreakingspace;00A0 +nonenthai;0E13 +nonuthai;0E19 +noonarabic;0646 +noonfinalarabic;FEE6 +noonghunnaarabic;06BA +noonghunnafinalarabic;FB9F +noonhehinitialarabic;FEE7 FEEC +nooninitialarabic;FEE7 +noonjeeminitialarabic;FCD2 +noonjeemisolatedarabic;FC4B +noonmedialarabic;FEE8 +noonmeeminitialarabic;FCD5 +noonmeemisolatedarabic;FC4E +noonnoonfinalarabic;FC8D +notcontains;220C +notelement;2209 +notelementof;2209 +notequal;2260 +notgreater;226F +notgreaternorequal;2271 +notgreaternorless;2279 +notidentical;2262 +notless;226E +notlessnorequal;2270 +notparallel;2226 +notprecedes;2280 +notsubset;2284 +notsucceeds;2281 +notsuperset;2285 +nowarmenian;0576 +nparen;24A9 +nssquare;33B1 +nsuperior;207F +ntilde;00F1 +nu;03BD +nuhiragana;306C +nukatakana;30CC +nukatakanahalfwidth;FF87 +nuktabengali;09BC +nuktadeva;093C +nuktagujarati;0ABC +nuktagurmukhi;0A3C +numbersign;0023 +numbersignmonospace;FF03 +numbersignsmall;FE5F +numeralsigngreek;0374 +numeralsignlowergreek;0375 +numero;2116 +nun;05E0 +nundagesh;FB40 +nundageshhebrew;FB40 +nunhebrew;05E0 +nvsquare;33B5 +nwsquare;33BB +nyabengali;099E +nyadeva;091E +nyagujarati;0A9E +nyagurmukhi;0A1E +o;006F +oacute;00F3 +oangthai;0E2D +obarred;0275 +obarredcyrillic;04E9 +obarreddieresiscyrillic;04EB +obengali;0993 +obopomofo;311B +obreve;014F +ocandradeva;0911 +ocandragujarati;0A91 +ocandravowelsigndeva;0949 +ocandravowelsigngujarati;0AC9 +ocaron;01D2 +ocircle;24DE +ocircumflex;00F4 +ocircumflexacute;1ED1 +ocircumflexdotbelow;1ED9 +ocircumflexgrave;1ED3 +ocircumflexhookabove;1ED5 +ocircumflextilde;1ED7 +ocyrillic;043E +odblacute;0151 +odblgrave;020D +odeva;0913 +odieresis;00F6 +odieresiscyrillic;04E7 +odotbelow;1ECD +oe;0153 +oekorean;315A +ogonek;02DB +ogonekcmb;0328 +ograve;00F2 +ogujarati;0A93 +oharmenian;0585 +ohiragana;304A +ohookabove;1ECF +ohorn;01A1 +ohornacute;1EDB +ohorndotbelow;1EE3 +ohorngrave;1EDD +ohornhookabove;1EDF +ohorntilde;1EE1 +ohungarumlaut;0151 +oi;01A3 +oinvertedbreve;020F +okatakana;30AA +okatakanahalfwidth;FF75 +okorean;3157 +olehebrew;05AB +omacron;014D +omacronacute;1E53 +omacrongrave;1E51 +omdeva;0950 +omega;03C9 +omega1;03D6 +omegacyrillic;0461 +omegalatinclosed;0277 +omegaroundcyrillic;047B +omegatitlocyrillic;047D +omegatonos;03CE +omgujarati;0AD0 +omicron;03BF +omicrontonos;03CC +omonospace;FF4F +one;0031 +onearabic;0661 +onebengali;09E7 +onecircle;2460 +onecircleinversesansserif;278A +onedeva;0967 +onedotenleader;2024 +oneeighth;215B +onefitted;F6DC +onegujarati;0AE7 +onegurmukhi;0A67 +onehackarabic;0661 +onehalf;00BD +onehangzhou;3021 +oneideographicparen;3220 +oneinferior;2081 +onemonospace;FF11 +onenumeratorbengali;09F4 +oneoldstyle;F731 +oneparen;2474 +oneperiod;2488 +onepersian;06F1 +onequarter;00BC +oneroman;2170 +onesuperior;00B9 +onethai;0E51 +onethird;2153 +oogonek;01EB +oogonekmacron;01ED +oogurmukhi;0A13 +oomatragurmukhi;0A4B +oopen;0254 +oparen;24AA +openbullet;25E6 +option;2325 +ordfeminine;00AA +ordmasculine;00BA +orthogonal;221F +oshortdeva;0912 +oshortvowelsigndeva;094A +oslash;00F8 +oslashacute;01FF +osmallhiragana;3049 +osmallkatakana;30A9 +osmallkatakanahalfwidth;FF6B +ostrokeacute;01FF +osuperior;F6F0 +otcyrillic;047F +otilde;00F5 +otildeacute;1E4D +otildedieresis;1E4F +oubopomofo;3121 +overline;203E +overlinecenterline;FE4A +overlinecmb;0305 +overlinedashed;FE49 +overlinedblwavy;FE4C +overlinewavy;FE4B +overscore;00AF +ovowelsignbengali;09CB +ovowelsigndeva;094B +ovowelsigngujarati;0ACB +p;0070 +paampssquare;3380 +paasentosquare;332B +pabengali;09AA +pacute;1E55 +padeva;092A +pagedown;21DF +pageup;21DE +pagujarati;0AAA +pagurmukhi;0A2A +pahiragana;3071 +paiyannoithai;0E2F +pakatakana;30D1 +palatalizationcyrilliccmb;0484 +palochkacyrillic;04C0 +pansioskorean;317F +paragraph;00B6 +parallel;2225 +parenleft;0028 +parenleftaltonearabic;FD3E +parenleftbt;F8ED +parenleftex;F8EC +parenleftinferior;208D +parenleftmonospace;FF08 +parenleftsmall;FE59 +parenleftsuperior;207D +parenlefttp;F8EB +parenleftvertical;FE35 +parenright;0029 +parenrightaltonearabic;FD3F +parenrightbt;F8F8 +parenrightex;F8F7 +parenrightinferior;208E +parenrightmonospace;FF09 +parenrightsmall;FE5A +parenrightsuperior;207E +parenrighttp;F8F6 +parenrightvertical;FE36 +partialdiff;2202 +paseqhebrew;05C0 +pashtahebrew;0599 +pasquare;33A9 +patah;05B7 +patah11;05B7 +patah1d;05B7 +patah2a;05B7 +patahhebrew;05B7 +patahnarrowhebrew;05B7 +patahquarterhebrew;05B7 +patahwidehebrew;05B7 +pazerhebrew;05A1 +pbopomofo;3106 +pcircle;24DF +pdotaccent;1E57 +pe;05E4 +pecyrillic;043F +pedagesh;FB44 +pedageshhebrew;FB44 +peezisquare;333B +pefinaldageshhebrew;FB43 +peharabic;067E +peharmenian;057A +pehebrew;05E4 +pehfinalarabic;FB57 +pehinitialarabic;FB58 +pehiragana;307A +pehmedialarabic;FB59 +pekatakana;30DA +pemiddlehookcyrillic;04A7 +perafehebrew;FB4E +percent;0025 +percentarabic;066A +percentmonospace;FF05 +percentsmall;FE6A +period;002E +periodarmenian;0589 +periodcentered;00B7 +periodhalfwidth;FF61 +periodinferior;F6E7 +periodmonospace;FF0E +periodsmall;FE52 +periodsuperior;F6E8 +perispomenigreekcmb;0342 +perpendicular;22A5 +perthousand;2030 +peseta;20A7 +pfsquare;338A +phabengali;09AB +phadeva;092B +phagujarati;0AAB +phagurmukhi;0A2B +phi;03C6 +phi1;03D5 +phieuphacirclekorean;327A +phieuphaparenkorean;321A +phieuphcirclekorean;326C +phieuphkorean;314D +phieuphparenkorean;320C +philatin;0278 +phinthuthai;0E3A +phisymbolgreek;03D5 +phook;01A5 +phophanthai;0E1E +phophungthai;0E1C +phosamphaothai;0E20 +pi;03C0 +pieupacirclekorean;3273 +pieupaparenkorean;3213 +pieupcieuckorean;3176 +pieupcirclekorean;3265 +pieupkiyeokkorean;3172 +pieupkorean;3142 +pieupparenkorean;3205 +pieupsioskiyeokkorean;3174 +pieupsioskorean;3144 +pieupsiostikeutkorean;3175 +pieupthieuthkorean;3177 +pieuptikeutkorean;3173 +pihiragana;3074 +pikatakana;30D4 +pisymbolgreek;03D6 +piwrarmenian;0583 +plus;002B +plusbelowcmb;031F +pluscircle;2295 +plusminus;00B1 +plusmod;02D6 +plusmonospace;FF0B +plussmall;FE62 +plussuperior;207A +pmonospace;FF50 +pmsquare;33D8 +pohiragana;307D +pointingindexdownwhite;261F +pointingindexleftwhite;261C +pointingindexrightwhite;261E +pointingindexupwhite;261D +pokatakana;30DD +poplathai;0E1B +postalmark;3012 +postalmarkface;3020 +pparen;24AB +precedes;227A +prescription;211E +primemod;02B9 +primereversed;2035 +product;220F +projective;2305 +prolongedkana;30FC +propellor;2318 +propersubset;2282 +propersuperset;2283 +proportion;2237 +proportional;221D +psi;03C8 +psicyrillic;0471 +psilipneumatacyrilliccmb;0486 +pssquare;33B0 +puhiragana;3077 +pukatakana;30D7 +pvsquare;33B4 +pwsquare;33BA +q;0071 +qadeva;0958 +qadmahebrew;05A8 +qafarabic;0642 +qaffinalarabic;FED6 +qafinitialarabic;FED7 +qafmedialarabic;FED8 +qamats;05B8 +qamats10;05B8 +qamats1a;05B8 +qamats1c;05B8 +qamats27;05B8 +qamats29;05B8 +qamats33;05B8 +qamatsde;05B8 +qamatshebrew;05B8 +qamatsnarrowhebrew;05B8 +qamatsqatanhebrew;05B8 +qamatsqatannarrowhebrew;05B8 +qamatsqatanquarterhebrew;05B8 +qamatsqatanwidehebrew;05B8 +qamatsquarterhebrew;05B8 +qamatswidehebrew;05B8 +qarneyparahebrew;059F +qbopomofo;3111 +qcircle;24E0 +qhook;02A0 +qmonospace;FF51 +qof;05E7 +qofdagesh;FB47 +qofdageshhebrew;FB47 +qofhatafpatah;05E7 05B2 +qofhatafpatahhebrew;05E7 05B2 +qofhatafsegol;05E7 05B1 +qofhatafsegolhebrew;05E7 05B1 +qofhebrew;05E7 +qofhiriq;05E7 05B4 +qofhiriqhebrew;05E7 05B4 +qofholam;05E7 05B9 +qofholamhebrew;05E7 05B9 +qofpatah;05E7 05B7 +qofpatahhebrew;05E7 05B7 +qofqamats;05E7 05B8 +qofqamatshebrew;05E7 05B8 +qofqubuts;05E7 05BB +qofqubutshebrew;05E7 05BB +qofsegol;05E7 05B6 +qofsegolhebrew;05E7 05B6 +qofsheva;05E7 05B0 +qofshevahebrew;05E7 05B0 +qoftsere;05E7 05B5 +qoftserehebrew;05E7 05B5 +qparen;24AC +quarternote;2669 +qubuts;05BB +qubuts18;05BB +qubuts25;05BB +qubuts31;05BB +qubutshebrew;05BB +qubutsnarrowhebrew;05BB +qubutsquarterhebrew;05BB +qubutswidehebrew;05BB +question;003F +questionarabic;061F +questionarmenian;055E +questiondown;00BF +questiondownsmall;F7BF +questiongreek;037E +questionmonospace;FF1F +questionsmall;F73F +quotedbl;0022 +quotedblbase;201E +quotedblleft;201C +quotedblmonospace;FF02 +quotedblprime;301E +quotedblprimereversed;301D +quotedblright;201D +quoteleft;2018 +quoteleftreversed;201B +quotereversed;201B +quoteright;2019 +quoterightn;0149 +quotesinglbase;201A +quotesingle;0027 +quotesinglemonospace;FF07 +r;0072 +raarmenian;057C +rabengali;09B0 +racute;0155 +radeva;0930 +radical;221A +radicalex;F8E5 +radoverssquare;33AE +radoverssquaredsquare;33AF +radsquare;33AD +rafe;05BF +rafehebrew;05BF +ragujarati;0AB0 +ragurmukhi;0A30 +rahiragana;3089 +rakatakana;30E9 +rakatakanahalfwidth;FF97 +ralowerdiagonalbengali;09F1 +ramiddlediagonalbengali;09F0 +ramshorn;0264 +ratio;2236 +rbopomofo;3116 +rcaron;0159 +rcedilla;0157 +rcircle;24E1 +rcommaaccent;0157 +rdblgrave;0211 +rdotaccent;1E59 +rdotbelow;1E5B +rdotbelowmacron;1E5D +referencemark;203B +reflexsubset;2286 +reflexsuperset;2287 +registered;00AE +registersans;F8E8 +registerserif;F6DA +reharabic;0631 +reharmenian;0580 +rehfinalarabic;FEAE +rehiragana;308C +rehyehaleflamarabic;0631 FEF3 FE8E 0644 +rekatakana;30EC +rekatakanahalfwidth;FF9A +resh;05E8 +reshdageshhebrew;FB48 +reshhatafpatah;05E8 05B2 +reshhatafpatahhebrew;05E8 05B2 +reshhatafsegol;05E8 05B1 +reshhatafsegolhebrew;05E8 05B1 +reshhebrew;05E8 +reshhiriq;05E8 05B4 +reshhiriqhebrew;05E8 05B4 +reshholam;05E8 05B9 +reshholamhebrew;05E8 05B9 +reshpatah;05E8 05B7 +reshpatahhebrew;05E8 05B7 +reshqamats;05E8 05B8 +reshqamatshebrew;05E8 05B8 +reshqubuts;05E8 05BB +reshqubutshebrew;05E8 05BB +reshsegol;05E8 05B6 +reshsegolhebrew;05E8 05B6 +reshsheva;05E8 05B0 +reshshevahebrew;05E8 05B0 +reshtsere;05E8 05B5 +reshtserehebrew;05E8 05B5 +reversedtilde;223D +reviahebrew;0597 +reviamugrashhebrew;0597 +revlogicalnot;2310 +rfishhook;027E +rfishhookreversed;027F +rhabengali;09DD +rhadeva;095D +rho;03C1 +rhook;027D +rhookturned;027B +rhookturnedsuperior;02B5 +rhosymbolgreek;03F1 +rhotichookmod;02DE +rieulacirclekorean;3271 +rieulaparenkorean;3211 +rieulcirclekorean;3263 +rieulhieuhkorean;3140 +rieulkiyeokkorean;313A +rieulkiyeoksioskorean;3169 +rieulkorean;3139 +rieulmieumkorean;313B +rieulpansioskorean;316C +rieulparenkorean;3203 +rieulphieuphkorean;313F +rieulpieupkorean;313C +rieulpieupsioskorean;316B +rieulsioskorean;313D +rieulthieuthkorean;313E +rieultikeutkorean;316A +rieulyeorinhieuhkorean;316D +rightangle;221F +righttackbelowcmb;0319 +righttriangle;22BF +rihiragana;308A +rikatakana;30EA +rikatakanahalfwidth;FF98 +ring;02DA +ringbelowcmb;0325 +ringcmb;030A +ringhalfleft;02BF +ringhalfleftarmenian;0559 +ringhalfleftbelowcmb;031C +ringhalfleftcentered;02D3 +ringhalfright;02BE +ringhalfrightbelowcmb;0339 +ringhalfrightcentered;02D2 +rinvertedbreve;0213 +rittorusquare;3351 +rlinebelow;1E5F +rlongleg;027C +rlonglegturned;027A +rmonospace;FF52 +rohiragana;308D +rokatakana;30ED +rokatakanahalfwidth;FF9B +roruathai;0E23 +rparen;24AD +rrabengali;09DC +rradeva;0931 +rragurmukhi;0A5C +rreharabic;0691 +rrehfinalarabic;FB8D +rrvocalicbengali;09E0 +rrvocalicdeva;0960 +rrvocalicgujarati;0AE0 +rrvocalicvowelsignbengali;09C4 +rrvocalicvowelsigndeva;0944 +rrvocalicvowelsigngujarati;0AC4 +rsuperior;F6F1 +rtblock;2590 +rturned;0279 +rturnedsuperior;02B4 +ruhiragana;308B +rukatakana;30EB +rukatakanahalfwidth;FF99 +rupeemarkbengali;09F2 +rupeesignbengali;09F3 +rupiah;F6DD +ruthai;0E24 +rvocalicbengali;098B +rvocalicdeva;090B +rvocalicgujarati;0A8B +rvocalicvowelsignbengali;09C3 +rvocalicvowelsigndeva;0943 +rvocalicvowelsigngujarati;0AC3 +s;0073 +sabengali;09B8 +sacute;015B +sacutedotaccent;1E65 +sadarabic;0635 +sadeva;0938 +sadfinalarabic;FEBA +sadinitialarabic;FEBB +sadmedialarabic;FEBC +sagujarati;0AB8 +sagurmukhi;0A38 +sahiragana;3055 +sakatakana;30B5 +sakatakanahalfwidth;FF7B +sallallahoualayhewasallamarabic;FDFA +samekh;05E1 +samekhdagesh;FB41 +samekhdageshhebrew;FB41 +samekhhebrew;05E1 +saraaathai;0E32 +saraaethai;0E41 +saraaimaimalaithai;0E44 +saraaimaimuanthai;0E43 +saraamthai;0E33 +saraathai;0E30 +saraethai;0E40 +saraiileftthai;F886 +saraiithai;0E35 +saraileftthai;F885 +saraithai;0E34 +saraothai;0E42 +saraueeleftthai;F888 +saraueethai;0E37 +saraueleftthai;F887 +sarauethai;0E36 +sarauthai;0E38 +sarauuthai;0E39 +sbopomofo;3119 +scaron;0161 +scarondotaccent;1E67 +scedilla;015F +schwa;0259 +schwacyrillic;04D9 +schwadieresiscyrillic;04DB +schwahook;025A +scircle;24E2 +scircumflex;015D +scommaaccent;0219 +sdotaccent;1E61 +sdotbelow;1E63 +sdotbelowdotaccent;1E69 +seagullbelowcmb;033C +second;2033 +secondtonechinese;02CA +section;00A7 +seenarabic;0633 +seenfinalarabic;FEB2 +seeninitialarabic;FEB3 +seenmedialarabic;FEB4 +segol;05B6 +segol13;05B6 +segol1f;05B6 +segol2c;05B6 +segolhebrew;05B6 +segolnarrowhebrew;05B6 +segolquarterhebrew;05B6 +segoltahebrew;0592 +segolwidehebrew;05B6 +seharmenian;057D +sehiragana;305B +sekatakana;30BB +sekatakanahalfwidth;FF7E +semicolon;003B +semicolonarabic;061B +semicolonmonospace;FF1B +semicolonsmall;FE54 +semivoicedmarkkana;309C +semivoicedmarkkanahalfwidth;FF9F +sentisquare;3322 +sentosquare;3323 +seven;0037 +sevenarabic;0667 +sevenbengali;09ED +sevencircle;2466 +sevencircleinversesansserif;2790 +sevendeva;096D +seveneighths;215E +sevengujarati;0AED +sevengurmukhi;0A6D +sevenhackarabic;0667 +sevenhangzhou;3027 +sevenideographicparen;3226 +seveninferior;2087 +sevenmonospace;FF17 +sevenoldstyle;F737 +sevenparen;247A +sevenperiod;248E +sevenpersian;06F7 +sevenroman;2176 +sevensuperior;2077 +seventeencircle;2470 +seventeenparen;2484 +seventeenperiod;2498 +seventhai;0E57 +sfthyphen;00AD +shaarmenian;0577 +shabengali;09B6 +shacyrillic;0448 +shaddaarabic;0651 +shaddadammaarabic;FC61 +shaddadammatanarabic;FC5E +shaddafathaarabic;FC60 +shaddafathatanarabic;0651 064B +shaddakasraarabic;FC62 +shaddakasratanarabic;FC5F +shade;2592 +shadedark;2593 +shadelight;2591 +shademedium;2592 +shadeva;0936 +shagujarati;0AB6 +shagurmukhi;0A36 +shalshelethebrew;0593 +shbopomofo;3115 +shchacyrillic;0449 +sheenarabic;0634 +sheenfinalarabic;FEB6 +sheeninitialarabic;FEB7 +sheenmedialarabic;FEB8 +sheicoptic;03E3 +sheqel;20AA +sheqelhebrew;20AA +sheva;05B0 +sheva115;05B0 +sheva15;05B0 +sheva22;05B0 +sheva2e;05B0 +shevahebrew;05B0 +shevanarrowhebrew;05B0 +shevaquarterhebrew;05B0 +shevawidehebrew;05B0 +shhacyrillic;04BB +shimacoptic;03ED +shin;05E9 +shindagesh;FB49 +shindageshhebrew;FB49 +shindageshshindot;FB2C +shindageshshindothebrew;FB2C +shindageshsindot;FB2D +shindageshsindothebrew;FB2D +shindothebrew;05C1 +shinhebrew;05E9 +shinshindot;FB2A +shinshindothebrew;FB2A +shinsindot;FB2B +shinsindothebrew;FB2B +shook;0282 +sigma;03C3 +sigma1;03C2 +sigmafinal;03C2 +sigmalunatesymbolgreek;03F2 +sihiragana;3057 +sikatakana;30B7 +sikatakanahalfwidth;FF7C +siluqhebrew;05BD +siluqlefthebrew;05BD +similar;223C +sindothebrew;05C2 +siosacirclekorean;3274 +siosaparenkorean;3214 +sioscieuckorean;317E +sioscirclekorean;3266 +sioskiyeokkorean;317A +sioskorean;3145 +siosnieunkorean;317B +siosparenkorean;3206 +siospieupkorean;317D +siostikeutkorean;317C +six;0036 +sixarabic;0666 +sixbengali;09EC +sixcircle;2465 +sixcircleinversesansserif;278F +sixdeva;096C +sixgujarati;0AEC +sixgurmukhi;0A6C +sixhackarabic;0666 +sixhangzhou;3026 +sixideographicparen;3225 +sixinferior;2086 +sixmonospace;FF16 +sixoldstyle;F736 +sixparen;2479 +sixperiod;248D +sixpersian;06F6 +sixroman;2175 +sixsuperior;2076 +sixteencircle;246F +sixteencurrencydenominatorbengali;09F9 +sixteenparen;2483 +sixteenperiod;2497 +sixthai;0E56 +slash;002F +slashmonospace;FF0F +slong;017F +slongdotaccent;1E9B +smileface;263A +smonospace;FF53 +sofpasuqhebrew;05C3 +softhyphen;00AD +softsigncyrillic;044C +sohiragana;305D +sokatakana;30BD +sokatakanahalfwidth;FF7F +soliduslongoverlaycmb;0338 +solidusshortoverlaycmb;0337 +sorusithai;0E29 +sosalathai;0E28 +sosothai;0E0B +sosuathai;0E2A +space;0020 +spacehackarabic;0020 +spade;2660 +spadesuitblack;2660 +spadesuitwhite;2664 +sparen;24AE +squarebelowcmb;033B +squarecc;33C4 +squarecm;339D +squarediagonalcrosshatchfill;25A9 +squarehorizontalfill;25A4 +squarekg;338F +squarekm;339E +squarekmcapital;33CE +squareln;33D1 +squarelog;33D2 +squaremg;338E +squaremil;33D5 +squaremm;339C +squaremsquared;33A1 +squareorthogonalcrosshatchfill;25A6 +squareupperlefttolowerrightfill;25A7 +squareupperrighttolowerleftfill;25A8 +squareverticalfill;25A5 +squarewhitewithsmallblack;25A3 +srsquare;33DB +ssabengali;09B7 +ssadeva;0937 +ssagujarati;0AB7 +ssangcieuckorean;3149 +ssanghieuhkorean;3185 +ssangieungkorean;3180 +ssangkiyeokkorean;3132 +ssangnieunkorean;3165 +ssangpieupkorean;3143 +ssangsioskorean;3146 +ssangtikeutkorean;3138 +ssuperior;F6F2 +sterling;00A3 +sterlingmonospace;FFE1 +strokelongoverlaycmb;0336 +strokeshortoverlaycmb;0335 +subset;2282 +subsetnotequal;228A +subsetorequal;2286 +succeeds;227B +suchthat;220B +suhiragana;3059 +sukatakana;30B9 +sukatakanahalfwidth;FF7D +sukunarabic;0652 +summation;2211 +sun;263C +superset;2283 +supersetnotequal;228B +supersetorequal;2287 +svsquare;33DC +syouwaerasquare;337C +t;0074 +tabengali;09A4 +tackdown;22A4 +tackleft;22A3 +tadeva;0924 +tagujarati;0AA4 +tagurmukhi;0A24 +taharabic;0637 +tahfinalarabic;FEC2 +tahinitialarabic;FEC3 +tahiragana;305F +tahmedialarabic;FEC4 +taisyouerasquare;337D +takatakana;30BF +takatakanahalfwidth;FF80 +tatweelarabic;0640 +tau;03C4 +tav;05EA +tavdages;FB4A +tavdagesh;FB4A +tavdageshhebrew;FB4A +tavhebrew;05EA +tbar;0167 +tbopomofo;310A +tcaron;0165 +tccurl;02A8 +tcedilla;0163 +tcheharabic;0686 +tchehfinalarabic;FB7B +tchehinitialarabic;FB7C +tchehmedialarabic;FB7D +tchehmeeminitialarabic;FB7C FEE4 +tcircle;24E3 +tcircumflexbelow;1E71 +tcommaaccent;0163 +tdieresis;1E97 +tdotaccent;1E6B +tdotbelow;1E6D +tecyrillic;0442 +tedescendercyrillic;04AD +teharabic;062A +tehfinalarabic;FE96 +tehhahinitialarabic;FCA2 +tehhahisolatedarabic;FC0C +tehinitialarabic;FE97 +tehiragana;3066 +tehjeeminitialarabic;FCA1 +tehjeemisolatedarabic;FC0B +tehmarbutaarabic;0629 +tehmarbutafinalarabic;FE94 +tehmedialarabic;FE98 +tehmeeminitialarabic;FCA4 +tehmeemisolatedarabic;FC0E +tehnoonfinalarabic;FC73 +tekatakana;30C6 +tekatakanahalfwidth;FF83 +telephone;2121 +telephoneblack;260E +telishagedolahebrew;05A0 +telishaqetanahebrew;05A9 +tencircle;2469 +tenideographicparen;3229 +tenparen;247D +tenperiod;2491 +tenroman;2179 +tesh;02A7 +tet;05D8 +tetdagesh;FB38 +tetdageshhebrew;FB38 +tethebrew;05D8 +tetsecyrillic;04B5 +tevirhebrew;059B +tevirlefthebrew;059B +thabengali;09A5 +thadeva;0925 +thagujarati;0AA5 +thagurmukhi;0A25 +thalarabic;0630 +thalfinalarabic;FEAC +thanthakhatlowleftthai;F898 +thanthakhatlowrightthai;F897 +thanthakhatthai;0E4C +thanthakhatupperleftthai;F896 +theharabic;062B +thehfinalarabic;FE9A +thehinitialarabic;FE9B +thehmedialarabic;FE9C +thereexists;2203 +therefore;2234 +theta;03B8 +theta1;03D1 +thetasymbolgreek;03D1 +thieuthacirclekorean;3279 +thieuthaparenkorean;3219 +thieuthcirclekorean;326B +thieuthkorean;314C +thieuthparenkorean;320B +thirteencircle;246C +thirteenparen;2480 +thirteenperiod;2494 +thonangmonthothai;0E11 +thook;01AD +thophuthaothai;0E12 +thorn;00FE +thothahanthai;0E17 +thothanthai;0E10 +thothongthai;0E18 +thothungthai;0E16 +thousandcyrillic;0482 +thousandsseparatorarabic;066C +thousandsseparatorpersian;066C +three;0033 +threearabic;0663 +threebengali;09E9 +threecircle;2462 +threecircleinversesansserif;278C +threedeva;0969 +threeeighths;215C +threegujarati;0AE9 +threegurmukhi;0A69 +threehackarabic;0663 +threehangzhou;3023 +threeideographicparen;3222 +threeinferior;2083 +threemonospace;FF13 +threenumeratorbengali;09F6 +threeoldstyle;F733 +threeparen;2476 +threeperiod;248A +threepersian;06F3 +threequarters;00BE +threequartersemdash;F6DE +threeroman;2172 +threesuperior;00B3 +threethai;0E53 +thzsquare;3394 +tihiragana;3061 +tikatakana;30C1 +tikatakanahalfwidth;FF81 +tikeutacirclekorean;3270 +tikeutaparenkorean;3210 +tikeutcirclekorean;3262 +tikeutkorean;3137 +tikeutparenkorean;3202 +tilde;02DC +tildebelowcmb;0330 +tildecmb;0303 +tildecomb;0303 +tildedoublecmb;0360 +tildeoperator;223C +tildeoverlaycmb;0334 +tildeverticalcmb;033E +timescircle;2297 +tipehahebrew;0596 +tipehalefthebrew;0596 +tippigurmukhi;0A70 +titlocyrilliccmb;0483 +tiwnarmenian;057F +tlinebelow;1E6F +tmonospace;FF54 +toarmenian;0569 +tohiragana;3068 +tokatakana;30C8 +tokatakanahalfwidth;FF84 +tonebarextrahighmod;02E5 +tonebarextralowmod;02E9 +tonebarhighmod;02E6 +tonebarlowmod;02E8 +tonebarmidmod;02E7 +tonefive;01BD +tonesix;0185 +tonetwo;01A8 +tonos;0384 +tonsquare;3327 +topatakthai;0E0F +tortoiseshellbracketleft;3014 +tortoiseshellbracketleftsmall;FE5D +tortoiseshellbracketleftvertical;FE39 +tortoiseshellbracketright;3015 +tortoiseshellbracketrightsmall;FE5E +tortoiseshellbracketrightvertical;FE3A +totaothai;0E15 +tpalatalhook;01AB +tparen;24AF +trademark;2122 +trademarksans;F8EA +trademarkserif;F6DB +tretroflexhook;0288 +triagdn;25BC +triaglf;25C4 +triagrt;25BA +triagup;25B2 +ts;02A6 +tsadi;05E6 +tsadidagesh;FB46 +tsadidageshhebrew;FB46 +tsadihebrew;05E6 +tsecyrillic;0446 +tsere;05B5 +tsere12;05B5 +tsere1e;05B5 +tsere2b;05B5 +tserehebrew;05B5 +tserenarrowhebrew;05B5 +tserequarterhebrew;05B5 +tserewidehebrew;05B5 +tshecyrillic;045B +tsuperior;F6F3 +ttabengali;099F +ttadeva;091F +ttagujarati;0A9F +ttagurmukhi;0A1F +tteharabic;0679 +ttehfinalarabic;FB67 +ttehinitialarabic;FB68 +ttehmedialarabic;FB69 +tthabengali;09A0 +tthadeva;0920 +tthagujarati;0AA0 +tthagurmukhi;0A20 +tturned;0287 +tuhiragana;3064 +tukatakana;30C4 +tukatakanahalfwidth;FF82 +tusmallhiragana;3063 +tusmallkatakana;30C3 +tusmallkatakanahalfwidth;FF6F +twelvecircle;246B +twelveparen;247F +twelveperiod;2493 +twelveroman;217B +twentycircle;2473 +twentyhangzhou;5344 +twentyparen;2487 +twentyperiod;249B +two;0032 +twoarabic;0662 +twobengali;09E8 +twocircle;2461 +twocircleinversesansserif;278B +twodeva;0968 +twodotenleader;2025 +twodotleader;2025 +twodotleadervertical;FE30 +twogujarati;0AE8 +twogurmukhi;0A68 +twohackarabic;0662 +twohangzhou;3022 +twoideographicparen;3221 +twoinferior;2082 +twomonospace;FF12 +twonumeratorbengali;09F5 +twooldstyle;F732 +twoparen;2475 +twoperiod;2489 +twopersian;06F2 +tworoman;2171 +twostroke;01BB +twosuperior;00B2 +twothai;0E52 +twothirds;2154 +u;0075 +uacute;00FA +ubar;0289 +ubengali;0989 +ubopomofo;3128 +ubreve;016D +ucaron;01D4 +ucircle;24E4 +ucircumflex;00FB +ucircumflexbelow;1E77 +ucyrillic;0443 +udattadeva;0951 +udblacute;0171 +udblgrave;0215 +udeva;0909 +udieresis;00FC +udieresisacute;01D8 +udieresisbelow;1E73 +udieresiscaron;01DA +udieresiscyrillic;04F1 +udieresisgrave;01DC +udieresismacron;01D6 +udotbelow;1EE5 +ugrave;00F9 +ugujarati;0A89 +ugurmukhi;0A09 +uhiragana;3046 +uhookabove;1EE7 +uhorn;01B0 +uhornacute;1EE9 +uhorndotbelow;1EF1 +uhorngrave;1EEB +uhornhookabove;1EED +uhorntilde;1EEF +uhungarumlaut;0171 +uhungarumlautcyrillic;04F3 +uinvertedbreve;0217 +ukatakana;30A6 +ukatakanahalfwidth;FF73 +ukcyrillic;0479 +ukorean;315C +umacron;016B +umacroncyrillic;04EF +umacrondieresis;1E7B +umatragurmukhi;0A41 +umonospace;FF55 +underscore;005F +underscoredbl;2017 +underscoremonospace;FF3F +underscorevertical;FE33 +underscorewavy;FE4F +union;222A +universal;2200 +uogonek;0173 +uparen;24B0 +upblock;2580 +upperdothebrew;05C4 +upsilon;03C5 +upsilondieresis;03CB +upsilondieresistonos;03B0 +upsilonlatin;028A +upsilontonos;03CD +uptackbelowcmb;031D +uptackmod;02D4 +uragurmukhi;0A73 +uring;016F +ushortcyrillic;045E +usmallhiragana;3045 +usmallkatakana;30A5 +usmallkatakanahalfwidth;FF69 +ustraightcyrillic;04AF +ustraightstrokecyrillic;04B1 +utilde;0169 +utildeacute;1E79 +utildebelow;1E75 +uubengali;098A +uudeva;090A +uugujarati;0A8A +uugurmukhi;0A0A +uumatragurmukhi;0A42 +uuvowelsignbengali;09C2 +uuvowelsigndeva;0942 +uuvowelsigngujarati;0AC2 +uvowelsignbengali;09C1 +uvowelsigndeva;0941 +uvowelsigngujarati;0AC1 +v;0076 +vadeva;0935 +vagujarati;0AB5 +vagurmukhi;0A35 +vakatakana;30F7 +vav;05D5 +vavdagesh;FB35 +vavdagesh65;FB35 +vavdageshhebrew;FB35 +vavhebrew;05D5 +vavholam;FB4B +vavholamhebrew;FB4B +vavvavhebrew;05F0 +vavyodhebrew;05F1 +vcircle;24E5 +vdotbelow;1E7F +vecyrillic;0432 +veharabic;06A4 +vehfinalarabic;FB6B +vehinitialarabic;FB6C +vehmedialarabic;FB6D +vekatakana;30F9 +venus;2640 +verticalbar;007C +verticallineabovecmb;030D +verticallinebelowcmb;0329 +verticallinelowmod;02CC +verticallinemod;02C8 +vewarmenian;057E +vhook;028B +vikatakana;30F8 +viramabengali;09CD +viramadeva;094D +viramagujarati;0ACD +visargabengali;0983 +visargadeva;0903 +visargagujarati;0A83 +vmonospace;FF56 +voarmenian;0578 +voicediterationhiragana;309E +voicediterationkatakana;30FE +voicedmarkkana;309B +voicedmarkkanahalfwidth;FF9E +vokatakana;30FA +vparen;24B1 +vtilde;1E7D +vturned;028C +vuhiragana;3094 +vukatakana;30F4 +w;0077 +wacute;1E83 +waekorean;3159 +wahiragana;308F +wakatakana;30EF +wakatakanahalfwidth;FF9C +wakorean;3158 +wasmallhiragana;308E +wasmallkatakana;30EE +wattosquare;3357 +wavedash;301C +wavyunderscorevertical;FE34 +wawarabic;0648 +wawfinalarabic;FEEE +wawhamzaabovearabic;0624 +wawhamzaabovefinalarabic;FE86 +wbsquare;33DD +wcircle;24E6 +wcircumflex;0175 +wdieresis;1E85 +wdotaccent;1E87 +wdotbelow;1E89 +wehiragana;3091 +weierstrass;2118 +wekatakana;30F1 +wekorean;315E +weokorean;315D +wgrave;1E81 +whitebullet;25E6 +whitecircle;25CB +whitecircleinverse;25D9 +whitecornerbracketleft;300E +whitecornerbracketleftvertical;FE43 +whitecornerbracketright;300F +whitecornerbracketrightvertical;FE44 +whitediamond;25C7 +whitediamondcontainingblacksmalldiamond;25C8 +whitedownpointingsmalltriangle;25BF +whitedownpointingtriangle;25BD +whiteleftpointingsmalltriangle;25C3 +whiteleftpointingtriangle;25C1 +whitelenticularbracketleft;3016 +whitelenticularbracketright;3017 +whiterightpointingsmalltriangle;25B9 +whiterightpointingtriangle;25B7 +whitesmallsquare;25AB +whitesmilingface;263A +whitesquare;25A1 +whitestar;2606 +whitetelephone;260F +whitetortoiseshellbracketleft;3018 +whitetortoiseshellbracketright;3019 +whiteuppointingsmalltriangle;25B5 +whiteuppointingtriangle;25B3 +wihiragana;3090 +wikatakana;30F0 +wikorean;315F +wmonospace;FF57 +wohiragana;3092 +wokatakana;30F2 +wokatakanahalfwidth;FF66 +won;20A9 +wonmonospace;FFE6 +wowaenthai;0E27 +wparen;24B2 +wring;1E98 +wsuperior;02B7 +wturned;028D +wynn;01BF +x;0078 +xabovecmb;033D +xbopomofo;3112 +xcircle;24E7 +xdieresis;1E8D +xdotaccent;1E8B +xeharmenian;056D +xi;03BE +xmonospace;FF58 +xparen;24B3 +xsuperior;02E3 +y;0079 +yaadosquare;334E +yabengali;09AF +yacute;00FD +yadeva;092F +yaekorean;3152 +yagujarati;0AAF +yagurmukhi;0A2F +yahiragana;3084 +yakatakana;30E4 +yakatakanahalfwidth;FF94 +yakorean;3151 +yamakkanthai;0E4E +yasmallhiragana;3083 +yasmallkatakana;30E3 +yasmallkatakanahalfwidth;FF6C +yatcyrillic;0463 +ycircle;24E8 +ycircumflex;0177 +ydieresis;00FF +ydotaccent;1E8F +ydotbelow;1EF5 +yeharabic;064A +yehbarreearabic;06D2 +yehbarreefinalarabic;FBAF +yehfinalarabic;FEF2 +yehhamzaabovearabic;0626 +yehhamzaabovefinalarabic;FE8A +yehhamzaaboveinitialarabic;FE8B +yehhamzaabovemedialarabic;FE8C +yehinitialarabic;FEF3 +yehmedialarabic;FEF4 +yehmeeminitialarabic;FCDD +yehmeemisolatedarabic;FC58 +yehnoonfinalarabic;FC94 +yehthreedotsbelowarabic;06D1 +yekorean;3156 +yen;00A5 +yenmonospace;FFE5 +yeokorean;3155 +yeorinhieuhkorean;3186 +yerahbenyomohebrew;05AA +yerahbenyomolefthebrew;05AA +yericyrillic;044B +yerudieresiscyrillic;04F9 +yesieungkorean;3181 +yesieungpansioskorean;3183 +yesieungsioskorean;3182 +yetivhebrew;059A +ygrave;1EF3 +yhook;01B4 +yhookabove;1EF7 +yiarmenian;0575 +yicyrillic;0457 +yikorean;3162 +yinyang;262F +yiwnarmenian;0582 +ymonospace;FF59 +yod;05D9 +yoddagesh;FB39 +yoddageshhebrew;FB39 +yodhebrew;05D9 +yodyodhebrew;05F2 +yodyodpatahhebrew;FB1F +yohiragana;3088 +yoikorean;3189 +yokatakana;30E8 +yokatakanahalfwidth;FF96 +yokorean;315B +yosmallhiragana;3087 +yosmallkatakana;30E7 +yosmallkatakanahalfwidth;FF6E +yotgreek;03F3 +yoyaekorean;3188 +yoyakorean;3187 +yoyakthai;0E22 +yoyingthai;0E0D +yparen;24B4 +ypogegrammeni;037A +ypogegrammenigreekcmb;0345 +yr;01A6 +yring;1E99 +ysuperior;02B8 +ytilde;1EF9 +yturned;028E +yuhiragana;3086 +yuikorean;318C +yukatakana;30E6 +yukatakanahalfwidth;FF95 +yukorean;3160 +yusbigcyrillic;046B +yusbigiotifiedcyrillic;046D +yuslittlecyrillic;0467 +yuslittleiotifiedcyrillic;0469 +yusmallhiragana;3085 +yusmallkatakana;30E5 +yusmallkatakanahalfwidth;FF6D +yuyekorean;318B +yuyeokorean;318A +yyabengali;09DF +yyadeva;095F +z;007A +zaarmenian;0566 +zacute;017A +zadeva;095B +zagurmukhi;0A5B +zaharabic;0638 +zahfinalarabic;FEC6 +zahinitialarabic;FEC7 +zahiragana;3056 +zahmedialarabic;FEC8 +zainarabic;0632 +zainfinalarabic;FEB0 +zakatakana;30B6 +zaqefgadolhebrew;0595 +zaqefqatanhebrew;0594 +zarqahebrew;0598 +zayin;05D6 +zayindagesh;FB36 +zayindageshhebrew;FB36 +zayinhebrew;05D6 +zbopomofo;3117 +zcaron;017E +zcircle;24E9 +zcircumflex;1E91 +zcurl;0291 +zdot;017C +zdotaccent;017C +zdotbelow;1E93 +zecyrillic;0437 +zedescendercyrillic;0499 +zedieresiscyrillic;04DF +zehiragana;305C +zekatakana;30BC +zero;0030 +zeroarabic;0660 +zerobengali;09E6 +zerodeva;0966 +zerogujarati;0AE6 +zerogurmukhi;0A66 +zerohackarabic;0660 +zeroinferior;2080 +zeromonospace;FF10 +zerooldstyle;F730 +zeropersian;06F0 +zerosuperior;2070 +zerothai;0E50 +zerowidthjoiner;FEFF +zerowidthnonjoiner;200C +zerowidthspace;200B +zeta;03B6 +zhbopomofo;3113 +zhearmenian;056A +zhebrevecyrillic;04C2 +zhecyrillic;0436 +zhedescendercyrillic;0497 +zhedieresiscyrillic;04DD +zihiragana;3058 +zikatakana;30B8 +zinorhebrew;05AE +zlinebelow;1E95 +zmonospace;FF5A +zohiragana;305E +zokatakana;30BE +zparen;24B5 +zretroflexhook;0290 +zstroke;01B6 +zuhiragana;305A +zukatakana;30BA +#--end +############################################################################### +# TeX extensions +Delta;2206;0394 +Ifractur;2111 +# FFsmall, FFIsmall, FFLsmall, FIsmall, FLsmall: invalid Unicode +FFsmall;D804 +FFIsmall;D807 +FFLsmall;D808 +FIsmall;D805 +FLsmall;D806 +# Germandbls: invalid Unicode, used for secondary replacement from EC.enc +Germandbls;1E9E +# Germandblssmall: invalid Unicode +Germandblssmall;D803 +Ng;014A +Omega;2126;03A9 +Rfractur;211C +# SS: invalid Unicode, used for secondary replacement from EC.enc +SS;0053 0053 +# SSsmall: invalid Unicode +SSsmall;D803 +# altselector: invalid Unicode +altselector;D802 +angbracketleft;27E8;2329 +angbracketright;27E9;232A +arrowbothv;2195 +arrowdblbothv;21D5 +arrowleftbothalf;21BD +arrowlefttophalf;21BC +arrownortheast;2197 +arrownorthwest;2196 +arrowrightbothalf;21C1 +arrowrighttophalf;21C0 +arrowsoutheast;2198 +arrowsouthwest;2199 +# ascendercompwordmark: taller cwm, invalid Unicode +ascendercompwordmark;D80A +asteriskcentered;2217 +bardbl;2225 +# capitalcompwordmark: taller cwm, invalid Unicode +capitalcompwordmark;D809 +ceilingleft;2308 +ceilingright;2309 +circlecopyrt;20DD;25CB +circledivide;2298 +circledot;2299 +circleminus;2296 +coproduct;2A3F +# cwm: Unicode for ZWNJ, used for secondary replacement from EC.enc +cwm;200C +dblbracketleft;27E6 +dblbracketright;27E7 +diamond;2662;2666 +diamondmath;22C4 +# 0237 is the proposed allocation for dotless j as of Unicode 4.0 +dotlessj;0237;F6BE +emptyset;2205;F638 +# emptyslot: invalid Unicode; the user explicitly wants this slot empty +emptyslot;D801 +epsilon1;03F5 +equivasymptotic;224D +flat;266D +floorleft;230A +floorright;230B +follows;227B +followsequal;227D +greatermuch;226B +heart;2661;2665 +interrobang;203D +# interrobangdown: inverted interrobang, invalid Unicode +interrobangdown;D80B +intersectionsq;2293 +latticetop;22A4 +lessmuch;226A +lscript;2113 +natural;266E +negationslash;0338 +ng;014B +owner;220B +pertenthousand;2031 +# TeX encodings such as 'texmital.enc' disagree with Unicode and the Adobe +# glyph list; in TeX, the "straight" form of phi takes the name "phi", +# whereas the more-common "loopy" form of phi, \varphi, takes the name "phi1". +phi;03D5;03C6 +phi1;03C6 +pi1;03D6 +precedesequal;227C +prime;2032;02B9 +rho1;03F1 +sharp;266F +similarequal;2243 +slurabove;2322 +slurbelow;2323 +star;22C6 +subsetsqequal;2291 +supersetsqequal;2292 +triangle;25B3 +triangleinv;25BD +triangleleft;25B9 +triangleright;25C3 +turnstileleft;22A2 +turnstileright;22A3 +# twelveudash: 2/3-em dash; invalid Unicode +twelveudash;F6DE +unionmulti;228E +unionsq;2294 +vector;20D7 +# visualspace: Unicode for OPEN BOX, used for secondary replacement from EC.enc +visualspace;2423 +wreathproduct;2240 +# -- Marco Kuhlmann reported annoying glyph neologisms in fontinst encodings +# Dbar = Dcroat +Dbar;0110 +# compwordmark = cwm +compwordmark;200C +# dbar = dcroat +dbar;0111 +# rangedash = endash +rangedash;2013 +# hyphenchar = hyphen +hyphenchar;002D +# punctdash = emdash +punctdash;2014 +# visiblespace = visualspace +visiblespace;2423 +# -- end TeX extensions +# additions by Michal Hoftich +tie;2040 +harpoonleftup;21BC +harpoonleftdown;21BD +harpoonrightup;21C0 +harpoonrightdown;21C1 +hookrightchar;21AA +hookleftchar;21A9 + +parenleftbig;0028 +parenrightbig;0029 +bracketleftbig;005B +bracketrightbig;005D +floorleftbig;230A +floorrightbig;230B +ceilingleftbig;2308 +ceilingrightbig;2309 +braceleftbig;007B +bracerightbig;007D +angleleftbig;2329 +anglerightbig;232A +barex;007C +bardblex;2016 +slashbig;2215 +backslashbig;005C +braceex;007C +parenleftex;007C +parenrightex;007C +unionsqtext;2294 +contintegraltext;222E +circledottext;2299 +circleplustext;2295 +circlemultiplytext;2297 +summationtext;2211 +producttext;220F +integraltext;222B +uniontext;22C3 +intersectiontext;22C2 +unionmultitext;228E +logicalandtext;22C0 +logicalortext;22C1 +coproducttext;2210 +hatwide;0302 +tildewide;0303 +radicalbig;221A +arrowtp;2191 +arrowbt;2193 +arrowdbltp;21D1 +braceleftmid;007C +bracerightmid;2016 +arrowvertex;2195 +arrowdblvertex;21D5 +parenleftBig;0028 +parenrightBig;0029 +slashBig;2215 +backslashBig;005C +angleleftBig;2329 +anglerightBig;232A +bracketleftBig;005B +bracketrightBig;005D +floorleftBig;230A +floorrightBig;230B +ceilingleftBig;2308 +ceilingrightBig;2309 +braceleftBig;007B +bracerightBig;007D +parenleftbigg;0028 +parenrightbigg;0029 +bracketleftbigg;005B +bracketrightbigg;005D +floorleftbigg;230A +floorrightbigg;230B +ceilingleftbigg;2308 +ceilingrightbigg;2309 +braceleftbigg;007B +bracerightbigg;007D +angleleftbigg;2329 +anglerightbigg;232A +slashbigg;2215 +backslashbigg;005C +parenleftBigg;0028 +parenrightBigg;0029 +bracketleftBigg;005B +bracketrightBigg;005D +floorleftBigg;230A +floorrightBigg;230B +ceilingleftBigg;2308 +ceilingrightBigg;2309 +braceleftBigg;007B +bracerightBigg;007D +angleleftBigg;2329 +anglerightBigg;232A +slashBigg;2215 +backslashBigg;005C +unionsqdisplay;2294 +contintegraldisplay;222E +circledotdisplay;2299 +circleplusdisplay;2295 +circlemultiplydisplay;2297 +summationdisplay;2211 +productdisplay;220F +integraldisplay;222B +uniondisplay;22C3 +intersectiondisplay;22C2 +unionmultidisplay;228E +logicalanddisplay;22C0 +logicalordisplay;22C1 +coproductdisplay;2210 +hatwider;0302 +hatwiderr;0302 +tildewider;0303 +tildewiderr;0303 +radicalBig;221A +radicalbigg;221A +radicalBigg;221A +radicalbt;221A +# fixes +a;0061 +b;0062 +c;0063 +d;0064 +e;0065 +f;0066 +Germandbls;0053 0053 +hyphen.alt;2010 diff --git a/crates/pdf-extract/src/glyphnames.rs b/crates/pdf-extract/src/glyphnames.rs new file mode 100644 index 0000000..b377e08 --- /dev/null +++ b/crates/pdf-extract/src/glyphnames.rs @@ -0,0 +1,4711 @@ +/* Autogenerated from: + https://github.com/michal-h21/htfgen/commits/master/glyphlist-extended.txt + https://github.com/kohler/lcdf-typetools/blob/master/texglyphlist.txt + https://github.com/apache/pdfbox/blob/trunk/pdfbox/src/main/resources/org/apache/pdfbox/resources/glyphlist/additional.txt +*/ +pub fn name_to_unicode(name: &str) -> Option { + const NAMES: [(&'static str, u16); 4700] = [ + ("A", 0x0041), + ("AE", 0x00c6), + ("AEacute", 0x01fc), + ("AEmacron", 0x01e2), + ("AEsmall", 0xf7e6), + ("Aacute", 0x00c1), + ("Aacutesmall", 0xf7e1), + ("Abreve", 0x0102), + ("Abreveacute", 0x1eae), + ("Abrevecyrillic", 0x04d0), + ("Abrevedotbelow", 0x1eb6), + ("Abrevegrave", 0x1eb0), + ("Abrevehookabove", 0x1eb2), + ("Abrevetilde", 0x1eb4), + ("Acaron", 0x01cd), + ("Acircle", 0x24b6), + ("Acircumflex", 0x00c2), + ("Acircumflexacute", 0x1ea4), + ("Acircumflexdotbelow", 0x1eac), + ("Acircumflexgrave", 0x1ea6), + ("Acircumflexhookabove", 0x1ea8), + ("Acircumflexsmall", 0xf7e2), + ("Acircumflextilde", 0x1eaa), + ("Acute", 0xf6c9), + ("Acutesmall", 0xf7b4), + ("Acyrillic", 0x0410), + ("Adblgrave", 0x0200), + ("Adieresis", 0x00c4), + ("Adieresiscyrillic", 0x04d2), + ("Adieresismacron", 0x01de), + ("Adieresissmall", 0xf7e4), + ("Adotbelow", 0x1ea0), + ("Adotmacron", 0x01e0), + ("Agrave", 0x00c0), + ("Agravesmall", 0xf7e0), + ("Ahookabove", 0x1ea2), + ("Aiecyrillic", 0x04d4), + ("Ainvertedbreve", 0x0202), + ("Alpha", 0x0391), + ("Alphatonos", 0x0386), + ("Amacron", 0x0100), + ("Amonospace", 0xff21), + ("Aogonek", 0x0104), + ("Aring", 0x00c5), + ("Aringacute", 0x01fa), + ("Aringbelow", 0x1e00), + ("Aringsmall", 0xf7e5), + ("Asmall", 0xf761), + ("Atilde", 0x00c3), + ("Atildesmall", 0xf7e3), + ("Aybarmenian", 0x0531), + ("B", 0x0042), + ("Bcircle", 0x24b7), + ("Bdotaccent", 0x1e02), + ("Bdotbelow", 0x1e04), + ("Becyrillic", 0x0411), + ("Benarmenian", 0x0532), + ("Beta", 0x0392), + ("Bhook", 0x0181), + ("Blinebelow", 0x1e06), + ("Bmonospace", 0xff22), + ("Brevesmall", 0xf6f4), + ("Bsmall", 0xf762), + ("Btopbar", 0x0182), + ("C", 0x0043), + ("Caarmenian", 0x053e), + ("Cacute", 0x0106), + ("Caron", 0xf6ca), + ("Caronsmall", 0xf6f5), + ("Ccaron", 0x010c), + ("Ccedilla", 0x00c7), + ("Ccedillaacute", 0x1e08), + ("Ccedillasmall", 0xf7e7), + ("Ccircle", 0x24b8), + ("Ccircumflex", 0x0108), + ("Cdot", 0x010a), + ("Cdotaccent", 0x010a), + ("Cedillasmall", 0xf7b8), + ("Chaarmenian", 0x0549), + ("Cheabkhasiancyrillic", 0x04bc), + ("Checyrillic", 0x0427), + ("Chedescenderabkhasiancyrillic", 0x04be), + ("Chedescendercyrillic", 0x04b6), + ("Chedieresiscyrillic", 0x04f4), + ("Cheharmenian", 0x0543), + ("Chekhakassiancyrillic", 0x04cb), + ("Cheverticalstrokecyrillic", 0x04b8), + ("Chi", 0x03a7), + ("Chook", 0x0187), + ("Circumflexsmall", 0xf6f6), + ("Cmonospace", 0xff23), + ("Coarmenian", 0x0551), + ("Csmall", 0xf763), + ("D", 0x0044), + ("DZ", 0x01f1), + ("DZcaron", 0x01c4), + ("Daarmenian", 0x0534), + ("Dafrican", 0x0189), + ("Dbar", 0x0110), + ("Dcaron", 0x010e), + ("Dcedilla", 0x1e10), + ("Dcircle", 0x24b9), + ("Dcircumflexbelow", 0x1e12), + ("Dcroat", 0x0110), + ("Ddotaccent", 0x1e0a), + ("Ddotbelow", 0x1e0c), + ("Decyrillic", 0x0414), + ("Deicoptic", 0x03ee), + ("Delta", 0x2206), + ("Deltagreek", 0x0394), + ("Dhook", 0x018a), + ("Dieresis", 0xf6cb), + ("DieresisAcute", 0xf6cc), + ("DieresisGrave", 0xf6cd), + ("Dieresissmall", 0xf7a8), + ("Digamma", 0x03dd), + ("Digammagreek", 0x03dc), + ("Djecyrillic", 0x0402), + ("Dlinebelow", 0x1e0e), + ("Dmonospace", 0xff24), + ("Dotaccentsmall", 0xf6f7), + ("Dslash", 0x0110), + ("Dsmall", 0xf764), + ("Dtopbar", 0x018b), + ("Dz", 0x01f2), + ("Dzcaron", 0x01c5), + ("Dzeabkhasiancyrillic", 0x04e0), + ("Dzecyrillic", 0x0405), + ("Dzhecyrillic", 0x040f), + ("E", 0x0045), + ("Eacute", 0x00c9), + ("Eacutesmall", 0xf7e9), + ("Ebreve", 0x0114), + ("Ecaron", 0x011a), + ("Ecedillabreve", 0x1e1c), + ("Echarmenian", 0x0535), + ("Ecircle", 0x24ba), + ("Ecircumflex", 0x00ca), + ("Ecircumflexacute", 0x1ebe), + ("Ecircumflexbelow", 0x1e18), + ("Ecircumflexdotbelow", 0x1ec6), + ("Ecircumflexgrave", 0x1ec0), + ("Ecircumflexhookabove", 0x1ec2), + ("Ecircumflexsmall", 0xf7ea), + ("Ecircumflextilde", 0x1ec4), + ("Ecyrillic", 0x0404), + ("Edblgrave", 0x0204), + ("Edieresis", 0x00cb), + ("Edieresissmall", 0xf7eb), + ("Edot", 0x0116), + ("Edotaccent", 0x0116), + ("Edotbelow", 0x1eb8), + ("Efcyrillic", 0x0424), + ("Egrave", 0x00c8), + ("Egravesmall", 0xf7e8), + ("Eharmenian", 0x0537), + ("Ehookabove", 0x1eba), + ("Eightroman", 0x2167), + ("Einvertedbreve", 0x0206), + ("Eiotifiedcyrillic", 0x0464), + ("Elcyrillic", 0x041b), + ("Elevenroman", 0x216a), + ("Emacron", 0x0112), + ("Emacronacute", 0x1e16), + ("Emacrongrave", 0x1e14), + ("Emcyrillic", 0x041c), + ("Emonospace", 0xff25), + ("Encyrillic", 0x041d), + ("Endescendercyrillic", 0x04a2), + ("Eng", 0x014a), + ("Enghecyrillic", 0x04a4), + ("Enhookcyrillic", 0x04c7), + ("Eogonek", 0x0118), + ("Eopen", 0x0190), + ("Epsilon", 0x0395), + ("Epsilontonos", 0x0388), + ("Ercyrillic", 0x0420), + ("Ereversed", 0x018e), + ("Ereversedcyrillic", 0x042d), + ("Escyrillic", 0x0421), + ("Esdescendercyrillic", 0x04aa), + ("Esh", 0x01a9), + ("Esmall", 0xf765), + ("Eta", 0x0397), + ("Etarmenian", 0x0538), + ("Etatonos", 0x0389), + ("Eth", 0x00d0), + ("Ethsmall", 0xf7f0), + ("Etilde", 0x1ebc), + ("Etildebelow", 0x1e1a), + ("Euro", 0x20ac), + ("Ezh", 0x01b7), + ("Ezhcaron", 0x01ee), + ("Ezhreversed", 0x01b8), + ("F", 0x0046), + ("FFIsmall", 0xd807), + ("FFLsmall", 0xd808), + ("FFsmall", 0xd804), + ("FIsmall", 0xd805), + ("FLsmall", 0xd806), + ("Fcircle", 0x24bb), + ("Fdotaccent", 0x1e1e), + ("Feharmenian", 0x0556), + ("Feicoptic", 0x03e4), + ("Fhook", 0x0191), + ("Finv", 0x2132), + ("Fitacyrillic", 0x0472), + ("Fiveroman", 0x2164), + ("Fmonospace", 0xff26), + ("Fourroman", 0x2163), + ("Fsmall", 0xf766), + ("G", 0x0047), + ("GBsquare", 0x3387), + ("Gacute", 0x01f4), + ("Gamma", 0x0393), + ("Gammaafrican", 0x0194), + ("Gangiacoptic", 0x03ea), + ("Gbreve", 0x011e), + ("Gcaron", 0x01e6), + ("Gcedilla", 0x0122), + ("Gcircle", 0x24bc), + ("Gcircumflex", 0x011c), + ("Gcommaaccent", 0x0122), + ("Gdot", 0x0120), + ("Gdotaccent", 0x0120), + ("Gecyrillic", 0x0413), + ("Germandbls", 0x1e9e), + ("Germandblssmall", 0xd803), + ("Ghadarmenian", 0x0542), + ("Ghemiddlehookcyrillic", 0x0494), + ("Ghestrokecyrillic", 0x0492), + ("Gheupturncyrillic", 0x0490), + ("Ghook", 0x0193), + ("Gimarmenian", 0x0533), + ("Gjecyrillic", 0x0403), + ("Gmacron", 0x1e20), + ("Gmir", 0x2141), + ("Gmonospace", 0xff27), + ("Grave", 0xf6ce), + ("Gravesmall", 0xf760), + ("Gsmall", 0xf767), + ("Gsmallhook", 0x029b), + ("Gstroke", 0x01e4), + ("H", 0x0048), + ("H18533", 0x25cf), + ("H18543", 0x25aa), + ("H18551", 0x25ab), + ("H22073", 0x25a1), + ("HPsquare", 0x33cb), + ("Haabkhasiancyrillic", 0x04a8), + ("Hadescendercyrillic", 0x04b2), + ("Hardsigncyrillic", 0x042a), + ("Hbar", 0x0126), + ("Hbrevebelow", 0x1e2a), + ("Hcedilla", 0x1e28), + ("Hcircle", 0x24bd), + ("Hcircumflex", 0x0124), + ("Hdieresis", 0x1e26), + ("Hdotaccent", 0x1e22), + ("Hdotbelow", 0x1e24), + ("Hmonospace", 0xff28), + ("Hoarmenian", 0x0540), + ("Horicoptic", 0x03e8), + ("Hsmall", 0xf768), + ("Hungarumlaut", 0xf6cf), + ("Hungarumlautsmall", 0xf6f8), + ("Hzsquare", 0x3390), + ("I", 0x0049), + ("IAcyrillic", 0x042f), + ("IJ", 0x0132), + ("IUcyrillic", 0x042e), + ("Iacute", 0x00cd), + ("Iacutesmall", 0xf7ed), + ("Ibreve", 0x012c), + ("Icaron", 0x01cf), + ("Icircle", 0x24be), + ("Icircumflex", 0x00ce), + ("Icircumflexsmall", 0xf7ee), + ("Icyrillic", 0x0406), + ("Idblgrave", 0x0208), + ("Idieresis", 0x00cf), + ("Idieresisacute", 0x1e2e), + ("Idieresiscyrillic", 0x04e4), + ("Idieresissmall", 0xf7ef), + ("Idot", 0x0130), + ("Idotaccent", 0x0130), + ("Idotbelow", 0x1eca), + ("Iebrevecyrillic", 0x04d6), + ("Iecyrillic", 0x0415), + ("Ifractur", 0x2111), + ("Ifraktur", 0x2111), + ("Igrave", 0x00cc), + ("Igravesmall", 0xf7ec), + ("Ihookabove", 0x1ec8), + ("Iicyrillic", 0x0418), + ("Iinvertedbreve", 0x020a), + ("Iishortcyrillic", 0x0419), + ("Imacron", 0x012a), + ("Imacroncyrillic", 0x04e2), + ("Imonospace", 0xff29), + ("Iniarmenian", 0x053b), + ("Iocyrillic", 0x0401), + ("Iogonek", 0x012e), + ("Iota", 0x0399), + ("Iotaafrican", 0x0196), + ("Iotadieresis", 0x03aa), + ("Iotatonos", 0x038a), + ("Ismall", 0xf769), + ("Istroke", 0x0197), + ("Itilde", 0x0128), + ("Itildebelow", 0x1e2c), + ("Izhitsacyrillic", 0x0474), + ("Izhitsadblgravecyrillic", 0x0476), + ("J", 0x004a), + ("Jaarmenian", 0x0541), + ("Jcircle", 0x24bf), + ("Jcircumflex", 0x0134), + ("Jecyrillic", 0x0408), + ("Jheharmenian", 0x054b), + ("Jmonospace", 0xff2a), + ("Jsmall", 0xf76a), + ("K", 0x004b), + ("KBsquare", 0x3385), + ("KKsquare", 0x33cd), + ("Kabashkircyrillic", 0x04a0), + ("Kacute", 0x1e30), + ("Kacyrillic", 0x041a), + ("Kadescendercyrillic", 0x049a), + ("Kahookcyrillic", 0x04c3), + ("Kappa", 0x039a), + ("Kastrokecyrillic", 0x049e), + ("Kaverticalstrokecyrillic", 0x049c), + ("Kcaron", 0x01e8), + ("Kcedilla", 0x0136), + ("Kcircle", 0x24c0), + ("Kcommaaccent", 0x0136), + ("Kdotbelow", 0x1e32), + ("Keharmenian", 0x0554), + ("Kenarmenian", 0x053f), + ("Khacyrillic", 0x0425), + ("Kheicoptic", 0x03e6), + ("Khook", 0x0198), + ("Kjecyrillic", 0x040c), + ("Klinebelow", 0x1e34), + ("Kmonospace", 0xff2b), + ("Koppacyrillic", 0x0480), + ("Koppagreek", 0x03de), + ("Ksicyrillic", 0x046e), + ("Ksmall", 0xf76b), + ("L", 0x004c), + ("LJ", 0x01c7), + ("LL", 0xf6bf), + ("Lacute", 0x0139), + ("Lambda", 0x039b), + ("Lcaron", 0x013d), + ("Lcedilla", 0x013b), + ("Lcircle", 0x24c1), + ("Lcircumflexbelow", 0x1e3c), + ("Lcommaaccent", 0x013b), + ("Ldot", 0x013f), + ("Ldotaccent", 0x013f), + ("Ldotbelow", 0x1e36), + ("Ldotbelowmacron", 0x1e38), + ("Liwnarmenian", 0x053c), + ("Lj", 0x01c8), + ("Ljecyrillic", 0x0409), + ("Llinebelow", 0x1e3a), + ("Lmonospace", 0xff2c), + ("Lslash", 0x0141), + ("Lslashsmall", 0xf6f9), + ("Lsmall", 0xf76c), + ("M", 0x004d), + ("MBsquare", 0x3386), + ("Macron", 0xf6d0), + ("Macronsmall", 0xf7af), + ("Macute", 0x1e3e), + ("Mcircle", 0x24c2), + ("Mdotaccent", 0x1e40), + ("Mdotbelow", 0x1e42), + ("Menarmenian", 0x0544), + ("Mmonospace", 0xff2d), + ("Msmall", 0xf76d), + ("Mturned", 0x019c), + ("Mu", 0x039c), + ("N", 0x004e), + ("NJ", 0x01ca), + ("Nacute", 0x0143), + ("Ncaron", 0x0147), + ("Ncedilla", 0x0145), + ("Ncircle", 0x24c3), + ("Ncircumflexbelow", 0x1e4a), + ("Ncommaaccent", 0x0145), + ("Ndotaccent", 0x1e44), + ("Ndotbelow", 0x1e46), + ("Ng", 0x014a), + ("Nhookleft", 0x019d), + ("Nineroman", 0x2168), + ("Nj", 0x01cb), + ("Njecyrillic", 0x040a), + ("Nlinebelow", 0x1e48), + ("Nmonospace", 0xff2e), + ("Nowarmenian", 0x0546), + ("Nsmall", 0xf76e), + ("Ntilde", 0x00d1), + ("Ntildesmall", 0xf7f1), + ("Nu", 0x039d), + ("O", 0x004f), + ("OE", 0x0152), + ("OEsmall", 0xf6fa), + ("Oacute", 0x00d3), + ("Oacutesmall", 0xf7f3), + ("Obarredcyrillic", 0x04e8), + ("Obarreddieresiscyrillic", 0x04ea), + ("Obreve", 0x014e), + ("Ocaron", 0x01d1), + ("Ocenteredtilde", 0x019f), + ("Ocircle", 0x24c4), + ("Ocircumflex", 0x00d4), + ("Ocircumflexacute", 0x1ed0), + ("Ocircumflexdotbelow", 0x1ed8), + ("Ocircumflexgrave", 0x1ed2), + ("Ocircumflexhookabove", 0x1ed4), + ("Ocircumflexsmall", 0xf7f4), + ("Ocircumflextilde", 0x1ed6), + ("Ocyrillic", 0x041e), + ("Odblacute", 0x0150), + ("Odblgrave", 0x020c), + ("Odieresis", 0x00d6), + ("Odieresiscyrillic", 0x04e6), + ("Odieresissmall", 0xf7f6), + ("Odotbelow", 0x1ecc), + ("Ogoneksmall", 0xf6fb), + ("Ograve", 0x00d2), + ("Ogravesmall", 0xf7f2), + ("Oharmenian", 0x0555), + ("Ohm", 0x2126), + ("Ohookabove", 0x1ece), + ("Ohorn", 0x01a0), + ("Ohornacute", 0x1eda), + ("Ohorndotbelow", 0x1ee2), + ("Ohorngrave", 0x1edc), + ("Ohornhookabove", 0x1ede), + ("Ohorntilde", 0x1ee0), + ("Ohungarumlaut", 0x0150), + ("Oi", 0x01a2), + ("Oinvertedbreve", 0x020e), + ("Omacron", 0x014c), + ("Omacronacute", 0x1e52), + ("Omacrongrave", 0x1e50), + ("Omega", 0x2126), + ("Omegacyrillic", 0x0460), + ("Omegagreek", 0x03a9), + ("Omegainv", 0x2127), + ("Omegaroundcyrillic", 0x047a), + ("Omegatitlocyrillic", 0x047c), + ("Omegatonos", 0x038f), + ("Omicron", 0x039f), + ("Omicrontonos", 0x038c), + ("Omonospace", 0xff2f), + ("Oneroman", 0x2160), + ("Oogonek", 0x01ea), + ("Oogonekmacron", 0x01ec), + ("Oopen", 0x0186), + ("Oslash", 0x00d8), + ("Oslashacute", 0x01fe), + ("Oslashsmall", 0xf7f8), + ("Osmall", 0xf76f), + ("Ostrokeacute", 0x01fe), + ("Otcyrillic", 0x047e), + ("Otilde", 0x00d5), + ("Otildeacute", 0x1e4c), + ("Otildedieresis", 0x1e4e), + ("Otildesmall", 0xf7f5), + ("P", 0x0050), + ("Pacute", 0x1e54), + ("Pcircle", 0x24c5), + ("Pdotaccent", 0x1e56), + ("Pecyrillic", 0x041f), + ("Peharmenian", 0x054a), + ("Pemiddlehookcyrillic", 0x04a6), + ("Phi", 0x03a6), + ("Phook", 0x01a4), + ("Pi", 0x03a0), + ("Piwrarmenian", 0x0553), + ("Pmonospace", 0xff30), + ("Psi", 0x03a8), + ("Psicyrillic", 0x0470), + ("Psmall", 0xf770), + ("Q", 0x0051), + ("Qcircle", 0x24c6), + ("Qmonospace", 0xff31), + ("Qsmall", 0xf771), + ("R", 0x0052), + ("Raarmenian", 0x054c), + ("Racute", 0x0154), + ("Rcaron", 0x0158), + ("Rcedilla", 0x0156), + ("Rcircle", 0x24c7), + ("Rcommaaccent", 0x0156), + ("Rdblgrave", 0x0210), + ("Rdotaccent", 0x1e58), + ("Rdotbelow", 0x1e5a), + ("Rdotbelowmacron", 0x1e5c), + ("Reharmenian", 0x0550), + ("Rfractur", 0x211c), + ("Rfraktur", 0x211c), + ("Rho", 0x03a1), + ("Ringsmall", 0xf6fc), + ("Rinvertedbreve", 0x0212), + ("Rlinebelow", 0x1e5e), + ("Rmonospace", 0xff32), + ("Rsmall", 0xf772), + ("Rsmallinverted", 0x0281), + ("Rsmallinvertedsuperior", 0x02b6), + ("S", 0x0053), + ("SF010000", 0x250c), + ("SF020000", 0x2514), + ("SF030000", 0x2510), + ("SF040000", 0x2518), + ("SF050000", 0x253c), + ("SF060000", 0x252c), + ("SF070000", 0x2534), + ("SF080000", 0x251c), + ("SF090000", 0x2524), + ("SF100000", 0x2500), + ("SF110000", 0x2502), + ("SF190000", 0x2561), + ("SF200000", 0x2562), + ("SF210000", 0x2556), + ("SF220000", 0x2555), + ("SF230000", 0x2563), + ("SF240000", 0x2551), + ("SF250000", 0x2557), + ("SF260000", 0x255d), + ("SF270000", 0x255c), + ("SF280000", 0x255b), + ("SF360000", 0x255e), + ("SF370000", 0x255f), + ("SF380000", 0x255a), + ("SF390000", 0x2554), + ("SF400000", 0x2569), + ("SF410000", 0x2566), + ("SF420000", 0x2560), + ("SF430000", 0x2550), + ("SF440000", 0x256c), + ("SF450000", 0x2567), + ("SF460000", 0x2568), + ("SF470000", 0x2564), + ("SF480000", 0x2565), + ("SF490000", 0x2559), + ("SF500000", 0x2558), + ("SF510000", 0x2552), + ("SF520000", 0x2553), + ("SF530000", 0x256b), + ("SF540000", 0x256a), + ("SS", 0x0053), + ("SSsmall", 0xd803), + ("Sacute", 0x015a), + ("Sacutedotaccent", 0x1e64), + ("Sampigreek", 0x03e0), + ("Scaron", 0x0160), + ("Scarondotaccent", 0x1e66), + ("Scaronsmall", 0xf6fd), + ("Scedilla", 0x015e), + ("Schwa", 0x018f), + ("Schwacyrillic", 0x04d8), + ("Schwadieresiscyrillic", 0x04da), + ("Scircle", 0x24c8), + ("Scircumflex", 0x015c), + ("Scommaaccent", 0x0218), + ("Sdotaccent", 0x1e60), + ("Sdotbelow", 0x1e62), + ("Sdotbelowdotaccent", 0x1e68), + ("Seharmenian", 0x054d), + ("Sevenroman", 0x2166), + ("Shaarmenian", 0x0547), + ("Shacyrillic", 0x0428), + ("Shchacyrillic", 0x0429), + ("Sheicoptic", 0x03e2), + ("Shhacyrillic", 0x04ba), + ("Shimacoptic", 0x03ec), + ("Sigma", 0x03a3), + ("Sixroman", 0x2165), + ("Smonospace", 0xff33), + ("Softsigncyrillic", 0x042c), + ("Ssmall", 0xf773), + ("Stigmagreek", 0x03da), + ("T", 0x0054), + ("Tau", 0x03a4), + ("Tbar", 0x0166), + ("Tcaron", 0x0164), + ("Tcedilla", 0x0162), + ("Tcircle", 0x24c9), + ("Tcircumflexbelow", 0x1e70), + ("Tcommaaccent", 0x0162), + ("Tdotaccent", 0x1e6a), + ("Tdotbelow", 0x1e6c), + ("Tecyrillic", 0x0422), + ("Tedescendercyrillic", 0x04ac), + ("Tenroman", 0x2169), + ("Tetsecyrillic", 0x04b4), + ("Theta", 0x0398), + ("Thook", 0x01ac), + ("Thorn", 0x00de), + ("Thornsmall", 0xf7fe), + ("Threeroman", 0x2162), + ("Tildesmall", 0xf6fe), + ("Tiwnarmenian", 0x054f), + ("Tlinebelow", 0x1e6e), + ("Tmonospace", 0xff34), + ("Toarmenian", 0x0539), + ("Tonefive", 0x01bc), + ("Tonesix", 0x0184), + ("Tonetwo", 0x01a7), + ("Tretroflexhook", 0x01ae), + ("Tsecyrillic", 0x0426), + ("Tshecyrillic", 0x040b), + ("Tsmall", 0xf774), + ("Twelveroman", 0x216b), + ("Tworoman", 0x2161), + ("U", 0x0055), + ("Uacute", 0x00da), + ("Uacutesmall", 0xf7fa), + ("Ubreve", 0x016c), + ("Ucaron", 0x01d3), + ("Ucircle", 0x24ca), + ("Ucircumflex", 0x00db), + ("Ucircumflexbelow", 0x1e76), + ("Ucircumflexsmall", 0xf7fb), + ("Ucyrillic", 0x0423), + ("Udblacute", 0x0170), + ("Udblgrave", 0x0214), + ("Udieresis", 0x00dc), + ("Udieresisacute", 0x01d7), + ("Udieresisbelow", 0x1e72), + ("Udieresiscaron", 0x01d9), + ("Udieresiscyrillic", 0x04f0), + ("Udieresisgrave", 0x01db), + ("Udieresismacron", 0x01d5), + ("Udieresissmall", 0xf7fc), + ("Udotbelow", 0x1ee4), + ("Ugrave", 0x00d9), + ("Ugravesmall", 0xf7f9), + ("Uhookabove", 0x1ee6), + ("Uhorn", 0x01af), + ("Uhornacute", 0x1ee8), + ("Uhorndotbelow", 0x1ef0), + ("Uhorngrave", 0x1eea), + ("Uhornhookabove", 0x1eec), + ("Uhorntilde", 0x1eee), + ("Uhungarumlaut", 0x0170), + ("Uhungarumlautcyrillic", 0x04f2), + ("Uinvertedbreve", 0x0216), + ("Ukcyrillic", 0x0478), + ("Umacron", 0x016a), + ("Umacroncyrillic", 0x04ee), + ("Umacrondieresis", 0x1e7a), + ("Umonospace", 0xff35), + ("Uogonek", 0x0172), + ("Upsilon", 0x03a5), + ("Upsilon1", 0x03d2), + ("Upsilonacutehooksymbolgreek", 0x03d3), + ("Upsilonafrican", 0x01b1), + ("Upsilondieresis", 0x03ab), + ("Upsilondieresishooksymbolgreek", 0x03d4), + ("Upsilonhooksymbol", 0x03d2), + ("Upsilontonos", 0x038e), + ("Uring", 0x016e), + ("Ushortcyrillic", 0x040e), + ("Usmall", 0xf775), + ("Ustraightcyrillic", 0x04ae), + ("Ustraightstrokecyrillic", 0x04b0), + ("Utilde", 0x0168), + ("Utildeacute", 0x1e78), + ("Utildebelow", 0x1e74), + ("V", 0x0056), + ("Vcircle", 0x24cb), + ("Vdotbelow", 0x1e7e), + ("Vecyrillic", 0x0412), + ("Vewarmenian", 0x054e), + ("Vhook", 0x01b2), + ("Vmonospace", 0xff36), + ("Voarmenian", 0x0548), + ("Vsmall", 0xf776), + ("Vtilde", 0x1e7c), + ("W", 0x0057), + ("Wacute", 0x1e82), + ("Wcircle", 0x24cc), + ("Wcircumflex", 0x0174), + ("Wdieresis", 0x1e84), + ("Wdotaccent", 0x1e86), + ("Wdotbelow", 0x1e88), + ("Wgrave", 0x1e80), + ("Wmonospace", 0xff37), + ("Wsmall", 0xf777), + ("X", 0x0058), + ("Xcircle", 0x24cd), + ("Xdieresis", 0x1e8c), + ("Xdotaccent", 0x1e8a), + ("Xeharmenian", 0x053d), + ("Xi", 0x039e), + ("Xmonospace", 0xff38), + ("Xsmall", 0xf778), + ("Y", 0x0059), + ("Yacute", 0x00dd), + ("Yacutesmall", 0xf7fd), + ("Yatcyrillic", 0x0462), + ("Ycircle", 0x24ce), + ("Ycircumflex", 0x0176), + ("Ydieresis", 0x0178), + ("Ydieresissmall", 0xf7ff), + ("Ydotaccent", 0x1e8e), + ("Ydotbelow", 0x1ef4), + ("Yen", 0x00a5), + ("Yericyrillic", 0x042b), + ("Yerudieresiscyrillic", 0x04f8), + ("Ygrave", 0x1ef2), + ("Yhook", 0x01b3), + ("Yhookabove", 0x1ef6), + ("Yiarmenian", 0x0545), + ("Yicyrillic", 0x0407), + ("Yiwnarmenian", 0x0552), + ("Ymonospace", 0xff39), + ("Ysmall", 0xf779), + ("Ytilde", 0x1ef8), + ("Yusbigcyrillic", 0x046a), + ("Yusbigiotifiedcyrillic", 0x046c), + ("Yuslittlecyrillic", 0x0466), + ("Yuslittleiotifiedcyrillic", 0x0468), + ("Z", 0x005a), + ("Zaarmenian", 0x0536), + ("Zacute", 0x0179), + ("Zcaron", 0x017d), + ("Zcaronsmall", 0xf6ff), + ("Zcircle", 0x24cf), + ("Zcircumflex", 0x1e90), + ("Zdot", 0x017b), + ("Zdotaccent", 0x017b), + ("Zdotbelow", 0x1e92), + ("Zecyrillic", 0x0417), + ("Zedescendercyrillic", 0x0498), + ("Zedieresiscyrillic", 0x04de), + ("Zeta", 0x0396), + ("Zhearmenian", 0x053a), + ("Zhebrevecyrillic", 0x04c1), + ("Zhecyrillic", 0x0416), + ("Zhedescendercyrillic", 0x0496), + ("Zhedieresiscyrillic", 0x04dc), + ("Zlinebelow", 0x1e94), + ("Zmonospace", 0xff3a), + ("Zsmall", 0xf77a), + ("Zstroke", 0x01b5), + ("a", 0x0061), + ("aabengali", 0x0986), + ("aacute", 0x00e1), + ("aadeva", 0x0906), + ("aagujarati", 0x0a86), + ("aagurmukhi", 0x0a06), + ("aamatragurmukhi", 0x0a3e), + ("aarusquare", 0x3303), + ("aavowelsignbengali", 0x09be), + ("aavowelsigndeva", 0x093e), + ("aavowelsigngujarati", 0x0abe), + ("abbreviationmarkarmenian", 0x055f), + ("abbreviationsigndeva", 0x0970), + ("abengali", 0x0985), + ("abopomofo", 0x311a), + ("abreve", 0x0103), + ("abreveacute", 0x1eaf), + ("abrevecyrillic", 0x04d1), + ("abrevedotbelow", 0x1eb7), + ("abrevegrave", 0x1eb1), + ("abrevehookabove", 0x1eb3), + ("abrevetilde", 0x1eb5), + ("acaron", 0x01ce), + ("acircle", 0x24d0), + ("acircumflex", 0x00e2), + ("acircumflexacute", 0x1ea5), + ("acircumflexdotbelow", 0x1ead), + ("acircumflexgrave", 0x1ea7), + ("acircumflexhookabove", 0x1ea9), + ("acircumflextilde", 0x1eab), + ("acute", 0x00b4), + ("acutebelowcmb", 0x0317), + ("acutecmb", 0x0301), + ("acutecomb", 0x0301), + ("acutedeva", 0x0954), + ("acutelowmod", 0x02cf), + ("acutetonecmb", 0x0341), + ("acyrillic", 0x0430), + ("adblgrave", 0x0201), + ("addakgurmukhi", 0x0a71), + ("adeva", 0x0905), + ("adieresis", 0x00e4), + ("adieresiscyrillic", 0x04d3), + ("adieresismacron", 0x01df), + ("adotbelow", 0x1ea1), + ("adotmacron", 0x01e1), + ("ae", 0x00e6), + ("aeacute", 0x01fd), + ("aekorean", 0x3150), + ("aemacron", 0x01e3), + ("afii00208", 0x2015), + ("afii08941", 0x20a4), + ("afii10017", 0x0410), + ("afii10018", 0x0411), + ("afii10019", 0x0412), + ("afii10020", 0x0413), + ("afii10021", 0x0414), + ("afii10022", 0x0415), + ("afii10023", 0x0401), + ("afii10024", 0x0416), + ("afii10025", 0x0417), + ("afii10026", 0x0418), + ("afii10027", 0x0419), + ("afii10028", 0x041a), + ("afii10029", 0x041b), + ("afii10030", 0x041c), + ("afii10031", 0x041d), + ("afii10032", 0x041e), + ("afii10033", 0x041f), + ("afii10034", 0x0420), + ("afii10035", 0x0421), + ("afii10036", 0x0422), + ("afii10037", 0x0423), + ("afii10038", 0x0424), + ("afii10039", 0x0425), + ("afii10040", 0x0426), + ("afii10041", 0x0427), + ("afii10042", 0x0428), + ("afii10043", 0x0429), + ("afii10044", 0x042a), + ("afii10045", 0x042b), + ("afii10046", 0x042c), + ("afii10047", 0x042d), + ("afii10048", 0x042e), + ("afii10049", 0x042f), + ("afii10050", 0x0490), + ("afii10051", 0x0402), + ("afii10052", 0x0403), + ("afii10053", 0x0404), + ("afii10054", 0x0405), + ("afii10055", 0x0406), + ("afii10056", 0x0407), + ("afii10057", 0x0408), + ("afii10058", 0x0409), + ("afii10059", 0x040a), + ("afii10060", 0x040b), + ("afii10061", 0x040c), + ("afii10062", 0x040e), + ("afii10063", 0xf6c4), + ("afii10064", 0xf6c5), + ("afii10065", 0x0430), + ("afii10066", 0x0431), + ("afii10067", 0x0432), + ("afii10068", 0x0433), + ("afii10069", 0x0434), + ("afii10070", 0x0435), + ("afii10071", 0x0451), + ("afii10072", 0x0436), + ("afii10073", 0x0437), + ("afii10074", 0x0438), + ("afii10075", 0x0439), + ("afii10076", 0x043a), + ("afii10077", 0x043b), + ("afii10078", 0x043c), + ("afii10079", 0x043d), + ("afii10080", 0x043e), + ("afii10081", 0x043f), + ("afii10082", 0x0440), + ("afii10083", 0x0441), + ("afii10084", 0x0442), + ("afii10085", 0x0443), + ("afii10086", 0x0444), + ("afii10087", 0x0445), + ("afii10088", 0x0446), + ("afii10089", 0x0447), + ("afii10090", 0x0448), + ("afii10091", 0x0449), + ("afii10092", 0x044a), + ("afii10093", 0x044b), + ("afii10094", 0x044c), + ("afii10095", 0x044d), + ("afii10096", 0x044e), + ("afii10097", 0x044f), + ("afii10098", 0x0491), + ("afii10099", 0x0452), + ("afii10100", 0x0453), + ("afii10101", 0x0454), + ("afii10102", 0x0455), + ("afii10103", 0x0456), + ("afii10104", 0x0457), + ("afii10105", 0x0458), + ("afii10106", 0x0459), + ("afii10107", 0x045a), + ("afii10108", 0x045b), + ("afii10109", 0x045c), + ("afii10110", 0x045e), + ("afii10145", 0x040f), + ("afii10146", 0x0462), + ("afii10147", 0x0472), + ("afii10148", 0x0474), + ("afii10192", 0xf6c6), + ("afii10193", 0x045f), + ("afii10194", 0x0463), + ("afii10195", 0x0473), + ("afii10196", 0x0475), + ("afii10831", 0xf6c7), + ("afii10832", 0xf6c8), + ("afii10846", 0x04d9), + ("afii299", 0x200e), + ("afii300", 0x200f), + ("afii301", 0x200d), + ("afii57381", 0x066a), + ("afii57388", 0x060c), + ("afii57392", 0x0660), + ("afii57393", 0x0661), + ("afii57394", 0x0662), + ("afii57395", 0x0663), + ("afii57396", 0x0664), + ("afii57397", 0x0665), + ("afii57398", 0x0666), + ("afii57399", 0x0667), + ("afii57400", 0x0668), + ("afii57401", 0x0669), + ("afii57403", 0x061b), + ("afii57407", 0x061f), + ("afii57409", 0x0621), + ("afii57410", 0x0622), + ("afii57411", 0x0623), + ("afii57412", 0x0624), + ("afii57413", 0x0625), + ("afii57414", 0x0626), + ("afii57415", 0x0627), + ("afii57416", 0x0628), + ("afii57417", 0x0629), + ("afii57418", 0x062a), + ("afii57419", 0x062b), + ("afii57420", 0x062c), + ("afii57421", 0x062d), + ("afii57422", 0x062e), + ("afii57423", 0x062f), + ("afii57424", 0x0630), + ("afii57425", 0x0631), + ("afii57426", 0x0632), + ("afii57427", 0x0633), + ("afii57428", 0x0634), + ("afii57429", 0x0635), + ("afii57430", 0x0636), + ("afii57431", 0x0637), + ("afii57432", 0x0638), + ("afii57433", 0x0639), + ("afii57434", 0x063a), + ("afii57440", 0x0640), + ("afii57441", 0x0641), + ("afii57442", 0x0642), + ("afii57443", 0x0643), + ("afii57444", 0x0644), + ("afii57445", 0x0645), + ("afii57446", 0x0646), + ("afii57448", 0x0648), + ("afii57449", 0x0649), + ("afii57450", 0x064a), + ("afii57451", 0x064b), + ("afii57452", 0x064c), + ("afii57453", 0x064d), + ("afii57454", 0x064e), + ("afii57455", 0x064f), + ("afii57456", 0x0650), + ("afii57457", 0x0651), + ("afii57458", 0x0652), + ("afii57470", 0x0647), + ("afii57505", 0x06a4), + ("afii57506", 0x067e), + ("afii57507", 0x0686), + ("afii57508", 0x0698), + ("afii57509", 0x06af), + ("afii57511", 0x0679), + ("afii57512", 0x0688), + ("afii57513", 0x0691), + ("afii57514", 0x06ba), + ("afii57519", 0x06d2), + ("afii57534", 0x06d5), + ("afii57636", 0x20aa), + ("afii57645", 0x05be), + ("afii57658", 0x05c3), + ("afii57664", 0x05d0), + ("afii57665", 0x05d1), + ("afii57666", 0x05d2), + ("afii57667", 0x05d3), + ("afii57668", 0x05d4), + ("afii57669", 0x05d5), + ("afii57670", 0x05d6), + ("afii57671", 0x05d7), + ("afii57672", 0x05d8), + ("afii57673", 0x05d9), + ("afii57674", 0x05da), + ("afii57675", 0x05db), + ("afii57676", 0x05dc), + ("afii57677", 0x05dd), + ("afii57678", 0x05de), + ("afii57679", 0x05df), + ("afii57680", 0x05e0), + ("afii57681", 0x05e1), + ("afii57682", 0x05e2), + ("afii57683", 0x05e3), + ("afii57684", 0x05e4), + ("afii57685", 0x05e5), + ("afii57686", 0x05e6), + ("afii57687", 0x05e7), + ("afii57688", 0x05e8), + ("afii57689", 0x05e9), + ("afii57690", 0x05ea), + ("afii57694", 0xfb2a), + ("afii57695", 0xfb2b), + ("afii57700", 0xfb4b), + ("afii57705", 0xfb1f), + ("afii57716", 0x05f0), + ("afii57717", 0x05f1), + ("afii57718", 0x05f2), + ("afii57723", 0xfb35), + ("afii57793", 0x05b4), + ("afii57794", 0x05b5), + ("afii57795", 0x05b6), + ("afii57796", 0x05bb), + ("afii57797", 0x05b8), + ("afii57798", 0x05b7), + ("afii57799", 0x05b0), + ("afii57800", 0x05b2), + ("afii57801", 0x05b1), + ("afii57802", 0x05b3), + ("afii57803", 0x05c2), + ("afii57804", 0x05c1), + ("afii57806", 0x05b9), + ("afii57807", 0x05bc), + ("afii57839", 0x05bd), + ("afii57841", 0x05bf), + ("afii57842", 0x05c0), + ("afii57929", 0x02bc), + ("afii61248", 0x2105), + ("afii61289", 0x2113), + ("afii61352", 0x2116), + ("afii61573", 0x202c), + ("afii61574", 0x202d), + ("afii61575", 0x202e), + ("afii61664", 0x200c), + ("afii63167", 0x066d), + ("afii64937", 0x02bd), + ("agrave", 0x00e0), + ("agujarati", 0x0a85), + ("agurmukhi", 0x0a05), + ("ahiragana", 0x3042), + ("ahookabove", 0x1ea3), + ("aibengali", 0x0990), + ("aibopomofo", 0x311e), + ("aideva", 0x0910), + ("aiecyrillic", 0x04d5), + ("aigujarati", 0x0a90), + ("aigurmukhi", 0x0a10), + ("aimatragurmukhi", 0x0a48), + ("ainarabic", 0x0639), + ("ainfinalarabic", 0xfeca), + ("aininitialarabic", 0xfecb), + ("ainmedialarabic", 0xfecc), + ("ainvertedbreve", 0x0203), + ("aivowelsignbengali", 0x09c8), + ("aivowelsigndeva", 0x0948), + ("aivowelsigngujarati", 0x0ac8), + ("akatakana", 0x30a2), + ("akatakanahalfwidth", 0xff71), + ("akorean", 0x314f), + ("alef", 0x05d0), + ("alefarabic", 0x0627), + ("alefdageshhebrew", 0xfb30), + ("aleffinalarabic", 0xfe8e), + ("alefhamzaabovearabic", 0x0623), + ("alefhamzaabovefinalarabic", 0xfe84), + ("alefhamzabelowarabic", 0x0625), + ("alefhamzabelowfinalarabic", 0xfe88), + ("alefhebrew", 0x05d0), + ("aleflamedhebrew", 0xfb4f), + ("alefmaddaabovearabic", 0x0622), + ("alefmaddaabovefinalarabic", 0xfe82), + ("alefmaksuraarabic", 0x0649), + ("alefmaksurafinalarabic", 0xfef0), + ("alefmaksurainitialarabic", 0xfef3), + ("alefmaksuramedialarabic", 0xfef4), + ("alefpatahhebrew", 0xfb2e), + ("alefqamatshebrew", 0xfb2f), + ("aleph", 0x2135), + ("allequal", 0x224c), + ("alpha", 0x03b1), + ("alphatonos", 0x03ac), + ("altselector", 0xd802), + ("amacron", 0x0101), + ("amonospace", 0xff41), + ("ampersand", 0x0026), + ("ampersandmonospace", 0xff06), + ("ampersandsmall", 0xf726), + ("amsquare", 0x33c2), + ("anbopomofo", 0x3122), + ("angbopomofo", 0x3124), + ("angbracketleft", 0x27e8), + ("angbracketleftBig", 0x2329), + ("angbracketleftBigg", 0x2329), + ("angbracketleftbig", 0x2329), + ("angbracketleftbigg", 0x2329), + ("angbracketright", 0x27e9), + ("angbracketrightBig", 0x232a), + ("angbracketrightBigg", 0x232a), + ("angbracketrightbig", 0x232a), + ("angbracketrightbigg", 0x232a), + ("angkhankhuthai", 0x0e5a), + ("angle", 0x2220), + ("anglebracketleft", 0x3008), + ("anglebracketleftvertical", 0xfe3f), + ("anglebracketright", 0x3009), + ("anglebracketrightvertical", 0xfe40), + ("angleleft", 0x2329), + ("angleleftBig", 0x2329), + ("angleleftBigg", 0x2329), + ("angleleftbig", 0x2329), + ("angleleftbigg", 0x2329), + ("angleright", 0x232a), + ("anglerightBig", 0x232a), + ("anglerightBigg", 0x232a), + ("anglerightbig", 0x232a), + ("anglerightbigg", 0x232a), + ("angstrom", 0x212b), + ("anoteleia", 0x0387), + ("anticlockwise", 0x27f2), + ("anudattadeva", 0x0952), + ("anusvarabengali", 0x0982), + ("anusvaradeva", 0x0902), + ("anusvaragujarati", 0x0a82), + ("aogonek", 0x0105), + ("apaatosquare", 0x3300), + ("aparen", 0x249c), + ("apostrophearmenian", 0x055a), + ("apostrophemod", 0x02bc), + ("apple", 0xf8ff), + ("approaches", 0x2250), + ("approxequal", 0x2248), + ("approxequalorimage", 0x2252), + ("approximatelyequal", 0x2245), + ("approxorequal", 0x224a), + ("araeaekorean", 0x318e), + ("araeakorean", 0x318d), + ("arc", 0x2312), + ("archleftdown", 0x21b6), + ("archrightdown", 0x21b7), + ("arighthalfring", 0x1e9a), + ("aring", 0x00e5), + ("aringacute", 0x01fb), + ("aringbelow", 0x1e01), + ("arrowboth", 0x2194), + ("arrowbothv", 0x2195), + ("arrowbt", 0x2193), + ("arrowdashdown", 0x21e3), + ("arrowdashleft", 0x21e0), + ("arrowdashright", 0x21e2), + ("arrowdashup", 0x21e1), + ("arrowdblboth", 0x21d4), + ("arrowdblbothv", 0x21d5), + ("arrowdbldown", 0x21d3), + ("arrowdblleft", 0x21d0), + ("arrowdblright", 0x21d2), + ("arrowdbltp", 0x21d1), + ("arrowdblup", 0x21d1), + ("arrowdblvertex", 0x21d5), + ("arrowdown", 0x2193), + ("arrowdownleft", 0x2199), + ("arrowdownright", 0x2198), + ("arrowdownwhite", 0x21e9), + ("arrowheaddownmod", 0x02c5), + ("arrowheadleftmod", 0x02c2), + ("arrowheadrightmod", 0x02c3), + ("arrowheadupmod", 0x02c4), + ("arrowhookleft", 0x21aa), + ("arrowhookright", 0x21a9), + ("arrowhorizex", 0xf8e7), + ("arrowleft", 0x2190), + ("arrowleftbothalf", 0x21bd), + ("arrowleftdbl", 0x21d0), + ("arrowleftdblstroke", 0x21cd), + ("arrowleftoverright", 0x21c6), + ("arrowlefttophalf", 0x21bc), + ("arrowleftwhite", 0x21e6), + ("arrownortheast", 0x2197), + ("arrownorthwest", 0x2196), + ("arrowparrleftright", 0x21c6), + ("arrowparrrightleft", 0x21c4), + ("arrowright", 0x2192), + ("arrowrightbothalf", 0x21c1), + ("arrowrightdblstroke", 0x21cf), + ("arrowrightheavy", 0x279e), + ("arrowrightoverleft", 0x21c4), + ("arrowrighttophalf", 0x21c0), + ("arrowrightwhite", 0x21e8), + ("arrowsoutheast", 0x2198), + ("arrowsouthwest", 0x2199), + ("arrowtableft", 0x21e4), + ("arrowtabright", 0x21e5), + ("arrowtailleft", 0x21a2), + ("arrowtailright", 0x21a3), + ("arrowtp", 0x2191), + ("arrowtripleleft", 0x21da), + ("arrowtripleright", 0x21db), + ("arrowup", 0x2191), + ("arrowupdn", 0x2195), + ("arrowupdnbse", 0x21a8), + ("arrowupdownbase", 0x21a8), + ("arrowupleft", 0x2196), + ("arrowupleftofdown", 0x21c5), + ("arrowupright", 0x2197), + ("arrowupwhite", 0x21e7), + ("arrowvertex", 0xf8e6), + ("ascendercompwordmark", 0xd80a), + ("asciicircum", 0x005e), + ("asciicircummonospace", 0xff3e), + ("asciitilde", 0x007e), + ("asciitildemonospace", 0xff5e), + ("ascript", 0x0251), + ("ascriptturned", 0x0252), + ("asmallhiragana", 0x3041), + ("asmallkatakana", 0x30a1), + ("asmallkatakanahalfwidth", 0xff67), + ("asterisk", 0x002a), + ("asteriskaltonearabic", 0x066d), + ("asteriskarabic", 0x066d), + ("asteriskcentered", 0x2217), + ("asteriskmath", 0x2217), + ("asteriskmonospace", 0xff0a), + ("asterisksmall", 0xfe61), + ("asterism", 0x2042), + ("asuperior", 0xf6e9), + ("asymptoticallyequal", 0x2243), + ("at", 0x0040), + ("atilde", 0x00e3), + ("atmonospace", 0xff20), + ("atsmall", 0xfe6b), + ("aturned", 0x0250), + ("aubengali", 0x0994), + ("aubopomofo", 0x3120), + ("audeva", 0x0914), + ("augujarati", 0x0a94), + ("augurmukhi", 0x0a14), + ("aulengthmarkbengali", 0x09d7), + ("aumatragurmukhi", 0x0a4c), + ("auvowelsignbengali", 0x09cc), + ("auvowelsigndeva", 0x094c), + ("auvowelsigngujarati", 0x0acc), + ("avagrahadeva", 0x093d), + ("aybarmenian", 0x0561), + ("ayin", 0x05e2), + ("ayinaltonehebrew", 0xfb20), + ("ayinhebrew", 0x05e2), + ("b", 0x0062), + ("babengali", 0x09ac), + ("backslash", 0x005c), + ("backslashBig", 0x005c), + ("backslashBigg", 0x005c), + ("backslashbig", 0x005c), + ("backslashbigg", 0x005c), + ("backslashmonospace", 0xff3c), + ("badeva", 0x092c), + ("bagujarati", 0x0aac), + ("bagurmukhi", 0x0a2c), + ("bahiragana", 0x3070), + ("bahtthai", 0x0e3f), + ("bakatakana", 0x30d0), + ("bar", 0x007c), + ("bardbl", 0x2225), + ("bardblex", 0x2016), + ("barex", 0x007c), + ("barmonospace", 0xff5c), + ("bbopomofo", 0x3105), + ("bcircle", 0x24d1), + ("bdotaccent", 0x1e03), + ("bdotbelow", 0x1e05), + ("beamedsixteenthnotes", 0x266c), + ("because", 0x2235), + ("becyrillic", 0x0431), + ("beharabic", 0x0628), + ("behfinalarabic", 0xfe90), + ("behinitialarabic", 0xfe91), + ("behiragana", 0x3079), + ("behmedialarabic", 0xfe92), + ("behmeeminitialarabic", 0xfc9f), + ("behmeemisolatedarabic", 0xfc08), + ("behnoonfinalarabic", 0xfc6d), + ("bekatakana", 0x30d9), + ("benarmenian", 0x0562), + ("bet", 0x05d1), + ("beta", 0x03b2), + ("betasymbolgreek", 0x03d0), + ("betdagesh", 0xfb31), + ("betdageshhebrew", 0xfb31), + ("beth", 0x2136), + ("bethebrew", 0x05d1), + ("betrafehebrew", 0xfb4c), + ("between", 0x226c), + ("bhabengali", 0x09ad), + ("bhadeva", 0x092d), + ("bhagujarati", 0x0aad), + ("bhagurmukhi", 0x0a2d), + ("bhook", 0x0253), + ("bihiragana", 0x3073), + ("bikatakana", 0x30d3), + ("bilabialclick", 0x0298), + ("bindigurmukhi", 0x0a02), + ("birusquare", 0x3331), + ("blackcircle", 0x25cf), + ("blackdiamond", 0x25c6), + ("blackdownpointingtriangle", 0x25bc), + ("blackleftpointingpointer", 0x25c4), + ("blackleftpointingtriangle", 0x25c0), + ("blacklenticularbracketleft", 0x3010), + ("blacklenticularbracketleftvertical", 0xfe3b), + ("blacklenticularbracketright", 0x3011), + ("blacklenticularbracketrightvertical", 0xfe3c), + ("blacklowerlefttriangle", 0x25e3), + ("blacklowerrighttriangle", 0x25e2), + ("blackrectangle", 0x25ac), + ("blackrightpointingpointer", 0x25ba), + ("blackrightpointingtriangle", 0x25b6), + ("blacksmallsquare", 0x25aa), + ("blacksmilingface", 0x263b), + ("blacksquare", 0x25a0), + ("blackstar", 0x2605), + ("blackupperlefttriangle", 0x25e4), + ("blackupperrighttriangle", 0x25e5), + ("blackuppointingsmalltriangle", 0x25b4), + ("blackuppointingtriangle", 0x25b2), + ("blank", 0x2423), + ("blinebelow", 0x1e07), + ("block", 0x2588), + ("bmonospace", 0xff42), + ("bobaimaithai", 0x0e1a), + ("bohiragana", 0x307c), + ("bokatakana", 0x30dc), + ("bparen", 0x249d), + ("bqsquare", 0x33c3), + ("braceex", 0xf8f4), + ("bracehtipdownleft", 0xfe37), + ("bracehtipdownright", 0xfe37), + ("bracehtipupleft", 0xfe38), + ("bracehtipupright", 0xfe38), + ("braceleft", 0x007b), + ("braceleftBig", 0x007b), + ("braceleftBigg", 0x007b), + ("braceleftbig", 0x007b), + ("braceleftbigg", 0x007b), + ("braceleftbt", 0xf8f3), + ("braceleftmid", 0xf8f2), + ("braceleftmonospace", 0xff5b), + ("braceleftsmall", 0xfe5b), + ("bracelefttp", 0xf8f1), + ("braceleftvertical", 0xfe37), + ("braceright", 0x007d), + ("bracerightBig", 0x007d), + ("bracerightBigg", 0x007d), + ("bracerightbig", 0x007d), + ("bracerightbigg", 0x007d), + ("bracerightbt", 0xf8fe), + ("bracerightmid", 0xf8fd), + ("bracerightmonospace", 0xff5d), + ("bracerightsmall", 0xfe5c), + ("bracerighttp", 0xf8fc), + ("bracerightvertical", 0xfe38), + ("bracketleft", 0x005b), + ("bracketleftBig", 0x005b), + ("bracketleftBigg", 0x005b), + ("bracketleftbig", 0x005b), + ("bracketleftbigg", 0x005b), + ("bracketleftbt", 0xf8f0), + ("bracketleftex", 0xf8ef), + ("bracketleftmath", 0x005b), + ("bracketleftmonospace", 0xff3b), + ("bracketlefttp", 0xf8ee), + ("bracketright", 0x005d), + ("bracketrightBig", 0x005d), + ("bracketrightBigg", 0x005d), + ("bracketrightbig", 0x005d), + ("bracketrightbigg", 0x005d), + ("bracketrightbt", 0xf8fb), + ("bracketrightex", 0xf8fa), + ("bracketrightmath", 0x005d), + ("bracketrightmonospace", 0xff3d), + ("bracketrighttp", 0xf8f9), + ("breve", 0x02d8), + ("brevebelowcmb", 0x032e), + ("brevecmb", 0x0306), + ("breveinvertedbelowcmb", 0x032f), + ("breveinvertedcmb", 0x0311), + ("breveinverteddoublecmb", 0x0361), + ("bridgebelowcmb", 0x032a), + ("bridgeinvertedbelowcmb", 0x033a), + ("brokenbar", 0x00a6), + ("bstroke", 0x0180), + ("bsuperior", 0xf6ea), + ("btopbar", 0x0183), + ("buhiragana", 0x3076), + ("bukatakana", 0x30d6), + ("bullet", 0x2022), + ("bulletinverse", 0x25d8), + ("bulletoperator", 0x2219), + ("bullseye", 0x25ce), + ("c", 0x0063), + ("caarmenian", 0x056e), + ("cabengali", 0x099a), + ("cacute", 0x0107), + ("cadeva", 0x091a), + ("cagujarati", 0x0a9a), + ("cagurmukhi", 0x0a1a), + ("calsquare", 0x3388), + ("candrabindubengali", 0x0981), + ("candrabinducmb", 0x0310), + ("candrabindudeva", 0x0901), + ("candrabindugujarati", 0x0a81), + ("capitalcompwordmark", 0xd809), + ("capslock", 0x21ea), + ("careof", 0x2105), + ("caron", 0x02c7), + ("caronbelowcmb", 0x032c), + ("caroncmb", 0x030c), + ("carriagereturn", 0x21b5), + ("cbopomofo", 0x3118), + ("ccaron", 0x010d), + ("ccedilla", 0x00e7), + ("ccedillaacute", 0x1e09), + ("ccircle", 0x24d2), + ("ccircumflex", 0x0109), + ("ccurl", 0x0255), + ("cdot", 0x010b), + ("cdotaccent", 0x010b), + ("cdsquare", 0x33c5), + ("cedilla", 0x00b8), + ("cedillacmb", 0x0327), + ("ceilingleft", 0x2308), + ("ceilingleftBig", 0x2308), + ("ceilingleftBigg", 0x2308), + ("ceilingleftbig", 0x2308), + ("ceilingleftbigg", 0x2308), + ("ceilingright", 0x2309), + ("ceilingrightBig", 0x2309), + ("ceilingrightBigg", 0x2309), + ("ceilingrightbig", 0x2309), + ("ceilingrightbigg", 0x2309), + ("cent", 0x00a2), + ("centigrade", 0x2103), + ("centinferior", 0xf6df), + ("centmonospace", 0xffe0), + ("centoldstyle", 0xf7a2), + ("centsuperior", 0xf6e0), + ("chaarmenian", 0x0579), + ("chabengali", 0x099b), + ("chadeva", 0x091b), + ("chagujarati", 0x0a9b), + ("chagurmukhi", 0x0a1b), + ("chbopomofo", 0x3114), + ("cheabkhasiancyrillic", 0x04bd), + ("check", 0x2713), + ("checkmark", 0x2713), + ("checyrillic", 0x0447), + ("chedescenderabkhasiancyrillic", 0x04bf), + ("chedescendercyrillic", 0x04b7), + ("chedieresiscyrillic", 0x04f5), + ("cheharmenian", 0x0573), + ("chekhakassiancyrillic", 0x04cc), + ("cheverticalstrokecyrillic", 0x04b9), + ("chi", 0x03c7), + ("chieuchacirclekorean", 0x3277), + ("chieuchaparenkorean", 0x3217), + ("chieuchcirclekorean", 0x3269), + ("chieuchkorean", 0x314a), + ("chieuchparenkorean", 0x3209), + ("chochangthai", 0x0e0a), + ("chochanthai", 0x0e08), + ("chochingthai", 0x0e09), + ("chochoethai", 0x0e0c), + ("chook", 0x0188), + ("cieucacirclekorean", 0x3276), + ("cieucaparenkorean", 0x3216), + ("cieuccirclekorean", 0x3268), + ("cieuckorean", 0x3148), + ("cieucparenkorean", 0x3208), + ("cieucuparenkorean", 0x321c), + ("circle", 0x25cb), + ("circleR", 0x00ae), + ("circleS", 0x24c8), + ("circleasterisk", 0x229b), + ("circlecopyrt", 0x20dd), + ("circledivide", 0x2298), + ("circledot", 0x2299), + ("circledotdisplay", 0x2299), + ("circledottext", 0x2299), + ("circleequal", 0x229c), + ("circleminus", 0x2296), + ("circlemultiply", 0x2297), + ("circlemultiplydisplay", 0x2297), + ("circlemultiplytext", 0x2297), + ("circleot", 0x2299), + ("circleplus", 0x2295), + ("circleplusdisplay", 0x2295), + ("circleplustext", 0x2295), + ("circlepostalmark", 0x3036), + ("circlering", 0x229a), + ("circlewithlefthalfblack", 0x25d0), + ("circlewithrighthalfblack", 0x25d1), + ("circumflex", 0x02c6), + ("circumflexbelowcmb", 0x032d), + ("circumflexcmb", 0x0302), + ("clear", 0x2327), + ("clickalveolar", 0x01c2), + ("clickdental", 0x01c0), + ("clicklateral", 0x01c1), + ("clickretroflex", 0x01c3), + ("clockwise", 0x27f3), + ("club", 0x2663), + ("clubsuitblack", 0x2663), + ("clubsuitwhite", 0x2667), + ("cmcubedsquare", 0x33a4), + ("cmonospace", 0xff43), + ("cmsquaredsquare", 0x33a0), + ("coarmenian", 0x0581), + ("colon", 0x003a), + ("colonmonetary", 0x20a1), + ("colonmonospace", 0xff1a), + ("colonsign", 0x20a1), + ("colonsmall", 0xfe55), + ("colontriangularhalfmod", 0x02d1), + ("colontriangularmod", 0x02d0), + ("comma", 0x002c), + ("commaabovecmb", 0x0313), + ("commaaboverightcmb", 0x0315), + ("commaaccent", 0xf6c3), + ("commaarabic", 0x060c), + ("commaarmenian", 0x055d), + ("commainferior", 0xf6e1), + ("commamonospace", 0xff0c), + ("commareversedabovecmb", 0x0314), + ("commareversedmod", 0x02bd), + ("commasmall", 0xfe50), + ("commasuperior", 0xf6e2), + ("commaturnedabovecmb", 0x0312), + ("commaturnedmod", 0x02bb), + ("compass", 0x263c), + ("complement", 0x2201), + ("compwordmark", 0x200c), + ("congruent", 0x2245), + ("contintegraldisplay", 0x222e), + ("contintegraltext", 0x222e), + ("contourintegral", 0x222e), + ("control", 0x2303), + ("controlACK", 0x0006), + ("controlBEL", 0x0007), + ("controlBS", 0x0008), + ("controlCAN", 0x0018), + ("controlCR", 0x000d), + ("controlDC1", 0x0011), + ("controlDC2", 0x0012), + ("controlDC3", 0x0013), + ("controlDC4", 0x0014), + ("controlDEL", 0x007f), + ("controlDLE", 0x0010), + ("controlEM", 0x0019), + ("controlENQ", 0x0005), + ("controlEOT", 0x0004), + ("controlESC", 0x001b), + ("controlETB", 0x0017), + ("controlETX", 0x0003), + ("controlFF", 0x000c), + ("controlFS", 0x001c), + ("controlGS", 0x001d), + ("controlHT", 0x0009), + ("controlLF", 0x000a), + ("controlNAK", 0x0015), + ("controlNULL", 0x0000), + ("controlRS", 0x001e), + ("controlSI", 0x000f), + ("controlSO", 0x000e), + ("controlSOT", 0x0002), + ("controlSTX", 0x0001), + ("controlSUB", 0x001a), + ("controlSYN", 0x0016), + ("controlUS", 0x001f), + ("controlVT", 0x000b), + ("coproduct", 0x2a3f), + ("coproductdisplay", 0x2210), + ("coproducttext", 0x2210), + ("copyright", 0x00a9), + ("copyrightsans", 0xf8e9), + ("copyrightserif", 0xf6d9), + ("cornerbracketleft", 0x300c), + ("cornerbracketlefthalfwidth", 0xff62), + ("cornerbracketleftvertical", 0xfe41), + ("cornerbracketright", 0x300d), + ("cornerbracketrighthalfwidth", 0xff63), + ("cornerbracketrightvertical", 0xfe42), + ("corporationsquare", 0x337f), + ("cosquare", 0x33c7), + ("coverkgsquare", 0x33c6), + ("cparen", 0x249e), + ("cruzeiro", 0x20a2), + ("cstretched", 0x0297), + ("ct", 0x0063), + ("curlyand", 0x22cf), + ("curlyleft", 0x21ab), + ("curlyor", 0x22ce), + ("curlyright", 0x21ac), + ("currency", 0x00a4), + ("cwm", 0x200c), + ("cyrBreve", 0xf6d1), + ("cyrFlex", 0xf6d2), + ("cyrbreve", 0xf6d4), + ("cyrflex", 0xf6d5), + ("d", 0x0064), + ("daarmenian", 0x0564), + ("dabengali", 0x09a6), + ("dadarabic", 0x0636), + ("dadeva", 0x0926), + ("dadfinalarabic", 0xfebe), + ("dadinitialarabic", 0xfebf), + ("dadmedialarabic", 0xfec0), + ("dagesh", 0x05bc), + ("dageshhebrew", 0x05bc), + ("dagger", 0x2020), + ("daggerdbl", 0x2021), + ("dagujarati", 0x0aa6), + ("dagurmukhi", 0x0a26), + ("dahiragana", 0x3060), + ("dakatakana", 0x30c0), + ("dalarabic", 0x062f), + ("dalet", 0x05d3), + ("daletdagesh", 0xfb33), + ("daletdageshhebrew", 0xfb33), + ("daleth", 0x2138), + ("dalethatafpatah", 0x05d3), + ("dalethatafpatahhebrew", 0x05d3), + ("dalethatafsegol", 0x05d3), + ("dalethatafsegolhebrew", 0x05d3), + ("dalethebrew", 0x05d3), + ("dalethiriq", 0x05d3), + ("dalethiriqhebrew", 0x05d3), + ("daletholam", 0x05d3), + ("daletholamhebrew", 0x05d3), + ("daletpatah", 0x05d3), + ("daletpatahhebrew", 0x05d3), + ("daletqamats", 0x05d3), + ("daletqamatshebrew", 0x05d3), + ("daletqubuts", 0x05d3), + ("daletqubutshebrew", 0x05d3), + ("daletsegol", 0x05d3), + ("daletsegolhebrew", 0x05d3), + ("daletsheva", 0x05d3), + ("daletshevahebrew", 0x05d3), + ("dalettsere", 0x05d3), + ("dalettserehebrew", 0x05d3), + ("dalfinalarabic", 0xfeaa), + ("dammaarabic", 0x064f), + ("dammalowarabic", 0x064f), + ("dammatanaltonearabic", 0x064c), + ("dammatanarabic", 0x064c), + ("danda", 0x0964), + ("dargahebrew", 0x05a7), + ("dargalefthebrew", 0x05a7), + ("dasiapneumatacyrilliccmb", 0x0485), + ("dbar", 0x0111), + ("dblGrave", 0xf6d3), + ("dblanglebracketleft", 0x300a), + ("dblanglebracketleftvertical", 0xfe3d), + ("dblanglebracketright", 0x300b), + ("dblanglebracketrightvertical", 0xfe3e), + ("dblarchinvertedbelowcmb", 0x032b), + ("dblarrowdwn", 0x21ca), + ("dblarrowheadleft", 0x219e), + ("dblarrowheadright", 0x21a0), + ("dblarrowleft", 0x21d4), + ("dblarrowright", 0x21d2), + ("dblarrowup", 0x21c8), + ("dblbracketleft", 0x27e6), + ("dblbracketright", 0x27e7), + ("dbldanda", 0x0965), + ("dblgrave", 0xf6d6), + ("dblgravecmb", 0x030f), + ("dblintegral", 0x222c), + ("dbllowline", 0x2017), + ("dbllowlinecmb", 0x0333), + ("dbloverlinecmb", 0x033f), + ("dblprimemod", 0x02ba), + ("dblverticalbar", 0x2016), + ("dblverticallineabovecmb", 0x030e), + ("dbopomofo", 0x3109), + ("dbsquare", 0x33c8), + ("dcaron", 0x010f), + ("dcedilla", 0x1e11), + ("dcircle", 0x24d3), + ("dcircumflexbelow", 0x1e13), + ("dcroat", 0x0111), + ("ddabengali", 0x09a1), + ("ddadeva", 0x0921), + ("ddagujarati", 0x0aa1), + ("ddagurmukhi", 0x0a21), + ("ddalarabic", 0x0688), + ("ddalfinalarabic", 0xfb89), + ("dddhadeva", 0x095c), + ("ddhabengali", 0x09a2), + ("ddhadeva", 0x0922), + ("ddhagujarati", 0x0aa2), + ("ddhagurmukhi", 0x0a22), + ("ddotaccent", 0x1e0b), + ("ddotbelow", 0x1e0d), + ("decimalseparatorarabic", 0x066b), + ("decimalseparatorpersian", 0x066b), + ("decyrillic", 0x0434), + ("defines", 0x225c), + ("degree", 0x00b0), + ("dehihebrew", 0x05ad), + ("dehiragana", 0x3067), + ("deicoptic", 0x03ef), + ("dekatakana", 0x30c7), + ("deleteleft", 0x232b), + ("deleteright", 0x2326), + ("delta", 0x03b4), + ("deltaturned", 0x018d), + ("denominatorminusonenumeratorbengali", 0x09f8), + ("dezh", 0x02a4), + ("dhabengali", 0x09a7), + ("dhadeva", 0x0927), + ("dhagujarati", 0x0aa7), + ("dhagurmukhi", 0x0a27), + ("dhook", 0x0257), + ("dialytikatonos", 0x0385), + ("dialytikatonoscmb", 0x0344), + ("diamond", 0x2666), + ("diamondmath", 0x22c4), + ("diamondsolid", 0x2666), + ("diamondsuitwhite", 0x2662), + ("dieresis", 0x00a8), + ("dieresisacute", 0xf6d7), + ("dieresisbelowcmb", 0x0324), + ("dieresiscmb", 0x0308), + ("dieresisgrave", 0xf6d8), + ("dieresistonos", 0x0385), + ("difference", 0x224f), + ("dihiragana", 0x3062), + ("dikatakana", 0x30c2), + ("dittomark", 0x3003), + ("divide", 0x00f7), + ("dividemultiply", 0x22c7), + ("divides", 0x2223), + ("divisionslash", 0x2215), + ("djecyrillic", 0x0452), + ("dkshade", 0x2593), + ("dlinebelow", 0x1e0f), + ("dlsquare", 0x3397), + ("dmacron", 0x0111), + ("dmonospace", 0xff44), + ("dnblock", 0x2584), + ("dochadathai", 0x0e0e), + ("dodekthai", 0x0e14), + ("dohiragana", 0x3069), + ("dokatakana", 0x30c9), + ("dollar", 0x0024), + ("dollarinferior", 0xf6e3), + ("dollarmonospace", 0xff04), + ("dollaroldstyle", 0xf724), + ("dollarsmall", 0xfe69), + ("dollarsuperior", 0xf6e4), + ("dong", 0x20ab), + ("dorusquare", 0x3326), + ("dotaccent", 0x02d9), + ("dotaccentcmb", 0x0307), + ("dotbelowcmb", 0x0323), + ("dotbelowcomb", 0x0323), + ("dotkatakana", 0x30fb), + ("dotlessi", 0x0131), + ("dotlessj", 0xf6be), + ("dotlessjstrokehook", 0x0284), + ("dotmath", 0x22c5), + ("dotplus", 0x2214), + ("dottedcircle", 0x25cc), + ("doubleyodpatah", 0xfb1f), + ("doubleyodpatahhebrew", 0xfb1f), + ("downfall", 0x22ce), + ("downslope", 0x29f9), + ("downtackbelowcmb", 0x031e), + ("downtackmod", 0x02d5), + ("dparen", 0x249f), + ("dsuperior", 0xf6eb), + ("dtail", 0x0256), + ("dtopbar", 0x018c), + ("duhiragana", 0x3065), + ("dukatakana", 0x30c5), + ("dz", 0x01f3), + ("dzaltone", 0x02a3), + ("dzcaron", 0x01c6), + ("dzcurl", 0x02a5), + ("dzeabkhasiancyrillic", 0x04e1), + ("dzecyrillic", 0x0455), + ("dzhecyrillic", 0x045f), + ("e", 0x0065), + ("eacute", 0x00e9), + ("earth", 0x2641), + ("ebengali", 0x098f), + ("ebopomofo", 0x311c), + ("ebreve", 0x0115), + ("ecandradeva", 0x090d), + ("ecandragujarati", 0x0a8d), + ("ecandravowelsigndeva", 0x0945), + ("ecandravowelsigngujarati", 0x0ac5), + ("ecaron", 0x011b), + ("ecedillabreve", 0x1e1d), + ("echarmenian", 0x0565), + ("echyiwnarmenian", 0x0587), + ("ecircle", 0x24d4), + ("ecircumflex", 0x00ea), + ("ecircumflexacute", 0x1ebf), + ("ecircumflexbelow", 0x1e19), + ("ecircumflexdotbelow", 0x1ec7), + ("ecircumflexgrave", 0x1ec1), + ("ecircumflexhookabove", 0x1ec3), + ("ecircumflextilde", 0x1ec5), + ("ecyrillic", 0x0454), + ("edblgrave", 0x0205), + ("edeva", 0x090f), + ("edieresis", 0x00eb), + ("edot", 0x0117), + ("edotaccent", 0x0117), + ("edotbelow", 0x1eb9), + ("eegurmukhi", 0x0a0f), + ("eematragurmukhi", 0x0a47), + ("efcyrillic", 0x0444), + ("egrave", 0x00e8), + ("egujarati", 0x0a8f), + ("eharmenian", 0x0567), + ("ehbopomofo", 0x311d), + ("ehiragana", 0x3048), + ("ehookabove", 0x1ebb), + ("eibopomofo", 0x311f), + ("eight", 0x0038), + ("eightarabic", 0x0668), + ("eightbengali", 0x09ee), + ("eightcircle", 0x2467), + ("eightcircleinversesansserif", 0x2791), + ("eightdeva", 0x096e), + ("eighteencircle", 0x2471), + ("eighteenparen", 0x2485), + ("eighteenperiod", 0x2499), + ("eightgujarati", 0x0aee), + ("eightgurmukhi", 0x0a6e), + ("eighthackarabic", 0x0668), + ("eighthangzhou", 0x3028), + ("eighthnotebeamed", 0x266b), + ("eightideographicparen", 0x3227), + ("eightinferior", 0x2088), + ("eightmonospace", 0xff18), + ("eightoldstyle", 0xf738), + ("eightparen", 0x247b), + ("eightperiod", 0x248f), + ("eightpersian", 0x06f8), + ("eightroman", 0x2177), + ("eightsuperior", 0x2078), + ("eightthai", 0x0e58), + ("einvertedbreve", 0x0207), + ("eiotifiedcyrillic", 0x0465), + ("ekatakana", 0x30a8), + ("ekatakanahalfwidth", 0xff74), + ("ekonkargurmukhi", 0x0a74), + ("ekorean", 0x3154), + ("elcyrillic", 0x043b), + ("element", 0x2208), + ("elevencircle", 0x246a), + ("elevenparen", 0x247e), + ("elevenperiod", 0x2492), + ("elevenroman", 0x217a), + ("ellipsis", 0x2026), + ("ellipsisvertical", 0x22ee), + ("emacron", 0x0113), + ("emacronacute", 0x1e17), + ("emacrongrave", 0x1e15), + ("emcyrillic", 0x043c), + ("emdash", 0x2014), + ("emdashvertical", 0xfe31), + ("emonospace", 0xff45), + ("emphasismarkarmenian", 0x055b), + ("emptyset", 0x2205), + ("emptyslot", 0xd801), + ("enbopomofo", 0x3123), + ("encyrillic", 0x043d), + ("endash", 0x2013), + ("endashvertical", 0xfe32), + ("endescendercyrillic", 0x04a3), + ("eng", 0x014b), + ("engbopomofo", 0x3125), + ("enghecyrillic", 0x04a5), + ("enhookcyrillic", 0x04c8), + ("enspace", 0x2002), + ("eogonek", 0x0119), + ("eokorean", 0x3153), + ("eopen", 0x025b), + ("eopenclosed", 0x029a), + ("eopenreversed", 0x025c), + ("eopenreversedclosed", 0x025e), + ("eopenreversedhook", 0x025d), + ("eparen", 0x24a0), + ("epsilon", 0x03b5), + ("epsilon1", 0x03f5), + ("epsiloninv", 0x03f6), + ("epsilontonos", 0x03ad), + ("equal", 0x003d), + ("equaldotleftright", 0x2252), + ("equaldotrightleft", 0x2253), + ("equalmath", 0x003d), + ("equalmonospace", 0xff1d), + ("equalorfollows", 0x22df), + ("equalorgreater", 0x2a96), + ("equalorless", 0x2a95), + ("equalorprecedes", 0x22de), + ("equalorsimilar", 0x2242), + ("equalsdots", 0x2251), + ("equalsmall", 0xfe66), + ("equalsuperior", 0x207c), + ("equivalence", 0x2261), + ("equivasymptotic", 0x224d), + ("erbopomofo", 0x3126), + ("ercyrillic", 0x0440), + ("ereversed", 0x0258), + ("ereversedcyrillic", 0x044d), + ("escyrillic", 0x0441), + ("esdescendercyrillic", 0x04ab), + ("esh", 0x0283), + ("eshcurl", 0x0286), + ("eshortdeva", 0x090e), + ("eshortvowelsigndeva", 0x0946), + ("eshreversedloop", 0x01aa), + ("eshsquatreversed", 0x0285), + ("esmallhiragana", 0x3047), + ("esmallkatakana", 0x30a7), + ("esmallkatakanahalfwidth", 0xff6a), + ("estimated", 0x212e), + ("esuperior", 0xf6ec), + ("eta", 0x03b7), + ("etarmenian", 0x0568), + ("etatonos", 0x03ae), + ("eth", 0x00f0), + ("etilde", 0x1ebd), + ("etildebelow", 0x1e1b), + ("etnahtafoukhhebrew", 0x0591), + ("etnahtafoukhlefthebrew", 0x0591), + ("etnahtahebrew", 0x0591), + ("etnahtalefthebrew", 0x0591), + ("eturned", 0x01dd), + ("eukorean", 0x3161), + ("euro", 0x20ac), + ("evowelsignbengali", 0x09c7), + ("evowelsigndeva", 0x0947), + ("evowelsigngujarati", 0x0ac7), + ("exclam", 0x0021), + ("exclamarmenian", 0x055c), + ("exclamdbl", 0x203c), + ("exclamdown", 0x00a1), + ("exclamdownsmall", 0xf7a1), + ("exclammonospace", 0xff01), + ("exclamsmall", 0xf721), + ("existential", 0x2203), + ("ezh", 0x0292), + ("ezhcaron", 0x01ef), + ("ezhcurl", 0x0293), + ("ezhreversed", 0x01b9), + ("ezhtail", 0x01ba), + ("f", 0x0066), + ("f_f", 0xfb00), + ("f_f_i", 0xfb03), + ("f_f_l", 0xfb04), + ("f_i", 0xfb01), + ("f_l", 0xfb02), + ("fadeva", 0x095e), + ("fagurmukhi", 0x0a5e), + ("fahrenheit", 0x2109), + ("fathaarabic", 0x064e), + ("fathalowarabic", 0x064e), + ("fathatanarabic", 0x064b), + ("fbopomofo", 0x3108), + ("fcircle", 0x24d5), + ("fdotaccent", 0x1e1f), + ("feharabic", 0x0641), + ("feharmenian", 0x0586), + ("fehfinalarabic", 0xfed2), + ("fehinitialarabic", 0xfed3), + ("fehmedialarabic", 0xfed4), + ("feicoptic", 0x03e5), + ("female", 0x2640), + ("ff", 0xfb00), + ("ffi", 0xfb03), + ("ffl", 0xfb04), + ("fi", 0xfb01), + ("fifteencircle", 0x246e), + ("fifteenparen", 0x2482), + ("fifteenperiod", 0x2496), + ("figuredash", 0x2012), + ("filledbox", 0x25a0), + ("filledrect", 0x25ac), + ("finalkaf", 0x05da), + ("finalkafdagesh", 0xfb3a), + ("finalkafdageshhebrew", 0xfb3a), + ("finalkafhebrew", 0x05da), + ("finalkafqamats", 0x05da), + ("finalkafqamatshebrew", 0x05da), + ("finalkafsheva", 0x05da), + ("finalkafshevahebrew", 0x05da), + ("finalmem", 0x05dd), + ("finalmemhebrew", 0x05dd), + ("finalnun", 0x05df), + ("finalnunhebrew", 0x05df), + ("finalpe", 0x05e3), + ("finalpehebrew", 0x05e3), + ("finaltsadi", 0x05e5), + ("finaltsadihebrew", 0x05e5), + ("firsttonechinese", 0x02c9), + ("fisheye", 0x25c9), + ("fitacyrillic", 0x0473), + ("five", 0x0035), + ("fivearabic", 0x0665), + ("fivebengali", 0x09eb), + ("fivecircle", 0x2464), + ("fivecircleinversesansserif", 0x278e), + ("fivedeva", 0x096b), + ("fiveeighths", 0x215d), + ("fivegujarati", 0x0aeb), + ("fivegurmukhi", 0x0a6b), + ("fivehackarabic", 0x0665), + ("fivehangzhou", 0x3025), + ("fiveideographicparen", 0x3224), + ("fiveinferior", 0x2085), + ("fivemonospace", 0xff15), + ("fiveoldstyle", 0xf735), + ("fiveparen", 0x2478), + ("fiveperiod", 0x248c), + ("fivepersian", 0x06f5), + ("fiveroman", 0x2174), + ("fivesuperior", 0x2075), + ("fivethai", 0x0e55), + ("fl", 0xfb02), + ("flat", 0x266d), + ("floorleft", 0x230a), + ("floorleftBig", 0x230a), + ("floorleftBigg", 0x230a), + ("floorleftbig", 0x230a), + ("floorleftbigg", 0x230a), + ("floorright", 0x230b), + ("floorrightBig", 0x230b), + ("floorrightBigg", 0x230b), + ("floorrightbig", 0x230b), + ("floorrightbigg", 0x230b), + ("florin", 0x0192), + ("fmonospace", 0xff46), + ("fmsquare", 0x3399), + ("fofanthai", 0x0e1f), + ("fofathai", 0x0e1d), + ("follownotdbleqv", 0x2aba), + ("follownotslnteql", 0x2ab6), + ("followornoteqvlnt", 0x22e9), + ("follows", 0x227b), + ("followsequal", 0x227d), + ("followsorcurly", 0x227d), + ("followsorequal", 0x227f), + ("fongmanthai", 0x0e4f), + ("forall", 0x2200), + ("forces", 0x22a9), + ("forcesbar", 0x22aa), + ("fork", 0x22d4), + ("four", 0x0034), + ("fourarabic", 0x0664), + ("fourbengali", 0x09ea), + ("fourcircle", 0x2463), + ("fourcircleinversesansserif", 0x278d), + ("fourdeva", 0x096a), + ("fourgujarati", 0x0aea), + ("fourgurmukhi", 0x0a6a), + ("fourhackarabic", 0x0664), + ("fourhangzhou", 0x3024), + ("fourideographicparen", 0x3223), + ("fourinferior", 0x2084), + ("fourmonospace", 0xff14), + ("fournumeratorbengali", 0x09f7), + ("fouroldstyle", 0xf734), + ("fourparen", 0x2477), + ("fourperiod", 0x248b), + ("fourpersian", 0x06f4), + ("fourroman", 0x2173), + ("foursuperior", 0x2074), + ("fourteencircle", 0x246d), + ("fourteenparen", 0x2481), + ("fourteenperiod", 0x2495), + ("fourthai", 0x0e54), + ("fourthtonechinese", 0x02cb), + ("fparen", 0x24a1), + ("fraction", 0x2044), + ("franc", 0x20a3), + ("frown", 0x2322), + ("g", 0x0067), + ("gabengali", 0x0997), + ("gacute", 0x01f5), + ("gadeva", 0x0917), + ("gafarabic", 0x06af), + ("gaffinalarabic", 0xfb93), + ("gafinitialarabic", 0xfb94), + ("gafmedialarabic", 0xfb95), + ("gagujarati", 0x0a97), + ("gagurmukhi", 0x0a17), + ("gahiragana", 0x304c), + ("gakatakana", 0x30ac), + ("gamma", 0x03b3), + ("gammalatinsmall", 0x0263), + ("gammasuperior", 0x02e0), + ("gangiacoptic", 0x03eb), + ("gbopomofo", 0x310d), + ("gbreve", 0x011f), + ("gcaron", 0x01e7), + ("gcedilla", 0x0123), + ("gcircle", 0x24d6), + ("gcircumflex", 0x011d), + ("gcommaaccent", 0x0123), + ("gdot", 0x0121), + ("gdotaccent", 0x0121), + ("gecyrillic", 0x0433), + ("gehiragana", 0x3052), + ("gekatakana", 0x30b2), + ("geomequivalent", 0x224e), + ("geometricallyequal", 0x2251), + ("gereshaccenthebrew", 0x059c), + ("gereshhebrew", 0x05f3), + ("gereshmuqdamhebrew", 0x059d), + ("germandbls", 0x00df), + ("gershayimaccenthebrew", 0x059e), + ("gershayimhebrew", 0x05f4), + ("getamark", 0x3013), + ("ghabengali", 0x0998), + ("ghadarmenian", 0x0572), + ("ghadeva", 0x0918), + ("ghagujarati", 0x0a98), + ("ghagurmukhi", 0x0a18), + ("ghainarabic", 0x063a), + ("ghainfinalarabic", 0xfece), + ("ghaininitialarabic", 0xfecf), + ("ghainmedialarabic", 0xfed0), + ("ghemiddlehookcyrillic", 0x0495), + ("ghestrokecyrillic", 0x0493), + ("gheupturncyrillic", 0x0491), + ("ghhadeva", 0x095a), + ("ghhagurmukhi", 0x0a5a), + ("ghook", 0x0260), + ("ghzsquare", 0x3393), + ("gihiragana", 0x304e), + ("gikatakana", 0x30ae), + ("gimarmenian", 0x0563), + ("gimel", 0x05d2), + ("gimeldagesh", 0xfb32), + ("gimeldageshhebrew", 0xfb32), + ("gimelhebrew", 0x05d2), + ("gjecyrillic", 0x0453), + ("glottalinvertedstroke", 0x01be), + ("glottalstop", 0x0294), + ("glottalstopinverted", 0x0296), + ("glottalstopmod", 0x02c0), + ("glottalstopreversed", 0x0295), + ("glottalstopreversedmod", 0x02c1), + ("glottalstopreversedsuperior", 0x02e4), + ("glottalstopstroke", 0x02a1), + ("glottalstopstrokereversed", 0x02a2), + ("gmacron", 0x1e21), + ("gmonospace", 0xff47), + ("gohiragana", 0x3054), + ("gokatakana", 0x30b4), + ("gparen", 0x24a2), + ("gpasquare", 0x33ac), + ("gradient", 0x2207), + ("grave", 0x0060), + ("gravebelowcmb", 0x0316), + ("gravecmb", 0x0300), + ("gravecomb", 0x0300), + ("gravedeva", 0x0953), + ("gravelowmod", 0x02ce), + ("gravemonospace", 0xff40), + ("gravetonecmb", 0x0340), + ("greater", 0x003e), + ("greaterdbleqlless", 0x2a8c), + ("greaterdblequal", 0x2267), + ("greaterdot", 0x22d7), + ("greaterequal", 0x2265), + ("greaterequalorless", 0x22db), + ("greaterlessequal", 0x22db), + ("greatermonospace", 0xff1e), + ("greatermuch", 0x226b), + ("greaternotdblequal", 0x2a8a), + ("greaternotequal", 0x2a88), + ("greaterorapproxeql", 0x2a86), + ("greaterorequalslant", 0x2a7e), + ("greaterorequivalent", 0x2273), + ("greaterorless", 0x2277), + ("greaterornotdbleql", 0x2269), + ("greaterornotequal", 0x2269), + ("greaterorsimilar", 0x2273), + ("greateroverequal", 0x2267), + ("greatersmall", 0xfe65), + ("gscript", 0x0261), + ("gstroke", 0x01e5), + ("guhiragana", 0x3050), + ("guillemotleft", 0x00ab), + ("guillemotright", 0x00bb), + ("guilsinglleft", 0x2039), + ("guilsinglright", 0x203a), + ("gukatakana", 0x30b0), + ("guramusquare", 0x3318), + ("gysquare", 0x33c9), + ("h", 0x0068), + ("haabkhasiancyrillic", 0x04a9), + ("haaltonearabic", 0x06c1), + ("habengali", 0x09b9), + ("hadescendercyrillic", 0x04b3), + ("hadeva", 0x0939), + ("hagujarati", 0x0ab9), + ("hagurmukhi", 0x0a39), + ("haharabic", 0x062d), + ("hahfinalarabic", 0xfea2), + ("hahinitialarabic", 0xfea3), + ("hahiragana", 0x306f), + ("hahmedialarabic", 0xfea4), + ("haitusquare", 0x332a), + ("hakatakana", 0x30cf), + ("hakatakanahalfwidth", 0xff8a), + ("halantgurmukhi", 0x0a4d), + ("hamzaarabic", 0x0621), + ("hamzadammaarabic", 0x0621), + ("hamzadammatanarabic", 0x0621), + ("hamzafathaarabic", 0x0621), + ("hamzafathatanarabic", 0x0621), + ("hamzalowarabic", 0x0621), + ("hamzalowkasraarabic", 0x0621), + ("hamzalowkasratanarabic", 0x0621), + ("hamzasukunarabic", 0x0621), + ("hangulfiller", 0x3164), + ("hardsigncyrillic", 0x044a), + ("harpoondownleft", 0x21c3), + ("harpoondownright", 0x21c2), + ("harpoonleftbarbup", 0x21bc), + ("harpoonleftdown", 0x21bd), + ("harpoonleftright", 0x21cc), + ("harpoonleftup", 0x21bc), + ("harpoonrightbarbup", 0x21c0), + ("harpoonrightdown", 0x21c1), + ("harpoonrightleft", 0x21cb), + ("harpoonrightup", 0x21c0), + ("harpoonupleft", 0x21bf), + ("harpoonupright", 0x21be), + ("hasquare", 0x33ca), + ("hatafpatah", 0x05b2), + ("hatafpatah16", 0x05b2), + ("hatafpatah23", 0x05b2), + ("hatafpatah2f", 0x05b2), + ("hatafpatahhebrew", 0x05b2), + ("hatafpatahnarrowhebrew", 0x05b2), + ("hatafpatahquarterhebrew", 0x05b2), + ("hatafpatahwidehebrew", 0x05b2), + ("hatafqamats", 0x05b3), + ("hatafqamats1b", 0x05b3), + ("hatafqamats28", 0x05b3), + ("hatafqamats34", 0x05b3), + ("hatafqamatshebrew", 0x05b3), + ("hatafqamatsnarrowhebrew", 0x05b3), + ("hatafqamatsquarterhebrew", 0x05b3), + ("hatafqamatswidehebrew", 0x05b3), + ("hatafsegol", 0x05b1), + ("hatafsegol17", 0x05b1), + ("hatafsegol24", 0x05b1), + ("hatafsegol30", 0x05b1), + ("hatafsegolhebrew", 0x05b1), + ("hatafsegolnarrowhebrew", 0x05b1), + ("hatafsegolquarterhebrew", 0x05b1), + ("hatafsegolwidehebrew", 0x05b1), + ("hatwide", 0x0302), + ("hatwider", 0x0302), + ("hatwiderr", 0x0302), + ("hatwidest", 0x0302), + ("hbar", 0x0127), + ("hbopomofo", 0x310f), + ("hbrevebelow", 0x1e2b), + ("hcedilla", 0x1e29), + ("hcircle", 0x24d7), + ("hcircumflex", 0x0125), + ("hdieresis", 0x1e27), + ("hdotaccent", 0x1e23), + ("hdotbelow", 0x1e25), + ("he", 0x05d4), + ("heart", 0x2665), + ("heartsuitblack", 0x2665), + ("heartsuitwhite", 0x2661), + ("hedagesh", 0xfb34), + ("hedageshhebrew", 0xfb34), + ("hehaltonearabic", 0x06c1), + ("heharabic", 0x0647), + ("hehebrew", 0x05d4), + ("hehfinalaltonearabic", 0xfba7), + ("hehfinalalttwoarabic", 0xfeea), + ("hehfinalarabic", 0xfeea), + ("hehhamzaabovefinalarabic", 0xfba5), + ("hehhamzaaboveisolatedarabic", 0xfba4), + ("hehinitialaltonearabic", 0xfba8), + ("hehinitialarabic", 0xfeeb), + ("hehiragana", 0x3078), + ("hehmedialaltonearabic", 0xfba9), + ("hehmedialarabic", 0xfeec), + ("heiseierasquare", 0x337b), + ("hekatakana", 0x30d8), + ("hekatakanahalfwidth", 0xff8d), + ("hekutaarusquare", 0x3336), + ("henghook", 0x0267), + ("herutusquare", 0x3339), + ("het", 0x05d7), + ("hethebrew", 0x05d7), + ("hhook", 0x0266), + ("hhooksuperior", 0x02b1), + ("hieuhacirclekorean", 0x327b), + ("hieuhaparenkorean", 0x321b), + ("hieuhcirclekorean", 0x326d), + ("hieuhkorean", 0x314e), + ("hieuhparenkorean", 0x320d), + ("hihiragana", 0x3072), + ("hikatakana", 0x30d2), + ("hikatakanahalfwidth", 0xff8b), + ("hiriq", 0x05b4), + ("hiriq14", 0x05b4), + ("hiriq21", 0x05b4), + ("hiriq2d", 0x05b4), + ("hiriqhebrew", 0x05b4), + ("hiriqnarrowhebrew", 0x05b4), + ("hiriqquarterhebrew", 0x05b4), + ("hiriqwidehebrew", 0x05b4), + ("hlinebelow", 0x1e96), + ("hmonospace", 0xff48), + ("hoarmenian", 0x0570), + ("hohipthai", 0x0e2b), + ("hohiragana", 0x307b), + ("hokatakana", 0x30db), + ("hokatakanahalfwidth", 0xff8e), + ("holam", 0x05b9), + ("holam19", 0x05b9), + ("holam26", 0x05b9), + ("holam32", 0x05b9), + ("holamhebrew", 0x05b9), + ("holamnarrowhebrew", 0x05b9), + ("holamquarterhebrew", 0x05b9), + ("holamwidehebrew", 0x05b9), + ("honokhukthai", 0x0e2e), + ("hookabovecomb", 0x0309), + ("hookcmb", 0x0309), + ("hookleftchar", 0x21a9), + ("hookpalatalizedbelowcmb", 0x0321), + ("hookretroflexbelowcmb", 0x0322), + ("hookrightchar", 0x21aa), + ("hoonsquare", 0x3342), + ("horicoptic", 0x03e9), + ("horizontalbar", 0x2015), + ("horncmb", 0x031b), + ("hotsprings", 0x2668), + ("house", 0x2302), + ("hparen", 0x24a3), + ("hsuperior", 0x02b0), + ("hturned", 0x0265), + ("huhiragana", 0x3075), + ("huiitosquare", 0x3333), + ("hukatakana", 0x30d5), + ("hukatakanahalfwidth", 0xff8c), + ("hungarumlaut", 0x02dd), + ("hungarumlautcmb", 0x030b), + ("hv", 0x0195), + ("hyphen", 0x002d), + ("hyphen.alt", 0x2010), + ("hyphenchar", 0x002d), + ("hypheninferior", 0xf6e5), + ("hyphenmonospace", 0xff0d), + ("hyphensmall", 0xfe63), + ("hyphensuperior", 0xf6e6), + ("hyphentwo", 0x2010), + ("i", 0x0069), + ("iacute", 0x00ed), + ("iacyrillic", 0x044f), + ("ibengali", 0x0987), + ("ibopomofo", 0x3127), + ("ibreve", 0x012d), + ("icaron", 0x01d0), + ("icircle", 0x24d8), + ("icircumflex", 0x00ee), + ("icyrillic", 0x0456), + ("idblgrave", 0x0209), + ("ideographearthcircle", 0x328f), + ("ideographfirecircle", 0x328b), + ("ideographicallianceparen", 0x323f), + ("ideographiccallparen", 0x323a), + ("ideographiccentrecircle", 0x32a5), + ("ideographicclose", 0x3006), + ("ideographiccomma", 0x3001), + ("ideographiccommaleft", 0xff64), + ("ideographiccongratulationparen", 0x3237), + ("ideographiccorrectcircle", 0x32a3), + ("ideographicearthparen", 0x322f), + ("ideographicenterpriseparen", 0x323d), + ("ideographicexcellentcircle", 0x329d), + ("ideographicfestivalparen", 0x3240), + ("ideographicfinancialcircle", 0x3296), + ("ideographicfinancialparen", 0x3236), + ("ideographicfireparen", 0x322b), + ("ideographichaveparen", 0x3232), + ("ideographichighcircle", 0x32a4), + ("ideographiciterationmark", 0x3005), + ("ideographiclaborcircle", 0x3298), + ("ideographiclaborparen", 0x3238), + ("ideographicleftcircle", 0x32a7), + ("ideographiclowcircle", 0x32a6), + ("ideographicmedicinecircle", 0x32a9), + ("ideographicmetalparen", 0x322e), + ("ideographicmoonparen", 0x322a), + ("ideographicnameparen", 0x3234), + ("ideographicperiod", 0x3002), + ("ideographicprintcircle", 0x329e), + ("ideographicreachparen", 0x3243), + ("ideographicrepresentparen", 0x3239), + ("ideographicresourceparen", 0x323e), + ("ideographicrightcircle", 0x32a8), + ("ideographicsecretcircle", 0x3299), + ("ideographicselfparen", 0x3242), + ("ideographicsocietyparen", 0x3233), + ("ideographicspace", 0x3000), + ("ideographicspecialparen", 0x3235), + ("ideographicstockparen", 0x3231), + ("ideographicstudyparen", 0x323b), + ("ideographicsunparen", 0x3230), + ("ideographicsuperviseparen", 0x323c), + ("ideographicwaterparen", 0x322c), + ("ideographicwoodparen", 0x322d), + ("ideographiczero", 0x3007), + ("ideographmetalcircle", 0x328e), + ("ideographmooncircle", 0x328a), + ("ideographnamecircle", 0x3294), + ("ideographsuncircle", 0x3290), + ("ideographwatercircle", 0x328c), + ("ideographwoodcircle", 0x328d), + ("ideva", 0x0907), + ("idieresis", 0x00ef), + ("idieresisacute", 0x1e2f), + ("idieresiscyrillic", 0x04e5), + ("idotbelow", 0x1ecb), + ("iebrevecyrillic", 0x04d7), + ("iecyrillic", 0x0435), + ("ieungacirclekorean", 0x3275), + ("ieungaparenkorean", 0x3215), + ("ieungcirclekorean", 0x3267), + ("ieungkorean", 0x3147), + ("ieungparenkorean", 0x3207), + ("igrave", 0x00ec), + ("igujarati", 0x0a87), + ("igurmukhi", 0x0a07), + ("ihiragana", 0x3044), + ("ihookabove", 0x1ec9), + ("iibengali", 0x0988), + ("iicyrillic", 0x0438), + ("iideva", 0x0908), + ("iigujarati", 0x0a88), + ("iigurmukhi", 0x0a08), + ("iimatragurmukhi", 0x0a40), + ("iinvertedbreve", 0x020b), + ("iishortcyrillic", 0x0439), + ("iivowelsignbengali", 0x09c0), + ("iivowelsigndeva", 0x0940), + ("iivowelsigngujarati", 0x0ac0), + ("ij", 0x0133), + ("ikatakana", 0x30a4), + ("ikatakanahalfwidth", 0xff72), + ("ikorean", 0x3163), + ("ilde", 0x02dc), + ("iluyhebrew", 0x05ac), + ("imacron", 0x012b), + ("imacroncyrillic", 0x04e3), + ("imageorapproximatelyequal", 0x2253), + ("imatragurmukhi", 0x0a3f), + ("imonospace", 0xff49), + ("increment", 0x2206), + ("infinity", 0x221e), + ("iniarmenian", 0x056b), + ("integerdivide", 0x2216), + ("integral", 0x222b), + ("integralbottom", 0x2321), + ("integralbt", 0x2321), + ("integraldisplay", 0x222b), + ("integralex", 0xf8f5), + ("integraltext", 0x222b), + ("integraltop", 0x2320), + ("integraltp", 0x2320), + ("intercal", 0x22ba), + ("interrobang", 0x203d), + ("interrobangdown", 0xd80b), + ("intersection", 0x2229), + ("intersectiondbl", 0x22d2), + ("intersectiondisplay", 0x22c2), + ("intersectionsq", 0x2293), + ("intersectiontext", 0x22c2), + ("intisquare", 0x3305), + ("invbullet", 0x25d8), + ("invcircle", 0x25d9), + ("invsmileface", 0x263b), + ("iocyrillic", 0x0451), + ("iogonek", 0x012f), + ("iota", 0x03b9), + ("iotadieresis", 0x03ca), + ("iotadieresistonos", 0x0390), + ("iotalatin", 0x0269), + ("iotatonos", 0x03af), + ("iparen", 0x24a4), + ("irigurmukhi", 0x0a72), + ("ismallhiragana", 0x3043), + ("ismallkatakana", 0x30a3), + ("ismallkatakanahalfwidth", 0xff68), + ("issharbengali", 0x09fa), + ("istroke", 0x0268), + ("isuperior", 0xf6ed), + ("iterationhiragana", 0x309d), + ("iterationkatakana", 0x30fd), + ("itilde", 0x0129), + ("itildebelow", 0x1e2d), + ("iubopomofo", 0x3129), + ("iucyrillic", 0x044e), + ("ivowelsignbengali", 0x09bf), + ("ivowelsigndeva", 0x093f), + ("ivowelsigngujarati", 0x0abf), + ("izhitsacyrillic", 0x0475), + ("izhitsadblgravecyrillic", 0x0477), + ("j", 0x006a), + ("jaarmenian", 0x0571), + ("jabengali", 0x099c), + ("jadeva", 0x091c), + ("jagujarati", 0x0a9c), + ("jagurmukhi", 0x0a1c), + ("jbopomofo", 0x3110), + ("jcaron", 0x01f0), + ("jcircle", 0x24d9), + ("jcircumflex", 0x0135), + ("jcrossedtail", 0x029d), + ("jdotlessstroke", 0x025f), + ("jecyrillic", 0x0458), + ("jeemarabic", 0x062c), + ("jeemfinalarabic", 0xfe9e), + ("jeeminitialarabic", 0xfe9f), + ("jeemmedialarabic", 0xfea0), + ("jeharabic", 0x0698), + ("jehfinalarabic", 0xfb8b), + ("jhabengali", 0x099d), + ("jhadeva", 0x091d), + ("jhagujarati", 0x0a9d), + ("jhagurmukhi", 0x0a1d), + ("jheharmenian", 0x057b), + ("jis", 0x3004), + ("jmonospace", 0xff4a), + ("jparen", 0x24a5), + ("jsuperior", 0x02b2), + ("k", 0x006b), + ("kabashkircyrillic", 0x04a1), + ("kabengali", 0x0995), + ("kacute", 0x1e31), + ("kacyrillic", 0x043a), + ("kadescendercyrillic", 0x049b), + ("kadeva", 0x0915), + ("kaf", 0x05db), + ("kafarabic", 0x0643), + ("kafdagesh", 0xfb3b), + ("kafdageshhebrew", 0xfb3b), + ("kaffinalarabic", 0xfeda), + ("kafhebrew", 0x05db), + ("kafinitialarabic", 0xfedb), + ("kafmedialarabic", 0xfedc), + ("kafrafehebrew", 0xfb4d), + ("kagujarati", 0x0a95), + ("kagurmukhi", 0x0a15), + ("kahiragana", 0x304b), + ("kahookcyrillic", 0x04c4), + ("kakatakana", 0x30ab), + ("kakatakanahalfwidth", 0xff76), + ("kappa", 0x03ba), + ("kappasymbolgreek", 0x03f0), + ("kapyeounmieumkorean", 0x3171), + ("kapyeounphieuphkorean", 0x3184), + ("kapyeounpieupkorean", 0x3178), + ("kapyeounssangpieupkorean", 0x3179), + ("karoriisquare", 0x330d), + ("kashidaautoarabic", 0x0640), + ("kashidaautonosidebearingarabic", 0x0640), + ("kasmallkatakana", 0x30f5), + ("kasquare", 0x3384), + ("kasraarabic", 0x0650), + ("kasratanarabic", 0x064d), + ("kastrokecyrillic", 0x049f), + ("katahiraprolongmarkhalfwidth", 0xff70), + ("kaverticalstrokecyrillic", 0x049d), + ("kbopomofo", 0x310e), + ("kcalsquare", 0x3389), + ("kcaron", 0x01e9), + ("kcedilla", 0x0137), + ("kcircle", 0x24da), + ("kcommaaccent", 0x0137), + ("kdotbelow", 0x1e33), + ("keharmenian", 0x0584), + ("kehiragana", 0x3051), + ("kekatakana", 0x30b1), + ("kekatakanahalfwidth", 0xff79), + ("kenarmenian", 0x056f), + ("kesmallkatakana", 0x30f6), + ("kgreenlandic", 0x0138), + ("khabengali", 0x0996), + ("khacyrillic", 0x0445), + ("khadeva", 0x0916), + ("khagujarati", 0x0a96), + ("khagurmukhi", 0x0a16), + ("khaharabic", 0x062e), + ("khahfinalarabic", 0xfea6), + ("khahinitialarabic", 0xfea7), + ("khahmedialarabic", 0xfea8), + ("kheicoptic", 0x03e7), + ("khhadeva", 0x0959), + ("khhagurmukhi", 0x0a59), + ("khieukhacirclekorean", 0x3278), + ("khieukhaparenkorean", 0x3218), + ("khieukhcirclekorean", 0x326a), + ("khieukhkorean", 0x314b), + ("khieukhparenkorean", 0x320a), + ("khokhaithai", 0x0e02), + ("khokhonthai", 0x0e05), + ("khokhuatthai", 0x0e03), + ("khokhwaithai", 0x0e04), + ("khomutthai", 0x0e5b), + ("khook", 0x0199), + ("khorakhangthai", 0x0e06), + ("khzsquare", 0x3391), + ("kihiragana", 0x304d), + ("kikatakana", 0x30ad), + ("kikatakanahalfwidth", 0xff77), + ("kiroguramusquare", 0x3315), + ("kiromeetorusquare", 0x3316), + ("kirosquare", 0x3314), + ("kiyeokacirclekorean", 0x326e), + ("kiyeokaparenkorean", 0x320e), + ("kiyeokcirclekorean", 0x3260), + ("kiyeokkorean", 0x3131), + ("kiyeokparenkorean", 0x3200), + ("kiyeoksioskorean", 0x3133), + ("kjecyrillic", 0x045c), + ("klinebelow", 0x1e35), + ("klsquare", 0x3398), + ("kmcubedsquare", 0x33a6), + ("kmonospace", 0xff4b), + ("kmsquaredsquare", 0x33a2), + ("kohiragana", 0x3053), + ("kohmsquare", 0x33c0), + ("kokaithai", 0x0e01), + ("kokatakana", 0x30b3), + ("kokatakanahalfwidth", 0xff7a), + ("kooposquare", 0x331e), + ("koppacyrillic", 0x0481), + ("koreanstandardsymbol", 0x327f), + ("koroniscmb", 0x0343), + ("kparen", 0x24a6), + ("kpasquare", 0x33aa), + ("ksicyrillic", 0x046f), + ("ktsquare", 0x33cf), + ("kturned", 0x029e), + ("kuhiragana", 0x304f), + ("kukatakana", 0x30af), + ("kukatakanahalfwidth", 0xff78), + ("kvsquare", 0x33b8), + ("kwsquare", 0x33be), + ("l", 0x006c), + ("labengali", 0x09b2), + ("lacute", 0x013a), + ("ladeva", 0x0932), + ("lagujarati", 0x0ab2), + ("lagurmukhi", 0x0a32), + ("lakkhangyaothai", 0x0e45), + ("lamaleffinalarabic", 0xfefc), + ("lamalefhamzaabovefinalarabic", 0xfef8), + ("lamalefhamzaaboveisolatedarabic", 0xfef7), + ("lamalefhamzabelowfinalarabic", 0xfefa), + ("lamalefhamzabelowisolatedarabic", 0xfef9), + ("lamalefisolatedarabic", 0xfefb), + ("lamalefmaddaabovefinalarabic", 0xfef6), + ("lamalefmaddaaboveisolatedarabic", 0xfef5), + ("lamarabic", 0x0644), + ("lambda", 0x03bb), + ("lambdastroke", 0x019b), + ("lamed", 0x05dc), + ("lameddagesh", 0xfb3c), + ("lameddageshhebrew", 0xfb3c), + ("lamedhebrew", 0x05dc), + ("lamedholam", 0x05dc), + ("lamedholamdagesh", 0x05dc), + ("lamedholamdageshhebrew", 0x05dc), + ("lamedholamhebrew", 0x05dc), + ("lamfinalarabic", 0xfede), + ("lamhahinitialarabic", 0xfcca), + ("laminitialarabic", 0xfedf), + ("lamjeeminitialarabic", 0xfcc9), + ("lamkhahinitialarabic", 0xfccb), + ("lamlamhehisolatedarabic", 0xfdf2), + ("lammedialarabic", 0xfee0), + ("lammeemhahinitialarabic", 0xfd88), + ("lammeeminitialarabic", 0xfccc), + ("lammeemjeeminitialarabic", 0xfedf), + ("lammeemkhahinitialarabic", 0xfedf), + ("largecircle", 0x25ef), + ("latticetop", 0x22a4), + ("lbar", 0x019a), + ("lbelt", 0x026c), + ("lbopomofo", 0x310c), + ("lcaron", 0x013e), + ("lcedilla", 0x013c), + ("lcircle", 0x24db), + ("lcircumflexbelow", 0x1e3d), + ("lcommaaccent", 0x013c), + ("ldot", 0x0140), + ("ldotaccent", 0x0140), + ("ldotbelow", 0x1e37), + ("ldotbelowmacron", 0x1e39), + ("leftangleabovecmb", 0x031a), + ("lefttackbelowcmb", 0x0318), + ("less", 0x003c), + ("lessdbleqlgreater", 0x2a8b), + ("lessdblequal", 0x2266), + ("lessdot", 0x22d6), + ("lessequal", 0x2264), + ("lessequalgreater", 0x22da), + ("lessequalorgreater", 0x22da), + ("lessmonospace", 0xff1c), + ("lessmuch", 0x226a), + ("lessnotdblequal", 0x2a89), + ("lessnotequal", 0x2a87), + ("lessorapproxeql", 0x2a85), + ("lessorequalslant", 0x2a7d), + ("lessorequivalent", 0x2272), + ("lessorgreater", 0x2276), + ("lessornotdbleql", 0x2268), + ("lessornotequal", 0x2268), + ("lessorsimilar", 0x2272), + ("lessoverequal", 0x2266), + ("lesssmall", 0xfe64), + ("lezh", 0x026e), + ("lfblock", 0x258c), + ("lhookretroflex", 0x026d), + ("lira", 0x20a4), + ("liwnarmenian", 0x056c), + ("lj", 0x01c9), + ("ljecyrillic", 0x0459), + ("ll", 0xf6c0), + ("lladeva", 0x0933), + ("llagujarati", 0x0ab3), + ("llinebelow", 0x1e3b), + ("llladeva", 0x0934), + ("llvocalicbengali", 0x09e1), + ("llvocalicdeva", 0x0961), + ("llvocalicvowelsignbengali", 0x09e3), + ("llvocalicvowelsigndeva", 0x0963), + ("lmiddletilde", 0x026b), + ("lmonospace", 0xff4c), + ("lmsquare", 0x33d0), + ("lochulathai", 0x0e2c), + ("logicaland", 0x2227), + ("logicalanddisplay", 0x22c0), + ("logicalandtext", 0x22c0), + ("logicalnot", 0x00ac), + ("logicalnotreversed", 0x2310), + ("logicalor", 0x2228), + ("logicalordisplay", 0x22c1), + ("logicalortext", 0x22c1), + ("lolingthai", 0x0e25), + ("longdbls", 0x017f), + ("longs", 0x017f), + ("longsh", 0x017f), + ("longsi", 0x017f), + ("longsl", 0x017f), + ("longst", 0xfb05), + ("lowlinecenterline", 0xfe4e), + ("lowlinecmb", 0x0332), + ("lowlinedashed", 0xfe4d), + ("lozenge", 0x25ca), + ("lparen", 0x24a7), + ("lscript", 0x2113), + ("lslash", 0x0142), + ("lsquare", 0x2113), + ("lsuperior", 0xf6ee), + ("ltshade", 0x2591), + ("luthai", 0x0e26), + ("lvocalicbengali", 0x098c), + ("lvocalicdeva", 0x090c), + ("lvocalicvowelsignbengali", 0x09e2), + ("lvocalicvowelsigndeva", 0x0962), + ("lxsquare", 0x33d3), + ("m", 0x006d), + ("mabengali", 0x09ae), + ("macron", 0x00af), + ("macronbelowcmb", 0x0331), + ("macroncmb", 0x0304), + ("macronlowmod", 0x02cd), + ("macronmonospace", 0xffe3), + ("macute", 0x1e3f), + ("madeva", 0x092e), + ("magujarati", 0x0aae), + ("magurmukhi", 0x0a2e), + ("mahapakhhebrew", 0x05a4), + ("mahapakhlefthebrew", 0x05a4), + ("mahiragana", 0x307e), + ("maichattawalowleftthai", 0xf895), + ("maichattawalowrightthai", 0xf894), + ("maichattawathai", 0x0e4b), + ("maichattawaupperleftthai", 0xf893), + ("maieklowleftthai", 0xf88c), + ("maieklowrightthai", 0xf88b), + ("maiekthai", 0x0e48), + ("maiekupperleftthai", 0xf88a), + ("maihanakatleftthai", 0xf884), + ("maihanakatthai", 0x0e31), + ("maitaikhuleftthai", 0xf889), + ("maitaikhuthai", 0x0e47), + ("maitholowleftthai", 0xf88f), + ("maitholowrightthai", 0xf88e), + ("maithothai", 0x0e49), + ("maithoupperleftthai", 0xf88d), + ("maitrilowleftthai", 0xf892), + ("maitrilowrightthai", 0xf891), + ("maitrithai", 0x0e4a), + ("maitriupperleftthai", 0xf890), + ("maiyamokthai", 0x0e46), + ("makatakana", 0x30de), + ("makatakanahalfwidth", 0xff8f), + ("male", 0x2642), + ("maltesecross", 0x2720), + ("mansyonsquare", 0x3347), + ("mapsto", 0x21a6), + ("maqafhebrew", 0x05be), + ("mars", 0x2642), + ("masoracirclehebrew", 0x05af), + ("masquare", 0x3383), + ("mbopomofo", 0x3107), + ("mbsquare", 0x33d4), + ("mcircle", 0x24dc), + ("mcubedsquare", 0x33a5), + ("mdotaccent", 0x1e41), + ("mdotbelow", 0x1e43), + ("measuredangle", 0x2221), + ("meemarabic", 0x0645), + ("meemfinalarabic", 0xfee2), + ("meeminitialarabic", 0xfee3), + ("meemmedialarabic", 0xfee4), + ("meemmeeminitialarabic", 0xfcd1), + ("meemmeemisolatedarabic", 0xfc48), + ("meetorusquare", 0x334d), + ("mehiragana", 0x3081), + ("meizierasquare", 0x337e), + ("mekatakana", 0x30e1), + ("mekatakanahalfwidth", 0xff92), + ("mem", 0x05de), + ("memdagesh", 0xfb3e), + ("memdageshhebrew", 0xfb3e), + ("memhebrew", 0x05de), + ("menarmenian", 0x0574), + ("merkhahebrew", 0x05a5), + ("merkhakefulahebrew", 0x05a6), + ("merkhakefulalefthebrew", 0x05a6), + ("merkhalefthebrew", 0x05a5), + ("mhook", 0x0271), + ("mhzsquare", 0x3392), + ("middledotkatakanahalfwidth", 0xff65), + ("middot", 0x00b7), + ("mieumacirclekorean", 0x3272), + ("mieumaparenkorean", 0x3212), + ("mieumcirclekorean", 0x3264), + ("mieumkorean", 0x3141), + ("mieumpansioskorean", 0x3170), + ("mieumparenkorean", 0x3204), + ("mieumpieupkorean", 0x316e), + ("mieumsioskorean", 0x316f), + ("mihiragana", 0x307f), + ("mikatakana", 0x30df), + ("mikatakanahalfwidth", 0xff90), + ("minus", 0x2212), + ("minusbelowcmb", 0x0320), + ("minuscircle", 0x2296), + ("minusmod", 0x02d7), + ("minusplus", 0x2213), + ("minute", 0x2032), + ("miribaarusquare", 0x334a), + ("mirisquare", 0x3349), + ("mlonglegturned", 0x0270), + ("mlsquare", 0x3396), + ("mmcubedsquare", 0x33a3), + ("mmonospace", 0xff4d), + ("mmsquaredsquare", 0x339f), + ("mohiragana", 0x3082), + ("mohmsquare", 0x33c1), + ("mokatakana", 0x30e2), + ("mokatakanahalfwidth", 0xff93), + ("molsquare", 0x33d6), + ("momathai", 0x0e21), + ("moverssquare", 0x33a7), + ("moverssquaredsquare", 0x33a8), + ("mparen", 0x24a8), + ("mpasquare", 0x33ab), + ("mssquare", 0x33b3), + ("msuperior", 0xf6ef), + ("mturned", 0x026f), + ("mu", 0x00b5), + ("mu1", 0x00b5), + ("muasquare", 0x3382), + ("muchgreater", 0x226b), + ("muchless", 0x226a), + ("mufsquare", 0x338c), + ("mugreek", 0x03bc), + ("mugsquare", 0x338d), + ("muhiragana", 0x3080), + ("mukatakana", 0x30e0), + ("mukatakanahalfwidth", 0xff91), + ("mulsquare", 0x3395), + ("multicloseleft", 0x22c9), + ("multicloseright", 0x22ca), + ("multimap", 0x22b8), + ("multiopenleft", 0x22cb), + ("multiopenright", 0x22cc), + ("multiply", 0x00d7), + ("mumsquare", 0x339b), + ("munahhebrew", 0x05a3), + ("munahlefthebrew", 0x05a3), + ("musicalnote", 0x266a), + ("musicalnotedbl", 0x266b), + ("musicflatsign", 0x266d), + ("musicsharpsign", 0x266f), + ("mussquare", 0x33b2), + ("muvsquare", 0x33b6), + ("muwsquare", 0x33bc), + ("mvmegasquare", 0x33b9), + ("mvsquare", 0x33b7), + ("mwmegasquare", 0x33bf), + ("mwsquare", 0x33bd), + ("n", 0x006e), + ("nabengali", 0x09a8), + ("nabla", 0x2207), + ("nacute", 0x0144), + ("nadeva", 0x0928), + ("nagujarati", 0x0aa8), + ("nagurmukhi", 0x0a28), + ("nahiragana", 0x306a), + ("nakatakana", 0x30ca), + ("nakatakanahalfwidth", 0xff85), + ("nand", 0x22bc), + ("napostrophe", 0x0149), + ("nasquare", 0x3381), + ("natural", 0x266e), + ("nbopomofo", 0x310b), + ("nbspace", 0x00a0), + ("ncaron", 0x0148), + ("ncedilla", 0x0146), + ("ncircle", 0x24dd), + ("ncircumflexbelow", 0x1e4b), + ("ncommaaccent", 0x0146), + ("ndotaccent", 0x1e45), + ("ndotbelow", 0x1e47), + ("negationslash", 0x0338), + ("nehiragana", 0x306d), + ("nekatakana", 0x30cd), + ("nekatakanahalfwidth", 0xff88), + ("newsheqelsign", 0x20aa), + ("nfsquare", 0x338b), + ("ng", 0x014b), + ("ngabengali", 0x0999), + ("ngadeva", 0x0919), + ("ngagujarati", 0x0a99), + ("ngagurmukhi", 0x0a19), + ("ngonguthai", 0x0e07), + ("nhiragana", 0x3093), + ("nhookleft", 0x0272), + ("nhookretroflex", 0x0273), + ("nieunacirclekorean", 0x326f), + ("nieunaparenkorean", 0x320f), + ("nieuncieuckorean", 0x3135), + ("nieuncirclekorean", 0x3261), + ("nieunhieuhkorean", 0x3136), + ("nieunkorean", 0x3134), + ("nieunpansioskorean", 0x3168), + ("nieunparenkorean", 0x3201), + ("nieunsioskorean", 0x3167), + ("nieuntikeutkorean", 0x3166), + ("nihiragana", 0x306b), + ("nikatakana", 0x30cb), + ("nikatakanahalfwidth", 0xff86), + ("nikhahitleftthai", 0xf899), + ("nikhahitthai", 0x0e4d), + ("nine", 0x0039), + ("ninearabic", 0x0669), + ("ninebengali", 0x09ef), + ("ninecircle", 0x2468), + ("ninecircleinversesansserif", 0x2792), + ("ninedeva", 0x096f), + ("ninegujarati", 0x0aef), + ("ninegurmukhi", 0x0a6f), + ("ninehackarabic", 0x0669), + ("ninehangzhou", 0x3029), + ("nineideographicparen", 0x3228), + ("nineinferior", 0x2089), + ("ninemonospace", 0xff19), + ("nineoldstyle", 0xf739), + ("nineparen", 0x247c), + ("nineperiod", 0x2490), + ("ninepersian", 0x06f9), + ("nineroman", 0x2178), + ("ninesuperior", 0x2079), + ("nineteencircle", 0x2472), + ("nineteenparen", 0x2486), + ("nineteenperiod", 0x249a), + ("ninethai", 0x0e59), + ("nj", 0x01cc), + ("njecyrillic", 0x045a), + ("nkatakana", 0x30f3), + ("nkatakanahalfwidth", 0xff9d), + ("nlegrightlong", 0x019e), + ("nlinebelow", 0x1e49), + ("nmonospace", 0xff4e), + ("nmsquare", 0x339a), + ("nnabengali", 0x09a3), + ("nnadeva", 0x0923), + ("nnagujarati", 0x0aa3), + ("nnagurmukhi", 0x0a23), + ("nnnadeva", 0x0929), + ("nohiragana", 0x306e), + ("nokatakana", 0x30ce), + ("nokatakanahalfwidth", 0xff89), + ("nonbreakingspace", 0x00a0), + ("nonenthai", 0x0e13), + ("nonuthai", 0x0e19), + ("noonarabic", 0x0646), + ("noonfinalarabic", 0xfee6), + ("noonghunnaarabic", 0x06ba), + ("noonghunnafinalarabic", 0xfb9f), + ("noonhehinitialarabic", 0xfee7), + ("nooninitialarabic", 0xfee7), + ("noonjeeminitialarabic", 0xfcd2), + ("noonjeemisolatedarabic", 0xfc4b), + ("noonmedialarabic", 0xfee8), + ("noonmeeminitialarabic", 0xfcd5), + ("noonmeemisolatedarabic", 0xfc4e), + ("noonnoonfinalarabic", 0xfc8d), + ("notapproxequal", 0x2247), + ("notarrowboth", 0x21ae), + ("notarrowleft", 0x219a), + ("notarrowright", 0x219b), + ("notbar", 0x2224), + ("notcontains", 0x220c), + ("notdblarrowboth", 0x21ce), + ("notdblarrowleft", 0x21cd), + ("notdblarrowright", 0x21cf), + ("notelement", 0x2209), + ("notelementof", 0x2209), + ("notequal", 0x2260), + ("notexistential", 0x2204), + ("notfollows", 0x2281), + ("notfollowsoreql", 0x2ab0), + ("notforces", 0x22ae), + ("notforcesextra", 0x22af), + ("notgreater", 0x226f), + ("notgreaterdblequal", 0x2267), + ("notgreaterequal", 0x2271), + ("notgreaternorequal", 0x2271), + ("notgreaternorless", 0x2279), + ("notgreaterorslnteql", 0x2a7e), + ("notidentical", 0x2262), + ("notless", 0x226e), + ("notlessdblequal", 0x2266), + ("notlessequal", 0x2270), + ("notlessnorequal", 0x2270), + ("notlessorslnteql", 0x2a7d), + ("notparallel", 0x2226), + ("notprecedes", 0x2280), + ("notprecedesoreql", 0x2aaf), + ("notsatisfies", 0x22ad), + ("notsimilar", 0x2241), + ("notsubset", 0x2284), + ("notsubseteql", 0x2288), + ("notsubsetordbleql", 0x2ac5), + ("notsubsetoreql", 0x228a), + ("notsucceeds", 0x2281), + ("notsuperset", 0x2285), + ("notsuperseteql", 0x2289), + ("notsupersetordbleql", 0x2ac6), + ("notsupersetoreql", 0x228b), + ("nottriangeqlleft", 0x22ec), + ("nottriangeqlright", 0x22ed), + ("nottriangleleft", 0x22ea), + ("nottriangleright", 0x22eb), + ("notturnstile", 0x22ac), + ("nowarmenian", 0x0576), + ("nparen", 0x24a9), + ("nssquare", 0x33b1), + ("nsuperior", 0x207f), + ("ntilde", 0x00f1), + ("nu", 0x03bd), + ("nuhiragana", 0x306c), + ("nukatakana", 0x30cc), + ("nukatakanahalfwidth", 0xff87), + ("nuktabengali", 0x09bc), + ("nuktadeva", 0x093c), + ("nuktagujarati", 0x0abc), + ("nuktagurmukhi", 0x0a3c), + ("numbersign", 0x0023), + ("numbersignmonospace", 0xff03), + ("numbersignsmall", 0xfe5f), + ("numeralsigngreek", 0x0374), + ("numeralsignlowergreek", 0x0375), + ("numero", 0x2116), + ("nun", 0x05e0), + ("nundagesh", 0xfb40), + ("nundageshhebrew", 0xfb40), + ("nunhebrew", 0x05e0), + ("nvsquare", 0x33b5), + ("nwsquare", 0x33bb), + ("nyabengali", 0x099e), + ("nyadeva", 0x091e), + ("nyagujarati", 0x0a9e), + ("nyagurmukhi", 0x0a1e), + ("o", 0x006f), + ("oacute", 0x00f3), + ("oangthai", 0x0e2d), + ("obarred", 0x0275), + ("obarredcyrillic", 0x04e9), + ("obarreddieresiscyrillic", 0x04eb), + ("obengali", 0x0993), + ("obopomofo", 0x311b), + ("obreve", 0x014f), + ("ocandradeva", 0x0911), + ("ocandragujarati", 0x0a91), + ("ocandravowelsigndeva", 0x0949), + ("ocandravowelsigngujarati", 0x0ac9), + ("ocaron", 0x01d2), + ("ocircle", 0x24de), + ("ocircumflex", 0x00f4), + ("ocircumflexacute", 0x1ed1), + ("ocircumflexdotbelow", 0x1ed9), + ("ocircumflexgrave", 0x1ed3), + ("ocircumflexhookabove", 0x1ed5), + ("ocircumflextilde", 0x1ed7), + ("ocyrillic", 0x043e), + ("odblacute", 0x0151), + ("odblgrave", 0x020d), + ("odeva", 0x0913), + ("odieresis", 0x00f6), + ("odieresiscyrillic", 0x04e7), + ("odotbelow", 0x1ecd), + ("oe", 0x0153), + ("oekorean", 0x315a), + ("ogonek", 0x02db), + ("ogonekcmb", 0x0328), + ("ograve", 0x00f2), + ("ogujarati", 0x0a93), + ("oharmenian", 0x0585), + ("ohiragana", 0x304a), + ("ohookabove", 0x1ecf), + ("ohorn", 0x01a1), + ("ohornacute", 0x1edb), + ("ohorndotbelow", 0x1ee3), + ("ohorngrave", 0x1edd), + ("ohornhookabove", 0x1edf), + ("ohorntilde", 0x1ee1), + ("ohungarumlaut", 0x0151), + ("oi", 0x01a3), + ("oinvertedbreve", 0x020f), + ("okatakana", 0x30aa), + ("okatakanahalfwidth", 0xff75), + ("okorean", 0x3157), + ("olehebrew", 0x05ab), + ("omacron", 0x014d), + ("omacronacute", 0x1e53), + ("omacrongrave", 0x1e51), + ("omdeva", 0x0950), + ("omega", 0x03c9), + ("omega1", 0x03d6), + ("omegacyrillic", 0x0461), + ("omegalatinclosed", 0x0277), + ("omegaroundcyrillic", 0x047b), + ("omegatitlocyrillic", 0x047d), + ("omegatonos", 0x03ce), + ("omgujarati", 0x0ad0), + ("omicron", 0x03bf), + ("omicrontonos", 0x03cc), + ("omonospace", 0xff4f), + ("one", 0x0031), + ("onearabic", 0x0661), + ("onebengali", 0x09e7), + ("onecircle", 0x2460), + ("onecircleinversesansserif", 0x278a), + ("onedeva", 0x0967), + ("onedotenleader", 0x2024), + ("oneeighth", 0x215b), + ("onefitted", 0xf6dc), + ("onegujarati", 0x0ae7), + ("onegurmukhi", 0x0a67), + ("onehackarabic", 0x0661), + ("onehalf", 0x00bd), + ("onehangzhou", 0x3021), + ("oneideographicparen", 0x3220), + ("oneinferior", 0x2081), + ("onemonospace", 0xff11), + ("onenumeratorbengali", 0x09f4), + ("oneoldstyle", 0xf731), + ("oneparen", 0x2474), + ("oneperiod", 0x2488), + ("onepersian", 0x06f1), + ("onequarter", 0x00bc), + ("oneroman", 0x2170), + ("onesuperior", 0x00b9), + ("onethai", 0x0e51), + ("onethird", 0x2153), + ("oogonek", 0x01eb), + ("oogonekmacron", 0x01ed), + ("oogurmukhi", 0x0a13), + ("oomatragurmukhi", 0x0a4b), + ("oopen", 0x0254), + ("oparen", 0x24aa), + ("openbullet", 0x25e6), + ("option", 0x2325), + ("ordfeminine", 0x00aa), + ("ordmasculine", 0x00ba), + ("orthogonal", 0x221f), + ("orunderscore", 0x22bb), + ("oshortdeva", 0x0912), + ("oshortvowelsigndeva", 0x094a), + ("oslash", 0x00f8), + ("oslashacute", 0x01ff), + ("osmallhiragana", 0x3049), + ("osmallkatakana", 0x30a9), + ("osmallkatakanahalfwidth", 0xff6b), + ("ostrokeacute", 0x01ff), + ("osuperior", 0xf6f0), + ("otcyrillic", 0x047f), + ("otilde", 0x00f5), + ("otildeacute", 0x1e4d), + ("otildedieresis", 0x1e4f), + ("oubopomofo", 0x3121), + ("overline", 0x203e), + ("overlinecenterline", 0xfe4a), + ("overlinecmb", 0x0305), + ("overlinedashed", 0xfe49), + ("overlinedblwavy", 0xfe4c), + ("overlinewavy", 0xfe4b), + ("overscore", 0x00af), + ("ovowelsignbengali", 0x09cb), + ("ovowelsigndeva", 0x094b), + ("ovowelsigngujarati", 0x0acb), + ("owner", 0x220b), + ("p", 0x0070), + ("paampssquare", 0x3380), + ("paasentosquare", 0x332b), + ("pabengali", 0x09aa), + ("pacute", 0x1e55), + ("padeva", 0x092a), + ("pagedown", 0x21df), + ("pageup", 0x21de), + ("pagujarati", 0x0aaa), + ("pagurmukhi", 0x0a2a), + ("pahiragana", 0x3071), + ("paiyannoithai", 0x0e2f), + ("pakatakana", 0x30d1), + ("palatalizationcyrilliccmb", 0x0484), + ("palochkacyrillic", 0x04c0), + ("pansioskorean", 0x317f), + ("paragraph", 0x00b6), + ("parallel", 0x2225), + ("parenleft", 0x0028), + ("parenleftBig", 0x0028), + ("parenleftBigg", 0x0028), + ("parenleftaltonearabic", 0xfd3e), + ("parenleftbig", 0x0028), + ("parenleftbigg", 0x0028), + ("parenleftbt", 0xf8ed), + ("parenleftex", 0xf8ec), + ("parenleftinferior", 0x208d), + ("parenleftmath", 0x0028), + ("parenleftmonospace", 0xff08), + ("parenleftsmall", 0xfe59), + ("parenleftsuperior", 0x207d), + ("parenlefttp", 0xf8eb), + ("parenleftvertical", 0xfe35), + ("parenright", 0x0029), + ("parenrightBig", 0x0029), + ("parenrightBigg", 0x0029), + ("parenrightaltonearabic", 0xfd3f), + ("parenrightbig", 0x0029), + ("parenrightbigg", 0x0029), + ("parenrightbt", 0xf8f8), + ("parenrightex", 0xf8f7), + ("parenrightinferior", 0x208e), + ("parenrightmath", 0x0029), + ("parenrightmonospace", 0xff09), + ("parenrightsmall", 0xfe5a), + ("parenrightsuperior", 0x207e), + ("parenrighttp", 0xf8f6), + ("parenrightvertical", 0xfe36), + ("partialdiff", 0x2202), + ("paseqhebrew", 0x05c0), + ("pashtahebrew", 0x0599), + ("pasquare", 0x33a9), + ("patah", 0x05b7), + ("patah11", 0x05b7), + ("patah1d", 0x05b7), + ("patah2a", 0x05b7), + ("patahhebrew", 0x05b7), + ("patahnarrowhebrew", 0x05b7), + ("patahquarterhebrew", 0x05b7), + ("patahwidehebrew", 0x05b7), + ("pazerhebrew", 0x05a1), + ("pbopomofo", 0x3106), + ("pcircle", 0x24df), + ("pdotaccent", 0x1e57), + ("pe", 0x05e4), + ("pecyrillic", 0x043f), + ("pedagesh", 0xfb44), + ("pedageshhebrew", 0xfb44), + ("peezisquare", 0x333b), + ("pefinaldageshhebrew", 0xfb43), + ("peharabic", 0x067e), + ("peharmenian", 0x057a), + ("pehebrew", 0x05e4), + ("pehfinalarabic", 0xfb57), + ("pehinitialarabic", 0xfb58), + ("pehiragana", 0x307a), + ("pehmedialarabic", 0xfb59), + ("pekatakana", 0x30da), + ("pemiddlehookcyrillic", 0x04a7), + ("perafehebrew", 0xfb4e), + ("percent", 0x0025), + ("percentarabic", 0x066a), + ("percentmonospace", 0xff05), + ("percentsmall", 0xfe6a), + ("period", 0x002e), + ("periodarmenian", 0x0589), + ("periodcentered", 0x00b7), + ("periodhalfwidth", 0xff61), + ("periodinferior", 0xf6e7), + ("periodmonospace", 0xff0e), + ("periodsmall", 0xfe52), + ("periodsuperior", 0xf6e8), + ("perispomenigreekcmb", 0x0342), + ("perpcorrespond", 0x2a5e), + ("perpendicular", 0x22a5), + ("pertenthousand", 0x2031), + ("perthousand", 0x2030), + ("peseta", 0x20a7), + ("pfsquare", 0x338a), + ("phabengali", 0x09ab), + ("phadeva", 0x092b), + ("phagujarati", 0x0aab), + ("phagurmukhi", 0x0a2b), + ("phi", 0x03c6), + ("phi1", 0x03d5), + ("phieuphacirclekorean", 0x327a), + ("phieuphaparenkorean", 0x321a), + ("phieuphcirclekorean", 0x326c), + ("phieuphkorean", 0x314d), + ("phieuphparenkorean", 0x320c), + ("philatin", 0x0278), + ("phinthuthai", 0x0e3a), + ("phisymbolgreek", 0x03d5), + ("phook", 0x01a5), + ("phophanthai", 0x0e1e), + ("phophungthai", 0x0e1c), + ("phosamphaothai", 0x0e20), + ("pi", 0x03c0), + ("pi1", 0x03d6), + ("pieupacirclekorean", 0x3273), + ("pieupaparenkorean", 0x3213), + ("pieupcieuckorean", 0x3176), + ("pieupcirclekorean", 0x3265), + ("pieupkiyeokkorean", 0x3172), + ("pieupkorean", 0x3142), + ("pieupparenkorean", 0x3205), + ("pieupsioskiyeokkorean", 0x3174), + ("pieupsioskorean", 0x3144), + ("pieupsiostikeutkorean", 0x3175), + ("pieupthieuthkorean", 0x3177), + ("pieuptikeutkorean", 0x3173), + ("pihiragana", 0x3074), + ("pikatakana", 0x30d4), + ("pisymbolgreek", 0x03d6), + ("piwrarmenian", 0x0583), + ("planckover2pi", 0x210f), + ("planckover2pi1", 0x210f), + ("plus", 0x002b), + ("plusbelowcmb", 0x031f), + ("pluscircle", 0x2295), + ("plusmath", 0x002b), + ("plusminus", 0x00b1), + ("plusmod", 0x02d6), + ("plusmonospace", 0xff0b), + ("plussmall", 0xfe62), + ("plussuperior", 0x207a), + ("pmonospace", 0xff50), + ("pmsquare", 0x33d8), + ("pohiragana", 0x307d), + ("pointingindexdownwhite", 0x261f), + ("pointingindexleftwhite", 0x261c), + ("pointingindexrightwhite", 0x261e), + ("pointingindexupwhite", 0x261d), + ("pokatakana", 0x30dd), + ("poplathai", 0x0e1b), + ("postalmark", 0x3012), + ("postalmarkface", 0x3020), + ("pparen", 0x24ab), + ("precedenotdbleqv", 0x2ab9), + ("precedenotslnteql", 0x2ab5), + ("precedeornoteqvlnt", 0x22e8), + ("precedes", 0x227a), + ("precedesequal", 0x227c), + ("precedesorcurly", 0x227c), + ("precedesorequal", 0x227e), + ("prescription", 0x211e), + ("prime", 0x2032), + ("primemod", 0x02b9), + ("primereverse", 0x2035), + ("primereversed", 0x2035), + ("product", 0x220f), + ("productdisplay", 0x220f), + ("producttext", 0x220f), + ("projective", 0x2305), + ("prolongedkana", 0x30fc), + ("propellor", 0x2318), + ("propersubset", 0x2282), + ("propersuperset", 0x2283), + ("proportion", 0x2237), + ("proportional", 0x221d), + ("psi", 0x03c8), + ("psicyrillic", 0x0471), + ("psilipneumatacyrilliccmb", 0x0486), + ("pssquare", 0x33b0), + ("puhiragana", 0x3077), + ("pukatakana", 0x30d7), + ("punctdash", 0x2014), + ("pvsquare", 0x33b4), + ("pwsquare", 0x33ba), + ("q", 0x0071), + ("qadeva", 0x0958), + ("qadmahebrew", 0x05a8), + ("qafarabic", 0x0642), + ("qaffinalarabic", 0xfed6), + ("qafinitialarabic", 0xfed7), + ("qafmedialarabic", 0xfed8), + ("qamats", 0x05b8), + ("qamats10", 0x05b8), + ("qamats1a", 0x05b8), + ("qamats1c", 0x05b8), + ("qamats27", 0x05b8), + ("qamats29", 0x05b8), + ("qamats33", 0x05b8), + ("qamatsde", 0x05b8), + ("qamatshebrew", 0x05b8), + ("qamatsnarrowhebrew", 0x05b8), + ("qamatsqatanhebrew", 0x05b8), + ("qamatsqatannarrowhebrew", 0x05b8), + ("qamatsqatanquarterhebrew", 0x05b8), + ("qamatsqatanwidehebrew", 0x05b8), + ("qamatsquarterhebrew", 0x05b8), + ("qamatswidehebrew", 0x05b8), + ("qarneyparahebrew", 0x059f), + ("qbopomofo", 0x3111), + ("qcircle", 0x24e0), + ("qhook", 0x02a0), + ("qmonospace", 0xff51), + ("qof", 0x05e7), + ("qofdagesh", 0xfb47), + ("qofdageshhebrew", 0xfb47), + ("qofhatafpatah", 0x05e7), + ("qofhatafpatahhebrew", 0x05e7), + ("qofhatafsegol", 0x05e7), + ("qofhatafsegolhebrew", 0x05e7), + ("qofhebrew", 0x05e7), + ("qofhiriq", 0x05e7), + ("qofhiriqhebrew", 0x05e7), + ("qofholam", 0x05e7), + ("qofholamhebrew", 0x05e7), + ("qofpatah", 0x05e7), + ("qofpatahhebrew", 0x05e7), + ("qofqamats", 0x05e7), + ("qofqamatshebrew", 0x05e7), + ("qofqubuts", 0x05e7), + ("qofqubutshebrew", 0x05e7), + ("qofsegol", 0x05e7), + ("qofsegolhebrew", 0x05e7), + ("qofsheva", 0x05e7), + ("qofshevahebrew", 0x05e7), + ("qoftsere", 0x05e7), + ("qoftserehebrew", 0x05e7), + ("qparen", 0x24ac), + ("quarternote", 0x2669), + ("qubuts", 0x05bb), + ("qubuts18", 0x05bb), + ("qubuts25", 0x05bb), + ("qubuts31", 0x05bb), + ("qubutshebrew", 0x05bb), + ("qubutsnarrowhebrew", 0x05bb), + ("qubutsquarterhebrew", 0x05bb), + ("qubutswidehebrew", 0x05bb), + ("question", 0x003f), + ("questionarabic", 0x061f), + ("questionarmenian", 0x055e), + ("questiondown", 0x00bf), + ("questiondownsmall", 0xf7bf), + ("questiongreek", 0x037e), + ("questionmonospace", 0xff1f), + ("questionsmall", 0xf73f), + ("quotedbl", 0x0022), + ("quotedblbase", 0x201e), + ("quotedblleft", 0x201c), + ("quotedblmonospace", 0xff02), + ("quotedblprime", 0x301e), + ("quotedblprimereversed", 0x301d), + ("quotedblright", 0x201d), + ("quoteleft", 0x2018), + ("quoteleftreversed", 0x201b), + ("quotereversed", 0x201b), + ("quoteright", 0x2019), + ("quoterightn", 0x0149), + ("quotesinglbase", 0x201a), + ("quotesingle", 0x0027), + ("quotesinglemonospace", 0xff07), + ("r", 0x0072), + ("raarmenian", 0x057c), + ("rabengali", 0x09b0), + ("racute", 0x0155), + ("radeva", 0x0930), + ("radical", 0x221a), + ("radicalBig", 0x221a), + ("radicalBigg", 0x221a), + ("radicalbig", 0x221a), + ("radicalbigg", 0x221a), + ("radicalbt", 0x221a), + ("radicalex", 0xf8e5), + ("radicaltp", 0x221a), + ("radicalvertex", 0x221a), + ("radoverssquare", 0x33ae), + ("radoverssquaredsquare", 0x33af), + ("radsquare", 0x33ad), + ("rafe", 0x05bf), + ("rafehebrew", 0x05bf), + ("ragujarati", 0x0ab0), + ("ragurmukhi", 0x0a30), + ("rahiragana", 0x3089), + ("rakatakana", 0x30e9), + ("rakatakanahalfwidth", 0xff97), + ("ralowerdiagonalbengali", 0x09f1), + ("ramiddlediagonalbengali", 0x09f0), + ("ramshorn", 0x0264), + ("rangedash", 0x2013), + ("ratio", 0x2236), + ("rbopomofo", 0x3116), + ("rcaron", 0x0159), + ("rcedilla", 0x0157), + ("rcircle", 0x24e1), + ("rcommaaccent", 0x0157), + ("rdblgrave", 0x0211), + ("rdotaccent", 0x1e59), + ("rdotbelow", 0x1e5b), + ("rdotbelowmacron", 0x1e5d), + ("referencemark", 0x203b), + ("reflexsubset", 0x2286), + ("reflexsuperset", 0x2287), + ("registered", 0x00ae), + ("registersans", 0xf8e8), + ("registerserif", 0xf6da), + ("reharabic", 0x0631), + ("reharmenian", 0x0580), + ("rehfinalarabic", 0xfeae), + ("rehiragana", 0x308c), + ("rehyehaleflamarabic", 0x0631), + ("rekatakana", 0x30ec), + ("rekatakanahalfwidth", 0xff9a), + ("resh", 0x05e8), + ("reshdageshhebrew", 0xfb48), + ("reshhatafpatah", 0x05e8), + ("reshhatafpatahhebrew", 0x05e8), + ("reshhatafsegol", 0x05e8), + ("reshhatafsegolhebrew", 0x05e8), + ("reshhebrew", 0x05e8), + ("reshhiriq", 0x05e8), + ("reshhiriqhebrew", 0x05e8), + ("reshholam", 0x05e8), + ("reshholamhebrew", 0x05e8), + ("reshpatah", 0x05e8), + ("reshpatahhebrew", 0x05e8), + ("reshqamats", 0x05e8), + ("reshqamatshebrew", 0x05e8), + ("reshqubuts", 0x05e8), + ("reshqubutshebrew", 0x05e8), + ("reshsegol", 0x05e8), + ("reshsegolhebrew", 0x05e8), + ("reshsheva", 0x05e8), + ("reshshevahebrew", 0x05e8), + ("reshtsere", 0x05e8), + ("reshtserehebrew", 0x05e8), + ("revasymptequal", 0x22cd), + ("reversedtilde", 0x223d), + ("reviahebrew", 0x0597), + ("reviamugrashhebrew", 0x0597), + ("revlogicalnot", 0x2310), + ("revsimilar", 0x223d), + ("rfishhook", 0x027e), + ("rfishhookreversed", 0x027f), + ("rhabengali", 0x09dd), + ("rhadeva", 0x095d), + ("rho", 0x03c1), + ("rho1", 0x03f1), + ("rhook", 0x027d), + ("rhookturned", 0x027b), + ("rhookturnedsuperior", 0x02b5), + ("rhosymbolgreek", 0x03f1), + ("rhotichookmod", 0x02de), + ("rieulacirclekorean", 0x3271), + ("rieulaparenkorean", 0x3211), + ("rieulcirclekorean", 0x3263), + ("rieulhieuhkorean", 0x3140), + ("rieulkiyeokkorean", 0x313a), + ("rieulkiyeoksioskorean", 0x3169), + ("rieulkorean", 0x3139), + ("rieulmieumkorean", 0x313b), + ("rieulpansioskorean", 0x316c), + ("rieulparenkorean", 0x3203), + ("rieulphieuphkorean", 0x313f), + ("rieulpieupkorean", 0x313c), + ("rieulpieupsioskorean", 0x316b), + ("rieulsioskorean", 0x313d), + ("rieulthieuthkorean", 0x313e), + ("rieultikeutkorean", 0x316a), + ("rieulyeorinhieuhkorean", 0x316d), + ("rightangle", 0x221f), + ("rightanglene", 0x231d), + ("rightanglenw", 0x231c), + ("rightanglese", 0x231f), + ("rightanglesw", 0x231e), + ("righttackbelowcmb", 0x0319), + ("righttriangle", 0x22bf), + ("rihiragana", 0x308a), + ("rikatakana", 0x30ea), + ("rikatakanahalfwidth", 0xff98), + ("ring", 0x02da), + ("ringbelowcmb", 0x0325), + ("ringcmb", 0x030a), + ("ringfitted", 0xd80d), + ("ringhalfleft", 0x02bf), + ("ringhalfleftarmenian", 0x0559), + ("ringhalfleftbelowcmb", 0x031c), + ("ringhalfleftcentered", 0x02d3), + ("ringhalfright", 0x02be), + ("ringhalfrightbelowcmb", 0x0339), + ("ringhalfrightcentered", 0x02d2), + ("ringinequal", 0x2256), + ("rinvertedbreve", 0x0213), + ("rittorusquare", 0x3351), + ("rlinebelow", 0x1e5f), + ("rlongleg", 0x027c), + ("rlonglegturned", 0x027a), + ("rmonospace", 0xff52), + ("rohiragana", 0x308d), + ("rokatakana", 0x30ed), + ("rokatakanahalfwidth", 0xff9b), + ("roruathai", 0x0e23), + ("rparen", 0x24ad), + ("rrabengali", 0x09dc), + ("rradeva", 0x0931), + ("rragurmukhi", 0x0a5c), + ("rreharabic", 0x0691), + ("rrehfinalarabic", 0xfb8d), + ("rrvocalicbengali", 0x09e0), + ("rrvocalicdeva", 0x0960), + ("rrvocalicgujarati", 0x0ae0), + ("rrvocalicvowelsignbengali", 0x09c4), + ("rrvocalicvowelsigndeva", 0x0944), + ("rrvocalicvowelsigngujarati", 0x0ac4), + ("rsuperior", 0xf6f1), + ("rtblock", 0x2590), + ("rturned", 0x0279), + ("rturnedsuperior", 0x02b4), + ("ruhiragana", 0x308b), + ("rukatakana", 0x30eb), + ("rukatakanahalfwidth", 0xff99), + ("rupeemarkbengali", 0x09f2), + ("rupeesignbengali", 0x09f3), + ("rupiah", 0xf6dd), + ("ruthai", 0x0e24), + ("rvocalicbengali", 0x098b), + ("rvocalicdeva", 0x090b), + ("rvocalicgujarati", 0x0a8b), + ("rvocalicvowelsignbengali", 0x09c3), + ("rvocalicvowelsigndeva", 0x0943), + ("rvocalicvowelsigngujarati", 0x0ac3), + ("s", 0x0073), + ("sabengali", 0x09b8), + ("sacute", 0x015b), + ("sacutedotaccent", 0x1e65), + ("sadarabic", 0x0635), + ("sadeva", 0x0938), + ("sadfinalarabic", 0xfeba), + ("sadinitialarabic", 0xfebb), + ("sadmedialarabic", 0xfebc), + ("sagujarati", 0x0ab8), + ("sagurmukhi", 0x0a38), + ("sahiragana", 0x3055), + ("sakatakana", 0x30b5), + ("sakatakanahalfwidth", 0xff7b), + ("sallallahoualayhewasallamarabic", 0xfdfa), + ("samekh", 0x05e1), + ("samekhdagesh", 0xfb41), + ("samekhdageshhebrew", 0xfb41), + ("samekhhebrew", 0x05e1), + ("saraaathai", 0x0e32), + ("saraaethai", 0x0e41), + ("saraaimaimalaithai", 0x0e44), + ("saraaimaimuanthai", 0x0e43), + ("saraamthai", 0x0e33), + ("saraathai", 0x0e30), + ("saraethai", 0x0e40), + ("saraiileftthai", 0xf886), + ("saraiithai", 0x0e35), + ("saraileftthai", 0xf885), + ("saraithai", 0x0e34), + ("saraothai", 0x0e42), + ("saraueeleftthai", 0xf888), + ("saraueethai", 0x0e37), + ("saraueleftthai", 0xf887), + ("sarauethai", 0x0e36), + ("sarauthai", 0x0e38), + ("sarauuthai", 0x0e39), + ("satisfies", 0x22a8), + ("sbopomofo", 0x3119), + ("scaron", 0x0161), + ("scarondotaccent", 0x1e67), + ("scedilla", 0x015f), + ("schwa", 0x0259), + ("schwacyrillic", 0x04d9), + ("schwadieresiscyrillic", 0x04db), + ("schwahook", 0x025a), + ("scircle", 0x24e2), + ("scircumflex", 0x015d), + ("scommaaccent", 0x0219), + ("sdotaccent", 0x1e61), + ("sdotbelow", 0x1e63), + ("sdotbelowdotaccent", 0x1e69), + ("seagullbelowcmb", 0x033c), + ("second", 0x2033), + ("secondtonechinese", 0x02ca), + ("section", 0x00a7), + ("seenarabic", 0x0633), + ("seenfinalarabic", 0xfeb2), + ("seeninitialarabic", 0xfeb3), + ("seenmedialarabic", 0xfeb4), + ("segol", 0x05b6), + ("segol13", 0x05b6), + ("segol1f", 0x05b6), + ("segol2c", 0x05b6), + ("segolhebrew", 0x05b6), + ("segolnarrowhebrew", 0x05b6), + ("segolquarterhebrew", 0x05b6), + ("segoltahebrew", 0x0592), + ("segolwidehebrew", 0x05b6), + ("seharmenian", 0x057d), + ("sehiragana", 0x305b), + ("sekatakana", 0x30bb), + ("sekatakanahalfwidth", 0xff7e), + ("semicolon", 0x003b), + ("semicolonarabic", 0x061b), + ("semicolonmonospace", 0xff1b), + ("semicolonsmall", 0xfe54), + ("semivoicedmarkkana", 0x309c), + ("semivoicedmarkkanahalfwidth", 0xff9f), + ("sentisquare", 0x3322), + ("sentosquare", 0x3323), + ("seven", 0x0037), + ("sevenarabic", 0x0667), + ("sevenbengali", 0x09ed), + ("sevencircle", 0x2466), + ("sevencircleinversesansserif", 0x2790), + ("sevendeva", 0x096d), + ("seveneighths", 0x215e), + ("sevengujarati", 0x0aed), + ("sevengurmukhi", 0x0a6d), + ("sevenhackarabic", 0x0667), + ("sevenhangzhou", 0x3027), + ("sevenideographicparen", 0x3226), + ("seveninferior", 0x2087), + ("sevenmonospace", 0xff17), + ("sevenoldstyle", 0xf737), + ("sevenparen", 0x247a), + ("sevenperiod", 0x248e), + ("sevenpersian", 0x06f7), + ("sevenroman", 0x2176), + ("sevensuperior", 0x2077), + ("seventeencircle", 0x2470), + ("seventeenparen", 0x2484), + ("seventeenperiod", 0x2498), + ("seventhai", 0x0e57), + ("sfthyphen", 0x00ad), + ("shaarmenian", 0x0577), + ("shabengali", 0x09b6), + ("shacyrillic", 0x0448), + ("shaddaarabic", 0x0651), + ("shaddadammaarabic", 0xfc61), + ("shaddadammatanarabic", 0xfc5e), + ("shaddafathaarabic", 0xfc60), + ("shaddafathatanarabic", 0x0651), + ("shaddakasraarabic", 0xfc62), + ("shaddakasratanarabic", 0xfc5f), + ("shade", 0x2592), + ("shadedark", 0x2593), + ("shadelight", 0x2591), + ("shademedium", 0x2592), + ("shadeva", 0x0936), + ("shagujarati", 0x0ab6), + ("shagurmukhi", 0x0a36), + ("shalshelethebrew", 0x0593), + ("sharp", 0x266f), + ("shbopomofo", 0x3115), + ("shchacyrillic", 0x0449), + ("sheenarabic", 0x0634), + ("sheenfinalarabic", 0xfeb6), + ("sheeninitialarabic", 0xfeb7), + ("sheenmedialarabic", 0xfeb8), + ("sheicoptic", 0x03e3), + ("sheqel", 0x20aa), + ("sheqelhebrew", 0x20aa), + ("sheva", 0x05b0), + ("sheva115", 0x05b0), + ("sheva15", 0x05b0), + ("sheva22", 0x05b0), + ("sheva2e", 0x05b0), + ("shevahebrew", 0x05b0), + ("shevanarrowhebrew", 0x05b0), + ("shevaquarterhebrew", 0x05b0), + ("shevawidehebrew", 0x05b0), + ("shhacyrillic", 0x04bb), + ("shiftleft", 0x21b0), + ("shiftright", 0x21b1), + ("shimacoptic", 0x03ed), + ("shin", 0x05e9), + ("shindagesh", 0xfb49), + ("shindageshhebrew", 0xfb49), + ("shindageshshindot", 0xfb2c), + ("shindageshshindothebrew", 0xfb2c), + ("shindageshsindot", 0xfb2d), + ("shindageshsindothebrew", 0xfb2d), + ("shindothebrew", 0x05c1), + ("shinhebrew", 0x05e9), + ("shinshindot", 0xfb2a), + ("shinshindothebrew", 0xfb2a), + ("shinsindot", 0xfb2b), + ("shinsindothebrew", 0xfb2b), + ("shook", 0x0282), + ("sigma", 0x03c3), + ("sigma1", 0x03c2), + ("sigmafinal", 0x03c2), + ("sigmalunatesymbolgreek", 0x03f2), + ("sihiragana", 0x3057), + ("sikatakana", 0x30b7), + ("sikatakanahalfwidth", 0xff7c), + ("siluqhebrew", 0x05bd), + ("siluqlefthebrew", 0x05bd), + ("similar", 0x223c), + ("similarequal", 0x2243), + ("sindothebrew", 0x05c2), + ("siosacirclekorean", 0x3274), + ("siosaparenkorean", 0x3214), + ("sioscieuckorean", 0x317e), + ("sioscirclekorean", 0x3266), + ("sioskiyeokkorean", 0x317a), + ("sioskorean", 0x3145), + ("siosnieunkorean", 0x317b), + ("siosparenkorean", 0x3206), + ("siospieupkorean", 0x317d), + ("siostikeutkorean", 0x317c), + ("six", 0x0036), + ("sixarabic", 0x0666), + ("sixbengali", 0x09ec), + ("sixcircle", 0x2465), + ("sixcircleinversesansserif", 0x278f), + ("sixdeva", 0x096c), + ("sixgujarati", 0x0aec), + ("sixgurmukhi", 0x0a6c), + ("sixhackarabic", 0x0666), + ("sixhangzhou", 0x3026), + ("sixideographicparen", 0x3225), + ("sixinferior", 0x2086), + ("sixmonospace", 0xff16), + ("sixoldstyle", 0xf736), + ("sixparen", 0x2479), + ("sixperiod", 0x248d), + ("sixpersian", 0x06f6), + ("sixroman", 0x2175), + ("sixsuperior", 0x2076), + ("sixteencircle", 0x246f), + ("sixteencurrencydenominatorbengali", 0x09f9), + ("sixteenparen", 0x2483), + ("sixteenperiod", 0x2497), + ("sixthai", 0x0e56), + ("slash", 0x002f), + ("slashBig", 0x2215), + ("slashBigg", 0x2215), + ("slashbig", 0x2215), + ("slashbigg", 0x2215), + ("slashmonospace", 0xff0f), + ("slong", 0x017f), + ("slongdotaccent", 0x1e9b), + ("slurabove", 0x2322), + ("slurbelow", 0x2323), + ("smile", 0x2323), + ("smileface", 0x263a), + ("smonospace", 0xff53), + ("sofpasuqhebrew", 0x05c3), + ("softhyphen", 0x00ad), + ("softsigncyrillic", 0x044c), + ("sohiragana", 0x305d), + ("sokatakana", 0x30bd), + ("sokatakanahalfwidth", 0xff7f), + ("soliduslongoverlaycmb", 0x0338), + ("solidusshortoverlaycmb", 0x0337), + ("sorusithai", 0x0e29), + ("sosalathai", 0x0e28), + ("sosothai", 0x0e0b), + ("sosuathai", 0x0e2a), + ("space", 0x0020), + ("spacehackarabic", 0x0020), + ("spade", 0x2660), + ("spadesuitblack", 0x2660), + ("spadesuitwhite", 0x2664), + ("sparen", 0x24ae), + ("sphericalangle", 0x2222), + ("square", 0x25a1), + ("squarebelowcmb", 0x033b), + ("squarecc", 0x33c4), + ("squarecm", 0x339d), + ("squarediagonalcrosshatchfill", 0x25a9), + ("squaredot", 0x22a1), + ("squarehorizontalfill", 0x25a4), + ("squareimage", 0x228f), + ("squarekg", 0x338f), + ("squarekm", 0x339e), + ("squarekmcapital", 0x33ce), + ("squareln", 0x33d1), + ("squarelog", 0x33d2), + ("squaremg", 0x338e), + ("squaremil", 0x33d5), + ("squareminus", 0x229f), + ("squaremm", 0x339c), + ("squaremsquared", 0x33a1), + ("squaremultiply", 0x22a0), + ("squareoriginal", 0x2290), + ("squareorthogonalcrosshatchfill", 0x25a6), + ("squareplus", 0x229e), + ("squaresolid", 0x25a0), + ("squareupperlefttolowerrightfill", 0x25a7), + ("squareupperrighttolowerleftfill", 0x25a8), + ("squareverticalfill", 0x25a5), + ("squarewhitewithsmallblack", 0x25a3), + ("squiggleleftright", 0x21ad), + ("squiggleright", 0x21dd), + ("srsquare", 0x33db), + ("ssabengali", 0x09b7), + ("ssadeva", 0x0937), + ("ssagujarati", 0x0ab7), + ("ssangcieuckorean", 0x3149), + ("ssanghieuhkorean", 0x3185), + ("ssangieungkorean", 0x3180), + ("ssangkiyeokkorean", 0x3132), + ("ssangnieunkorean", 0x3165), + ("ssangpieupkorean", 0x3143), + ("ssangsioskorean", 0x3146), + ("ssangtikeutkorean", 0x3138), + ("ssuperior", 0xf6f2), + ("st", 0xfb06), + ("star", 0x22c6), + ("sterling", 0x00a3), + ("sterlingmonospace", 0xffe1), + ("strokelongoverlaycmb", 0x0336), + ("strokeshortoverlaycmb", 0x0335), + ("subset", 0x2282), + ("subsetdbl", 0x22d0), + ("subsetdblequal", 0x2ac5), + ("subsetnoteql", 0x228a), + ("subsetnotequal", 0x228a), + ("subsetorequal", 0x2286), + ("subsetornotdbleql", 0x2acb), + ("subsetsqequal", 0x2291), + ("succeeds", 0x227b), + ("suchthat", 0x220b), + ("suhiragana", 0x3059), + ("sukatakana", 0x30b9), + ("sukatakanahalfwidth", 0xff7d), + ("sukunarabic", 0x0652), + ("summation", 0x2211), + ("summationdisplay", 0x2211), + ("summationtext", 0x2211), + ("sun", 0x263c), + ("superset", 0x2283), + ("supersetdbl", 0x22d1), + ("supersetdblequal", 0x2ac6), + ("supersetnoteql", 0x228b), + ("supersetnotequal", 0x228b), + ("supersetorequal", 0x2287), + ("supersetornotdbleql", 0x2acc), + ("supersetsqequal", 0x2292), + ("svsquare", 0x33dc), + ("syouwaerasquare", 0x337c), + ("t", 0x0074), + ("tabengali", 0x09a4), + ("tackdown", 0x22a4), + ("tackleft", 0x22a3), + ("tadeva", 0x0924), + ("tagujarati", 0x0aa4), + ("tagurmukhi", 0x0a24), + ("taharabic", 0x0637), + ("tahfinalarabic", 0xfec2), + ("tahinitialarabic", 0xfec3), + ("tahiragana", 0x305f), + ("tahmedialarabic", 0xfec4), + ("taisyouerasquare", 0x337d), + ("takatakana", 0x30bf), + ("takatakanahalfwidth", 0xff80), + ("tatweelarabic", 0x0640), + ("tau", 0x03c4), + ("tav", 0x05ea), + ("tavdages", 0xfb4a), + ("tavdagesh", 0xfb4a), + ("tavdageshhebrew", 0xfb4a), + ("tavhebrew", 0x05ea), + ("tbar", 0x0167), + ("tbopomofo", 0x310a), + ("tcaron", 0x0165), + ("tccurl", 0x02a8), + ("tcedilla", 0x0163), + ("tcheharabic", 0x0686), + ("tchehfinalarabic", 0xfb7b), + ("tchehinitialarabic", 0xfb7c), + ("tchehmedialarabic", 0xfb7d), + ("tchehmeeminitialarabic", 0xfb7c), + ("tcircle", 0x24e3), + ("tcircumflexbelow", 0x1e71), + ("tcommaaccent", 0x0163), + ("tdieresis", 0x1e97), + ("tdotaccent", 0x1e6b), + ("tdotbelow", 0x1e6d), + ("tecyrillic", 0x0442), + ("tedescendercyrillic", 0x04ad), + ("teharabic", 0x062a), + ("tehfinalarabic", 0xfe96), + ("tehhahinitialarabic", 0xfca2), + ("tehhahisolatedarabic", 0xfc0c), + ("tehinitialarabic", 0xfe97), + ("tehiragana", 0x3066), + ("tehjeeminitialarabic", 0xfca1), + ("tehjeemisolatedarabic", 0xfc0b), + ("tehmarbutaarabic", 0x0629), + ("tehmarbutafinalarabic", 0xfe94), + ("tehmedialarabic", 0xfe98), + ("tehmeeminitialarabic", 0xfca4), + ("tehmeemisolatedarabic", 0xfc0e), + ("tehnoonfinalarabic", 0xfc73), + ("tekatakana", 0x30c6), + ("tekatakanahalfwidth", 0xff83), + ("telephone", 0x2121), + ("telephoneblack", 0x260e), + ("telishagedolahebrew", 0x05a0), + ("telishaqetanahebrew", 0x05a9), + ("tencircle", 0x2469), + ("tenideographicparen", 0x3229), + ("tenparen", 0x247d), + ("tenperiod", 0x2491), + ("tenroman", 0x2179), + ("tesh", 0x02a7), + ("tet", 0x05d8), + ("tetdagesh", 0xfb38), + ("tetdageshhebrew", 0xfb38), + ("tethebrew", 0x05d8), + ("tetsecyrillic", 0x04b5), + ("tevirhebrew", 0x059b), + ("tevirlefthebrew", 0x059b), + ("thabengali", 0x09a5), + ("thadeva", 0x0925), + ("thagujarati", 0x0aa5), + ("thagurmukhi", 0x0a25), + ("thalarabic", 0x0630), + ("thalfinalarabic", 0xfeac), + ("thanthakhatlowleftthai", 0xf898), + ("thanthakhatlowrightthai", 0xf897), + ("thanthakhatthai", 0x0e4c), + ("thanthakhatupperleftthai", 0xf896), + ("theharabic", 0x062b), + ("thehfinalarabic", 0xfe9a), + ("thehinitialarabic", 0xfe9b), + ("thehmedialarabic", 0xfe9c), + ("thereexists", 0x2203), + ("therefore", 0x2234), + ("theta", 0x03b8), + ("theta1", 0x03d1), + ("thetasymbolgreek", 0x03d1), + ("thieuthacirclekorean", 0x3279), + ("thieuthaparenkorean", 0x3219), + ("thieuthcirclekorean", 0x326b), + ("thieuthkorean", 0x314c), + ("thieuthparenkorean", 0x320b), + ("thirteencircle", 0x246c), + ("thirteenparen", 0x2480), + ("thirteenperiod", 0x2494), + ("thonangmonthothai", 0x0e11), + ("thook", 0x01ad), + ("thophuthaothai", 0x0e12), + ("thorn", 0x00fe), + ("thothahanthai", 0x0e17), + ("thothanthai", 0x0e10), + ("thothongthai", 0x0e18), + ("thothungthai", 0x0e16), + ("thousandcyrillic", 0x0482), + ("thousandsseparatorarabic", 0x066c), + ("thousandsseparatorpersian", 0x066c), + ("three", 0x0033), + ("threearabic", 0x0663), + ("threebengali", 0x09e9), + ("threecircle", 0x2462), + ("threecircleinversesansserif", 0x278c), + ("threedeva", 0x0969), + ("threeeighths", 0x215c), + ("threegujarati", 0x0ae9), + ("threegurmukhi", 0x0a69), + ("threehackarabic", 0x0663), + ("threehangzhou", 0x3023), + ("threeideographicparen", 0x3222), + ("threeinferior", 0x2083), + ("threemonospace", 0xff13), + ("threenumeratorbengali", 0x09f6), + ("threeoldstyle", 0xf733), + ("threeparen", 0x2476), + ("threeperiod", 0x248a), + ("threepersian", 0x06f3), + ("threequarters", 0x00be), + ("threequartersemdash", 0xf6de), + ("threeroman", 0x2172), + ("threesuperior", 0x00b3), + ("threethai", 0x0e53), + ("thzsquare", 0x3394), + ("tie", 0x2040), + ("tihiragana", 0x3061), + ("tikatakana", 0x30c1), + ("tikatakanahalfwidth", 0xff81), + ("tikeutacirclekorean", 0x3270), + ("tikeutaparenkorean", 0x3210), + ("tikeutcirclekorean", 0x3262), + ("tikeutkorean", 0x3137), + ("tikeutparenkorean", 0x3202), + ("tilde", 0x02dc), + ("tildebelowcmb", 0x0330), + ("tildecmb", 0x0303), + ("tildecomb", 0x0303), + ("tildedoublecmb", 0x0360), + ("tildeoperator", 0x223c), + ("tildeoverlaycmb", 0x0334), + ("tildeverticalcmb", 0x033e), + ("tildewide", 0x0303), + ("tildewider", 0x0303), + ("tildewiderr", 0x0303), + ("tildewidest", 0x02dc), + ("timescircle", 0x2297), + ("tipehahebrew", 0x0596), + ("tipehalefthebrew", 0x0596), + ("tippigurmukhi", 0x0a70), + ("titlocyrilliccmb", 0x0483), + ("tiwnarmenian", 0x057f), + ("tlinebelow", 0x1e6f), + ("tmonospace", 0xff54), + ("toarmenian", 0x0569), + ("tohiragana", 0x3068), + ("tokatakana", 0x30c8), + ("tokatakanahalfwidth", 0xff84), + ("tonebarextrahighmod", 0x02e5), + ("tonebarextralowmod", 0x02e9), + ("tonebarhighmod", 0x02e6), + ("tonebarlowmod", 0x02e8), + ("tonebarmidmod", 0x02e7), + ("tonefive", 0x01bd), + ("tonesix", 0x0185), + ("tonetwo", 0x01a8), + ("tonos", 0x0384), + ("tonsquare", 0x3327), + ("topatakthai", 0x0e0f), + ("tortoiseshellbracketleft", 0x3014), + ("tortoiseshellbracketleftsmall", 0xfe5d), + ("tortoiseshellbracketleftvertical", 0xfe39), + ("tortoiseshellbracketright", 0x3015), + ("tortoiseshellbracketrightsmall", 0xfe5e), + ("tortoiseshellbracketrightvertical", 0xfe3a), + ("totaothai", 0x0e15), + ("tpalatalhook", 0x01ab), + ("tparen", 0x24af), + ("trademark", 0x2122), + ("trademarksans", 0xf8ea), + ("trademarkserif", 0xf6db), + ("tretroflexhook", 0x0288), + ("triagdn", 0x25bc), + ("triaglf", 0x25c4), + ("triagrt", 0x25ba), + ("triagup", 0x25b2), + ("triangle", 0x25b3), + ("triangledownsld", 0x25bc), + ("triangleinv", 0x25bd), + ("triangleleft", 0x25b9), + ("triangleleftequal", 0x22b4), + ("triangleleftsld", 0x25c0), + ("triangleright", 0x25c3), + ("trianglerightequal", 0x22b5), + ("trianglerightsld", 0x25b6), + ("trianglesolid", 0x25b2), + ("ts", 0x02a6), + ("tsadi", 0x05e6), + ("tsadidagesh", 0xfb46), + ("tsadidageshhebrew", 0xfb46), + ("tsadihebrew", 0x05e6), + ("tsecyrillic", 0x0446), + ("tsere", 0x05b5), + ("tsere12", 0x05b5), + ("tsere1e", 0x05b5), + ("tsere2b", 0x05b5), + ("tserehebrew", 0x05b5), + ("tserenarrowhebrew", 0x05b5), + ("tserequarterhebrew", 0x05b5), + ("tserewidehebrew", 0x05b5), + ("tshecyrillic", 0x045b), + ("tsuperior", 0xf6f3), + ("ttabengali", 0x099f), + ("ttadeva", 0x091f), + ("ttagujarati", 0x0a9f), + ("ttagurmukhi", 0x0a1f), + ("tteharabic", 0x0679), + ("ttehfinalarabic", 0xfb67), + ("ttehinitialarabic", 0xfb68), + ("ttehmedialarabic", 0xfb69), + ("tthabengali", 0x09a0), + ("tthadeva", 0x0920), + ("tthagujarati", 0x0aa0), + ("tthagurmukhi", 0x0a20), + ("tturned", 0x0287), + ("tuhiragana", 0x3064), + ("tukatakana", 0x30c4), + ("tukatakanahalfwidth", 0xff82), + ("turnstileleft", 0x22a2), + ("turnstileright", 0x22a3), + ("tusmallhiragana", 0x3063), + ("tusmallkatakana", 0x30c3), + ("tusmallkatakanahalfwidth", 0xff6f), + ("twelvecircle", 0x246b), + ("twelveparen", 0x247f), + ("twelveperiod", 0x2493), + ("twelveroman", 0x217b), + ("twelveudash", 0xf6de), + ("twentycircle", 0x2473), + ("twentyhangzhou", 0x5344), + ("twentyparen", 0x2487), + ("twentyperiod", 0x249b), + ("two", 0x0032), + ("twoarabic", 0x0662), + ("twobengali", 0x09e8), + ("twocircle", 0x2461), + ("twocircleinversesansserif", 0x278b), + ("twodeva", 0x0968), + ("twodotenleader", 0x2025), + ("twodotleader", 0x2025), + ("twodotleadervertical", 0xfe30), + ("twogujarati", 0x0ae8), + ("twogurmukhi", 0x0a68), + ("twohackarabic", 0x0662), + ("twohangzhou", 0x3022), + ("twoideographicparen", 0x3221), + ("twoinferior", 0x2082), + ("twomonospace", 0xff12), + ("twonumeratorbengali", 0x09f5), + ("twooldstyle", 0xf732), + ("twoparen", 0x2475), + ("twoperiod", 0x2489), + ("twopersian", 0x06f2), + ("tworoman", 0x2171), + ("twostroke", 0x01bb), + ("twosuperior", 0x00b2), + ("twothai", 0x0e52), + ("twothirds", 0x2154), + ("u", 0x0075), + ("uacute", 0x00fa), + ("ubar", 0x0289), + ("ubengali", 0x0989), + ("ubopomofo", 0x3128), + ("ubreve", 0x016d), + ("ucaron", 0x01d4), + ("ucircle", 0x24e4), + ("ucircumflex", 0x00fb), + ("ucircumflexbelow", 0x1e77), + ("ucyrillic", 0x0443), + ("udattadeva", 0x0951), + ("udblacute", 0x0171), + ("udblgrave", 0x0215), + ("udeva", 0x0909), + ("udieresis", 0x00fc), + ("udieresisacute", 0x01d8), + ("udieresisbelow", 0x1e73), + ("udieresiscaron", 0x01da), + ("udieresiscyrillic", 0x04f1), + ("udieresisgrave", 0x01dc), + ("udieresismacron", 0x01d6), + ("udotbelow", 0x1ee5), + ("ugrave", 0x00f9), + ("ugujarati", 0x0a89), + ("ugurmukhi", 0x0a09), + ("uhiragana", 0x3046), + ("uhookabove", 0x1ee7), + ("uhorn", 0x01b0), + ("uhornacute", 0x1ee9), + ("uhorndotbelow", 0x1ef1), + ("uhorngrave", 0x1eeb), + ("uhornhookabove", 0x1eed), + ("uhorntilde", 0x1eef), + ("uhungarumlaut", 0x0171), + ("uhungarumlautcyrillic", 0x04f3), + ("uinvertedbreve", 0x0217), + ("ukatakana", 0x30a6), + ("ukatakanahalfwidth", 0xff73), + ("ukcyrillic", 0x0479), + ("ukorean", 0x315c), + ("umacron", 0x016b), + ("umacroncyrillic", 0x04ef), + ("umacrondieresis", 0x1e7b), + ("umatragurmukhi", 0x0a41), + ("umonospace", 0xff55), + ("underscore", 0x005f), + ("underscoredbl", 0x2017), + ("underscoremonospace", 0xff3f), + ("underscorevertical", 0xfe33), + ("underscorewavy", 0xfe4f), + ("union", 0x222a), + ("uniondbl", 0x22d3), + ("uniondisplay", 0x22c3), + ("unionmulti", 0x228e), + ("unionmultidisplay", 0x228e), + ("unionmultitext", 0x228e), + ("unionsq", 0x2294), + ("unionsqdisplay", 0x2294), + ("unionsqtext", 0x2294), + ("uniontext", 0x22c3), + ("universal", 0x2200), + ("uogonek", 0x0173), + ("uparen", 0x24b0), + ("upblock", 0x2580), + ("upperdothebrew", 0x05c4), + ("uprise", 0x22cf), + ("upsilon", 0x03c5), + ("upsilondieresis", 0x03cb), + ("upsilondieresistonos", 0x03b0), + ("upsilonlatin", 0x028a), + ("upsilontonos", 0x03cd), + ("upslope", 0x29f8), + ("uptackbelowcmb", 0x031d), + ("uptackmod", 0x02d4), + ("uragurmukhi", 0x0a73), + ("uring", 0x016f), + ("ushortcyrillic", 0x045e), + ("usmallhiragana", 0x3045), + ("usmallkatakana", 0x30a5), + ("usmallkatakanahalfwidth", 0xff69), + ("ustraightcyrillic", 0x04af), + ("ustraightstrokecyrillic", 0x04b1), + ("utilde", 0x0169), + ("utildeacute", 0x1e79), + ("utildebelow", 0x1e75), + ("uubengali", 0x098a), + ("uudeva", 0x090a), + ("uugujarati", 0x0a8a), + ("uugurmukhi", 0x0a0a), + ("uumatragurmukhi", 0x0a42), + ("uuvowelsignbengali", 0x09c2), + ("uuvowelsigndeva", 0x0942), + ("uuvowelsigngujarati", 0x0ac2), + ("uvowelsignbengali", 0x09c1), + ("uvowelsigndeva", 0x0941), + ("uvowelsigngujarati", 0x0ac1), + ("v", 0x0076), + ("vadeva", 0x0935), + ("vagujarati", 0x0ab5), + ("vagurmukhi", 0x0a35), + ("vakatakana", 0x30f7), + ("vav", 0x05d5), + ("vavdagesh", 0xfb35), + ("vavdagesh65", 0xfb35), + ("vavdageshhebrew", 0xfb35), + ("vavhebrew", 0x05d5), + ("vavholam", 0xfb4b), + ("vavholamhebrew", 0xfb4b), + ("vavvavhebrew", 0x05f0), + ("vavyodhebrew", 0x05f1), + ("vcircle", 0x24e5), + ("vdotbelow", 0x1e7f), + ("vector", 0x20d7), + ("vecyrillic", 0x0432), + ("veharabic", 0x06a4), + ("vehfinalarabic", 0xfb6b), + ("vehinitialarabic", 0xfb6c), + ("vehmedialarabic", 0xfb6d), + ("vekatakana", 0x30f9), + ("venus", 0x2640), + ("verticalbar", 0x007c), + ("verticallineabovecmb", 0x030d), + ("verticallinebelowcmb", 0x0329), + ("verticallinelowmod", 0x02cc), + ("verticallinemod", 0x02c8), + ("vewarmenian", 0x057e), + ("vextenddouble", 0x2225), + ("vextendsingle", 0x2223), + ("vhook", 0x028b), + ("vikatakana", 0x30f8), + ("viramabengali", 0x09cd), + ("viramadeva", 0x094d), + ("viramagujarati", 0x0acd), + ("visargabengali", 0x0983), + ("visargadeva", 0x0903), + ("visargagujarati", 0x0a83), + ("visiblespace", 0x2423), + ("visualspace", 0x2423), + ("vmonospace", 0xff56), + ("voarmenian", 0x0578), + ("voicediterationhiragana", 0x309e), + ("voicediterationkatakana", 0x30fe), + ("voicedmarkkana", 0x309b), + ("voicedmarkkanahalfwidth", 0xff9e), + ("vokatakana", 0x30fa), + ("vparen", 0x24b1), + ("vtilde", 0x1e7d), + ("vturned", 0x028c), + ("vuhiragana", 0x3094), + ("vukatakana", 0x30f4), + ("w", 0x0077), + ("wacute", 0x1e83), + ("waekorean", 0x3159), + ("wahiragana", 0x308f), + ("wakatakana", 0x30ef), + ("wakatakanahalfwidth", 0xff9c), + ("wakorean", 0x3158), + ("wasmallhiragana", 0x308e), + ("wasmallkatakana", 0x30ee), + ("wattosquare", 0x3357), + ("wavedash", 0x301c), + ("wavyunderscorevertical", 0xfe34), + ("wawarabic", 0x0648), + ("wawfinalarabic", 0xfeee), + ("wawhamzaabovearabic", 0x0624), + ("wawhamzaabovefinalarabic", 0xfe86), + ("wbsquare", 0x33dd), + ("wcircle", 0x24e6), + ("wcircumflex", 0x0175), + ("wdieresis", 0x1e85), + ("wdotaccent", 0x1e87), + ("wdotbelow", 0x1e89), + ("wehiragana", 0x3091), + ("weierstrass", 0x2118), + ("wekatakana", 0x30f1), + ("wekorean", 0x315e), + ("weokorean", 0x315d), + ("wgrave", 0x1e81), + ("whitebullet", 0x25e6), + ("whitecircle", 0x25cb), + ("whitecircleinverse", 0x25d9), + ("whitecornerbracketleft", 0x300e), + ("whitecornerbracketleftvertical", 0xfe43), + ("whitecornerbracketright", 0x300f), + ("whitecornerbracketrightvertical", 0xfe44), + ("whitediamond", 0x25c7), + ("whitediamondcontainingblacksmalldiamond", 0x25c8), + ("whitedownpointingsmalltriangle", 0x25bf), + ("whitedownpointingtriangle", 0x25bd), + ("whiteleftpointingsmalltriangle", 0x25c3), + ("whiteleftpointingtriangle", 0x25c1), + ("whitelenticularbracketleft", 0x3016), + ("whitelenticularbracketright", 0x3017), + ("whiterightpointingsmalltriangle", 0x25b9), + ("whiterightpointingtriangle", 0x25b7), + ("whitesmallsquare", 0x25ab), + ("whitesmilingface", 0x263a), + ("whitesquare", 0x25a1), + ("whitestar", 0x2606), + ("whitetelephone", 0x260f), + ("whitetortoiseshellbracketleft", 0x3018), + ("whitetortoiseshellbracketright", 0x3019), + ("whiteuppointingsmalltriangle", 0x25b5), + ("whiteuppointingtriangle", 0x25b3), + ("wihiragana", 0x3090), + ("wikatakana", 0x30f0), + ("wikorean", 0x315f), + ("wmonospace", 0xff57), + ("wohiragana", 0x3092), + ("wokatakana", 0x30f2), + ("wokatakanahalfwidth", 0xff66), + ("won", 0x20a9), + ("wonmonospace", 0xffe6), + ("wowaenthai", 0x0e27), + ("wparen", 0x24b2), + ("wreathproduct", 0x2240), + ("wring", 0x1e98), + ("wsuperior", 0x02b7), + ("wturned", 0x028d), + ("wynn", 0x01bf), + ("x", 0x0078), + ("xabovecmb", 0x033d), + ("xbopomofo", 0x3112), + ("xcircle", 0x24e7), + ("xdieresis", 0x1e8d), + ("xdotaccent", 0x1e8b), + ("xeharmenian", 0x056d), + ("xi", 0x03be), + ("xmonospace", 0xff58), + ("xparen", 0x24b3), + ("xsuperior", 0x02e3), + ("y", 0x0079), + ("yaadosquare", 0x334e), + ("yabengali", 0x09af), + ("yacute", 0x00fd), + ("yadeva", 0x092f), + ("yaekorean", 0x3152), + ("yagujarati", 0x0aaf), + ("yagurmukhi", 0x0a2f), + ("yahiragana", 0x3084), + ("yakatakana", 0x30e4), + ("yakatakanahalfwidth", 0xff94), + ("yakorean", 0x3151), + ("yamakkanthai", 0x0e4e), + ("yasmallhiragana", 0x3083), + ("yasmallkatakana", 0x30e3), + ("yasmallkatakanahalfwidth", 0xff6c), + ("yatcyrillic", 0x0463), + ("ycircle", 0x24e8), + ("ycircumflex", 0x0177), + ("ydieresis", 0x00ff), + ("ydotaccent", 0x1e8f), + ("ydotbelow", 0x1ef5), + ("yeharabic", 0x064a), + ("yehbarreearabic", 0x06d2), + ("yehbarreefinalarabic", 0xfbaf), + ("yehfinalarabic", 0xfef2), + ("yehhamzaabovearabic", 0x0626), + ("yehhamzaabovefinalarabic", 0xfe8a), + ("yehhamzaaboveinitialarabic", 0xfe8b), + ("yehhamzaabovemedialarabic", 0xfe8c), + ("yehinitialarabic", 0xfef3), + ("yehmedialarabic", 0xfef4), + ("yehmeeminitialarabic", 0xfcdd), + ("yehmeemisolatedarabic", 0xfc58), + ("yehnoonfinalarabic", 0xfc94), + ("yehthreedotsbelowarabic", 0x06d1), + ("yekorean", 0x3156), + ("yen", 0x00a5), + ("yenmonospace", 0xffe5), + ("yeokorean", 0x3155), + ("yeorinhieuhkorean", 0x3186), + ("yerahbenyomohebrew", 0x05aa), + ("yerahbenyomolefthebrew", 0x05aa), + ("yericyrillic", 0x044b), + ("yerudieresiscyrillic", 0x04f9), + ("yesieungkorean", 0x3181), + ("yesieungpansioskorean", 0x3183), + ("yesieungsioskorean", 0x3182), + ("yetivhebrew", 0x059a), + ("ygrave", 0x1ef3), + ("yhook", 0x01b4), + ("yhookabove", 0x1ef7), + ("yiarmenian", 0x0575), + ("yicyrillic", 0x0457), + ("yikorean", 0x3162), + ("yinyang", 0x262f), + ("yiwnarmenian", 0x0582), + ("ymonospace", 0xff59), + ("yod", 0x05d9), + ("yoddagesh", 0xfb39), + ("yoddageshhebrew", 0xfb39), + ("yodhebrew", 0x05d9), + ("yodyodhebrew", 0x05f2), + ("yodyodpatahhebrew", 0xfb1f), + ("yohiragana", 0x3088), + ("yoikorean", 0x3189), + ("yokatakana", 0x30e8), + ("yokatakanahalfwidth", 0xff96), + ("yokorean", 0x315b), + ("yosmallhiragana", 0x3087), + ("yosmallkatakana", 0x30e7), + ("yosmallkatakanahalfwidth", 0xff6e), + ("yotgreek", 0x03f3), + ("yoyaekorean", 0x3188), + ("yoyakorean", 0x3187), + ("yoyakthai", 0x0e22), + ("yoyingthai", 0x0e0d), + ("yparen", 0x24b4), + ("ypogegrammeni", 0x037a), + ("ypogegrammenigreekcmb", 0x0345), + ("yr", 0x01a6), + ("yring", 0x1e99), + ("ysuperior", 0x02b8), + ("ytilde", 0x1ef9), + ("yturned", 0x028e), + ("yuhiragana", 0x3086), + ("yuikorean", 0x318c), + ("yukatakana", 0x30e6), + ("yukatakanahalfwidth", 0xff95), + ("yukorean", 0x3160), + ("yusbigcyrillic", 0x046b), + ("yusbigiotifiedcyrillic", 0x046d), + ("yuslittlecyrillic", 0x0467), + ("yuslittleiotifiedcyrillic", 0x0469), + ("yusmallhiragana", 0x3085), + ("yusmallkatakana", 0x30e5), + ("yusmallkatakanahalfwidth", 0xff6d), + ("yuyekorean", 0x318b), + ("yuyeokorean", 0x318a), + ("yyabengali", 0x09df), + ("yyadeva", 0x095f), + ("z", 0x007a), + ("zaarmenian", 0x0566), + ("zacute", 0x017a), + ("zadeva", 0x095b), + ("zagurmukhi", 0x0a5b), + ("zaharabic", 0x0638), + ("zahfinalarabic", 0xfec6), + ("zahinitialarabic", 0xfec7), + ("zahiragana", 0x3056), + ("zahmedialarabic", 0xfec8), + ("zainarabic", 0x0632), + ("zainfinalarabic", 0xfeb0), + ("zakatakana", 0x30b6), + ("zaqefgadolhebrew", 0x0595), + ("zaqefqatanhebrew", 0x0594), + ("zarqahebrew", 0x0598), + ("zayin", 0x05d6), + ("zayindagesh", 0xfb36), + ("zayindageshhebrew", 0xfb36), + ("zayinhebrew", 0x05d6), + ("zbopomofo", 0x3117), + ("zcaron", 0x017e), + ("zcircle", 0x24e9), + ("zcircumflex", 0x1e91), + ("zcurl", 0x0291), + ("zdot", 0x017c), + ("zdotaccent", 0x017c), + ("zdotbelow", 0x1e93), + ("zecyrillic", 0x0437), + ("zedescendercyrillic", 0x0499), + ("zedieresiscyrillic", 0x04df), + ("zehiragana", 0x305c), + ("zekatakana", 0x30bc), + ("zero", 0x0030), + ("zeroarabic", 0x0660), + ("zerobengali", 0x09e6), + ("zerodeva", 0x0966), + ("zerogujarati", 0x0ae6), + ("zerogurmukhi", 0x0a66), + ("zerohackarabic", 0x0660), + ("zeroinferior", 0x2080), + ("zeromonospace", 0xff10), + ("zerooldstyle", 0xf730), + ("zeropersian", 0x06f0), + ("zerosuperior", 0x2070), + ("zerothai", 0x0e50), + ("zerowidthjoiner", 0xfeff), + ("zerowidthnonjoiner", 0x200c), + ("zerowidthspace", 0x200b), + ("zeta", 0x03b6), + ("zhbopomofo", 0x3113), + ("zhearmenian", 0x056a), + ("zhebrevecyrillic", 0x04c2), + ("zhecyrillic", 0x0436), + ("zhedescendercyrillic", 0x0497), + ("zhedieresiscyrillic", 0x04dd), + ("zihiragana", 0x3058), + ("zikatakana", 0x30b8), + ("zinorhebrew", 0x05ae), + ("zlinebelow", 0x1e95), + ("zmonospace", 0xff5a), + ("zohiragana", 0x305e), + ("zokatakana", 0x30be), + ("zparen", 0x24b5), + ("zretroflexhook", 0x0290), + ("zstroke", 0x01b6), + ("zuhiragana", 0x305a), + ("zukatakana", 0x30ba), + ]; + let result = NAMES.binary_search_by_key(&name, |&(name, _code)| &name); + result.ok().map(|indx| NAMES[indx].1) +} diff --git a/crates/pdf-extract/src/lib.rs b/crates/pdf-extract/src/lib.rs new file mode 100644 index 0000000..00b1acf --- /dev/null +++ b/crates/pdf-extract/src/lib.rs @@ -0,0 +1,3356 @@ +// Vendored from pdf-extract 0.10.0 (https://github.com/jrmuizel/pdf-extract), +// MIT license, authors Jeff Muizelaar and Mozilla contributors. +// batdoc fork changes (see git log): panic-free font decoding; glyph +// recovery for CID/simple fonts with broken or missing /ToUnicode. +// Public API is unchanged from upstream 0.10. +// +// Vendored third-party code (upstream pdf-extract 0.10): not lint-gated. +// batdoc's own fork changes still follow the repo clippy policy by hand. +#![allow(warnings)] +#![allow(clippy::all, clippy::pedantic, clippy::nursery)] + +extern crate lopdf; + +use adobe_cmap_parser::{ByteMapping, CIDRange, CodeRange}; +use encoding_rs::UTF_16BE; +use euclid::*; +use lopdf::content::Content; +use lopdf::encryption::DecryptionError; +pub use lopdf::*; +use std::fmt::{Debug, Formatter}; +extern crate adobe_cmap_parser; +extern crate encoding_rs; +extern crate euclid; +extern crate type1_encoding_parser; +extern crate unicode_normalization; +use euclid::vec2; +use log::{debug, error, warn}; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::fmt; +use std::fs::File; +use std::marker::PhantomData; +use std::rc::Rc; +use std::result::Result; +use std::slice::Iter; +use std::str; +use unicode_normalization::UnicodeNormalization; +mod core_fonts; +mod encodings; +mod font_cmap; +mod glyphnames; +mod zapfglyphnames; + +pub struct Space; +pub type Transform = Transform2D; + +#[derive(Debug)] +pub enum OutputError { + FormatError(std::fmt::Error), + IoError(std::io::Error), + PdfError(lopdf::Error), +} + +impl std::fmt::Display for OutputError { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> { + match self { + OutputError::FormatError(e) => write!(f, "Formating error: {}", e), + OutputError::IoError(e) => write!(f, "IO error: {}", e), + OutputError::PdfError(e) => write!(f, "PDF error: {}", e), + } + } +} + +impl std::error::Error for OutputError {} + +impl From for OutputError { + fn from(e: std::fmt::Error) -> Self { + OutputError::FormatError(e) + } +} + +impl From for OutputError { + fn from(e: std::io::Error) -> Self { + OutputError::IoError(e) + } +} + +impl From for OutputError { + fn from(e: lopdf::Error) -> Self { + OutputError::PdfError(e) + } +} + +macro_rules! dlog { + ($($e:expr),*) => { {$(let _ = $e;)*} } + //($($t:tt)*) => { println!($($t)*) } +} + +fn get_info(doc: &Document) -> Option<&Dictionary> { + match doc.trailer.get(b"Info") { + Ok(&Object::Reference(ref id)) => match doc.get_object(*id) { + Ok(&Object::Dictionary(ref info)) => { + return Some(info); + } + _ => {} + }, + _ => {} + } + None +} + +fn get_catalog(doc: &Document) -> &Dictionary { + match doc.trailer.get(b"Root").unwrap() { + &Object::Reference(ref id) => match doc.get_object(*id) { + Ok(&Object::Dictionary(ref catalog)) => { + return catalog; + } + _ => {} + }, + _ => {} + } + panic!(); +} + +fn get_pages(doc: &Document) -> &Dictionary { + let catalog = get_catalog(doc); + match catalog.get(b"Pages").unwrap() { + &Object::Reference(ref id) => match doc.get_object(*id) { + Ok(&Object::Dictionary(ref pages)) => { + return pages; + } + other => { + dlog!("pages: {:?}", other) + } + }, + other => { + dlog!("pages: {:?}", other) + } + } + dlog!("catalog {:?}", catalog); + panic!(); +} + +#[allow(non_upper_case_globals)] +const PDFDocEncoding: &'static [u16] = &[ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x02d8, 0x02c7, 0x02c6, 0x02d9, 0x02dd, 0x02db, 0x02da, 0x02dc, 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x0000, 0x2022, 0x2020, 0x2021, 0x2026, + 0x2014, 0x2013, 0x0192, 0x2044, 0x2039, 0x203a, 0x2212, 0x2030, 0x201e, 0x201c, 0x201d, 0x2018, + 0x2019, 0x201a, 0x2122, 0xfb01, 0xfb02, 0x0141, 0x0152, 0x0160, 0x0178, 0x017d, 0x0131, 0x0142, + 0x0153, 0x0161, 0x017e, 0x0000, 0x20ac, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x0000, 0x00ae, 0x00af, 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, + 0x00cc, 0x00cd, 0x00ce, 0x00cf, 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, 0x00e0, 0x00e1, 0x00e2, 0x00e3, + 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, 0x00fa, 0x00fb, + 0x00fc, 0x00fd, 0x00fe, 0x00ff, +]; + +fn pdf_to_utf8(s: &[u8]) -> String { + // Fork: invalid UTF-16BE (e.g. lone surrogates) decodes to an empty + // string instead of panicking. + if s.len() > 2 && s[0] == 0xfe && s[1] == 0xff { + return UTF_16BE + .decode_without_bom_handling_and_without_replacement(&s[2..]) + .map_or_else(String::new, |s| s.into_owned()); + } else { + let r: Vec = s + .iter() + .map(|x| *x) + .flat_map(|x| { + let k = PDFDocEncoding[x as usize]; + vec![(k >> 8) as u8, k as u8].into_iter() + }) + .collect(); + return UTF_16BE + .decode_without_bom_handling_and_without_replacement(&r) + .map_or_else(String::new, |s| s.into_owned()); + } +} + +fn to_utf8(encoding: &[u16], s: &[u8]) -> String { + // Fork: invalid UTF-16BE decodes to an empty string instead of + // panicking. + if s.len() > 2 && s[0] == 0xfe && s[1] == 0xff { + return UTF_16BE + .decode_without_bom_handling_and_without_replacement(&s[2..]) + .map_or_else(String::new, |s| s.into_owned()); + } else { + let r: Vec = s + .iter() + .map(|x| *x) + .flat_map(|x| { + let k = encoding[x as usize]; + vec![(k >> 8) as u8, k as u8].into_iter() + }) + .collect(); + return UTF_16BE + .decode_without_bom_handling_and_without_replacement(&r) + .map_or_else(String::new, |s| s.into_owned()); + } +} + +fn maybe_deref<'a>(doc: &'a Document, o: &'a Object) -> &'a Object { + match o { + &Object::Reference(r) => doc.get_object(r).expect("missing object reference"), + _ => o, + } +} + +fn maybe_get_obj<'a>(doc: &'a Document, dict: &'a Dictionary, key: &[u8]) -> Option<&'a Object> { + dict.get(key).map(|o| maybe_deref(doc, o)).ok() +} + +// an intermediate trait that can be used to chain conversions that may have failed +trait FromOptObj<'a> { + fn from_opt_obj(doc: &'a Document, obj: Option<&'a Object>, key: &[u8]) -> Self; +} + +// conditionally convert to Self returns None if the conversion failed +trait FromObj<'a> +where + Self: std::marker::Sized, +{ + fn from_obj(doc: &'a Document, obj: &'a Object) -> Option; +} + +impl<'a, T: FromObj<'a>> FromOptObj<'a> for Option { + fn from_opt_obj(doc: &'a Document, obj: Option<&'a Object>, _key: &[u8]) -> Self { + obj.and_then(|x| T::from_obj(doc, x)) + } +} + +impl<'a, T: FromObj<'a>> FromOptObj<'a> for T { + fn from_opt_obj(doc: &'a Document, obj: Option<&'a Object>, key: &[u8]) -> Self { + T::from_obj(doc, obj.expect(&String::from_utf8_lossy(key))).expect("wrong type") + } +} + +// we follow the same conventions as pdfium for when to support indirect objects: +// on arrays, streams and dicts +impl<'a, T: FromObj<'a>> FromObj<'a> for Vec { + fn from_obj(doc: &'a Document, obj: &'a Object) -> Option { + maybe_deref(doc, obj) + .as_array() + .map(|x| { + x.iter() + .map(|x| T::from_obj(doc, x).expect("wrong type")) + .collect() + }) + .ok() + } +} + +// XXX: These will panic if we don't have the right number of items +// we don't want to do that +impl<'a, T: FromObj<'a>> FromObj<'a> for [T; 4] { + fn from_obj(doc: &'a Document, obj: &'a Object) -> Option { + maybe_deref(doc, obj) + .as_array() + .map(|x| { + let mut all = x.iter().map(|x| T::from_obj(doc, x).expect("wrong type")); + [ + all.next().unwrap(), + all.next().unwrap(), + all.next().unwrap(), + all.next().unwrap(), + ] + }) + .ok() + } +} + +impl<'a, T: FromObj<'a>> FromObj<'a> for [T; 3] { + fn from_obj(doc: &'a Document, obj: &'a Object) -> Option { + maybe_deref(doc, obj) + .as_array() + .map(|x| { + let mut all = x.iter().map(|x| T::from_obj(doc, x).expect("wrong type")); + [ + all.next().unwrap(), + all.next().unwrap(), + all.next().unwrap(), + ] + }) + .ok() + } +} + +impl<'a> FromObj<'a> for f64 { + fn from_obj(_doc: &Document, obj: &Object) -> Option { + match obj { + &Object::Integer(i) => Some(i as f64), + &Object::Real(f) => Some(f.into()), + _ => None, + } + } +} + +impl<'a> FromObj<'a> for i64 { + fn from_obj(_doc: &Document, obj: &Object) -> Option { + match obj { + &Object::Integer(i) => Some(i), + _ => None, + } + } +} + +impl<'a> FromObj<'a> for &'a Dictionary { + fn from_obj(doc: &'a Document, obj: &'a Object) -> Option<&'a Dictionary> { + maybe_deref(doc, obj).as_dict().ok() + } +} + +impl<'a> FromObj<'a> for &'a Stream { + fn from_obj(doc: &'a Document, obj: &'a Object) -> Option<&'a Stream> { + maybe_deref(doc, obj).as_stream().ok() + } +} + +impl<'a> FromObj<'a> for &'a Object { + fn from_obj(doc: &'a Document, obj: &'a Object) -> Option<&'a Object> { + Some(maybe_deref(doc, obj)) + } +} + +fn get<'a, T: FromOptObj<'a>>(doc: &'a Document, dict: &'a Dictionary, key: &[u8]) -> T { + T::from_opt_obj(doc, dict.get(key).ok(), key) +} + +fn maybe_get<'a, T: FromObj<'a>>(doc: &'a Document, dict: &'a Dictionary, key: &[u8]) -> Option { + maybe_get_obj(doc, dict, key).and_then(|o| T::from_obj(doc, o)) +} + +fn get_name_string<'a>(doc: &'a Document, dict: &'a Dictionary, key: &[u8]) -> String { + // Fork: a missing/mistyped name yields an empty string instead of a + // panic (font dictionaries with broken BaseFont/Subtype/Encoding). + dict.get(key) + .ok() + .map(|o| maybe_deref(doc, o)) + .and_then(|o| o.as_name().ok()) + .map_or_else(String::new, pdf_to_utf8) +} + +#[allow(dead_code)] +fn maybe_get_name_string<'a>( + doc: &'a Document, + dict: &'a Dictionary, + key: &[u8], +) -> Option { + maybe_get_obj(doc, dict, key) + .and_then(|n| n.as_name().ok()) + .map(|n| pdf_to_utf8(n)) +} + +fn maybe_get_name<'a>(doc: &'a Document, dict: &'a Dictionary, key: &[u8]) -> Option<&'a [u8]> { + maybe_get_obj(doc, dict, key).and_then(|n| n.as_name().ok()) +} + +fn maybe_get_array<'a>( + doc: &'a Document, + dict: &'a Dictionary, + key: &[u8], +) -> Option<&'a Vec> { + maybe_get_obj(doc, dict, key).and_then(|n| n.as_array().ok()) +} + +/// Glyph-recovery state for one font, built at font construction (where +/// FontFile2/FontFile3 streams are reachable) and consulted by +/// `decode_char`. Stored in a `RefCell` because `decode_char` takes +/// `&self`. +#[derive(Clone)] +struct FontRecovery { + /// Inverted cmap from the embedded font program, if one is embedded. + reverse_cmap: Option, + /// Chars decoded so far via /ToUnicode. + decoded: u32, + /// Decoded chars that look like garbage (control chars, PUA, + /// unmapped/empty). + suspicious: u32, + /// Latched: true once suspicious >= 10% of decoded (liteparse + /// extract.rs criterion) — the font's /ToUnicode is garbage. + recovering: bool, +} + +impl FontRecovery { + fn new(reverse_cmap: Option) -> Self { + FontRecovery { + reverse_cmap, + decoded: 0, + suspicious: 0, + recovering: false, + } + } + + /// Record one decoded string; flip `recovering` once the garbage ratio + /// crosses 10% with at least 20 decoded chars (avoids flapping on tiny + /// samples). Latched: clean chars never flip it back. + fn observe(&mut self, s: &str) { + self.decoded += 1; + let garbage = s.is_empty() + || s.chars().any(|c| { + c.is_control() || ('\u{E000}'..='\u{F8FF}').contains(&c) || c == '\u{FFFD}' + }); + if garbage { + self.suspicious += 1; + } + if self.decoded >= 20 && self.suspicious * 10 >= self.decoded { + self.recovering = true; + } + } +} + +/// Expand Unicode alphabetic presentation forms U+FB00..=U+FB06 to their +/// ASCII decompositions. Applied ONLY to recovered glyphs (recovery +/// paths), never to clean /ToUnicode output (batdoc D2 byte-stability). +fn expand_ligatures(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '\u{FB00}' => out.push_str("ff"), + '\u{FB01}' => out.push_str("fi"), + '\u{FB02}' => out.push_str("fl"), + '\u{FB03}' => out.push_str("ffi"), + '\u{FB04}' => out.push_str("ffl"), + '\u{FB05}' | '\u{FB06}' => out.push_str("st"), + _ => out.push(c), + } + } + out +} + +/// Decode one character code with glyph recovery (batdoc spec §5.2 order): +/// +/// 1. Healthy /ToUnicode hit (font not flipped to recovery): returned +/// verbatim after feeding the garbage prescan (D2 byte-stability). +/// 2. Otherwise — garbage /ToUnicode (flipped) or a miss — consult the +/// reverse cmap from the embedded font program. The raw code is only a +/// valid lookup key for CID fonts (code == glyph id under Identity-H); +/// simple fonts must not route raw codes here (their char codes are +/// encoding-table indices, not glyph ids). +/// 3. Ligatures/presentation forms in the recovered glyph are expanded. +/// 4. Recovery produced nothing: return "" and let the caller apply its +/// existing miss behavior (CID: empty string; simple fonts: encoding +/// table). +fn decode_char_with( + to_unicode_hit: Option, + recovery: &mut FontRecovery, + code: u32, +) -> String { + match to_unicode_hit { + Some(hit) if !recovery.recovering => { + recovery.observe(&hit); + hit + } + hit => { + // A miss counts as a suspicious decode for the prescan; a hit + // on a flipped font was already judged garbage and is dropped. + if hit.is_none() { + recovery.observe(""); + } + if let Some(ref rc) = recovery.reverse_cmap { + if let Some(c) = rc.lookup(code) { + return expand_ligatures(&c.to_string()); + } + } + String::new() + } + } +} + +/// Build a reverse cmap from a font descriptor's embedded font program +/// (`FontFile2` TrueType or `FontFile3` sfnt-wrapped OpenType), if one is +/// embedded and parseable. Bare CFF programs (`Type1C`/`CIDFontType0C`) +/// are not sfnt and yield `None`; recovery then relies on the glyph-name +/// fallback (Task 5). +fn reverse_cmap_from_descriptor( + doc: &Document, + descriptor: &Dictionary, +) -> Option { + for key in [&b"FontFile2"[..], &b"FontFile3"[..]] { + if let Some(&Object::Stream(ref s)) = maybe_get_obj(doc, descriptor, key) { + if let Some(rc) = font_cmap::ReverseCmap::from_font_program(&get_contents(s)) { + return Some(rc); + } + } + } + None +} + +#[derive(Clone)] +struct PdfSimpleFont<'a> { + font: &'a Dictionary, + doc: &'a Document, + encoding: Option>, + unicode_map: Option>, + widths: HashMap, // should probably just use i32 here + missing_width: f64, + /// Glyph-recovery state; consulted by `decode_char`. + recovery: std::cell::RefCell, +} + +#[derive(Clone)] +struct PdfType3Font<'a> { + font: &'a Dictionary, + doc: &'a Document, + encoding: Option>, + unicode_map: Option>, + widths: HashMap, // should probably just use i32 here +} + +// Fork: returns None when a font is too broken to construct (upstream +// panicked); callers leave the current font unchanged. +fn make_font<'a>(doc: &'a Document, font: &'a Dictionary) -> Option> { + let subtype = get_name_string(doc, font, b"Subtype"); + dlog!("MakeFont({})", subtype); + if subtype == "Type0" { + PdfCIDFont::new(doc, font).map(|f| Rc::new(f) as Rc) + } else if subtype == "Type3" { + Some(Rc::new(PdfType3Font::new(doc, font))) + } else { + Some(Rc::new(PdfSimpleFont::new(doc, font))) + } +} + +fn is_core_font(name: &str) -> bool { + match name { + "Courier-Bold" + | "Courier-BoldOblique" + | "Courier-Oblique" + | "Courier" + | "Helvetica-Bold" + | "Helvetica-BoldOblique" + | "Helvetica-Oblique" + | "Helvetica" + | "Symbol" + | "Times-Bold" + | "Times-BoldItalic" + | "Times-Italic" + | "Times-Roman" + | "ZapfDingbats" => true, + _ => false, + } +} + +fn encoding_to_unicode_table(name: &[u8]) -> Vec { + let encoding = match &name[..] { + b"MacRomanEncoding" => encodings::MAC_ROMAN_ENCODING, + b"MacExpertEncoding" => encodings::MAC_EXPERT_ENCODING, + b"WinAnsiEncoding" => encodings::WIN_ANSI_ENCODING, + // Fork: unknown encoding names fall back to PDFDocEncoding instead + // of panicking (malformed documents). + _ => { + warn!( + "unexpected encoding {:?}; falling back to PDFDocEncoding", + pdf_to_utf8(name) + ); + return Vec::from(PDFDocEncoding); + } + }; + let encoding_table = encoding + .iter() + .map(|x| { + if let &Some(x) = x { + // Fork: unknown glyph names map to unmapped (0), not panic. + glyphnames::name_to_unicode(x).unwrap_or(0) + } else { + 0 + } + }) + .collect(); + encoding_table +} + +/* "Glyphs in the font are selected by single-byte character codes obtained from a string that + is shown by the text-showing operators. Logically, these codes index into a table of 256 + glyphs; the mapping from codes to glyphs is called the font’s encoding. Each font program + has a built-in encoding. Under some circumstances, the encoding can be altered by means + described in Section 5.5.5, “Character Encoding.” +*/ +impl<'a> PdfSimpleFont<'a> { + fn new(doc: &'a Document, font: &'a Dictionary) -> PdfSimpleFont<'a> { + let base_name = get_name_string(doc, font, b"BaseFont"); + let subtype = get_name_string(doc, font, b"Subtype"); + + let encoding: Option<&Object> = get(doc, font, b"Encoding"); + dlog!( + "base_name {} {} enc:{:?} {:?}", + base_name, + subtype, + encoding, + font + ); + let descriptor: Option<&Dictionary> = get(doc, font, b"FontDescriptor"); + let mut type1_encoding = None; + let mut unicode_map = None; + let mut reverse_cmap = None; + if let Some(descriptor) = descriptor { + dlog!("descriptor {:?}", descriptor); + reverse_cmap = reverse_cmap_from_descriptor(doc, descriptor); + if subtype == "Type1" { + let file = maybe_get_obj(doc, descriptor, b"FontFile"); + match file { + Some(&Object::Stream(ref s)) => { + let s = get_contents(s); + //dlog!("font contents {:?}", pdf_to_utf8(&s)); + // Fork: a malformed embedded Type1 program loses + // its encoding fallback instead of panicking. + type1_encoding = match type1_encoding_parser::get_encoding_map(&s) { + Ok(map) => Some(map), + Err(e) => { + dlog!("failed to parse Type1 encoding: {:?}", e); + None + } + }; + } + _ => { + dlog!("font file {:?}", file) + } + } + } else if subtype == "TrueType" { + let file = maybe_get_obj(doc, descriptor, b"FontFile2"); + match file { + Some(&Object::Stream(ref s)) => { + let _s = get_contents(s); + //File::create(format!("/tmp/{}", base_name)).unwrap().write_all(&s); + } + _ => { + dlog!("font file {:?}", file) + } + } + } + + let font_file3 = get::>(doc, descriptor, b"FontFile3"); + match font_file3 { + Some(&Object::Stream(ref s)) => { + let subtype = get_name_string(doc, &s.dict, b"Subtype"); + dlog!("font file {}, {:?}", subtype, s); + let s = get_contents(s); + if subtype == "Type1C" { + // Fork: a malformed CFF program yields no unicode + // map instead of panicking. + match cff_parser::Table::parse(&s) { + Some(table) => { + let charset = table.charset.get_table(); + let encoding = table.encoding.get_table(); + let mut mapping = HashMap::new(); + for i in 0..encoding.len().min(charset.len()) { + let cid = encoding[i]; + let sid = charset[i]; + let Some(name) = cff_parser::string_by_id(&table, sid) else { + continue; + }; + let unicode = glyphnames::name_to_unicode(name).or_else(|| { + zapfglyphnames::zapfdigbats_names_to_unicode(name) + }); + if let Some(unicode) = unicode { + if let Ok(str) = String::from_utf16(&[unicode]) { + mapping.insert(cid as u32, str); + } + } + } + unicode_map = Some(mapping); + } + None => dlog!("failed to parse CFF font program"), + } + // + //File::create(format!("/tmp/{}", base_name)).unwrap().write_all(&s); + } + + // + //File::create(format!("/tmp/{}", base_name)).unwrap().write_all(&s); + } + None => {} + _ => { + dlog!("unexpected") + } + } + + let charset = maybe_get_obj(doc, descriptor, b"CharSet"); + let _charset = match charset { + Some(&Object::String(ref s, _)) => Some(pdf_to_utf8(&s)), + _ => None, + }; + //dlog!("charset {:?}", charset); + } + + let mut unicode_map = match unicode_map { + Some(mut unicode_map) => { + unicode_map.extend(get_unicode_map(doc, font).unwrap_or(HashMap::new())); + Some(unicode_map) + } + None => get_unicode_map(doc, font), + }; + + let mut encoding_table = None; + match encoding { + Some(&Object::Name(ref encoding_name)) => { + dlog!("encoding {:?}", pdf_to_utf8(encoding_name)); + encoding_table = Some(encoding_to_unicode_table(encoding_name)); + } + Some(&Object::Dictionary(ref encoding)) => { + //dlog!("Encoding {:?}", encoding); + let mut table = + if let Some(base_encoding) = maybe_get_name(doc, encoding, b"BaseEncoding") { + dlog!("BaseEncoding {:?}", base_encoding); + encoding_to_unicode_table(base_encoding) + } else { + Vec::from(PDFDocEncoding) + }; + let differences = maybe_get_array(doc, encoding, b"Differences"); + if let Some(differences) = differences { + dlog!("Differences"); + let mut code = 0; + for o in differences { + let o = maybe_deref(doc, o); + match o { + &Object::Integer(i) => { + code = i; + } + &Object::Name(ref n) => { + let name = pdf_to_utf8(&n); + // Fork: ignore out-of-range Differences + // codes instead of indexing past the table. + if code < 0 || code as usize >= table.len() { + warn!( + "Differences code {} out of range for font {}", + code, base_name + ); + code += 1; + continue; + } + // XXX: names of Type1 fonts can map to arbitrary strings instead of real + // unicode names, so we should probably handle this differently + let unicode = glyphnames::name_to_unicode(&name); + if let Some(unicode) = unicode { + table[code as usize] = unicode; + if let Some(ref mut unicode_map) = unicode_map { + let be = [unicode]; + match unicode_map.entry(code as u32) { + // If there's a unicode table entry missing use one based on the name + Entry::Vacant(v) => { + // Fork: from_utf16 never + // panics (fallback: ""). + v.insert( + String::from_utf16(&be).unwrap_or_default(), + ); + } + Entry::Occupied(e) => { + if e.get() + != &String::from_utf16(&be).unwrap_or_default() + { + let normal_match = + e.get().nfkc().eq(String::from_utf16(&be) + .unwrap_or_default() + .nfkc()); + if !normal_match { + warn!( + "Unicode mismatch {} {} {:?} {:?} {:?}", + normal_match, + name, + e.get(), + String::from_utf16(&be), + be + ); + } + } + } + } + } + } else { + match unicode_map { + Some(ref mut unicode_map) + if base_name.contains("FontAwesome") => + { + // the fontawesome tex package will use glyph names that don't have a corresponding unicode + // code point, so we'll use an empty string instead. See issue #76 + match unicode_map.entry(code as u32) { + Entry::Vacant(v) => { + v.insert("".to_owned()); + } + Entry::Occupied(_e) => { + // Fork: warn instead of panicking. + warn!("unexpected entry in unicode map"); + } + } + } + _ => { + warn!( + "unknown glyph name '{}' for font {}", + name, base_name + ); + } + } + } + dlog!("{} = {} ({:?})", code, name, unicode); + if let Some(ref mut unicode_map) = unicode_map { + // The unicode map might not have the code in it, but the code might + // not be used so we don't want to panic here. + // An example of this is the 'suppress' character in the TeX Latin Modern font. + // This shows up in https://arxiv.org/pdf/2405.01295v1.pdf + dlog!("{} {:?}", code, unicode_map.get(&(code as u32))); + } + code += 1; + } + _ => { + // Fork: skip non-integer/non-name + // Differences entries instead of panicking. + warn!("unexpected Differences entry {:?}", o); + } + } + } + } + // "Type" is optional + let name = encoding + .get(b"Type") + .and_then(|x| x.as_name()) + .and_then(|x| Ok(pdf_to_utf8(x))); + dlog!("name: {}", name); + + encoding_table = Some(table); + } + None => { + if let Some(type1_encoding) = type1_encoding { + let mut table = Vec::from(PDFDocEncoding); + dlog!("type1encoding"); + for (code, name) in type1_encoding { + let unicode = glyphnames::name_to_unicode(&pdf_to_utf8(&name)); + if let Some(unicode) = unicode { + table[code as usize] = unicode; + } else { + dlog!("unknown character {}", pdf_to_utf8(&name)); + } + } + encoding_table = Some(table) + } else if subtype == "TrueType" { + encoding_table = Some( + encodings::WIN_ANSI_ENCODING + .iter() + .map(|x| { + if let &Some(x) = x { + // Fork: unknown glyph names map to + // unmapped (0), not panic. + glyphnames::name_to_unicode(x).unwrap_or(0) + } else { + 0 + } + }) + .collect(), + ); + } + } + other => { + // Fork: an Encoding that is neither name nor dictionary + // leaves no encoding table instead of panicking. + warn!("unexpected Encoding object {:?}", other); + } + } + + let mut width_map = HashMap::new(); + /* "Ordinarily, a font dictionary that refers to one of the standard fonts + should omit the FirstChar, LastChar, Widths, and FontDescriptor entries. + However, it is permissible to override a standard font by including these + entries and embedding the font program in the PDF file." + + Note: some PDFs include a descriptor but still don't include these entries */ + + // If we have widths prefer them over the core font widths. Needed for https://dkp.de/wp-content/uploads/parteitage/Sozialismusvorstellungen-der-DKP.pdf + if let (Some(first_char), Some(last_char), Some(widths)) = ( + maybe_get::(doc, font, b"FirstChar"), + maybe_get::(doc, font, b"LastChar"), + maybe_get::>(doc, font, b"Widths"), + ) { + // Some PDF's don't have these like fips-197.pdf + let mut i: i64 = 0; + dlog!( + "first_char {:?}, last_char: {:?}, widths: {} {:?}", + first_char, + last_char, + widths.len(), + widths + ); + + for w in widths { + width_map.insert((first_char + i) as CharCode, w); + i += 1; + } + // Fork: a FirstChar/LastChar/Widths length mismatch is logged, + // not asserted (malformed documents). + if first_char + i - 1 != last_char { + warn!( + "Widths length mismatch: first_char {} + {} widths - 1 != last_char {}", + first_char, i, last_char + ); + } + } else { + let _name = if is_core_font(&base_name) { + &base_name + } else { + warn!("no widths and not core font {:?}", base_name); + + // This situation is handled differently by different readers + // but basically we try to substitute the best font that we can. + + // Poppler/Xpdf: + // this is technically an error -- the Widths entry is required + // for all but the Base-14 fonts -- but certain PDF generators + // apparently don't include widths for Arial and TimesNewRoman + + // Pdfium: CFX_FontMapper::FindSubstFont + + // mupdf: pdf_load_substitute_font + + // We can try to do a better job guessing at a font by looking at the flags + // or the basename but for now we'll just use Helvetica + "Helvetica" + }; + for font_metrics in core_fonts::metrics().iter() { + if font_metrics.0 == base_name { + if let Some(ref encoding) = encoding_table { + dlog!("has encoding"); + for w in font_metrics.2 { + // Fork: unknown glyph names are skipped instead + // of panicking (they cannot map to a codepoint). + let Some(c) = glyphnames::name_to_unicode(w.2) else { + continue; + }; + for i in 0..encoding.len() { + if encoding[i] == c { + width_map.insert(i as CharCode, w.1 as f64); + } + } + } + } else { + // Instead of using the encoding from the core font we'll just look up all + // of the character names. We should probably verify that this produces the + // same result. + + let mut table = vec![0; 256]; + for w in font_metrics.2 { + dlog!("{} {}", w.0, w.2); + // -1 is "not encoded" + if w.0 != -1 { + // Fork: unknown glyph names decode to + // U+FFFD instead of panicking. + table[w.0 as usize] = if base_name == "ZapfDingbats" { + zapfglyphnames::zapfdigbats_names_to_unicode(w.2) + .unwrap_or(0xFFFD) + } else { + glyphnames::name_to_unicode(w.2).unwrap_or(0xFFFD) + } + } + } + + let encoding = &table[..]; + for w in font_metrics.2 { + width_map.insert(w.0 as CharCode, w.1 as f64); + // -1 is "not encoded" + } + encoding_table = Some(encoding.to_vec()); + } + /* "Ordinarily, a font dictionary that refers to one of the standard fonts + should omit the FirstChar, LastChar, Widths, and FontDescriptor entries. + However, it is permissible to override a standard font by including these + entries and embedding the font program in the PDF file." + + Note: some PDFs include a descriptor but still don't include these entries */ + // assert!(maybe_get_obj(doc, font, b"FirstChar").is_none()); + // assert!(maybe_get_obj(doc, font, b"LastChar").is_none()); + // assert!(maybe_get_obj(doc, font, b"Widths").is_none()); + } + } + } + + let missing_width = get::>(doc, font, b"MissingWidth").unwrap_or(0.); + PdfSimpleFont { + doc, + font, + widths: width_map, + encoding: encoding_table, + missing_width, + unicode_map, + recovery: std::cell::RefCell::new(FontRecovery::new(reverse_cmap)), + } + } + + #[allow(dead_code)] + fn get_type(&self) -> String { + get_name_string(self.doc, self.font, b"Type") + } + #[allow(dead_code)] + fn get_basefont(&self) -> String { + get_name_string(self.doc, self.font, b"BaseFont") + } + #[allow(dead_code)] + fn get_subtype(&self) -> String { + get_name_string(self.doc, self.font, b"Subtype") + } + #[allow(dead_code)] + fn get_widths(&self) -> Option<&Vec> { + maybe_get_obj(self.doc, self.font, b"Widths") + .map(|widths| widths.as_array().expect("Widths should be an array")) + } + /* For type1: This entry is obsolescent and its use is no longer recommended. (See + * implementation note 42 in Appendix H.) */ + #[allow(dead_code)] + fn get_name(&self) -> Option { + maybe_get_name_string(self.doc, self.font, b"Name") + } + + #[allow(dead_code)] + fn get_descriptor(&self) -> Option> { + maybe_get_obj(self.doc, self.font, b"FontDescriptor") + .and_then(|desc| desc.as_dict().ok()) + .map(|desc| PdfFontDescriptor { + desc: desc, + doc: self.doc, + }) + } +} + +impl<'a> PdfType3Font<'a> { + fn new(doc: &'a Document, font: &'a Dictionary) -> PdfType3Font<'a> { + let unicode_map = get_unicode_map(doc, font); + let encoding: Option<&Object> = get(doc, font, b"Encoding"); + + let encoding_table; + match encoding { + Some(&Object::Name(ref encoding_name)) => { + dlog!("encoding {:?}", pdf_to_utf8(encoding_name)); + encoding_table = Some(encoding_to_unicode_table(encoding_name)); + } + Some(&Object::Dictionary(ref encoding)) => { + //dlog!("Encoding {:?}", encoding); + let mut table = + if let Some(base_encoding) = maybe_get_name(doc, encoding, b"BaseEncoding") { + dlog!("BaseEncoding {:?}", base_encoding); + encoding_to_unicode_table(base_encoding) + } else { + Vec::from(PDFDocEncoding) + }; + let differences = maybe_get_array(doc, encoding, b"Differences"); + if let Some(differences) = differences { + dlog!("Differences"); + let mut code = 0; + for o in differences { + match o { + &Object::Integer(i) => { + code = i; + } + &Object::Name(ref n) => { + let name = pdf_to_utf8(&n); + // Fork: ignore out-of-range Differences + // codes instead of indexing past the table. + if code < 0 || code as usize >= table.len() { + warn!("Differences code {} out of range", code); + code += 1; + continue; + } + // XXX: names of Type1 fonts can map to arbitrary strings instead of real + // unicode names, so we should probably handle this differently + let unicode = glyphnames::name_to_unicode(&name); + if let Some(unicode) = unicode { + table[code as usize] = unicode; + } + dlog!("{} = {} ({:?})", code, name, unicode); + if let Some(ref unicode_map) = unicode_map { + dlog!("{} {:?}", code, unicode_map.get(&(code as u32))); + } + code += 1; + } + _ => { + // Fork: skip bad entries instead of + // panicking. + warn!("unexpected Differences entry {:?}", o); + } + } + } + } + let name_encoded = encoding.get(b"Type"); + if let Ok(Object::Name(name)) = name_encoded { + dlog!("name: {}", pdf_to_utf8(name)); + } else { + dlog!("name not found"); + } + + encoding_table = Some(table); + } + other => { + // Fork: a Type3 font without a usable Encoding gets no + // encoding table instead of panicking. + warn!("unexpected Type3 Encoding object {:?}", other); + encoding_table = None; + } + } + + // Fork: Type3 metrics fall back to defaults instead of panicking + // on malformed font dictionaries. + let first_char: i64 = maybe_get(doc, font, b"FirstChar").unwrap_or(0); + let last_char: i64 = maybe_get(doc, font, b"LastChar").unwrap_or(0); + let widths: Vec = maybe_get(doc, font, b"Widths").unwrap_or_default(); + + let mut width_map = HashMap::new(); + + let mut i = 0; + dlog!( + "first_char {:?}, last_char: {:?}, widths: {} {:?}", + first_char, + last_char, + widths.len(), + widths + ); + + for w in widths { + width_map.insert((first_char + i) as CharCode, w); + i += 1; + } + // Fork: mismatch logged, not asserted (malformed documents). + if first_char + i - 1 != last_char { + warn!( + "Widths length mismatch: first_char {} + {} widths - 1 != last_char {}", + first_char, i, last_char + ); + } + PdfType3Font { + doc, + font, + widths: width_map, + encoding: encoding_table, + unicode_map, + } + } +} + +type CharCode = u32; + +struct PdfFontIter<'a> { + i: Iter<'a, u8>, + font: &'a dyn PdfFont, +} + +impl<'a> Iterator for PdfFontIter<'a> { + type Item = (CharCode, u8); + fn next(&mut self) -> Option<(CharCode, u8)> { + self.font.next_char(&mut self.i) + } +} + +trait PdfFont: Debug { + fn get_width(&self, id: CharCode) -> f64; + fn next_char(&self, iter: &mut Iter) -> Option<(CharCode, u8)>; + fn decode_char(&self, char: CharCode) -> String; + + /*fn char_codes<'a>(&'a self, chars: &'a [u8]) -> PdfFontIter { + let p = self; + PdfFontIter{i: chars.iter(), font: p as &PdfFont} + }*/ +} + +impl<'a> dyn PdfFont + 'a { + fn char_codes(&'a self, chars: &'a [u8]) -> PdfFontIter<'a> { + PdfFontIter { + i: chars.iter(), + font: self, + } + } + fn decode(&self, chars: &[u8]) -> String { + let strings = self + .char_codes(chars) + .map(|x| self.decode_char(x.0)) + .collect::>(); + strings.join("") + } +} + +impl<'a> PdfFont for PdfSimpleFont<'a> { + fn get_width(&self, id: CharCode) -> f64 { + let width = self.widths.get(&id); + if let Some(width) = width { + return *width; + } else { + let mut widths = self.widths.iter().collect::>(); + widths.sort_by_key(|x| x.0); + dlog!( + "missing width for {} len(widths) = {}, {:?} falling back to missing_width {:?}", + id, + self.widths.len(), + widths, + self.font + ); + return self.missing_width; + } + } + /*fn decode(&self, chars: &[u8]) -> String { + let encoding = self.encoding.as_ref().map(|x| &x[..]).unwrap_or(&PDFDocEncoding); + to_utf8(encoding, chars) + }*/ + + fn next_char(&self, iter: &mut Iter) -> Option<(CharCode, u8)> { + iter.next().map(|x| (*x as CharCode, 1)) + } + fn decode_char(&self, char: CharCode) -> String { + let slice = [char as u8]; + let encoding = self + .encoding + .as_ref() + .map(|x| &x[..]) + .unwrap_or(&PDFDocEncoding); + if let Some(ref unicode_map) = self.unicode_map { + if self.recovery.borrow().recovering { + // Flipped: the /ToUnicode is garbage — recover via the + // glyph-name-derived encoding table (spec §5.2b), with + // ligature expansion on the recovered text. A raw + // simple-font char code is NOT a glyph id, so the CID + // reverse cmap is not consulted here. + return expand_ligatures(&to_utf8(encoding, &slice)); + } + let hit = unicode_map.get(&char).cloned(); + return match hit { + // Healthy path (D2): verbatim, feeding the prescan. + Some(s) => { + self.recovery.borrow_mut().observe(&s); + s + } + None => { + debug!( + "missing char {:?} in unicode map {:?} for {:?}", + char, unicode_map, self.font + ); + // some pdf's like http://arxiv.org/pdf/2312.00064v1 are missing entries in their unicode map but do have + // entries in the encoding. + // Fork: fall back to PDFDocEncoding instead of + // panicking when both are missing. + // A miss also counts as a suspicious decode for the + // garbage prescan. + self.recovery.borrow_mut().observe(""); + let s = to_utf8(encoding, &slice); + debug!("falling back to encoding {} -> {:?}", char, s); + s + } + }; + } + //dlog!("char_code {:?} {:?}", char, self.encoding); + let s = to_utf8(encoding, &slice); + s + } +} + +impl<'a> fmt::Debug for PdfSimpleFont<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.font.fmt(f) + } +} + +impl<'a> PdfFont for PdfType3Font<'a> { + fn get_width(&self, id: CharCode) -> f64 { + let width = self.widths.get(&id); + if let Some(width) = width { + return *width; + } else { + // Fork: missing Type3 widths degrade to 0 instead of panicking. + dlog!("missing width for {} {:?}", id, self.font); + return 0.0; + } + } + /*fn decode(&self, chars: &[u8]) -> String { + let encoding = self.encoding.as_ref().map(|x| &x[..]).unwrap_or(&PDFDocEncoding); + to_utf8(encoding, chars) + }*/ + + fn next_char(&self, iter: &mut Iter) -> Option<(CharCode, u8)> { + iter.next().map(|x| (*x as CharCode, 1)) + } + fn decode_char(&self, char: CharCode) -> String { + let slice = [char as u8]; + if let Some(ref unicode_map) = self.unicode_map { + let s = unicode_map.get(&char); + let s = match s { + None => { + debug!( + "missing char {:?} in unicode map {:?} for {:?}", + char, unicode_map, self.font + ); + // some pdf's like http://arxiv.org/pdf/2312.00577v1 are missing entries in their unicode map but do have + // entries in the encoding. + // Fork: fall back to PDFDocEncoding instead of + // panicking when both are missing. + let encoding = self + .encoding + .as_ref() + .map(|x| &x[..]) + .unwrap_or(PDFDocEncoding); + let s = to_utf8(encoding, &slice); + debug!("falling back to encoding {} -> {:?}", char, s); + s + } + Some(s) => s.clone(), + }; + return s; + } + let encoding = self + .encoding + .as_ref() + .map(|x| &x[..]) + .unwrap_or(&PDFDocEncoding); + //dlog!("char_code {:?} {:?}", char, self.encoding); + let s = to_utf8(encoding, &slice); + s + } +} + +impl<'a> fmt::Debug for PdfType3Font<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.font.fmt(f) + } +} + +struct PdfCIDFont<'a> { + font: &'a Dictionary, + #[allow(dead_code)] + doc: &'a Document, + #[allow(dead_code)] + encoding: ByteMapping, + to_unicode: Option>, + widths: HashMap, // should probably just use i32 here + default_width: Option, // only used for CID fonts and we should probably brake out the different font types + /// Glyph-recovery state; consulted by `decode_char`. + recovery: std::cell::RefCell, +} + +fn get_unicode_map<'a>(doc: &'a Document, font: &'a Dictionary) -> Option> { + let to_unicode = maybe_get_obj(doc, font, b"ToUnicode"); + dlog!("ToUnicode: {:?}", to_unicode); + let mut unicode_map = None; + match to_unicode { + Some(&Object::Stream(ref stream)) => { + let contents = get_contents(stream); + dlog!("Stream: {}", String::from_utf8_lossy(&contents)); + + // Fork: an unparseable ToUnicode CMap means no unicode map, + // not a panic. + let Ok(cmap) = adobe_cmap_parser::get_unicode_map(&contents) else { + dlog!("failed to parse ToUnicode CMap"); + return None; + }; + let mut unicode = HashMap::new(); + // "It must use the beginbfchar, endbfchar, beginbfrange, and endbfrange operators to + // define the mapping from character codes to Unicode character sequences expressed in + // UTF-16BE encoding." + for (&k, v) in cmap.iter() { + let mut be: Vec = Vec::new(); + let mut i = 0; + // Fork: skip malformed (odd-length) UTF-16BE values + // instead of asserting. + if v.len() % 2 != 0 { + warn!("odd-length ToUnicode value for code {}", k); + continue; + } + while i < v.len() { + be.push(((v[i] as u16) << 8) | v[i + 1] as u16); + i += 2; + } + match &be[..] { + [0xd800..=0xdfff] => { + // this range is not specified as not being encoded + // we ignore them so we don't an error from from_utt16 + continue; + } + _ => {} + } + // Fork: skip values that aren't valid UTF-16. + let Ok(s) = String::from_utf16(&be) else { + continue; + }; + + unicode.insert(k, s); + } + unicode_map = Some(unicode); + + dlog!("map: {:?}", unicode_map); + } + None => {} + Some(&Object::Name(ref name)) => { + let name = pdf_to_utf8(name); + if name != "Identity-H" { + // Fork: unsupported ToUnicode names degrade to no map. + warn!("unsupported ToUnicode name: {:?}", name); + } + } + other => { + // Fork: non-stream, non-name ToUnicode values degrade to no map. + warn!("unsupported ToUnicode value {:?}", other); + } + } + unicode_map +} + +impl<'a> PdfCIDFont<'a> { + // Fork: returns None instead of panicking when the font dictionary is + // structurally broken (missing DescendantFonts/Encoding, unparseable + // CMap streams); make_font disables the font in that case. + fn new(doc: &'a Document, font: &'a Dictionary) -> Option> { + let base_name = get_name_string(doc, font, b"BaseFont"); + let descendants = maybe_get_array(doc, font, b"DescendantFonts")?; + let ciddict = maybe_deref(doc, descendants.first()?).as_dict().ok()?; + let encoding = maybe_get_obj(doc, font, b"Encoding")?; + dlog!("base_name {} {:?}", base_name, font); + + let encoding = match encoding { + &Object::Name(ref name) => { + let name = pdf_to_utf8(name); + dlog!("encoding {:?}", name); + if name == "Identity-H" || name == "Identity-V" { + ByteMapping { + codespace: vec![CodeRange { + width: 2, + start: 0, + end: 0xffff, + }], + cid: vec![CIDRange { + src_code_lo: 0, + src_code_hi: 0xffff, + dst_CID_lo: 0, + }], + } + } else { + // Fork: unsupported CID encodings disable the font + // instead of panicking. + dlog!("unsupported encoding {}", name); + return None; + } + } + &Object::Stream(ref stream) => { + let contents = get_contents(stream); + dlog!("Stream: {}", String::from_utf8_lossy(&contents)); + // Fork: an unparseable CID encoding CMap disables the font. + adobe_cmap_parser::get_byte_mapping(&contents).ok()? + } + other => { + dlog!("unsupported encoding {:?}", other); + return None; + } + }; + + // Sometimes a Type0 font might refer to the same underlying data as regular font. In this case we may be able to extract some encoding + // data. + // We should also look inside the truetype data to see if there's a cmap table. It will help us convert as well. + // This won't work if the cmap has been subsetted. A better approach might be to hash glyph contents and use that against + // a global library of glyph hashes + let unicode_map = get_unicode_map(doc, font); + + dlog!("descendents {:?} {:?}", descendants, ciddict); + + // Fork: a missing FontDescriptor is logged, not panicked on. + // When present, its embedded font program (FontFile2/FontFile3) + // seeds the reverse cmap used for glyph recovery. + let font_dict = maybe_get_obj(doc, ciddict, b"FontDescriptor"); + dlog!("{:?}", font_dict); + let reverse_cmap = font_dict + .and_then(|o| o.as_dict().ok()) + .and_then(|desc| reverse_cmap_from_descriptor(doc, desc)); + let default_width = get::>(doc, ciddict, b"DW").unwrap_or(1000); + let w: Option> = get(doc, ciddict, b"W"); + dlog!("widths {:?}", w); + let mut widths = HashMap::new(); + let mut i = 0; + if let Some(w) = w { + // Fork: malformed W arrays stop width parsing instead of + // panicking (both the index and the element types are + // untrusted). Upstream's (likely accidental) w[i] reads for + // c_first/c_last/c_width are preserved verbatim for + // byte-stability. + while i + 1 < w.len() { + if let &Object::Array(ref wa) = w[i + 1] { + let cid = match w[i].as_i64() { + Ok(cid) => cid, + Err(_) => { + dlog!("non-integer W array element"); + break; + } + }; + let mut j = 0; + dlog!("wa: {:?} -> {:?}", cid, wa); + for w in wa { + widths.insert((cid + j) as CharCode, as_num(w)); + j += 1; + } + i += 2; + } else { + let (c_first, c_last) = match (w[i].as_i64(), w[i].as_i64()) { + (Ok(first), Ok(last)) => (first, last), + _ => { + dlog!("non-integer W range bounds"); + break; + } + }; + let c_width = as_num(&w[i]); + for id in c_first..c_last { + widths.insert(id as CharCode, c_width); + } + i += 3; + } + } + } + Some(PdfCIDFont { + doc, + font, + widths, + to_unicode: unicode_map, + encoding, + default_width: Some(default_width as f64), + recovery: std::cell::RefCell::new(FontRecovery::new(reverse_cmap)), + }) + } +} + +impl<'a> PdfFont for PdfCIDFont<'a> { + fn get_width(&self, id: CharCode) -> f64 { + let width = self.widths.get(&id); + if let Some(width) = width { + dlog!("GetWidth {} -> {}", id, *width); + return *width; + } else { + dlog!("missing width for {} falling back to default_width", id); + // Fork: a missing default width degrades to 1000 (the PDF + // default) instead of panicking. + return self.default_width.unwrap_or(1000.0); + } + } /* + fn decode(&self, chars: &[u8]) -> String { + self.char_codes(chars); + + //let utf16 = Vec::new(); + + let encoding = self.encoding.as_ref().map(|x| &x[..]).unwrap_or(&PDFDocEncoding); + to_utf8(encoding, chars) + }*/ + + fn next_char(&self, iter: &mut Iter) -> Option<(CharCode, u8)> { + let mut c = *iter.next()? as u32; + let mut code = None; + 'outer: for width in 1..=4 { + for range in &self.encoding.codespace { + if c as u32 >= range.start && c as u32 <= range.end && range.width == width { + code = Some((c as u32, width)); + break 'outer; + } + } + let next = *iter.next()?; + c = ((c as u32) << 8) | next as u32; + } + let code = code?; + for range in &self.encoding.cid { + if code.0 >= range.src_code_lo && code.0 <= range.src_code_hi { + return Some((code.0 + range.dst_CID_lo, code.1 as u8)); + } + } + None + } + fn decode_char(&self, char: CharCode) -> String { + let hit = self.to_unicode.as_ref().and_then(|x| x.get(&char)).cloned(); + // Thin adapter over the shared recovery state machine; valid for + // CID fonts because the char code is the glyph id (Identity-H). + let out = decode_char_with(hit, &mut self.recovery.borrow_mut(), char); + if out.is_empty() { + dlog!( + "Unknown character {:?} in {:?} {:?}", + char, + self.font, + self.to_unicode + ); + } + out + } +} + +impl<'a> fmt::Debug for PdfCIDFont<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.font.fmt(f) + } +} + +#[derive(Copy, Clone)] +struct PdfFontDescriptor<'a> { + desc: &'a Dictionary, + doc: &'a Document, +} + +impl<'a> PdfFontDescriptor<'a> { + #[allow(dead_code)] + fn get_file(&self) -> Option<&'a Object> { + maybe_get_obj(self.doc, self.desc, b"FontFile") + } +} + +impl<'a> fmt::Debug for PdfFontDescriptor<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.desc.fmt(f) + } +} + +#[derive(Clone, Debug)] +struct Type0Func { + domain: Vec, + range: Vec, + contents: Vec, + size: Vec, + bits_per_sample: i64, + encode: Vec, + decode: Vec, +} + +#[allow(dead_code)] +fn interpolate(x: f64, x_min: f64, _x_max: f64, y_min: f64, y_max: f64) -> f64 { + let divisor = x - x_min; + if divisor != 0. { + y_min + (x - x_min) * ((y_max - y_min) / divisor) + } else { + // (x - x_min) will be 0 which means we want to discard the interpolation + // and arbitrarily choose y_min to match pdfium + y_min + } +} + +impl Type0Func { + #[allow(dead_code)] + fn eval(&self, _input: &[f64], _output: &mut [f64]) { + let _n_inputs = self.domain.len() / 2; + let _n_ouputs = self.range.len() / 2; + } +} + +#[derive(Clone, Debug)] +struct Type2Func { + c0: Option>, + c1: Option>, + n: f64, +} + +#[derive(Clone, Debug)] +enum Function { + Type0(Type0Func), + Type2(Type2Func), + #[allow(dead_code)] + Type3, + #[allow(dead_code)] + Type4(Vec), +} + +impl Function { + fn new(doc: &Document, obj: &Object) -> Function { + let dict = match obj { + &Object::Dictionary(ref dict) => dict, + &Object::Stream(ref stream) => &stream.dict, + _ => panic!(), + }; + let function_type: i64 = get(doc, dict, b"FunctionType"); + let f = match function_type { + 0 => { + // Sampled function + let stream = match obj { + &Object::Stream(ref stream) => stream, + _ => panic!(), + }; + let range: Vec = get(doc, dict, b"Range"); + let domain: Vec = get(doc, dict, b"Domain"); + let contents = get_contents(stream); + let size: Vec = get(doc, dict, b"Size"); + let bits_per_sample = get(doc, dict, b"BitsPerSample"); + // We ignore 'Order' like pdfium, poppler and pdf.js + + let encode = get::>>(doc, dict, b"Encode"); + // maybe there's some better way to write this. + let encode = encode.unwrap_or_else(|| { + let mut default = Vec::new(); + for i in &size { + default.extend([0., (i - 1) as f64].iter()); + } + default + }); + let decode = + get::>>(doc, dict, b"Decode").unwrap_or_else(|| range.clone()); + + Function::Type0(Type0Func { + domain, + range, + size, + contents, + bits_per_sample, + encode, + decode, + }) + } + 2 => { + // Exponential interpolation function + let c0 = get::>>(doc, dict, b"C0"); + let c1 = get::>>(doc, dict, b"C1"); + let n = get::(doc, dict, b"N"); + Function::Type2(Type2Func { c0, c1, n }) + } + 3 => { + // Stitching function + Function::Type3 + } + 4 => { + // PostScript calculator function + let contents = match obj { + &Object::Stream(ref stream) => { + let contents = get_contents(stream); + warn!("unhandled type-4 function"); + warn!("Stream: {}", String::from_utf8(contents.clone()).unwrap()); + contents + } + _ => { + panic!("type 4 functions should be streams") + } + }; + Function::Type4(contents) + } + _ => { + panic!("unhandled function type {}", function_type) + } + }; + f + } +} + +fn as_num(o: &Object) -> f64 { + match o { + &Object::Integer(i) => i as f64, + &Object::Real(f) => f.into(), + _ => { + panic!("not a number") + } + } +} + +#[derive(Clone)] +struct TextState<'a> { + font: Option>, + font_size: f64, + character_spacing: f64, + word_spacing: f64, + horizontal_scaling: f64, + leading: f64, + rise: f64, + tm: Transform, +} + +// XXX: We'd ideally implement this without having to copy the uncompressed data +fn get_contents(contents: &Stream) -> Vec { + if contents.filters().is_ok() { + contents + .decompressed_content() + .unwrap_or_else(|_| contents.content.clone()) + } else { + contents.content.clone() + } +} + +#[derive(Clone)] +struct GraphicsState<'a> { + ctm: Transform, + ts: TextState<'a>, + smask: Option, + fill_colorspace: ColorSpace, + fill_color: Vec, + stroke_colorspace: ColorSpace, + stroke_color: Vec, + line_width: f64, +} + +fn show_text( + gs: &mut GraphicsState, + s: &[u8], + _tlm: &Transform, + _flip_ctm: &Transform, + output: &mut dyn OutputDev, +) -> Result<(), OutputError> { + let ts = &mut gs.ts; + // Fork: text with no font selected (e.g. a failed font load) is + // skipped instead of panicking. + let Some(font) = ts.font.as_ref() else { + dlog!("show_text with no font set"); + return Ok(()); + }; + //let encoding = font.encoding.as_ref().map(|x| &x[..]).unwrap_or(&PDFDocEncoding); + dlog!("{:?}", font.decode(s)); + dlog!("{:?}", font.decode(s).as_bytes()); + dlog!("{:?}", s); + output.begin_word()?; + + for (c, length) in font.char_codes(s) { + // 5.3.3 Text Space Details + let tsm = Transform2D::row_major(ts.horizontal_scaling, 0., 0., 1.0, 0., ts.rise); + // Trm = Tsm × Tm × CTM + let trm = tsm.post_transform(&ts.tm.post_transform(&gs.ctm)); + //dlog!("ctm: {:?} tm {:?}", gs.ctm, tm); + //dlog!("current pos: {:?}", position); + // 5.9 Extraction of Text Content + + //dlog!("w: {}", font.widths[&(*c as i64)]); + let w0 = font.get_width(c) / 1000.; + + let mut spacing = ts.character_spacing; + // "Word spacing is applied to every occurrence of the single-byte character code 32 in a + // string when using a simple font or a composite font that defines code 32 as a + // single-byte code. It does not apply to occurrences of the byte value 32 in + // multiple-byte codes." + let is_space = c == 32 && length == 1; + if is_space { + spacing += ts.word_spacing + } + + output.output_character(&trm, w0, spacing, ts.font_size, &font.decode_char(c))?; + let tj = 0.; + let ty = 0.; + let tx = ts.horizontal_scaling * ((w0 - tj / 1000.) * ts.font_size + spacing); + dlog!( + "horizontal {} adjust {} {} {} {}", + ts.horizontal_scaling, + tx, + w0, + ts.font_size, + spacing + ); + // dlog!("w0: {}, tx: {}", w0, tx); + ts.tm = ts + .tm + .pre_transform(&Transform2D::create_translation(tx, ty)); + let _trm = ts.tm.pre_transform(&gs.ctm); + //dlog!("post pos: {:?}", trm); + } + output.end_word()?; + Ok(()) +} + +#[derive(Debug, Clone, Copy)] +pub struct MediaBox { + pub llx: f64, + pub lly: f64, + pub urx: f64, + pub ury: f64, +} + +fn apply_state(doc: &Document, gs: &mut GraphicsState, state: &Dictionary) { + for (k, v) in state.iter() { + let k: &[u8] = k.as_ref(); + match k { + b"SMask" => match maybe_deref(doc, v) { + &Object::Name(ref name) => { + if name == b"None" { + gs.smask = None; + } else { + panic!("unexpected smask name") + } + } + &Object::Dictionary(ref dict) => { + gs.smask = Some(dict.clone()); + } + _ => { + panic!("unexpected smask type {:?}", v) + } + }, + b"Type" => match v { + &Object::Name(ref name) => { + assert_eq!(name, b"ExtGState") + } + _ => { + panic!("unexpected type") + } + }, + _ => { + dlog!("unapplied state: {:?} {:?}", k, v); + } + } + } +} + +#[derive(Debug)] +pub enum PathOp { + MoveTo(f64, f64), + LineTo(f64, f64), + // XXX: is it worth distinguishing the different kinds of curve ops? + CurveTo(f64, f64, f64, f64, f64, f64), + Rect(f64, f64, f64, f64), + Close, +} + +#[derive(Debug)] +pub struct Path { + pub ops: Vec, +} + +impl Path { + fn new() -> Path { + Path { ops: Vec::new() } + } + fn current_point(&self) -> (f64, f64) { + match self.ops.last().unwrap() { + &PathOp::MoveTo(x, y) => (x, y), + &PathOp::LineTo(x, y) => (x, y), + &PathOp::CurveTo(_, _, _, _, x, y) => (x, y), + _ => { + panic!() + } + } + } +} + +#[derive(Clone, Debug)] +pub struct CalGray { + white_point: [f64; 3], + black_point: Option<[f64; 3]>, + gamma: Option, +} + +#[derive(Clone, Debug)] +pub struct CalRGB { + white_point: [f64; 3], + black_point: Option<[f64; 3]>, + gamma: Option<[f64; 3]>, + matrix: Option>, +} + +#[derive(Clone, Debug)] +pub struct Lab { + white_point: [f64; 3], + black_point: Option<[f64; 3]>, + range: Option<[f64; 4]>, +} + +#[derive(Clone, Debug)] +pub enum AlternateColorSpace { + DeviceGray, + DeviceRGB, + DeviceCMYK, + CalRGB(CalRGB), + CalGray(CalGray), + Lab(Lab), + ICCBased(Vec), +} + +#[derive(Clone)] +pub struct Separation { + name: String, + alternate_space: AlternateColorSpace, + tint_transform: Box, +} + +#[derive(Clone)] +pub enum ColorSpace { + DeviceGray, + DeviceRGB, + DeviceCMYK, + DeviceN, + Pattern, + CalRGB(CalRGB), + CalGray(CalGray), + Lab(Lab), + Separation(Separation), + ICCBased(Vec), +} + +fn make_colorspace<'a>(doc: &'a Document, name: &[u8], resources: &'a Dictionary) -> ColorSpace { + match name { + b"DeviceGray" => ColorSpace::DeviceGray, + b"DeviceRGB" => ColorSpace::DeviceRGB, + b"DeviceCMYK" => ColorSpace::DeviceCMYK, + b"Pattern" => ColorSpace::Pattern, + _ => { + let colorspaces: &Dictionary = get(&doc, resources, b"ColorSpace"); + let cs: &Object = maybe_get_obj(doc, colorspaces, &name[..]) + .unwrap_or_else(|| panic!("missing colorspace {:?}", &name[..])); + if let Ok(cs) = cs.as_array() { + let cs_name = pdf_to_utf8(cs[0].as_name().expect("first arg must be a name")); + match cs_name.as_ref() { + "Separation" => { + let name = pdf_to_utf8(cs[1].as_name().expect("second arg must be a name")); + let alternate_space = match &maybe_deref(doc, &cs[2]) { + Object::Name(name) => match &name[..] { + b"DeviceGray" => AlternateColorSpace::DeviceGray, + b"DeviceRGB" => AlternateColorSpace::DeviceRGB, + b"DeviceCMYK" => AlternateColorSpace::DeviceCMYK, + _ => panic!("unexpected color space name"), + }, + Object::Array(cs) => { + let cs_name = + pdf_to_utf8(cs[0].as_name().expect("first arg must be a name")); + match cs_name.as_ref() { + "ICCBased" => { + let stream = maybe_deref(doc, &cs[1]).as_stream().unwrap(); + dlog!("ICCBased {:?}", stream); + // XXX: we're going to be continually decompressing everytime this object is referenced + AlternateColorSpace::ICCBased(get_contents(stream)) + } + "CalGray" => { + let dict = + cs[1].as_dict().expect("second arg must be a dict"); + AlternateColorSpace::CalGray(CalGray { + white_point: get(&doc, dict, b"WhitePoint"), + black_point: get(&doc, dict, b"BackPoint"), + gamma: get(&doc, dict, b"Gamma"), + }) + } + "CalRGB" => { + let dict = + cs[1].as_dict().expect("second arg must be a dict"); + AlternateColorSpace::CalRGB(CalRGB { + white_point: get(&doc, dict, b"WhitePoint"), + black_point: get(&doc, dict, b"BackPoint"), + gamma: get(&doc, dict, b"Gamma"), + matrix: get(&doc, dict, b"Matrix"), + }) + } + "Lab" => { + let dict = + cs[1].as_dict().expect("second arg must be a dict"); + AlternateColorSpace::Lab(Lab { + white_point: get(&doc, dict, b"WhitePoint"), + black_point: get(&doc, dict, b"BackPoint"), + range: get(&doc, dict, b"Range"), + }) + } + _ => panic!("Unexpected color space name"), + } + } + _ => panic!("Alternate space should be name or array {:?}", cs[2]), + }; + let tint_transform = Box::new(Function::new(doc, maybe_deref(doc, &cs[3]))); + + dlog!("{:?} {:?} {:?}", name, alternate_space, tint_transform); + ColorSpace::Separation(Separation { + name, + alternate_space, + tint_transform, + }) + } + "ICCBased" => { + let stream = maybe_deref(doc, &cs[1]).as_stream().unwrap(); + dlog!("ICCBased {:?}", stream); + // XXX: we're going to be continually decompressing everytime this object is referenced + ColorSpace::ICCBased(get_contents(stream)) + } + "CalGray" => { + let dict = cs[1].as_dict().expect("second arg must be a dict"); + ColorSpace::CalGray(CalGray { + white_point: get(&doc, dict, b"WhitePoint"), + black_point: get(&doc, dict, b"BackPoint"), + gamma: get(&doc, dict, b"Gamma"), + }) + } + "CalRGB" => { + let dict = cs[1].as_dict().expect("second arg must be a dict"); + ColorSpace::CalRGB(CalRGB { + white_point: get(&doc, dict, b"WhitePoint"), + black_point: get(&doc, dict, b"BackPoint"), + gamma: get(&doc, dict, b"Gamma"), + matrix: get(&doc, dict, b"Matrix"), + }) + } + "Lab" => { + let dict = cs[1].as_dict().expect("second arg must be a dict"); + ColorSpace::Lab(Lab { + white_point: get(&doc, dict, b"WhitePoint"), + black_point: get(&doc, dict, b"BackPoint"), + range: get(&doc, dict, b"Range"), + }) + } + "Pattern" => ColorSpace::Pattern, + "DeviceGray" => ColorSpace::DeviceGray, + "DeviceRGB" => ColorSpace::DeviceRGB, + "DeviceCMYK" => ColorSpace::DeviceCMYK, + "DeviceN" => ColorSpace::DeviceN, + _ => { + panic!("color_space {:?} {:?} {:?}", name, cs_name, cs) + } + } + } else if let Ok(cs) = cs.as_name() { + match pdf_to_utf8(cs).as_ref() { + "DeviceRGB" => ColorSpace::DeviceRGB, + "DeviceGray" => ColorSpace::DeviceGray, + _ => panic!(), + } + } else { + panic!(); + } + } + } +} + +struct Processor<'a> { + _none: PhantomData<&'a ()>, +} + +impl<'a> Processor<'a> { + fn new() -> Processor<'a> { + Processor { _none: PhantomData } + } + + fn process_stream( + &mut self, + doc: &'a Document, + content: Vec, + resources: &'a Dictionary, + media_box: &MediaBox, + output: &mut dyn OutputDev, + page_num: u32, + ) -> Result<(), OutputError> { + // Fork: a content stream that fails to lex is skipped (the page + // yields no text) instead of panicking. + let content = match Content::decode(&content) { + Ok(content) => content, + Err(e) => { + warn!("failed to decode content stream: {:?}", e); + return Ok(()); + } + }; + let mut font_table = HashMap::new(); + let mut gs: GraphicsState = GraphicsState { + ts: TextState { + font: None, + font_size: std::f64::NAN, + character_spacing: 0., + word_spacing: 0., + horizontal_scaling: 100. / 100., + leading: 0., + rise: 0., + tm: Transform2D::identity(), + }, + fill_color: Vec::new(), + fill_colorspace: ColorSpace::DeviceGray, + stroke_color: Vec::new(), + stroke_colorspace: ColorSpace::DeviceGray, + line_width: 1., + ctm: Transform2D::identity(), + smask: None, + }; + //let mut ts = &mut gs.ts; + let mut gs_stack = Vec::new(); + let mut mc_stack = Vec::new(); + // XXX: replace tlm with a point for text start + let mut tlm = Transform2D::identity(); + let mut path = Path::new(); + let flip_ctm = Transform2D::row_major(1., 0., 0., -1., 0., media_box.ury - media_box.lly); + dlog!("MediaBox {:?}", media_box); + for operation in &content.operations { + //dlog!("op: {:?}", operation); + + match operation.operator.as_ref() { + "BT" => { + tlm = Transform2D::identity(); + gs.ts.tm = tlm; + } + "ET" => { + tlm = Transform2D::identity(); + gs.ts.tm = tlm; + } + "cm" => { + assert!(operation.operands.len() == 6); + let m = Transform2D::row_major( + as_num(&operation.operands[0]), + as_num(&operation.operands[1]), + as_num(&operation.operands[2]), + as_num(&operation.operands[3]), + as_num(&operation.operands[4]), + as_num(&operation.operands[5]), + ); + gs.ctm = gs.ctm.pre_transform(&m); + dlog!("matrix {:?}", gs.ctm); + } + "CS" => { + // Fork: missing/mistyped operands are skipped instead + // of panicking (make_colorspace internals remain + // upstream — graphics-only path). + let Some(name) = operation.operands.first().and_then(|o| o.as_name().ok()) + else { + warn!("CS without a colorspace name operand"); + continue; + }; + gs.stroke_colorspace = make_colorspace(doc, name, resources); + } + "cs" => { + let Some(name) = operation.operands.first().and_then(|o| o.as_name().ok()) + else { + warn!("cs without a colorspace name operand"); + continue; + }; + gs.fill_colorspace = make_colorspace(doc, name, resources); + } + "SC" | "SCN" => { + gs.stroke_color = match gs.stroke_colorspace { + ColorSpace::Pattern => { + dlog!("unhandled pattern color"); + Vec::new() + } + _ => operation.operands.iter().map(|x| as_num(x)).collect(), + }; + } + "sc" | "scn" => { + gs.fill_color = match gs.fill_colorspace { + ColorSpace::Pattern => { + dlog!("unhandled pattern color"); + Vec::new() + } + _ => operation.operands.iter().map(|x| as_num(x)).collect(), + }; + } + "G" | "g" | "RG" | "rg" | "K" | "k" => { + dlog!("unhandled color operation {:?}", operation); + } + // Fork: a missing/mistyped TJ operand falls through to the + // catch-all instead of panicking on the index. + "TJ" => match operation.operands.first() { + Some(&Object::Array(ref array)) => { + for e in array { + match e { + &Object::String(ref s, _) => { + show_text(&mut gs, s, &tlm, &flip_ctm, output)?; + } + &Object::Integer(i) => { + let ts = &mut gs.ts; + let w0 = 0.; + let tj = i as f64; + let ty = 0.; + let tx = + ts.horizontal_scaling * ((w0 - tj / 1000.) * ts.font_size); + ts.tm = ts + .tm + .pre_transform(&Transform2D::create_translation(tx, ty)); + dlog!("adjust text by: {} {:?}", i, ts.tm); + } + &Object::Real(i) => { + let ts = &mut gs.ts; + let w0 = 0.; + let tj = i as f64; + let ty = 0.; + let tx = + ts.horizontal_scaling * ((w0 - tj / 1000.) * ts.font_size); + ts.tm = ts + .tm + .pre_transform(&Transform2D::create_translation(tx, ty)); + dlog!("adjust text by: {} {:?}", i, ts.tm); + } + _ => { + dlog!("kind of {:?}", e); + } + } + } + } + other => { + warn!("unexpected TJ operand {:?}", other) + } + }, + "Tj" => match operation.operands.first() { + Some(&Object::String(ref s, _)) => { + show_text(&mut gs, s, &tlm, &flip_ctm, output)?; + } + // Fork: malformed Tj operands are skipped, not + // panicked on. + other => { + warn!("unexpected Tj operand {:?}", other) + } + }, + "Tc" => { + gs.ts.character_spacing = as_num(&operation.operands[0]); + } + "Tw" => { + gs.ts.word_spacing = as_num(&operation.operands[0]); + } + "Tz" => { + gs.ts.horizontal_scaling = as_num(&operation.operands[0]) / 100.; + } + "TL" => { + gs.ts.leading = as_num(&operation.operands[0]); + } + "Tf" => { + // Fork: a missing/invalid Font resource or operand, or + // a font that fails to load, leaves the current font + // unchanged instead of panicking. + let name = operation.operands.first().and_then(|o| o.as_name().ok()); + let font_dict = name + .and_then(|name| { + maybe_get_obj(&doc, resources, b"Font")? + .as_dict() + .ok() + .and_then(|fonts| maybe_get_obj(&doc, fonts, name)) + }) + .and_then(|f| f.as_dict().ok()); + match (name, font_dict) { + (Some(name), Some(font_dict)) => { + if !font_table.contains_key(name) { + if let Some(font) = make_font(doc, font_dict) { + font_table.insert(name.to_owned(), font); + } else { + warn!("failed to load font {:?}", pdf_to_utf8(name)); + } + } + if let Some(font) = font_table.get(name) { + gs.ts.font = Some(font.clone()); + } + if let Some(size) = operation.operands.get(1) { + gs.ts.font_size = as_num(size); + } + dlog!( + "font {} size: {} {:?}", + pdf_to_utf8(name), + gs.ts.font_size, + operation + ); + } + _ => warn!("Tf with missing font resource or operand"), + } + } + "Ts" => { + gs.ts.rise = as_num(&operation.operands[0]); + } + "Tm" => { + // Fork: malformed Tm operators are skipped, not + // asserted on. + if operation.operands.len() != 6 { + warn!("Tm with {} operands", operation.operands.len()); + continue; + } + tlm = Transform2D::row_major( + as_num(&operation.operands[0]), + as_num(&operation.operands[1]), + as_num(&operation.operands[2]), + as_num(&operation.operands[3]), + as_num(&operation.operands[4]), + as_num(&operation.operands[5]), + ); + gs.ts.tm = tlm; + dlog!("Tm: matrix {:?}", gs.ts.tm); + output.end_line()?; + } + "Td" => { + /* Move to the start of the next line, offset from the start of the current line by (tx , ty ). + tx and ty are numbers expressed in unscaled text space units. + More precisely, this operator performs the following assignments: + */ + assert!(operation.operands.len() == 2); + let tx = as_num(&operation.operands[0]); + let ty = as_num(&operation.operands[1]); + dlog!("translation: {} {}", tx, ty); + + tlm = tlm.pre_transform(&Transform2D::create_translation(tx, ty)); + gs.ts.tm = tlm; + dlog!("Td matrix {:?}", gs.ts.tm); + output.end_line()?; + } + + "TD" => { + /* Move to the start of the next line, offset from the start of the current line by (tx , ty ). + As a side effect, this operator sets the leading parameter in the text state. + */ + assert!(operation.operands.len() == 2); + let tx = as_num(&operation.operands[0]); + let ty = as_num(&operation.operands[1]); + dlog!("translation: {} {}", tx, ty); + gs.ts.leading = -ty; + + tlm = tlm.pre_transform(&Transform2D::create_translation(tx, ty)); + gs.ts.tm = tlm; + dlog!("TD matrix {:?}", gs.ts.tm); + output.end_line()?; + } + + "T*" => { + let tx = 0.0; + let ty = -gs.ts.leading; + + tlm = tlm.pre_transform(&Transform2D::create_translation(tx, ty)); + gs.ts.tm = tlm; + dlog!("T* matrix {:?}", gs.ts.tm); + output.end_line()?; + } + "q" => { + gs_stack.push(gs.clone()); + } + "Q" => { + let s = gs_stack.pop(); + if let Some(s) = s { + gs = s; + } else { + warn!("No state to pop"); + } + } + "gs" => { + // Fork: malformed ExtGState references are skipped + // instead of panicking. + let name = operation.operands.first().and_then(|o| o.as_name().ok()); + let state = maybe_get_obj(&doc, resources, b"ExtGState") + .and_then(|g| g.as_dict().ok()) + .and_then(|g| name.and_then(|name| maybe_get_obj(&doc, g, name))) + .and_then(|s| s.as_dict().ok()); + match state { + Some(state) => apply_state(doc, &mut gs, state), + None => warn!("gs operator with missing ExtGState or operand"), + } + } + "i" => { + dlog!( + "unhandled graphics state flattness operator {:?}", + operation + ); + } + "w" => { + gs.line_width = as_num(&operation.operands[0]); + } + "J" | "j" | "M" | "d" | "ri" => { + dlog!("unknown graphics state operator {:?}", operation); + } + "m" => path.ops.push(PathOp::MoveTo( + as_num(&operation.operands[0]), + as_num(&operation.operands[1]), + )), + "l" => path.ops.push(PathOp::LineTo( + as_num(&operation.operands[0]), + as_num(&operation.operands[1]), + )), + "c" => path.ops.push(PathOp::CurveTo( + as_num(&operation.operands[0]), + as_num(&operation.operands[1]), + as_num(&operation.operands[2]), + as_num(&operation.operands[3]), + as_num(&operation.operands[4]), + as_num(&operation.operands[5]), + )), + "v" => { + let (x, y) = path.current_point(); + path.ops.push(PathOp::CurveTo( + x, + y, + as_num(&operation.operands[0]), + as_num(&operation.operands[1]), + as_num(&operation.operands[2]), + as_num(&operation.operands[3]), + )) + } + "y" => path.ops.push(PathOp::CurveTo( + as_num(&operation.operands[0]), + as_num(&operation.operands[1]), + as_num(&operation.operands[2]), + as_num(&operation.operands[3]), + as_num(&operation.operands[2]), + as_num(&operation.operands[3]), + )), + "h" => path.ops.push(PathOp::Close), + "re" => path.ops.push(PathOp::Rect( + as_num(&operation.operands[0]), + as_num(&operation.operands[1]), + as_num(&operation.operands[2]), + as_num(&operation.operands[3]), + )), + "s" | "f*" | "B" | "B*" | "b" => { + dlog!("unhandled path op {:?}", operation); + } + "S" => { + output.stroke(&gs.ctm, &gs.stroke_colorspace, &gs.stroke_color, &path)?; + path.ops.clear(); + } + "F" | "f" => { + output.fill(&gs.ctm, &gs.fill_colorspace, &gs.fill_color, &path)?; + path.ops.clear(); + } + "W" | "w*" => { + dlog!("unhandled clipping operation {:?}", operation); + } + "n" => { + dlog!("discard {:?}", path); + path.ops.clear(); + } + "BMC" | "BDC" => { + mc_stack.push(operation); + } + "EMC" => { + mc_stack.pop(); + } + "Do" => { + // `Do` process an entire subdocument, so we do a recursive call to `process_stream` + // with the subdocument content and resources + // Fork: missing/mistyped XObjects are skipped instead + // of panicking. + let xf = operation + .operands + .first() + .and_then(|o| o.as_name().ok()) + .and_then(|name| { + maybe_get_obj(&doc, resources, b"XObject")? + .as_dict() + .ok() + .and_then(|xobject| maybe_get_obj(&doc, xobject, name)) + }) + .and_then(|x| x.as_stream().ok()); + match xf { + Some(xf) => { + let resources = maybe_get_obj(&doc, &xf.dict, b"Resources") + .and_then(|n| n.as_dict().ok()) + .unwrap_or(resources); + let contents = get_contents(xf); + self.process_stream( + &doc, contents, resources, &media_box, output, page_num, + )?; + } + None => warn!("Do with missing XObject or operand"), + } + } + _ => { + dlog!("unknown operation {:?}", operation); + } + } + } + Ok(()) + } +} + +pub trait OutputDev { + fn begin_page( + &mut self, + page_num: u32, + media_box: &MediaBox, + art_box: Option<(f64, f64, f64, f64)>, + ) -> Result<(), OutputError>; + fn end_page(&mut self) -> Result<(), OutputError>; + fn output_character( + &mut self, + trm: &Transform, + width: f64, + spacing: f64, + font_size: f64, + char: &str, + ) -> Result<(), OutputError>; + fn begin_word(&mut self) -> Result<(), OutputError>; + fn end_word(&mut self) -> Result<(), OutputError>; + fn end_line(&mut self) -> Result<(), OutputError>; + fn stroke( + &mut self, + _ctm: &Transform, + _colorspace: &ColorSpace, + _color: &[f64], + _path: &Path, + ) -> Result<(), OutputError> { + Ok(()) + } + fn fill( + &mut self, + _ctm: &Transform, + _colorspace: &ColorSpace, + _color: &[f64], + _path: &Path, + ) -> Result<(), OutputError> { + Ok(()) + } +} + +pub struct HTMLOutput<'a> { + file: &'a mut dyn std::io::Write, + flip_ctm: Transform, + last_ctm: Transform, + buf_ctm: Transform, + buf_font_size: f64, + buf: String, +} + +fn insert_nbsp(input: &str) -> String { + let mut result = String::new(); + let mut word_end = false; + let mut chars = input.chars().peekable(); + while let Some(c) = chars.next() { + if c == ' ' { + if !word_end || chars.peek().filter(|x| **x != ' ').is_none() { + result += " "; + } else { + result += " "; + } + word_end = false; + } else { + word_end = true; + result.push(c); + } + } + result +} + +impl<'a> HTMLOutput<'a> { + pub fn new(file: &mut dyn std::io::Write) -> HTMLOutput<'_> { + HTMLOutput { + file, + flip_ctm: Transform2D::identity(), + last_ctm: Transform2D::identity(), + buf_ctm: Transform2D::identity(), + buf: String::new(), + buf_font_size: 0., + } + } + fn flush_string(&mut self) -> Result<(), OutputError> { + if self.buf.len() != 0 { + let position = self.buf_ctm.post_transform(&self.flip_ctm); + let transformed_font_size_vec = self + .buf_ctm + .transform_vector(vec2(self.buf_font_size, self.buf_font_size)); + // get the length of one sized of the square with the same area with a rectangle of size (x, y) + let transformed_font_size = + (transformed_font_size_vec.x * transformed_font_size_vec.y).sqrt(); + let (x, y) = (position.m31, position.m32); + warn!("flush {} {:?}", self.buf, (x, y)); + + write!(self.file, "
{}
\n", + x, y, transformed_font_size, insert_nbsp(&self.buf))?; + } + Ok(()) + } +} + +type ArtBox = (f64, f64, f64, f64); + +impl<'a> OutputDev for HTMLOutput<'a> { + fn begin_page( + &mut self, + page_num: u32, + media_box: &MediaBox, + _: Option, + ) -> Result<(), OutputError> { + write!(self.file, " ")?; + write!(self.file, "", page_num)?; + write!(self.file, "
", page_num, media_box.ury - media_box.lly, media_box.urx - media_box.llx)?; + self.flip_ctm = Transform::row_major(1., 0., 0., -1., 0., media_box.ury - media_box.lly); + Ok(()) + } + fn end_page(&mut self) -> Result<(), OutputError> { + self.flush_string()?; + self.buf = String::new(); + self.last_ctm = Transform::identity(); + write!(self.file, "
")?; + Ok(()) + } + fn output_character( + &mut self, + trm: &Transform, + width: f64, + spacing: f64, + font_size: f64, + char: &str, + ) -> Result<(), OutputError> { + if trm.approx_eq(&self.last_ctm) { + let position = trm.post_transform(&self.flip_ctm); + let (x, y) = (position.m31, position.m32); + + warn!("accum {} {:?}", char, (x, y)); + self.buf += char; + } else { + warn!( + "flush {} {:?} {:?} {} {} {}", + char, trm, self.last_ctm, width, font_size, spacing + ); + self.flush_string()?; + self.buf = char.to_owned(); + self.buf_font_size = font_size; + self.buf_ctm = *trm; + } + let position = trm.post_transform(&self.flip_ctm); + let transformed_font_size_vec = trm.transform_vector(vec2(font_size, font_size)); + // get the length of one sized of the square with the same area with a rectangle of size (x, y) + let transformed_font_size = + (transformed_font_size_vec.x * transformed_font_size_vec.y).sqrt(); + let (x, y) = (position.m31, position.m32); + write!(self.file, "
{}
", + x, y, transformed_font_size, char)?; + self.last_ctm = trm.pre_transform(&Transform2D::create_translation( + width * font_size + spacing, + 0., + )); + + Ok(()) + } + fn begin_word(&mut self) -> Result<(), OutputError> { + Ok(()) + } + fn end_word(&mut self) -> Result<(), OutputError> { + Ok(()) + } + fn end_line(&mut self) -> Result<(), OutputError> { + Ok(()) + } +} + +pub struct SVGOutput<'a> { + file: &'a mut dyn std::io::Write, +} +impl<'a> SVGOutput<'a> { + pub fn new(file: &mut dyn std::io::Write) -> SVGOutput<'_> { + SVGOutput { file } + } +} + +impl<'a> OutputDev for SVGOutput<'a> { + fn begin_page( + &mut self, + _page_num: u32, + media_box: &MediaBox, + art_box: Option<(f64, f64, f64, f64)>, + ) -> Result<(), OutputError> { + let ver = 1.1; + write!(self.file, "\n")?; + if ver == 1.1 { + write!( + self.file, + r#""# + )?; + } else { + write!( + self.file, + r#""# + )?; + } + if let Some(art_box) = art_box { + let width = art_box.2 - art_box.0; + let height = art_box.3 - art_box.1; + let y = media_box.ury - art_box.1 - height; + write!(self.file, "", width, height, ver, art_box.0, y, width, height)?; + } else { + let width = media_box.urx - media_box.llx; + let height = media_box.ury - media_box.lly; + write!(self.file, "", width, height, ver, media_box.llx, media_box.lly, width, height)?; + } + write!(self.file, "\n")?; + type Mat = Transform; + + let ctm = Mat::create_scale(1., -1.).post_translate(vec2(0., media_box.ury)); + write!( + self.file, + "\n", + ctm.m11, ctm.m12, ctm.m21, ctm.m22, ctm.m31, ctm.m32, + )?; + Ok(()) + } + fn end_page(&mut self) -> Result<(), OutputError> { + write!(self.file, "\n")?; + write!(self.file, "")?; + Ok(()) + } + fn output_character( + &mut self, + _trm: &Transform, + _width: f64, + _spacing: f64, + _font_size: f64, + _char: &str, + ) -> Result<(), OutputError> { + Ok(()) + } + fn begin_word(&mut self) -> Result<(), OutputError> { + Ok(()) + } + fn end_word(&mut self) -> Result<(), OutputError> { + Ok(()) + } + fn end_line(&mut self) -> Result<(), OutputError> { + Ok(()) + } + fn fill( + &mut self, + ctm: &Transform, + _colorspace: &ColorSpace, + _color: &[f64], + path: &Path, + ) -> Result<(), OutputError> { + write!( + self.file, + "", + ctm.m11, ctm.m12, ctm.m21, ctm.m22, ctm.m31, ctm.m32, + )?; + + /*if path.ops.len() == 1 { + if let PathOp::Rect(x, y, width, height) = path.ops[0] { + write!(self.file, "\n", x, y, width, height); + write!(self.file, ""); + return; + } + }*/ + let mut d = Vec::new(); + for op in &path.ops { + match op { + &PathOp::MoveTo(x, y) => d.push(format!("M{} {}", x, y)), + &PathOp::LineTo(x, y) => d.push(format!("L{} {}", x, y)), + &PathOp::CurveTo(x1, y1, x2, y2, x, y) => { + d.push(format!("C{} {} {} {} {} {}", x1, y1, x2, y2, x, y)) + } + &PathOp::Close => d.push(format!("Z")), + &PathOp::Rect(x, y, width, height) => { + d.push(format!("M{} {}", x, y)); + d.push(format!("L{} {}", x + width, y)); + d.push(format!("L{} {}", x + width, y + height)); + d.push(format!("L{} {}", x, y + height)); + d.push(format!("Z")); + } + } + } + write!(self.file, "", d.join(" "))?; + write!(self.file, "")?; + write!(self.file, "\n")?; + Ok(()) + } +} + +/* +File doesn't implement std::fmt::Write so we have +to do some gymnastics to accept a File or String +See https://github.com/rust-lang/rust/issues/51305 +*/ + +pub trait ConvertToFmt { + type Writer: std::fmt::Write; + fn convert(self) -> Self::Writer; +} + +impl<'a> ConvertToFmt for &'a mut String { + type Writer = &'a mut String; + fn convert(self) -> Self::Writer { + self + } +} + +pub struct WriteAdapter { + f: W, +} + +impl std::fmt::Write for WriteAdapter { + fn write_str(&mut self, s: &str) -> Result<(), std::fmt::Error> { + self.f.write_all(s.as_bytes()).map_err(|_| fmt::Error) + } +} + +impl<'a> ConvertToFmt for &'a mut dyn std::io::Write { + type Writer = WriteAdapter; + fn convert(self) -> Self::Writer { + WriteAdapter { f: self } + } +} + +impl<'a> ConvertToFmt for &'a mut File { + type Writer = WriteAdapter; + fn convert(self) -> Self::Writer { + WriteAdapter { f: self } + } +} + +pub struct PlainTextOutput { + writer: W::Writer, + last_end: f64, + last_y: f64, + first_char: bool, + flip_ctm: Transform, +} + +impl PlainTextOutput { + pub fn new(writer: W) -> PlainTextOutput { + PlainTextOutput { + writer: writer.convert(), + last_end: 100000., + first_char: false, + last_y: 0., + flip_ctm: Transform2D::identity(), + } + } +} + +/* There are some structural hints that PDFs can use to signal word and line endings: + * however relying on these is not likely to be sufficient. */ +impl OutputDev for PlainTextOutput { + fn begin_page( + &mut self, + _page_num: u32, + media_box: &MediaBox, + _: Option, + ) -> Result<(), OutputError> { + self.flip_ctm = Transform2D::row_major(1., 0., 0., -1., 0., media_box.ury - media_box.lly); + Ok(()) + } + fn end_page(&mut self) -> Result<(), OutputError> { + Ok(()) + } + fn output_character( + &mut self, + trm: &Transform, + width: f64, + _spacing: f64, + font_size: f64, + char: &str, + ) -> Result<(), OutputError> { + let position = trm.post_transform(&self.flip_ctm); + let transformed_font_size_vec = trm.transform_vector(vec2(font_size, font_size)); + // get the length of one sized of the square with the same area with a rectangle of size (x, y) + let transformed_font_size = + (transformed_font_size_vec.x * transformed_font_size_vec.y).sqrt(); + let (x, y) = (position.m31, position.m32); + use std::fmt::Write; + //dlog!("last_end: {} x: {}, width: {}", self.last_end, x, width); + if self.first_char { + if (y - self.last_y).abs() > transformed_font_size * 1.5 { + write!(self.writer, "\n")?; + } + + // we've moved to the left and down + if x < self.last_end && (y - self.last_y).abs() > transformed_font_size * 0.5 { + write!(self.writer, "\n")?; + } + + if x > self.last_end + transformed_font_size * 0.1 { + dlog!( + "width: {}, space: {}, thresh: {}", + width, + x - self.last_end, + transformed_font_size * 0.1 + ); + write!(self.writer, " ")?; + } + } + //let norm = unicode_normalization::UnicodeNormalization::nfkc(char); + write!(self.writer, "{}", char)?; + self.first_char = false; + self.last_y = y; + self.last_end = x + width * transformed_font_size; + Ok(()) + } + fn begin_word(&mut self) -> Result<(), OutputError> { + self.first_char = true; + Ok(()) + } + fn end_word(&mut self) -> Result<(), OutputError> { + Ok(()) + } + fn end_line(&mut self) -> Result<(), OutputError> { + //write!(self.file, "\n"); + Ok(()) + } +} + +pub fn print_metadata(doc: &Document) { + dlog!("Version: {}", doc.version); + if let Some(ref info) = get_info(&doc) { + for (k, v) in *info { + match v { + &Object::String(ref s, StringFormat::Literal) => { + dlog!("{}: {}", pdf_to_utf8(k), pdf_to_utf8(s)); + } + _ => {} + } + } + } + dlog!( + "Page count: {}", + get::(&doc, &get_pages(&doc), b"Count") + ); + dlog!("Pages: {:?}", get_pages(&doc)); + dlog!( + "Type: {:?}", + get_pages(&doc) + .get(b"Type") + .and_then(|x| x.as_name()) + .unwrap() + ); +} + +/// Extract the text from a pdf at `path` and return a `String` with the results +pub fn extract_text>( + path: P, +) -> Result { + let mut s = String::new(); + { + let mut output = PlainTextOutput::new(&mut s); + let mut doc = Document::load(path)?; + maybe_decrypt(&mut doc)?; + output_doc(&doc, &mut output)?; + } + Ok(s) +} + +fn maybe_decrypt(doc: &mut Document) -> Result<(), OutputError> { + if !doc.is_encrypted() { + return Ok(()); + } + + if let Err(e) = doc.decrypt("") { + if let Error::Decryption(DecryptionError::IncorrectPassword) = e { + error!("Encrypted documents must be decrypted with a password using {{extract_text|extract_text_from_mem|output_doc}}_encrypted") + } + + return Err(OutputError::PdfError(e)); + } + + Ok(()) +} + +pub fn extract_text_encrypted>( + path: P, + password: &str, +) -> Result { + let mut s = String::new(); + { + let mut output = PlainTextOutput::new(&mut s); + let mut doc = Document::load(path)?; + output_doc_encrypted(&mut doc, &mut output, password)?; + } + Ok(s) +} + +pub fn extract_text_from_mem(buffer: &[u8]) -> Result { + let mut s = String::new(); + { + let mut output = PlainTextOutput::new(&mut s); + let mut doc = Document::load_mem(buffer)?; + maybe_decrypt(&mut doc)?; + output_doc(&doc, &mut output)?; + } + Ok(s) +} + +pub fn extract_text_from_mem_encrypted( + buffer: &[u8], + password: &str, +) -> Result { + let mut s = String::new(); + { + let mut output = PlainTextOutput::new(&mut s); + let mut doc = Document::load_mem(buffer)?; + output_doc_encrypted(&mut doc, &mut output, password)?; + } + Ok(s) +} + +fn extract_text_by_page(doc: &Document, page_num: u32) -> Result { + let mut s = String::new(); + { + let mut output = PlainTextOutput::new(&mut s); + output_doc_page(doc, &mut output, page_num)?; + } + Ok(s) +} + +/// Extract the text from a pdf at `path` and return a `Vec` with the results separately by page + +pub fn extract_text_by_pages>( + path: P, +) -> Result, OutputError> { + let mut v = Vec::new(); + { + let mut doc = Document::load(path)?; + maybe_decrypt(&mut doc)?; + let mut page_num = 1; + while let Ok(content) = extract_text_by_page(&doc, page_num) { + v.push(content); + page_num += 1; + } + } + Ok(v) +} + +pub fn extract_text_by_pages_encrypted>( + path: P, + password: &str, +) -> Result, OutputError> { + let mut v = Vec::new(); + { + let mut doc = Document::load(path)?; + doc.decrypt(password)?; + let mut page_num = 1; + while let Ok(content) = extract_text_by_page(&mut doc, page_num) { + v.push(content); + page_num += 1; + } + } + Ok(v) +} + +pub fn extract_text_from_mem_by_pages(buffer: &[u8]) -> Result, OutputError> { + let mut v = Vec::new(); + { + let mut doc = Document::load_mem(buffer)?; + maybe_decrypt(&mut doc)?; + let mut page_num = 1; + while let Ok(content) = extract_text_by_page(&doc, page_num) { + v.push(content); + page_num += 1; + } + } + Ok(v) +} + +pub fn extract_text_from_mem_by_pages_encrypted( + buffer: &[u8], + password: &str, +) -> Result, OutputError> { + let mut v = Vec::new(); + { + let mut doc = Document::load_mem(buffer)?; + doc.decrypt(password)?; + let mut page_num = 1; + while let Ok(content) = extract_text_by_page(&doc, page_num) { + v.push(content); + page_num += 1; + } + } + Ok(v) +} + +fn get_inherited<'a, T: FromObj<'a>>( + doc: &'a Document, + dict: &'a Dictionary, + key: &[u8], +) -> Option { + let o: Option = get(doc, dict, key); + if let Some(o) = o { + Some(o) + } else { + let parent = dict + .get(b"Parent") + .and_then(|parent| parent.as_reference()) + .and_then(|id| doc.get_dictionary(id)) + .ok()?; + get_inherited(doc, parent, key) + } +} + +pub fn output_doc_encrypted( + doc: &mut Document, + output: &mut dyn OutputDev, + password: &str, +) -> Result<(), OutputError> { + doc.decrypt(password)?; + output_doc(doc, output) +} + +/// Parse a given document and output it to `output` +pub fn output_doc(doc: &Document, output: &mut dyn OutputDev) -> Result<(), OutputError> { + if doc.is_encrypted() { + error!("Encrypted documents must be decrypted with a password using {{extract_text|extract_text_from_mem|output_doc}}_encrypted"); + } + let empty_resources = Dictionary::new(); + let pages = doc.get_pages(); + let mut p = Processor::new(); + for dict in pages { + let page_num = dict.0; + let object_id = dict.1; + output_doc_inner(page_num, object_id, doc, &mut p, output, &empty_resources)?; + } + Ok(()) +} + +pub fn output_doc_page( + doc: &Document, + output: &mut dyn OutputDev, + page_num: u32, +) -> Result<(), OutputError> { + if doc.is_encrypted() { + error!("Encrypted documents must be decrypted with a password using {{extract_text|extract_text_from_mem|output_doc}}_encrypted"); + } + let empty_resources = Dictionary::new(); + let pages = doc.get_pages(); + let object_id = pages + .get(&page_num) + .ok_or(lopdf::Error::PageNumberNotFound(page_num))?; + let mut p = Processor::new(); + output_doc_inner(page_num, *object_id, doc, &mut p, output, &empty_resources)?; + Ok(()) +} + +fn output_doc_inner<'a>( + page_num: u32, + object_id: ObjectId, + doc: &'a Document, + p: &mut Processor<'a>, + output: &mut dyn OutputDev, + empty_resources: &'a Dictionary, +) -> Result<(), OutputError> { + // Fork: malformed pages produce OutputError, not panics. + let page_dict = doc + .get_object(object_id) + .ok() + .and_then(|o| o.as_dict().ok()) + .ok_or(OutputError::PdfError(lopdf::Error::ObjectType { + expected: "Dictionary", + found: "other", + }))?; + dlog!("page {} {:?}", page_num, page_dict); + // XXX: Some pdfs lack a Resources directory + let resources = get_inherited(doc, page_dict, b"Resources").unwrap_or(empty_resources); + dlog!("resources {:?}", resources); + // pdfium searches up the page tree for MediaBoxes as needed + let media_box: Vec = get_inherited(doc, page_dict, b"MediaBox").ok_or( + OutputError::PdfError(lopdf::Error::DictKey("MediaBox".into())), + )?; + let media_box = match media_box.as_slice() { + [llx, lly, urx, ury] => MediaBox { + llx: *llx, + lly: *lly, + urx: *urx, + ury: *ury, + }, + _ => { + return Err(OutputError::PdfError(lopdf::Error::ObjectType { + expected: "MediaBox [llx lly urx ury]", + found: "other", + })) + } + }; + let art_box = + get::>>(&doc, page_dict, b"ArtBox").and_then(|x| match x.as_slice() { + [a, b, c, d] => Some((*a, *b, *c, *d)), + _ => None, + }); + output.begin_page(page_num, &media_box, art_box)?; + p.process_stream( + &doc, + doc.get_page_content(object_id)?, + resources, + &media_box, + output, + page_num, + )?; + output.end_page()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::font_cmap::ReverseCmap; + use super::{decode_char_with, expand_ligatures, FontRecovery}; + + #[test] + fn expand_ligatures_expands_alphabetic_forms() { + assert_eq!(expand_ligatures("\u{FB00}"), "ff"); + assert_eq!(expand_ligatures("\u{FB01}"), "fi"); + assert_eq!(expand_ligatures("\u{FB02}"), "fl"); + assert_eq!(expand_ligatures("\u{FB03}"), "ffi"); + assert_eq!(expand_ligatures("\u{FB04}"), "ffl"); + assert_eq!(expand_ligatures("\u{FB05}"), "st"); + assert_eq!(expand_ligatures("\u{FB06}"), "st"); + // Brief said o+FB01+ce == "office"; that's a plan typo — FB01 is + // "fi", so o+fi+ce = "ofice". "office" needs the ffi ligature. + assert_eq!(expand_ligatures("o\u{FB01}ce"), "ofice"); + assert_eq!(expand_ligatures("o\u{FB03}ce"), "office"); + assert_eq!(expand_ligatures("plain"), "plain"); + } + + #[test] + fn decode_char_prefers_to_unicode_when_healthy() { + // Healthy font: the /ToUnicode hit wins verbatim even when a + // reverse cmap disagrees (D2 byte-stability). + let rc = ReverseCmap::from_pairs_for_test(&[(65, 'X')]); + let mut r = FontRecovery::new(Some(rc)); + assert_eq!(decode_char_with(Some("A".into()), &mut r, 65), "A"); + assert!(!r.recovering); + } + + #[test] + fn decode_char_recovers_via_reverse_cmap_when_flipped() { + let rc = ReverseCmap::from_pairs_for_test(&[(1, 'A')]); + let mut r = FontRecovery::new(Some(rc)); + r.recovering = true; + // Flipped: the (garbage) /ToUnicode hit is not trusted. + assert_eq!(decode_char_with(Some("\u{E001}".into()), &mut r, 1), "A"); + // Miss with flipped font also recovers. + assert_eq!(decode_char_with(None, &mut r, 1), "A"); + } + + #[test] + fn decode_char_miss_recovers_via_reverse_cmap_without_flip() { + // No /ToUnicode entry (or none at all): the reverse cmap is + // consulted even before the prescan flips. + let rc = ReverseCmap::from_pairs_for_test(&[(7, 'q')]); + let mut r = FontRecovery::new(Some(rc)); + assert_eq!(decode_char_with(None, &mut r, 7), "q"); + } + + #[test] + fn decode_char_expands_ligatures_only_on_recovery_paths() { + // Healthy /ToUnicode hit of U+FB01 stays U+FB01 (D2). + let rc = ReverseCmap::from_pairs_for_test(&[(7, '\u{FB01}')]); + let mut healthy = FontRecovery::new(Some(rc.clone())); + assert_eq!( + decode_char_with(Some("\u{FB01}".into()), &mut healthy, 7), + "\u{FB01}" + ); + // Recovered U+FB01 expands to "fi". + let mut recovering = FontRecovery::new(Some(rc)); + assert_eq!(decode_char_with(None, &mut recovering, 7), "fi"); + } + + #[test] + fn decode_char_recovery_failure_returns_empty_for_miss_behavior() { + // No reverse cmap, miss: "" — callers apply their existing miss + // behavior (CID: empty string; simple font: encoding table). + let mut r = FontRecovery::new(None); + assert_eq!(decode_char_with(None, &mut r, 42), ""); + // Flipped with a hit but no recovery source: "" too — the garbage + // hit is not passed through. + r.recovering = true; + assert_eq!(decode_char_with(Some("\u{E000}".into()), &mut r, 42), ""); + } + + #[test] + fn decode_char_miss_feeds_prescan() { + // Every miss counts as a suspicious decode: 20/20 garbage flips. + let mut r = FontRecovery::new(None); + for _ in 0..20 { + let _ = decode_char_with(None, &mut r, 999); + } + assert!(r.recovering); + } + + #[test] + fn recovery_flips_at_ten_percent_garbage() { + let mut r = FontRecovery::new(None); + for _ in 0..19 { + r.observe("a"); + } + r.observe("\u{E001}"); // 1/20 = 5% — not yet + assert!(!r.recovering); + r.observe("\u{E002}"); // 2/21 ≈ 9.5% — not yet + assert!(!r.recovering); + r.observe("\u{E003}"); // 3/22 ≈ 13.6% — flip + assert!(r.recovering); + // latched: clean chars never flip it back + for _ in 0..100 { + r.observe("b"); + } + assert!(r.recovering); + } + + #[test] + fn recovery_never_flips_on_clean_text() { + let mut r = FontRecovery::new(None); + for _ in 0..1000 { + r.observe("x"); + } + assert!(!r.recovering); + } + + #[test] + fn recovery_counts_empty_and_control_and_replacement_as_suspicious() { + // 20 clean, then one of each garbage class. Flip math: after the + // third garbage char, 3/23 decoded → 3*10=30 >= 23 → recovering. + let mut r = FontRecovery::new(None); + for _ in 0..20 { + r.observe("ok"); + } + assert!(!r.recovering); + r.observe(""); // unmapped code + r.observe("\u{0007}"); // control char + assert!(!r.recovering); // 2/22 ≈ 9.1% — just under + r.observe("\u{FFFD}"); // replacement char + assert!(r.recovering); // 3/23 ≈ 13% — over + } +} diff --git a/crates/pdf-extract/src/zapfglyphnames.rs b/crates/pdf-extract/src/zapfglyphnames.rs new file mode 100644 index 0000000..774f005 --- /dev/null +++ b/crates/pdf-extract/src/zapfglyphnames.rs @@ -0,0 +1,209 @@ +pub fn zapfdigbats_names_to_unicode(name: &str) -> Option { + let names = [ + ("a1", 0x2701), + ("a10", 0x2721), + ("a100", 0x275e), + ("a101", 0x2761), + ("a102", 0x2762), + ("a103", 0x2763), + ("a104", 0x2764), + ("a105", 0x2710), + ("a106", 0x2765), + ("a107", 0x2766), + ("a108", 0x2767), + ("a109", 0x2660), + ("a11", 0x261b), + ("a110", 0x2665), + ("a111", 0x2666), + ("a112", 0x2663), + ("a117", 0x2709), + ("a118", 0x2708), + ("a119", 0x2707), + ("a12", 0x261e), + ("a120", 0x2460), + ("a121", 0x2461), + ("a122", 0x2462), + ("a123", 0x2463), + ("a124", 0x2464), + ("a125", 0x2465), + ("a126", 0x2466), + ("a127", 0x2467), + ("a128", 0x2468), + ("a129", 0x2469), + ("a13", 0x270c), + ("a130", 0x2776), + ("a131", 0x2777), + ("a132", 0x2778), + ("a133", 0x2779), + ("a134", 0x277a), + ("a135", 0x277b), + ("a136", 0x277c), + ("a137", 0x277d), + ("a138", 0x277e), + ("a139", 0x277f), + ("a14", 0x270d), + ("a140", 0x2780), + ("a141", 0x2781), + ("a142", 0x2782), + ("a143", 0x2783), + ("a144", 0x2784), + ("a145", 0x2785), + ("a146", 0x2786), + ("a147", 0x2787), + ("a148", 0x2788), + ("a149", 0x2789), + ("a15", 0x270e), + ("a150", 0x278a), + ("a151", 0x278b), + ("a152", 0x278c), + ("a153", 0x278d), + ("a154", 0x278e), + ("a155", 0x278f), + ("a156", 0x2790), + ("a157", 0x2791), + ("a158", 0x2792), + ("a159", 0x2793), + ("a16", 0x270f), + ("a160", 0x2794), + ("a161", 0x2192), + ("a162", 0x27a3), + ("a163", 0x2194), + ("a164", 0x2195), + ("a165", 0x2799), + ("a166", 0x279b), + ("a167", 0x279c), + ("a168", 0x279d), + ("a169", 0x279e), + ("a17", 0x2711), + ("a170", 0x279f), + ("a171", 0x27a0), + ("a172", 0x27a1), + ("a173", 0x27a2), + ("a174", 0x27a4), + ("a175", 0x27a5), + ("a176", 0x27a6), + ("a177", 0x27a7), + ("a178", 0x27a8), + ("a179", 0x27a9), + ("a18", 0x2712), + ("a180", 0x27ab), + ("a181", 0x27ad), + ("a182", 0x27af), + ("a183", 0x27b2), + ("a184", 0x27b3), + ("a185", 0x27b5), + ("a186", 0x27b8), + ("a187", 0x27ba), + ("a188", 0x27bb), + ("a189", 0x27bc), + ("a19", 0x2713), + ("a190", 0x27bd), + ("a191", 0x27be), + ("a192", 0x279a), + ("a193", 0x27aa), + ("a194", 0x27b6), + ("a195", 0x27b9), + ("a196", 0x2798), + ("a197", 0x27b4), + ("a198", 0x27b7), + ("a199", 0x27ac), + ("a2", 0x2702), + ("a20", 0x2714), + ("a200", 0x27ae), + ("a201", 0x27b1), + ("a202", 0x2703), + ("a203", 0x2750), + ("a204", 0x2752), + ("a205", 0x276e), + ("a206", 0x2770), + ("a21", 0x2715), + ("a22", 0x2716), + ("a23", 0x2717), + ("a24", 0x2718), + ("a25", 0x2719), + ("a26", 0x271a), + ("a27", 0x271b), + ("a28", 0x271c), + ("a29", 0x2722), + ("a3", 0x2704), + ("a30", 0x2723), + ("a31", 0x2724), + ("a32", 0x2725), + ("a33", 0x2726), + ("a34", 0x2727), + ("a35", 0x2605), + ("a36", 0x2729), + ("a37", 0x272a), + ("a38", 0x272b), + ("a39", 0x272c), + ("a4", 0x260e), + ("a40", 0x272d), + ("a41", 0x272e), + ("a42", 0x272f), + ("a43", 0x2730), + ("a44", 0x2731), + ("a45", 0x2732), + ("a46", 0x2733), + ("a47", 0x2734), + ("a48", 0x2735), + ("a49", 0x2736), + ("a5", 0x2706), + ("a50", 0x2737), + ("a51", 0x2738), + ("a52", 0x2739), + ("a53", 0x273a), + ("a54", 0x273b), + ("a55", 0x273c), + ("a56", 0x273d), + ("a57", 0x273e), + ("a58", 0x273f), + ("a59", 0x2740), + ("a6", 0x271d), + ("a60", 0x2741), + ("a61", 0x2742), + ("a62", 0x2743), + ("a63", 0x2744), + ("a64", 0x2745), + ("a65", 0x2746), + ("a66", 0x2747), + ("a67", 0x2748), + ("a68", 0x2749), + ("a69", 0x274a), + ("a7", 0x271e), + ("a70", 0x274b), + ("a71", 0x25cf), + ("a72", 0x274d), + ("a73", 0x25a0), + ("a74", 0x274f), + ("a75", 0x2751), + ("a76", 0x25b2), + ("a77", 0x25bc), + ("a78", 0x25c6), + ("a79", 0x2756), + ("a8", 0x271f), + ("a81", 0x25d7), + ("a82", 0x2758), + ("a83", 0x2759), + ("a84", 0x275a), + ("a85", 0x276f), + ("a86", 0x2771), + ("a87", 0x2772), + ("a88", 0x2773), + ("a89", 0x2768), + ("a9", 0x2720), + ("a90", 0x2769), + ("a91", 0x276c), + ("a92", 0x276d), + ("a93", 0x276a), + ("a94", 0x276b), + ("a95", 0x2774), + ("a96", 0x2775), + ("a97", 0x275b), + ("a98", 0x275c), + ("a99", 0x275d), + ("space", 0x0020), + ]; + + let result = names.binary_search_by_key(&name, |&(name, _code)| &name); + result.ok().map(|indx| names[indx].1) +} diff --git a/crates/pdf-extract/tests/panic_free.rs b/crates/pdf-extract/tests/panic_free.rs new file mode 100644 index 0000000..c360564 --- /dev/null +++ b/crates/pdf-extract/tests/panic_free.rs @@ -0,0 +1,216 @@ +//! The fork must never panic on malformed fonts/pages — errors or +//! replacement characters instead (batdoc spec §5.1). + +use lopdf::{dictionary, Document, Object, Stream}; + +/// One-page PDF whose simple font's Differences array references a glyph +/// name that exists in no table (`/not_a_real_glyph_name`). +fn unknown_glyph_name_pdf() -> Vec { + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + "Encoding" => dictionary! { + "Type" => "Encoding", + "BaseEncoding" => "WinAnsiEncoding", + "Differences" => vec![Object::Integer(65), Object::Name(b"not_a_real_glyph_name".to_vec())], + }, + }); + let resources_id = doc.add_object(dictionary! { + "Font" => dictionary! { "F1" => font_id }, + }); + let content_id = doc.add_object(Stream::new( + lopdf::Dictionary::new(), + b"BT /F1 12 Tf 72 720 Td (A) Tj ET".to_vec(), + )); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => 1_i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + buf +} + +#[test] +fn unknown_glyph_name_does_not_panic() { + // Upstream 0.10 panics here ("invalid glyph name"); the fork must + // return Ok with a replacement char or skip the glyph. + let result = pdf_extract::extract_text_from_mem(&unknown_glyph_name_pdf()); + result.expect("extraction must not error/panic"); +} + +#[test] +fn page_with_missing_contents_does_not_panic() { + // A page dict with no /Contents: output_doc_inner must return Err, + // not unwrap-panic. + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => 1_i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + let result = pdf_extract::extract_text_from_mem(&buf); + // Either an Ok with empty text or an Err — anything but a panic. + let _ = result; +} + +/// A page whose /Contents is a stream of bytes that do not parse as a +/// content stream: Content::decode fails, extraction must not panic. +#[test] +fn undecodable_content_stream_does_not_panic() { + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + // 0xFF and other non-token bytes defeat the content lexer. + let content_id = doc.add_object(Stream::new( + lopdf::Dictionary::new(), + vec![0xFF, 0xD8, 0xFF, 0xE0, 0x00], + )); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => 1_i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + let result = pdf_extract::extract_text_from_mem(&buf); + // Ok (page skipped) or Err — anything but a panic. + let _ = result; +} + +/// A text-showing operator with a non-string operand must not panic. +#[test] +fn malformed_tj_operand_does_not_panic() { + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type1", + "BaseFont" => "Helvetica", + "Encoding" => "WinAnsiEncoding", + }); + let resources_id = doc.add_object(dictionary! { + "Font" => dictionary! { "F1" => font_id }, + }); + // `42 Tj` — Tj with an integer operand. + let content_id = doc.add_object(Stream::new( + lopdf::Dictionary::new(), + b"BT /F1 12 Tf 72 720 Td 42 Tj ET".to_vec(), + )); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => 1_i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + let result = pdf_extract::extract_text_from_mem(&buf); + let _ = result; +} + +/// A Type0 font missing its /DescendantFonts must not panic. +#[test] +fn cid_font_without_descendants_does_not_panic() { + let mut doc = Document::with_version("1.4"); + let pages_id = doc.new_object_id(); + let font_id = doc.add_object(dictionary! { + "Type" => "Font", + "Subtype" => "Type0", + "BaseFont" => "Broken", + "Encoding" => "Identity-H", + }); + let resources_id = doc.add_object(dictionary! { + "Font" => dictionary! { "F1" => font_id }, + }); + let content_id = doc.add_object(Stream::new( + lopdf::Dictionary::new(), + b"BT /F1 12 Tf 72 720 Td (AB) Tj ET".to_vec(), + )); + let page_id = doc.add_object(dictionary! { + "Type" => "Page", + "Parent" => pages_id, + "Contents" => content_id, + "Resources" => resources_id, + "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()], + }); + doc.objects.insert( + pages_id, + Object::Dictionary(dictionary! { + "Type" => "Pages", + "Kids" => vec![Object::Reference(page_id)], + "Count" => 1_i64, + }), + ); + let catalog_id = doc.add_object(dictionary! { + "Type" => "Catalog", + "Pages" => pages_id, + }); + doc.trailer.set("Root", catalog_id); + let mut buf = Vec::new(); + doc.save_to(&mut buf).unwrap(); + let result = pdf_extract::extract_text_from_mem(&buf); + let _ = result; +} diff --git a/src/main.rs b/src/main.rs index 5452e53..9f63afa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,8 @@ //! //! Reads legacy OLE2 `.doc` and `.xls`, modern OOXML `.docx`, `.xlsx`, and //! `.pptx`, PDF, and raster image files and dumps their text to stdout. Image -//! files are always OCR'd; embedded images and textless PDF pages are OCR'd +//! files are always OCR'd; textless PDF pages are OCR'd automatically as a +//! fallback (a textless PDF is a scan). Embedded images in DOCX/PPTX are OCR'd //! with `--ocr`. When stdout is a terminal the output is pretty-printed as //! syntax-highlighted markdown via `bat`; when piped, plain text is emitted. @@ -24,7 +25,7 @@ Options: -p, --plain Force plain text output (no colors, no decorations) -m, --markdown Output as markdown (default when terminal detected) -i, --images Embed images as inline base64 data URIs in markdown - --ocr OCR text from images (embedded doc images, textless PDF pages) + --ocr OCR embedded images (docx/pptx); textless PDFs already auto-OCR -h, --help Show this help When stdout is a terminal, output is pretty-printed as syntax-highlighted @@ -37,8 +38,9 @@ Ignored in plain text mode and for formats without image support (.doc, .xls, .p --ocr uses the ocrs engine (models downloaded on first use to $BATDOC_MODELS_DIR, $XDG_CACHE_HOME/batdoc/models, or ~/.cache/batdoc/models). -For .docx/.pptx, embedded images are OCR'd; for .pdf, pages without a text -layer are OCR'd from their embedded images. Image files (.png/.jpg/.gif/ +For .docx/.pptx, embedded images are OCR'd. PDFs need no flag: any page +without a text layer is OCR'd automatically from its embedded images as a +fallback (a textless PDF is a scan). Image files (.png/.jpg/.gif/ .webp/.bmp) are always OCR'd, with or without --ocr. Multiple files can be specified and will be processed in order. @@ -156,7 +158,11 @@ fn run( io::stdout().write_all(b"\n")?; } - let opts = ExtractOptions { images, ocr }; + let opts = ExtractOptions { + images, + ocr, + ..Default::default() + }; // OCR input (flagged, or image input which is always OCR'd) downloads // models on first use; say so once per process, before it happens. @@ -172,15 +178,16 @@ fn run( match mode { Mode::Plain => { - let text = batdoc_core::extract_plain_with(data, format, opts)?; - io::stdout().write_all(text.as_bytes())?; + let mut sink = batdoc_core::IoSink(io::stdout()); + batdoc_core::extract_plain_to(data, format, opts, &mut sink)?; } Mode::Markdown => { - let md = batdoc_core::extract_markdown_with(data, format, opts)?; if is_tty && format != Format::Image { + let md = batdoc_core::extract_markdown_with(data, format, opts)?; pretty_print(&md, filename)?; } else { - io::stdout().write_all(md.as_bytes())?; + let mut sink = batdoc_core::IoSink(io::stdout()); + batdoc_core::extract_markdown_to(data, format, opts, &mut sink)?; } } Mode::Auto => { @@ -188,8 +195,8 @@ fn run( let md = batdoc_core::extract_markdown_with(data, format, opts)?; pretty_print(&md, filename)?; } else { - let text = batdoc_core::extract_plain_with(data, format, opts)?; - io::stdout().write_all(text.as_bytes())?; + let mut sink = batdoc_core::IoSink(io::stdout()); + batdoc_core::extract_plain_to(data, format, opts, &mut sink)?; } } } diff --git a/web/build.sh b/web/build.sh new file mode 100755 index 0000000..2faca56 --- /dev/null +++ b/web/build.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Build the wasm module and generate the browser JS glue into web/pkg/. +set -euo pipefail +cd "$(dirname "$0")" +ROOT="$(cd .. && pwd)" + +cargo build \ + --target wasm32-unknown-unknown \ + --release \ + -p batdoc-core \ + --no-default-features \ + --manifest-path "$ROOT/Cargo.toml" + +wasm-bindgen \ + --target web \ + --out-dir pkg \ + "$ROOT/target/wasm32-unknown-unknown/release/batdoc_core.wasm" + +echo "Built web/pkg/ — serve this directory over HTTP to demo (e.g. python3 -m http.server)." diff --git a/web/demo.js b/web/demo.js new file mode 100644 index 0000000..bdde270 --- /dev/null +++ b/web/demo.js @@ -0,0 +1,48 @@ +import init, { detect, to_plain, to_markdown } from './pkg/batdoc_core.js'; + +const fileInput = document.getElementById('file'); +const modeSel = document.getElementById('mode'); +const imagesCb = document.getElementById('images'); +const ocrCb = document.getElementById('ocr'); +const output = document.getElementById('output'); +const status = document.getElementById('status'); + +function run(bytes) { + const label = `detected ${detect(bytes)}`; + let out; + if (modeSel.value === 'plain') { + out = to_plain(bytes); + } else { + out = to_markdown(bytes, imagesCb.checked, ocrCb.checked); + } + output.textContent = out; + status.textContent = `${label} · ${bytes.length} bytes`; +} + +async function load(path) { + try { + const res = await fetch(path); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + run(new Uint8Array(await res.arrayBuffer())); + } catch (e) { + output.textContent = ''; + status.textContent = `error: ${e}`; + } +} + +fileInput.addEventListener('change', () => { + const file = fileInput.files[0]; + if (!file) return; + file.arrayBuffer().then((buf) => run(new Uint8Array(buf))) + .catch((e) => { status.textContent = `read error: ${e}`; }); +}); + +for (const a of document.querySelectorAll('a[data-sample]')) { + a.addEventListener('click', (ev) => { + ev.preventDefault(); + load(a.dataset.sample); + }); +} + +await init(); +status.textContent = 'Ready — choose a file or click a sample.'; diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..e4dc094 --- /dev/null +++ b/web/index.html @@ -0,0 +1,44 @@ + + + + + + batdoc — wasm demo + + + +

batdoc — wasm demo

+

Pick a document (or click a sample) and it is converted to markdown/plain text + entirely in the browser via batdoc-core compiled to WebAssembly.

+ +

+ + +

+

+ Mode: + + + + (ocr needs model files; this demo build has net disabled) +

+

+ Samples: + sample.docx · + sample.pdf +

+ +

Loading wasm module…

+ 

+
+  
+
+
diff --git a/web/make_samples.py b/web/make_samples.py
new file mode 100644
index 0000000..379c6e2
--- /dev/null
+++ b/web/make_samples.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""Generate tiny sample documents for the wasm demo.
+
+Outputs: web/samples/sample.docx and web/samples/sample.pdf.
+No third-party libraries — only the stdlib (zip + manual PDF xref).
+"""
+import zipfile
+import os
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+OUT = os.path.join(HERE, "samples")
+os.makedirs(OUT, exist_ok=True)
+
+# ---------------------------------------------------------------------------
+# DOCX (minimal OOXML package)
+# ---------------------------------------------------------------------------
+content_types = """
+
+  
+  
+  
+
+"""
+rels = """
+
+  
+
+"""
+document = """
+
+  
+    Sample Report
+    Hello from batdoc compiled to WebAssembly.
+    This text was extracted in the browser, no server involved.
+  
+
+"""
+
+with zipfile.ZipFile(os.path.join(OUT, "sample.docx"), "w", zipfile.ZIP_DEFLATED) as z:
+    z.writestr("[Content_Types].xml", content_types)
+    z.writestr("_rels/.rels", rels)
+    z.writestr("word/document.xml", document)
+
+# ---------------------------------------------------------------------------
+# PDF (one page, one line of text, hand-built xref)
+# ---------------------------------------------------------------------------
+def pdf_object(n, body):
+    return f"{n} 0 obj\n{body}\nendobj\n".encode()
+
+stream = b"BT /F1 24 Tf 72 720 Td (Hello from batdoc wasm!) Tj ET"
+
+objs = []
+objs.append(pdf_object(1, "<< /Type /Catalog /Pages 2 0 R >>"))
+objs.append(pdf_object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"))
+objs.append(pdf_object(
+    3,
+    "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
+    "/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>",
+))
+objs.append(pdf_object(4, f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream"))
+objs.append(pdf_object(5, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"))
+
+pdf = bytearray(b"%PDF-1.4\n")
+offsets = []
+for o in objs:
+    offsets.append(len(pdf))
+    pdf += o
+
+xref_pos = len(pdf)
+pdf += f"xref\n0 {len(objs)+1}\n".encode()
+pdf += b"0000000000 65535 f \n"
+for off in offsets:
+    pdf += f"{off:010d} 00000 n \n".encode()
+pdf += f"trailer\n<< /Size {len(objs)+1} /Root 1 0 R >>\nstartxref\n{xref_pos}\n%%EOF\n".encode()
+
+with open(os.path.join(OUT, "sample.pdf"), "wb") as f:
+    f.write(pdf)
+
+print(f"wrote {os.path.join(OUT, 'sample.docx')}")
+print(f"wrote {os.path.join(OUT, 'sample.pdf')}")