diff --git a/Makefile b/Makefile index 0451c3e39..1806728e8 100644 --- a/Makefile +++ b/Makefile @@ -583,14 +583,19 @@ ifeq ($(BROTLI),0) BROTLI_CFLAGS := BROTLI_LDFLAGS := else - BROTLI_CFLAGS := $(shell pkg-config --cflags libbrotlienc 2>/dev/null) + BROTLI_CFLAGS := $(shell pkg-config --cflags libbrotlienc libbrotlidec 2>/dev/null) # --static so libbrotlicommon comes with it. It is a Requires.private of - # libbrotlienc, so a plain --libs omits it -- fine when the loader resolves - # a shared object, fatal on a static link: MinGW's ld then reports a wall of - # undefined BrotliGetDictionary / BrotliDefaultAllocFunc / _kBrotli* - # references from inside libbrotlienc.a itself. Found on a real MSYS2 box; - # a Linux shared link hides it entirely. - BROTLI_LDFLAGS := $(shell pkg-config --static --libs libbrotlienc 2>/dev/null) + # libbrotlienc and libbrotlidec, so a plain --libs omits it -- fine when the + # loader resolves a shared object, fatal on a static link: MinGW's ld then + # reports a wall of undefined BrotliGetDictionary / BrotliDefaultAllocFunc / + # _kBrotli* references from inside libbrotlienc.a itself. Found on a real + # MSYS2 box; a Linux shared link hides it entirely. Asking for both + # libraries names -lbrotlicommon twice, and the link line must name each + # library once (optional_library_linking checks), so keep the LAST of each: + # a static link resolves left to right, and -lbrotlicommon has to follow + # libbrotlidec as well as libbrotlienc. + BROTLI_LDFLAGS := $(shell pkg-config --static --libs libbrotlienc libbrotlidec 2>/dev/null \ + | awk '{ o = ""; for (i = NF; i > 0; i--) if (!seen[$$i]++) o = $$i " " o; print o }') endif ifneq ($(BROTLI_LDFLAGS),) BROTLI_CFLAGS += -DAETHER_HAS_BROTLI diff --git a/compiler/codegen/codegen.c b/compiler/codegen/codegen.c index 7965f09da..682070df2 100644 --- a/compiler/codegen/codegen.c +++ b/compiler/codegen/codegen.c @@ -6673,7 +6673,7 @@ static const AetherLinkReq g_link_reqs[] = { /* Base64 entry points share aether_cryptography.o with OpenSSL users. */ { "std.encoding", "-lssl -lcrypto" }, { "std.zlib", "-lz" }, - { "std.brotli", "-lbrotlienc -lbrotlicommon" }, + { "std.brotli", "-lbrotlienc -lbrotlidec -lbrotlicommon" }, { "std.zstd", "-lzstd" }, { "std.yaml", "-lfyaml" }, /* These veneers reference the HTTP server/client objects from C, not diff --git a/docs/api/brotli.html b/docs/api/brotli.html index 3606ec8c4..570134da1 100644 --- a/docs/api/brotli.html +++ b/docs/api/brotli.html @@ -24,13 +24,27 @@

Exports

  • get_compress_bytes
  • get_compress_length
  • release_compress
    • +
  • get_decompress_bytes
    • +
  • get_decompress_length
    • +
  • release_decompress
    • +
  • try_decompress_stream_new
    • +
  • get_decompress_stream_bytes
    • +
  • get_decompress_stream_length
    • +
  • release_decompress_stream
  • available
    • +
  • decoder_available
  • compress
    • +
  • decompress
    • +
  • decompress_max
  • stream_new
  • stream_write
  • stream_flush
  • stream_finish
  • stream_free
    • +
  • decompress_stream_new
    • +
  • decompress_stream_write
    • +
  • decompress_stream_finish
    • +
  • decompress_stream_free
  • MIN_QUALITY
  • MAX_QUALITY
  • DEFAULT_QUALITY
    • @@ -132,7 +146,7 @@

    Modules

    brotli

    -

    std.brotli — Brotli compression (RFC 7932), streaming and one-shot.

    +

    std.brotli — Brotli (RFC 7932) encoding and decoding, streaming and one-shot.

    import std.brotli

    @@ -178,10 +192,47 @@

    get_compress_length externrelease_compress extern

    release_compress()
    +
    +

    get_decompress_bytes extern

    + 
    get_decompress_bytes() -> ptr
    +
    +
    +

    get_decompress_length extern

    + 
    get_decompress_length() -> int
    +
    +
    +

    release_decompress extern

    + 
    release_decompress()
    +
    +
    +

    try_decompress_stream_new extern

    + 
    try_decompress_stream_new(max_bytes) -> ptr
    +

    max_bytes: int

    +
    +
    +

    get_decompress_stream_bytes extern

    + 
    get_decompress_stream_bytes(handle) -> ptr
    +

    handle: ptr

    +
    +
    +

    get_decompress_stream_length extern

    + 
    get_decompress_stream_length(handle) -> int
    +

    handle: ptr

    +
    +
    +

    release_decompress_stream extern

    + 
    release_decompress_stream(handle)
    +

    handle: ptr

    +

    available fn

    available() -> int
    -

    1 when the toolchain was built with libbrotlienc detected. Use it to skip a test or pick a fallback encoding rather than threading an error everywhere.

    +

    1 when the toolchain was built with libbrotli detected. Use it to skip a test or pick a fallback encoding rather than threading an error everywhere.

    +
    +
    +

    decoder_available fn

    + 
    decoder_available() -> int
    +

    1 when decoding works here: brotli is built in AND the system libbrotlidec carries the CVE-2026-111497 fix (an out-of-bounds write on malformed input, libbrotli 1.0.3 to 1.2.0). Without the fix every decode refuses with an error rather than run a decoder that untrusted input can exploit; encoding is unaffected. The check asks the library itself, so a distro backport that keeps the old version number passes.

    compress fn

    @@ -189,6 +240,18 @@

    compress fn

    data: byte[], quality: int

    One-shot: compress a body already in memory. Returns (bytes, count, "") on success, ("", 0, error) otherwise. Quality out of range is clamped.

    +
    +

    decompress fn

    + 
    decompress(data)
    +

    data: byte[]

    +

    One-shot: decompress with the default 1 GiB ceiling. Returns (bytes, count, "") on success, ("", 0, error) otherwise. For input you did not produce, use decompress_max with a cap that fits the job.

    +
    +
    +

    decompress_max fn

    + 
    decompress_max(data, max_bytes)
    +

    data: byte[], max_bytes: int

    +

    One-shot with a cap: decoding stops, and returns an "output limit exceeded" error, as soon as the output would pass `max_bytes` (0 = the 1 GiB ceiling). Truncated input, corrupt input and bytes after the end of the stream are errors too.

    +

    stream_new fn

    stream_new(quality, window) -> (ptr, string)
    @@ -219,6 +282,30 @@

    stream_free fn

    handle: ptr

    Release the handle. Safe on null.

    +
    +

    decompress_stream_new fn

    + 
    decompress_stream_new(max_bytes) -> (ptr, string)
    +

    max_bytes: int

    +

    Open a decoding stream whose total output is capped at `max_bytes` (0 = the 1 GiB ceiling). Returns (handle, "") or (null, error).

    +
    +
    +

    decompress_stream_write fn

    + 
    decompress_stream_write(handle, data)
    +

    handle: ptr, data: byte[]

    +

    Feed compressed bytes in any size of chunk. Returns (bytes, count, "") with whatever they decoded to. After an error the handle refuses further writes.

    +
    +
    +

    decompress_stream_finish fn

    + 
    decompress_stream_finish(handle) -> (string, int, string)
    +

    handle: ptr

    +

    Confirm the stream ended. Returns ("", 0, "") when it did, and a "truncated stream" error when the input stopped short of the end, so a cut connection is not mistaken for a shorter body. Still needs decompress_stream_free.

    +
    +
    +

    decompress_stream_free fn

    + 
    decompress_stream_free(handle)
    +

    handle: ptr

    +

    Release a decoding stream. Safe on null.

    +

    MIN_QUALITY const

    MIN_QUALITY = 0
    diff --git a/docs/api/index.html b/docs/api/index.html index 19fea2194..4469d85b4 100644 --- a/docs/api/index.html +++ b/docs/api/index.html @@ -148,8 +148,8 @@

    bits

    brotli

    -

    std.brotli — Brotli compression (RFC 7932), streaming and one-shot.

    - 20 exports +

    std.brotli — Brotli (RFC 7932) encoding and decoding, streaming and one-shot.

    + 34 exports

    bytes

    @@ -679,13 +679,27 @@

    Search Results

  • get_compress_bytesbrotli
  • get_compress_lengthbrotli
  • release_compressbrotli
    • +
  • get_decompress_bytesbrotli
    • +
  • get_decompress_lengthbrotli
    • +
  • release_decompressbrotli
    • +
  • try_decompress_stream_newbrotli
    • +
  • get_decompress_stream_bytesbrotli
    • +
  • get_decompress_stream_lengthbrotli
    • +
  • release_decompress_streambrotli
  • availablebrotli
    • +
  • decoder_availablebrotli
  • compressbrotli
    • +
  • decompressbrotli
    • +
  • decompress_maxbrotli
  • stream_newbrotli
  • stream_writebrotli
  • stream_flushbrotli
  • stream_finishbrotli
  • stream_freebrotli
    • +
  • decompress_stream_newbrotli
    • +
  • decompress_stream_writebrotli
    • +
  • decompress_stream_finishbrotli
    • +
  • decompress_stream_freebrotli
  • MIN_QUALITYbrotli
  • MAX_QUALITYbrotli
  • DEFAULT_QUALITYbrotli
    • diff --git a/docs/stdlib-reference.md b/docs/stdlib-reference.md index fa32982fc..511ab1906 100644 --- a/docs/stdlib-reference.md +++ b/docs/stdlib-reference.md @@ -18,7 +18,7 @@ header comment is the authoritative description. | `std.audit` | Query the sandbox audit trail. | 4 | [guide](../std/audit/README.md) · [source](../std/audit/module.ae) | | `std.bignum` | Arbitrary-precision integers. | 29 | [guide](../std/bignum/README.md) · [source](../std/bignum/module.ae) | | `std.bits` | Unsigned bit operations: rotates, shifts, popcount, leading zeros, unsigned divide. | 17 | [guide](../std/bits/README.md) · [source](../std/bits/module.ae) | -| `std.brotli` | Brotli compression, streaming and one-shot, for `Content-Encoding: br`. | 20 | [guide](../std/brotli/README.md) · [source](../std/brotli/module.ae) | +| `std.brotli` | Brotli encoding and capped decoding, streaming and one-shot, for `Content-Encoding: br`. | 34 | [guide](../std/brotli/README.md) · [source](../std/brotli/module.ae) | | `std.bytes` | Mutable byte buffer with random access and overlap-safe copies. | 32 | [guide](../std/bytes/README.md) · [source](../std/bytes/module.ae) | | `std.capsicum` | FreeBSD Capsicum capability-mode bindings. | 28 | [guide](../std/capsicum/README.md) · [source](../std/capsicum/module.ae) | | `std.cas` | Content-addressed store keyed by the sha256 of file contents. | 7 | [guide](../std/cas/README.md) · [source](../std/cas/module.ae) | diff --git a/new_changelogs/20261024T153948Z-added-brotli-decoder.md b/new_changelogs/20261024T153948Z-added-brotli-decoder.md new file mode 100644 index 000000000..6ace03bab --- /dev/null +++ b/new_changelogs/20261024T153948Z-added-brotli-decoder.md @@ -0,0 +1,12 @@ +- **`std.brotli` decodes.** `decompress` / `decompress_max` (one-shot) and + `decompress_stream_new` / `_write` / `_finish` / `_free` (streaming) wrap the + system libbrotlidec. Every decode takes a cap on what it will produce + (`max_bytes`, 0 = a 1 GiB ceiling), so a few hundred bytes of `br` cannot + expand into gigabytes (CVE-2025-6176, urllib3's CVE-2025-66471). Truncated + input, corrupt input and bytes after the end of the stream are errors, and + `decompress_stream_finish` reports a stream that was cut short. The decoder + refuses to run on a libbrotlidec without the CVE-2026-111497 + ("broccoli2shell") fix for its out-of-bounds write: `decoder_available()` + asks the library itself, so distro backports that keep the old version number + pass. Large-window streams stay refused. Building with brotli now needs + libbrotlidec beside libbrotlienc; both come in the same distro package. diff --git a/std/brotli/README.md b/std/brotli/README.md index 1473b614f..cba332bc8 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -1,16 +1,63 @@ # std.brotli -Brotli compression (RFC 7932): streaming and one-shot. +Brotli (RFC 7932) encoding and decoding, streaming and one-shot. `br` is what current browsers actually prefer: every modern browser lists it ahead of `gzip` in `Accept-Encoding`, and it typically beats gzip by 15-25% on -text at comparable speed. This module wraps the system **libbrotlienc** -(`apt install libbrotli-dev`, `brew install brotli`). +text at comparable speed. This module wraps the system **libbrotlienc** and +**libbrotlidec** (`apt install libbrotli-dev`, `brew install brotli`). -Compression only. An HTTP server compresses responses; nothing in-tree needs to -*decode* `br`, and a decoder is separate work. +## Decoding -## Streaming +Decoding is where brotli's CVEs are, because the input is whatever a peer +sent. Every decode takes a cap on what it will produce: + +```aether,fragment +body, n, err = brotli.decompress_max(string.bytes(req_body), 1048576) // 1 MiB +``` + +Past the cap the decoder stops and returns `"... output limit exceeded"`. +`decompress(data)` is `decompress_max(data, 0)`, and 0 means a 1 GiB ceiling: +fine for data you produced, too generous for a request body. + +The streaming decoder takes chunks of any size, and its cap covers the total: + +```aether,fragment +ds, err = brotli.decompress_stream_new(1048576) +chunk, n, err = brotli.decompress_stream_write(ds, string.bytes(part)) +_, _, err = brotli.decompress_stream_finish(ds) // "truncated stream" if cut short +brotli.decompress_stream_free(ds) +``` + +Truncated input, corrupt input and bytes after the end of the stream are all +errors, on both paths. A failed stream refuses further writes. + +### What it defends against + +| Issue | Where | What this module does | +|---|---|---| +| CVE-2026-111497 ("broccoli2shell" / Node2Shell): out-of-bounds write on a malformed block switch, RCE | libbrotlidec 1.0.3-1.2.0; Node.js, Bun, Python `brotli` | Refuses to decode on an unfixed libbrotlidec (below). | +| CVE-2025-6176: decompression bomb, no output limit | Python `brotli` < 1.2.0, Scrapy | `max_bytes` on every decode; tested with 8 MiB packed into a few hundred bytes. | +| CVE-2025-66471: a small streaming read decodes a whole highly-compressed chunk | urllib3 < 2.6.0 | The stream cap counts every byte a write produces, so one write cannot exceed it. | +| CVE-2020-8927: overflow when one-shot decoding > 2 GiB | libbrotli < 1.0.8 | The one-shot path uses the streaming decoder, lengths are `int`, and the fix check below rules out pre-1.0.8 libraries. | +| Large-window streams: the header can ask for a 1 GiB window outside any output cap | non-RFC 7932 extension | Left off; a test pins that such a stream is refused. | +| Truncation read as a complete body | a common decoder-wrapper bug | `decompress_stream_finish` reports `"truncated stream"`. | + +`std.http`'s client does not decode `Content-Encoding` itself, so the chained +encodings of urllib3's CVE-2025-66418 (`br, br, br, ...`) do not arise: a +caller decodes once, explicitly. + +### An unfixed system libbrotlidec is refused + +CVE-2026-111497 is in the system library, so linking against an old one +would make every decode exploitable. `decoder_available()` is 1 only when the +library carries the fix (upstream 1.0.10, 1.1.1, 1.2.1, or a distro backport +such as Debian's `1.0.9-2+deb12u1`); otherwise every decode returns an error +naming the CVE and encoding keeps working. The check asks the library to name +error code -17, which the fix added. A backport that keeps the old version +number still passes, where a version comparison would refuse it. + +## Streaming Encoding The surface deliberately mirrors `std.zlib`'s streaming deflate, so a server negotiating `br` against `gzip` writes the same shape either way: @@ -47,7 +94,7 @@ packed, n, err = brotli.compress(string.bytes(body), brotli.FAST_QUALITY) ## When the backend is absent -Built without libbrotlienc, `available()` returns 0 and `stream_new` returns +Built without libbrotli, `available()` returns 0 and `stream_new` returns `(null, "brotli unavailable")`. Every other stream call needs the handle, so a program fails where it *asks for* a stream rather than silently emitting nothing. A server negotiating content-encoding should check `available()` and @@ -55,6 +102,8 @@ fall back to gzip. ## Exports -`available`, `compress`, `stream_new`, `stream_write`, `stream_flush`, -`stream_finish`, `stream_free` (and the `MIN_QUALITY` / `MAX_QUALITY` / +`available`, `decoder_available`, `compress`, `decompress`, `decompress_max`, +`stream_new`, `stream_write`, `stream_flush`, `stream_finish`, `stream_free`, +`decompress_stream_new`, `decompress_stream_write`, `decompress_stream_finish`, +`decompress_stream_free` (and the `MIN_QUALITY` / `MAX_QUALITY` / `DEFAULT_QUALITY` / `FAST_QUALITY` constants), plus the `brotli_*` raw forms. diff --git a/std/brotli/aether_brotli.c b/std/brotli/aether_brotli.c index 03bd1ce16..210f08fbd 100644 --- a/std/brotli/aether_brotli.c +++ b/std/brotli/aether_brotli.c @@ -7,6 +7,7 @@ #ifdef AETHER_HAS_BROTLI #include +#include #endif /* The input bytes and their count. `data` is always raw: the std module @@ -201,6 +202,266 @@ int brotli_try_compress(const char* data, int length, int quality) { return 1; } +/* CVE-2026-111497 ("broccoli2shell"): libbrotlidec 1.0.3 to 1.2.0 can write + * out of bounds on a malformed block switch, which is remote code execution + * for anything decoding untrusted `br`. The fix (1.0.10, 1.1.1, 1.2.1, and + * distro backports that keep the old version number, Debian's + * 1.0.9-2+deb12u1 among them) gave the reserved error code -17 a name, + * BLOCK_SWITCH, in the same commit. The library's own error-string table is + * compiled from that list, so asking it to name -17 tells a fixed decoder + * from an unfixed one where the version number cannot. An unfixed library + * is refused: decoding fails closed, encoding is unaffected. */ +int brotli_decoder_patched(void) { + const char* name = BrotliDecoderErrorString((BrotliDecoderErrorCode)-17); + return name && strstr(name, "BLOCK_SWITCH") ? 1 : 0; +} + +/* One-shot decompression output slot, thread-local */ +static _Thread_local unsigned char* tls_decomp_buf = NULL; +static _Thread_local size_t tls_decomp_cap = 0; +static _Thread_local int tls_decomp_len = 0; + +static void free_decomp_tls(void) { + if (tls_decomp_buf) { + aether_caps_free(tls_decomp_buf, tls_decomp_cap); + tls_decomp_buf = NULL; + } + tls_decomp_cap = 0; + tls_decomp_len = 0; +} + +int brotli_try_decompress(const char* data, int length, int max_bytes) { + free_decomp_tls(); + if (length < 0 || max_bytes < 0) return 0; + if (!brotli_decoder_patched()) return -4; + + size_t in_len; + const unsigned char* in = brotli_unwrap_bytes(data, length, &in_len); + if (in_len > 0 && !in) return 0; + + BrotliDecoderState* dec = BrotliDecoderCreateInstance(NULL, NULL, NULL); + if (!dec) return 0; + + size_t limit = (max_bytes > 0) ? (size_t)max_bytes : ((size_t)1 << 30); + size_t initial_cap = (in_len > 0 && in_len * 4 > 256) ? (in_len * 4) : 256; + if (initial_cap > limit) initial_cap = limit; + if (initial_cap == 0) initial_cap = 1; + + size_t cap = initial_cap; + unsigned char* out = (unsigned char*)aether_caps_malloc(cap); + if (!out) { + BrotliDecoderDestroyInstance(dec); + return 0; + } + + size_t avail_in = in_len; + const uint8_t* next_in = (const uint8_t*)in; + size_t total_out = 0; + int status = 0; + + for (;;) { + size_t avail_out = cap - total_out; + uint8_t* next_out = (uint8_t*)(out + total_out); + + BrotliDecoderResult res = BrotliDecoderDecompressStream( + dec, &avail_in, &next_in, &avail_out, &next_out, NULL); + + total_out = cap - avail_out; + + if (res == BROTLI_DECODER_RESULT_SUCCESS) { + if (avail_in > 0) { + status = 0; /* Trailing garbage after valid stream */ + } else { + status = 1; + } + break; + } else if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + if (total_out >= limit) { + status = -2; + break; + } + size_t ncap = cap * 2; + if (ncap > limit) ncap = limit; + if (ncap <= cap) { + status = -2; + break; + } + unsigned char* nb = (unsigned char*)aether_caps_malloc(ncap); + if (!nb) { + status = 0; + break; + } + memcpy(nb, out, total_out); + aether_caps_free(out, cap); + out = nb; + cap = ncap; + } else if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + status = -3; /* the input ends before the stream does */ + break; + } else { + status = 0; + break; + } + } + + BrotliDecoderDestroyInstance(dec); + + if (status == 1) { + tls_decomp_buf = out; + tls_decomp_cap = cap; + tls_decomp_len = (int)total_out; + return 1; + } else { + aether_caps_free(out, cap); + return status; + } +} + +const char* brotli_get_decompress_bytes(void) { + return (const char*)(tls_decomp_buf ? tls_decomp_buf : (unsigned char*)""); +} + +int brotli_get_decompress_length(void) { + return tls_decomp_len; +} + +void brotli_release_decompress(void) { + free_decomp_tls(); +} + +typedef struct { + BrotliDecoderState* dec; + unsigned char* out; + size_t out_cap; + int out_len; + size_t total_out; + size_t limit; + int finished; + int broken; + int limit_exceeded; +} BrotliDecompressStream; + +void* brotli_try_decompress_stream_new(int max_bytes) { + if (max_bytes < 0 || !brotli_decoder_patched()) return NULL; + BrotliDecompressStream* z = (BrotliDecompressStream*)aether_caps_calloc(1, sizeof(BrotliDecompressStream)); + if (!z) return NULL; + z->dec = BrotliDecoderCreateInstance(NULL, NULL, NULL); + if (!z->dec) { + aether_caps_free(z, sizeof(BrotliDecompressStream)); + return NULL; + } + z->limit = (max_bytes > 0) ? (size_t)max_bytes : ((size_t)1 << 30); + return z; +} + +int brotli_try_decompress_stream_write(void* handle, const char* data, int length) { + BrotliDecompressStream* z = (BrotliDecompressStream*)handle; + if (!z || z->finished || z->broken || length < 0) return 0; + + size_t in_len; + const unsigned char* in = brotli_unwrap_bytes(data, length, &in_len); + if (in_len > 0 && !in) return 0; + + z->out_len = 0; + size_t used = 0; + + if (z->out_cap < 256) { + size_t cap = 256; + unsigned char* nb = (unsigned char*)aether_caps_malloc(cap); + if (!nb) { z->broken = 1; return 0; } + if (z->out) aether_caps_free(z->out, z->out_cap); + z->out = nb; + z->out_cap = cap; + } + + size_t avail_in = in_len; + const uint8_t* next_in = (const uint8_t*)in; + + for (;;) { + size_t avail_out = z->out_cap - used; + uint8_t* next_out = (uint8_t*)(z->out + used); + + BrotliDecoderResult res = BrotliDecoderDecompressStream( + z->dec, &avail_in, &next_in, &avail_out, &next_out, NULL); + + used = z->out_cap - avail_out; + + if (z->total_out + used > z->limit) { + z->broken = 1; + z->limit_exceeded = 1; + return -2; + } + + if (res == BROTLI_DECODER_RESULT_SUCCESS) { + /* Bytes after the end of the stream are corruption, not a + * second stream: the one-shot path refuses them too. */ + if (avail_in > 0) { z->broken = 1; return 0; } + z->finished = 1; + break; + } else if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + if (avail_in == 0) break; + } else if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + size_t remaining_limit = z->limit - z->total_out; + if (used >= remaining_limit) { + z->broken = 1; + z->limit_exceeded = 1; + return -2; + } + size_t ncap = z->out_cap * 2; + if (ncap > remaining_limit) ncap = remaining_limit; + if (ncap <= z->out_cap) { + z->broken = 1; + z->limit_exceeded = 1; + return -2; + } + unsigned char* nb = (unsigned char*)aether_caps_malloc(ncap); + if (!nb) { z->broken = 1; return 0; } + memcpy(nb, z->out, used); + aether_caps_free(z->out, z->out_cap); + z->out = nb; + z->out_cap = ncap; + } else { + z->broken = 1; + return 0; + } + } + + z->total_out += used; + z->out_len = (int)used; + return 1; +} + +/* The stream must have reached its end by now: a decoder still waiting for + * input means the body was cut short, and a truncated body is an error + * rather than the shorter plaintext decoded so far. Returns -3 for that. */ +int brotli_try_decompress_stream_finish(void* handle) { + BrotliDecompressStream* z = (BrotliDecompressStream*)handle; + if (!z || z->broken) return 0; + z->out_len = 0; + if (z->finished) return 1; + z->broken = 1; + return -3; +} + +const char* brotli_get_decompress_stream_bytes(void* handle) { + BrotliDecompressStream* z = (BrotliDecompressStream*)handle; + if (!z || !z->out) return ""; + return (const char*)z->out; +} + +int brotli_get_decompress_stream_length(void* handle) { + BrotliDecompressStream* z = (BrotliDecompressStream*)handle; + return z ? z->out_len : 0; +} + +void brotli_release_decompress_stream(void* handle) { + BrotliDecompressStream* z = (BrotliDecompressStream*)handle; + if (!z) return; + if (z->dec) BrotliDecoderDestroyInstance(z->dec); + if (z->out) aether_caps_free(z->out, z->out_cap); + aether_caps_free(z, sizeof(BrotliDecompressStream)); +} + #else /* !AETHER_HAS_BROTLI */ /* stream_new returning NULL is the one signal a caller cannot ignore: every @@ -224,6 +485,32 @@ int brotli_try_compress(const char* data, int length, int quality) { (void)data; (void)length; (void)quality; return 0; } +int brotli_decoder_patched(void) { return 0; } + +int brotli_try_decompress(const char* data, int length, int max_bytes) { + (void)data; (void)length; (void)max_bytes; return 0; +} + +const char* brotli_get_decompress_bytes(void) { return ""; } +int brotli_get_decompress_length(void) { return 0; } +void brotli_release_decompress(void) {} + +void* brotli_try_decompress_stream_new(int max_bytes) { + (void)max_bytes; return NULL; +} + +int brotli_try_decompress_stream_write(void* handle, const char* data, int length) { + (void)handle; (void)data; (void)length; return 0; +} + +int brotli_try_decompress_stream_finish(void* handle) { + (void)handle; return 0; +} + +const char* brotli_get_decompress_stream_bytes(void* handle) { (void)handle; return ""; } +int brotli_get_decompress_stream_length(void* handle) { (void)handle; return 0; } +void brotli_release_decompress_stream(void* handle) { (void)handle; } + #endif /* AETHER_HAS_BROTLI */ const char* brotli_get_compress_bytes(void) { diff --git a/std/brotli/aether_brotli.h b/std/brotli/aether_brotli.h index bb5810e4d..a4c11a838 100644 --- a/std/brotli/aether_brotli.h +++ b/std/brotli/aether_brotli.h @@ -58,6 +58,35 @@ const char* brotli_get_compress_bytes(void); int brotli_get_compress_length(void); void brotli_release_compress(void); +/* 1 when the linked libbrotlidec carries the CVE-2026-111497 fix, which + * every decode below requires. 0 when it does not, or brotli is absent. */ +int brotli_decoder_patched(void); + +/* One-shot decompression. `max_bytes` caps the decoded size (0 = a 1 GiB + * ceiling). Returns 1 on success, -2 when the output would pass `max_bytes`, + * -3 when the input ends before the stream does, -4 when the decoder is + * refused (see brotli_decoder_patched), and 0 for anything else + * (corrupt input, bytes after the end, out of memory). The cap is what lets + * a server decode a request body it did not write: a few hundred bytes of + * brotli can describe gigabytes. */ +int brotli_try_decompress(const char* data, int length, int max_bytes); +const char* brotli_get_decompress_bytes(void); +int brotli_get_decompress_length(void); +void brotli_release_decompress(void); + +/* Streaming decompression. `max_bytes` caps the total decoded across every + * write (0 = a 1 GiB ceiling). write returns 1, -2 past the cap, or 0; + * finish returns 1 once the stream has ended and -3 when it has not. Any + * failure leaves the handle refusing further writes. Each call's output is + * held on the handle until the next call, as for the encoder. */ +void* brotli_try_decompress_stream_new(int max_bytes); +int brotli_try_decompress_stream_write(void* handle, const char* data, int length); +int brotli_try_decompress_stream_finish(void* handle); + +const char* brotli_get_decompress_stream_bytes(void* handle); +int brotli_get_decompress_stream_length(void* handle); +void brotli_release_decompress_stream(void* handle); + /* Why a system library rather than a vendored implementation: * libbrotlienc ships in every mainstream distro (apt libbrotli-dev, brew * brotli) and exposes exactly the streaming encoder this needs diff --git a/std/brotli/module.ae b/std/brotli/module.ae index 34991400a..5bf3a1ef7 100644 --- a/std/brotli/module.ae +++ b/std/brotli/module.ae @@ -1,9 +1,9 @@ -// std.brotli — Brotli compression (RFC 7932), streaming and one-shot. +// std.brotli — Brotli (RFC 7932) encoding and decoding, streaming and one-shot. // // `br` is what current browsers actually prefer over gzip: every modern // browser lists it ahead of gzip in Accept-Encoding, and it typically beats // gzip by 15-25% on text at comparable speed. This module wraps the system -// libbrotlienc (apt: libbrotli-dev, brew: brotli). +// libbrotlienc and libbrotlidec (apt: libbrotli-dev, brew: brotli). // // The streaming surface deliberately mirrors std.zlib's, so a server // negotiating `br` vs `gzip` writes the same shape either way: @@ -14,8 +14,11 @@ // tail, n, err = brotli.stream_finish(s) // at connection close // brotli.stream_free(s) // -// Compression only. An HTTP server compresses responses; nothing in-tree -// needs to DECODE br, and a decoder is a separate piece of work. +// Decoding has the same two shapes, and every decode takes a cap on how much +// it will produce, because a few hundred bytes of brotli can describe +// gigabytes: +// +// body, n, err = brotli.decompress_max(string.bytes(req_body), 1048576) exports( brotli_try_stream_new, brotli_try_stream_flush, @@ -23,8 +26,14 @@ exports( brotli_release_stream, brotli_get_compress_bytes, brotli_get_compress_length, brotli_release_compress, - available, compress, + brotli_get_decompress_bytes, brotli_get_decompress_length, + brotli_release_decompress, + brotli_try_decompress_stream_new, brotli_get_decompress_stream_bytes, + brotli_get_decompress_stream_length, brotli_release_decompress_stream, + available, decoder_available, compress, decompress, decompress_max, stream_new, stream_write, stream_flush, stream_finish, stream_free, + decompress_stream_new, decompress_stream_write, decompress_stream_finish, + decompress_stream_free, // Quality levels for compress / stream_new MIN_QUALITY, MAX_QUALITY, DEFAULT_QUALITY, FAST_QUALITY ) @@ -49,6 +58,19 @@ extern brotli_get_compress_bytes() -> ptr extern brotli_get_compress_length() -> int extern brotli_release_compress() +extern brotli_decoder_patched() -> int +extern brotli_try_decompress(data: ptr, length: int, max_bytes: int) -> int +extern brotli_get_decompress_bytes() -> ptr +extern brotli_get_decompress_length() -> int +extern brotli_release_decompress() + +extern brotli_try_decompress_stream_new(max_bytes: int) -> ptr +extern brotli_try_decompress_stream_write(handle: ptr, data: ptr, length: int) -> int +extern brotli_try_decompress_stream_finish(handle: ptr) -> int +extern brotli_get_decompress_stream_bytes(handle: ptr) -> ptr +extern brotli_get_decompress_stream_length(handle: ptr) -> int +extern brotli_release_decompress_stream(handle: ptr) + // Length-preserving string constructor from std.string — takes an explicit // byte count so binary payloads with embedded NULs survive the copy out of // the borrowed C buffer into an owned AetherString. @@ -66,7 +88,7 @@ fn unbounded(fname: string) { panic("brotli.${fname}: the slice has no bound (.len -1); give it one, as `(p as byte[])[0..n]`") } -// 1 when the toolchain was built with libbrotlienc detected. Use it to skip a +// 1 when the toolchain was built with libbrotli detected. Use it to skip a // test or pick a fallback encoding rather than threading an error everywhere. available() -> int { return brotli_backend_available() } @@ -146,3 +168,104 @@ stream_take(handle: ptr) -> { stream_free(handle: ptr) { if handle != null { brotli_release_stream(handle) } } + +const UNPATCHED = "brotli decoder refused: the system libbrotlidec lacks the CVE-2026-111497 fix (1.0.10, 1.1.1, 1.2.1 or a distro backport)" + +// 1 when decoding works here: brotli is built in AND the system libbrotlidec +// carries the CVE-2026-111497 fix (an out-of-bounds write on malformed input, +// libbrotli 1.0.3 to 1.2.0). Without the fix every decode refuses with an +// error rather than run a decoder that untrusted input can exploit; encoding +// is unaffected. The check asks the library itself, so a distro backport +// that keeps the old version number passes. +decoder_available() -> int { + if brotli_backend_available() == 0 { return 0 } + return brotli_decoder_patched() +} + +// The message for a decoder status: -2 is the cap, -3 a stream cut short, +// -4 an unpatched decoder. +fn decode_error(fname: string, rc: int) -> string { + if rc == -2 { return "brotli ${fname}: output limit exceeded" } + if rc == -3 { return "brotli ${fname}: truncated stream" } + if rc == -4 { return UNPATCHED } + return "brotli ${fname}: corrupt input" +} + +// One-shot: decompress with the default 1 GiB ceiling. Returns +// (bytes, count, "") on success, ("", 0, error) otherwise. For input you did +// not produce, use decompress_max with a cap that fits the job. +decompress(data: byte[]) -> { + if data.len < 0 { unbounded("decompress") } + return decompress_max(data, 0) +} + +// One-shot with a cap: decoding stops, and returns an "output limit +// exceeded" error, as soon as the output would pass `max_bytes` (0 = the +// 1 GiB ceiling). Truncated input, corrupt input and bytes after the end of +// the stream are errors too. +decompress_max(data: byte[], max_bytes: int) -> { + if data.len < 0 { unbounded("decompress_max") } + if max_bytes < 0 { return "", 0, "brotli decompress: max_bytes must be >= 0" } + if brotli_backend_available() == 0 { + return "", 0, "brotli unavailable" + } + rc = brotli_try_decompress(data, data.len, max_bytes) + if rc != 1 { return "", 0, decode_error("decompress", rc) } + raw = brotli_get_decompress_bytes() + n = brotli_get_decompress_length() + owned = string_new_with_length(raw, n) + brotli_release_decompress() + return owned, n, "" +} + +// Open a decoding stream whose total output is capped at `max_bytes` +// (0 = the 1 GiB ceiling). Returns (handle, "") or (null, error). +decompress_stream_new(max_bytes: int) -> { + if brotli_backend_available() == 0 { + return null, "brotli unavailable" + } + if max_bytes < 0 { + return null, "brotli decompress_stream_new: max_bytes must be >= 0" + } + if brotli_decoder_patched() == 0 { return null, UNPATCHED } + h = brotli_try_decompress_stream_new(max_bytes) + if h == null { + return null, "brotli decompress_stream_new failed" + } + return h, "" +} + +// Feed compressed bytes in any size of chunk. Returns (bytes, count, "") with +// whatever they decoded to. After an error the handle refuses further writes. +decompress_stream_write(handle: ptr, data: byte[]) -> { + if data.len < 0 { unbounded("decompress_stream_write") } + if handle == null { return "", 0, "brotli stream: null handle" } + rc = brotli_try_decompress_stream_write(handle, data, data.len) + if rc != 1 { return "", 0, decode_error("decompress_stream_write", rc) } + return decompress_stream_take(handle) +} + +// Confirm the stream ended. Returns ("", 0, "") when it did, and a +// "truncated stream" error when the input stopped short of the end, so a cut +// connection is not mistaken for a shorter body. Still needs +// decompress_stream_free. +decompress_stream_finish(handle: ptr) -> { + if handle == null { return "", 0, "brotli stream: null handle" } + rc = brotli_try_decompress_stream_finish(handle) + if rc != 1 { return "", 0, decode_error("decompress_stream_finish", rc) } + return decompress_stream_take(handle) +} + +// Copy the borrowed C buffer into an owned AetherString, as stream_take does. +decompress_stream_take(handle: ptr) -> { + raw = brotli_get_decompress_stream_bytes(handle) + n = brotli_get_decompress_stream_length(handle) + if n == 0 { return "", 0, "" } + owned = string_new_with_length(raw, n) + return owned, n, "" +} + +// Release a decoding stream. Safe on null. +decompress_stream_free(handle: ptr) { + if handle != null { brotli_release_decompress_stream(handle) } +} diff --git a/std/brotli/test_brotli.ae b/std/brotli/test_brotli.ae new file mode 100644 index 000000000..ee8a4d1f7 --- /dev/null +++ b/std/brotli/test_brotli.ae @@ -0,0 +1,396 @@ +// std.brotli unit & security regression tests — co-located with std/brotli/ module. +// +// Includes test vector patterns and edge cases adapted from Google Brotli +// (https://github.com/google/brotli), subject to the MIT License: +// Copyright 2016 The Brotli Authors. All rights reserved. + +import std.spec +import std.brotli +import std.string +import std.bytes + +// "hello large window" encoded by the brotli 1.2.1 CLI with +// --large_window=30: the non-RFC 7932 large-window format, whose header may ask +// the decoder for a window of up to 1 GiB before a byte of output exists. +const LARGE_WINDOW_BR: int[] = [0x11, 0x5e, 0x44, 0x00, 0xe0, 0x97, 0x2e, 0x43, 0x63, 0x7b, + 0x53, 0x84, 0xac, 0x52, 0x66, 0x38, 0x47, 0xa9, 0xeb, 0x00] + +const SAMPLE = "the quick brown fox jumps over the lazy dog, and then does it again and again" + +// `n` bytes cycling through 0..250, NULs included, so a decoder that stops +// at a NUL or mixes up chunk order is caught. +binary_payload(n: int) -> string { + b = bytes.new(n) + i = 0 + while i < n { + bytes.set(b, i, i % 251) + i = i + 1 + } + return bytes.finish(b, n) +} + +// `mb` megabytes of one byte, compressed through the streaming encoder so the +// plaintext never sits in memory at once: a small input that expands a +// thousandfold, the shape a decompression bomb takes. +bomb(mb: int) -> (string, int) { + chunk = bytes.new(65536) + i = 0 + while i < 65536 { + bytes.set(chunk, i, 0) + i = i + 1 + } + block = bytes.finish(chunk, 65536) + s, _e = brotli.stream_new(brotli.FAST_QUALITY, 0) + out = bytes.new(64) + total = 0 + k = 0 + while k < mb * 16 { + c, n, _e2 = brotli.stream_write(s, string.bytes(block)[0..65536]) + if n > 0 { + bytes.copy_from_string(out, total, c, n) + total = total + n + } + k = k + 1 + } + c, n, _e3 = brotli.stream_finish(s) + brotli.stream_free(s) + if n > 0 { bytes.copy_from_string(out, total, c, n) } + total = total + n + return bytes.finish(out, total), total +} + +main() { + fw = spec.init() + + if brotli.available() == 1 && brotli.decoder_available() == 0 { + // An unpatched system decoder: every decode must refuse (CVE-2026-111497). + spec.describe(fw, "std.brotli on an unpatched libbrotlidec") { + spec.it("refuses to decode") callback { + packed, plen, _e = brotli.compress(string.bytes(SAMPLE)[0..string.length(SAMPLE)], 5) + _o, _n, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_err_contains(derr, "CVE-2026-111497", "one-shot refuses") + _h, serr = brotli.decompress_stream_new(0) + spec.assert_err_contains(serr, "CVE-2026-111497", "streaming refuses") + } + } + return spec.run_summary(fw) + } + + unavailable = spec.skip_all_if(fw, brotli.decoder_available() == 0, + "brotli backend not built in") + + if unavailable == 0 { + spec.describe(fw, "std.brotli one-shot compress/decompress") { + spec.it("round-trips a string byte for byte") callback { + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + spec.assert_true(plen > 0, "compressed length is positive") + + back, blen, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_ok(derr, "decompress succeeds") + spec.assert_eq(blen, n, "length restored") + spec.assert_str_eq(back, SAMPLE, "content restored") + } + + spec.it("shrinks repetitive input") callback { + original = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + n = string.length(original) + packed, plen, err = brotli.compress(string.bytes(original)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + spec.assert_true(plen < n, "compressed ${plen} < original ${n}") + + back, blen, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_ok(derr, "decompress succeeds") + spec.assert_eq(blen, n, "length restored") + spec.assert_str_eq(back, original, "content restored") + } + + spec.it("round-trips at various quality levels") callback { + n = string.length(SAMPLE) + q = brotli.MIN_QUALITY + while q <= 9 { + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], q) + spec.assert_ok(err, "compress at quality ${q}") + back, blen, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_ok(derr, "decompress at quality ${q}") + spec.assert_str_eq(back, SAMPLE, "content restored at quality ${q}") + q = q + 3 + } + } + } + + spec.describe(fw, "std.brotli decompress_max security limits") { + spec.it("succeeds when output fits within max_bytes limit") callback { + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + + back, blen, derr = brotli.decompress_max(string.bytes(packed)[0..plen], n) + spec.assert_ok(derr, "decompress_max succeeds with exact limit") + spec.assert_eq(blen, n, "length matches") + spec.assert_str_eq(back, SAMPLE, "content restored") + } + + spec.it("refuses decompression bomb exceeding max_bytes limit") callback { + // Highly compressible payload (decompression bomb pattern) + b = bytes.new(2000) + i = 0 + while i < 2000 { + bytes.set(b, i, 120) // 'x' + i = i + 1 + } + bomb = bytes.finish(b, 2000) + + packed, plen, err = brotli.compress(string.bytes(bomb)[0..2000], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress bomb payload") + + _out, _olen, derr = brotli.decompress_max(string.bytes(packed)[0..plen], 500) + spec.assert_err_contains(derr, "output limit exceeded", "strictly halts decompression bomb when limit exceeded") + } + + spec.it("rejects negative max_bytes parameter") callback { + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + + _out, _olen, derr = brotli.decompress_max(string.bytes(packed)[0..plen], 0 - 5) + spec.assert_err_contains(derr, "max_bytes must be >= 0", "rejects negative limit") + } + } + + spec.describe(fw, "std.brotli large and hostile input") { + spec.it("round-trips a binary payload larger than its first buffer") callback { + n = 300000 + plain = binary_payload(n) + packed, plen, err = brotli.compress(string.bytes(plain)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + back, blen, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_ok(derr, "decompress succeeds") + spec.assert_eq(blen, n, "length restored, NULs and all") + spec.assert_true(string.string_equals(back, plain) == 1, "content restored byte for byte") + _exact, elen, eerr = brotli.decompress_max(string.bytes(packed)[0..plen], n) + spec.assert_ok(eerr, "a limit equal to the output size is enough") + spec.assert_eq(elen, n, "length at the exact limit") + _o, _l, oerr = brotli.decompress_max(string.bytes(packed)[0..plen], n - 1) + spec.assert_err_contains(oerr, "output limit exceeded", "one byte under the size is refused") + } + + spec.it("stops a thousandfold bomb at the limit") callback { + packed, plen = bomb(8) + spec.assert_true(plen < 65536, "8 MiB packs into ${plen} bytes") + _o, _l, derr = brotli.decompress_max(string.bytes(packed)[0..plen], 1048576) + spec.assert_err_contains(derr, "output limit exceeded", "one-shot refuses past 1 MiB") + ds, _e = brotli.decompress_stream_new(1048576) + _d, _dn, serr = brotli.decompress_stream_write(ds, string.bytes(packed)[0..plen]) + spec.assert_err_contains(serr, "output limit exceeded", "a stream refuses past 1 MiB") + brotli.decompress_stream_free(ds) + } + } + + spec.describe(fw, "std.brotli edge cases & invalid input") { + spec.it("round-trips an empty body") callback { + packed, plen, err = brotli.compress(string.bytes("")[0..0], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress of nothing") + spec.assert_true(plen > 0, "an empty stream still has a header") + back, blen, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_ok(derr, "decompress of an empty stream") + spec.assert_eq(blen, 0, "nothing comes back") + spec.assert_str_eq(back, "", "as an empty string") + } + + spec.it("refuses the large-window format") callback { + // Large windows are off in libbrotlidec unless a caller turns + // them on; this pins that nobody does, since the window is + // allocated from the header, outside max_bytes. + n = LARGE_WINDOW_BR.len + b = bytes.new(n) + i = 0 + while i < n { + bytes.set(b, i, LARGE_WINDOW_BR[i]) + i = i + 1 + } + lw = bytes.finish(b, n) + _o, _l, derr = brotli.decompress(string.bytes(lw)[0..n]) + spec.assert_err(derr, "one-shot refuses a large-window stream") + ds, _e = brotli.decompress_stream_new(0) + _d, _dn, serr = brotli.decompress_stream_write(ds, string.bytes(lw)[0..n]) + spec.assert_err(serr, "a stream refuses it too") + brotli.decompress_stream_free(ds) + } + + spec.it("rejects garbage input on decompress") callback { + _out, _olen, derr = brotli.decompress(string.bytes("this is not brotli data")) + spec.assert_err(derr, "garbage input is rejected") + } + + spec.it("rejects empty buffer on decompress") callback { + _out, _olen, derr = brotli.decompress(string.bytes("")) + spec.assert_err(derr, "empty input buffer is rejected") + } + + spec.it("rejects trailing garbage appended to valid stream") callback { + // Adapted from Google Brotli decompress_test.py (test_garbage_appended) + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + + buf = bytes.new(plen + 5) + bytes.copy_from_string(buf, 0, packed, plen) + bytes.copy_from_string(buf, plen, "extra", 5) + corrupt = bytes.finish(buf, plen + 5) + + _out, _olen, derr = brotli.decompress(string.bytes(corrupt)[0..plen + 5]) + spec.assert_err(derr, "trailing garbage appended to stream is rejected") + } + + spec.it("rejects truncated bitstream") callback { + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + + // Truncate stream midway + trunc_len = plen / 2 + _out, _olen, derr = brotli.decompress(string.bytes(packed)[0..trunc_len]) + spec.assert_err(derr, "truncated bitstream is rejected") + } + } + + spec.describe(fw, "std.brotli streaming decompression") { + spec.it("decompresses streamed chunks accurately") callback { + s, err = brotli.stream_new(brotli.FAST_QUALITY, 0) + spec.assert_ok(err, "stream_new encoder") + + n = string.length(SAMPLE) + c1, n1, e1 = brotli.stream_write(s, string.bytes(SAMPLE)[0..n]) + spec.assert_ok(e1, "stream_write") + c2, n2, e2 = brotli.stream_finish(s) + spec.assert_ok(e2, "stream_finish") + + // Concatenate encoded output chunks + tot = n1 + n2 + buf = bytes.new(tot) + if n1 > 0 { bytes.copy_from_string(buf, 0, c1, n1) } + if n2 > 0 { bytes.copy_from_string(buf, n1, c2, n2) } + compressed = bytes.finish(buf, tot) + brotli.stream_free(s) + + // Decompress via streaming decoder + ds, derr = brotli.decompress_stream_new(0) + spec.assert_ok(derr, "decompress_stream_new") + + d1, dn1, de1 = brotli.decompress_stream_write(ds, string.bytes(compressed)[0..tot]) + spec.assert_ok(de1, "decompress_stream_write") + + d2, dn2, de2 = brotli.decompress_stream_finish(ds) + spec.assert_ok(de2, "decompress_stream_finish") + + full_decomp = "${d1}${d2}" + spec.assert_eq(dn1 + dn2, n, "decompressed total length matches original") + spec.assert_str_eq(full_decomp, SAMPLE, "decompressed stream content restored") + + brotli.decompress_stream_free(ds) + } + + spec.it("decodes a stream fed one byte at a time") callback { + n = 5000 + plain = binary_payload(n) + packed, plen, err = brotli.compress(string.bytes(plain)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + ds, _e = brotli.decompress_stream_new(0) + out = bytes.new(n) + got = 0 + i = 0 + while i < plen { + c, cn, werr = brotli.decompress_stream_write(ds, string.bytes(packed)[i..i + 1]) + spec.assert_ok(werr, "write of byte ${i}") + if cn > 0 { + bytes.copy_from_string(out, got, c, cn) + got = got + cn + } + i = i + 1 + } + _t, tn, ferr = brotli.decompress_stream_finish(ds) + spec.assert_ok(ferr, "finish after the last byte") + spec.assert_eq(got + tn, n, "every byte came out") + spec.assert_true(string.string_equals(bytes.finish(out, got), plain) == 1, "in order") + brotli.decompress_stream_free(ds) + } + + spec.it("finish refuses a truncated stream") callback { + n = 5000 + plain = binary_payload(n) + packed, plen, err = brotli.compress(string.bytes(plain)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + ds, _e = brotli.decompress_stream_new(0) + _d, _dn, werr = brotli.decompress_stream_write(ds, string.bytes(packed)[0..plen - 3]) + spec.assert_ok(werr, "a prefix decodes so far") + _t, _tn, ferr = brotli.decompress_stream_finish(ds) + spec.assert_err_contains(ferr, "truncated", "the missing tail is reported") + brotli.decompress_stream_free(ds) + } + + spec.it("refuses bytes after the end of the stream") callback { + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + buf = bytes.new(plen + 5) + bytes.copy_from_string(buf, 0, packed, plen) + bytes.copy_from_string(buf, plen, "extra", 5) + padded = bytes.finish(buf, plen + 5) + ds, _e = brotli.decompress_stream_new(0) + _d, _dn, werr = brotli.decompress_stream_write(ds, string.bytes(padded)[0..plen + 5]) + spec.assert_err(werr, "trailing bytes in the same write are refused") + brotli.decompress_stream_free(ds) + + ds2, _e2 = brotli.decompress_stream_new(0) + _d2, _dn2, w2 = brotli.decompress_stream_write(ds2, string.bytes(packed)[0..plen]) + spec.assert_ok(w2, "the whole stream decodes") + _d3, _dn3, w3 = brotli.decompress_stream_write(ds2, string.bytes("extra")[0..5]) + spec.assert_err(w3, "a later write past the end is refused") + brotli.decompress_stream_free(ds2) + } + + spec.it("allows output exactly at a stream's limit") callback { + n = 5000 + plain = binary_payload(n) + packed, plen, err = brotli.compress(string.bytes(plain)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + ds, _e = brotli.decompress_stream_new(n) + _d, dn, werr = brotli.decompress_stream_write(ds, string.bytes(packed)[0..plen]) + spec.assert_ok(werr, "output equal to the limit") + _t, tn, ferr = brotli.decompress_stream_finish(ds) + spec.assert_ok(ferr, "finish") + spec.assert_eq(dn + tn, n, "all of it") + brotli.decompress_stream_free(ds) + } + + spec.it("enforces max_bytes limit during streaming decompression") callback { + b = bytes.new(2000) + i = 0 + while i < 2000 { + bytes.set(b, i, 121) // 'y' + i = i + 1 + } + bomb = bytes.finish(b, 2000) + + packed, plen, err = brotli.compress(string.bytes(bomb)[0..2000], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress bomb") + + ds, derr = brotli.decompress_stream_new(300) + spec.assert_ok(derr, "decompress_stream_new with max_bytes=300") + + _d1, _dn1, de1 = brotli.decompress_stream_write(ds, string.bytes(packed)[0..plen]) + spec.assert_err_contains(de1, "output limit exceeded", "streaming decompress halts bomb exceeding limit") + + // Subsequent write on broken handle fails safely + _d2, _dn2, de2 = brotli.decompress_stream_write(ds, string.bytes("more")[0..4]) + spec.assert_err(de2, "broken stream handle rejects further writes") + + brotli.decompress_stream_free(ds) + } + } + } + + return spec.run_summary(fw) +} diff --git a/tests/integration/optional_library_linking/test_optional_library_linking.sh b/tests/integration/optional_library_linking/test_optional_library_linking.sh index 3f8b17e74..becc988ce 100755 --- a/tests/integration/optional_library_linking/test_optional_library_linking.sh +++ b/tests/integration/optional_library_linking/test_optional_library_linking.sh @@ -21,7 +21,7 @@ cat > cc-probe <<'EOF' for arg do printf '%s\n' "$arg" >> "$LINK_ARGS" case "$arg" in - -lssl|-lcrypto|-lz|-lnghttp2|-lpcre2-8|-lbrotlienc|-lbrotlicommon|-lzstd|-lfyaml) + -lssl|-lcrypto|-lz|-lnghttp2|-lpcre2-8|-lbrotlienc|-lbrotlidec|-lbrotlicommon|-lzstd|-lfyaml) case " $ALLOWED_LIBS " in *" $arg "*) ;; *) echo "unexpected optional dependency: $arg" >&2; exit 1 ;; @@ -43,7 +43,7 @@ check_args() { # Windows uses the native compiler; inspect its verbose command instead. if [ "$WINDOWS" = 1 ]; then tr ' ' '\n' < "$1" > "$LINK_ARGS"; fi [ -s "$LINK_ARGS" ] || fail 'C compiler command missing' - for lib in -lssl -lcrypto -lz -lnghttp2 -lpcre2-8 -lbrotlienc -lbrotlicommon -lzstd -lfyaml; do + for lib in -lssl -lcrypto -lz -lnghttp2 -lpcre2-8 -lbrotlienc -lbrotlidec -lbrotlicommon -lzstd -lfyaml; do if grep -Fxq -- "$lib" "$LINK_ARGS"; then case " $ALLOWED_LIBS " in *" $lib "*) ;; @@ -81,7 +81,7 @@ esac for entry in \ 'regex|-lpcre2-8' \ 'zlib|-lz' \ - 'brotli|-lbrotlienc -lbrotlicommon' \ + 'brotli|-lbrotlienc -lbrotlidec -lbrotlicommon' \ 'zstd|-lzstd' \ 'yaml|-lfyaml' \ 'cryptography|-lssl -lcrypto' \ @@ -150,7 +150,7 @@ main() { else { println("excluded") } } EOF -ALLOWED_LIBS='-lbrotlienc -lbrotlicommon' +ALLOWED_LIBS='-lbrotlienc -lbrotlidec -lbrotlicommon' export ALLOWED_LIBS : > "$LINK_ARGS" "$AE" build conditional.ae -D WITH_CODEC -o conditional-on --verbose > conditional-on.log 2>&1 \ diff --git a/tools/ae.c b/tools/ae.c index 9ea44f23b..86aa25d60 100644 --- a/tools/ae.c +++ b/tools/ae.c @@ -3019,7 +3019,8 @@ static unsigned optional_link_requirement(const char* tok, size_t len) { { "-lssl", LINK_OPENSSL }, { "-lcrypto", LINK_OPENSSL }, { "-lz", LINK_ZLIB }, { "-lnghttp2", LINK_NGHTTP2 }, { "-lpcre2-8", LINK_PCRE2 }, - { "-lbrotlienc", LINK_BROTLI }, { "-lbrotlicommon", LINK_BROTLI }, + { "-lbrotlienc", LINK_BROTLI }, { "-lbrotlidec", LINK_BROTLI }, + { "-lbrotlicommon", LINK_BROTLI }, { "-lzstd", LINK_ZSTD }, { "-lfyaml", LINK_YAML }, }; for (size_t i = 0; i < sizeof(libs) / sizeof(libs[0]); i++) { @@ -4615,9 +4616,9 @@ void build_gcc_cmd(char* cmd, size_t size, const char* pcre2_libs = ""; #endif #ifdef AETHER_BROTLI_LIBS - /* libbrotlienc powers std.brotli. Empty when not detected, and that - * emptiness is load-bearing: a box without the library must not get a - * -lbrotlienc it cannot resolve. The std.brotli surface stays valid and + /* libbrotlienc and libbrotlidec power std.brotli. Empty when not + * detected, and that emptiness is load-bearing: a box without the + * libraries must not get a -lbrotlienc it cannot resolve. The std.brotli surface stays valid and * reports "brotli unavailable". */ const char* brotli_libs = (required_libs & LINK_BROTLI) ? AETHER_BROTLI_LIBS : ""; #else @@ -4935,9 +4936,9 @@ void build_gcc_cmd(char* cmd, size_t size, #else const char* pcre2_libs = ""; #endif - // libbrotlienc powers std.brotli. Empty when not detected, and that - // emptiness is load-bearing: a box without the library must not be handed - // a -lbrotlienc it cannot resolve. + // libbrotlienc and libbrotlidec power std.brotli. Empty when not detected, + // and that emptiness is load-bearing: a box without the libraries must not + // be handed a -lbrotlienc it cannot resolve. #ifdef AETHER_BROTLI_LIBS const char* brotli_libs = (required_libs & LINK_BROTLI) ? AETHER_BROTLI_LIBS : ""; #else