From 432269bbe0b21d5355f14ac177112991731e7070 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:46:43 +0100 Subject: [PATCH 1/2] std.brotli: add one-shot and streaming decoder with size limits Co-authored-by: paul-hammant <82182+paul-hammant@users.noreply.github.com> --- Makefile | 9 +- compiler/codegen/codegen.c | 2 +- docs/api/brotli.html | 81 ++++++ docs/api/index.html | 15 +- docs/stdlib-reference.md | 2 +- .../20261024T153948Z-added-brotli-decoder.md | 1 + std/brotli/README.md | 32 ++- std/brotli/aether_brotli.c | 262 ++++++++++++++++++ std/brotli/aether_brotli.h | 20 ++ std/brotli/module.ae | 101 ++++++- std/brotli/test_brotli.ae | 202 ++++++++++++++ .../test_optional_library_linking.sh | 8 +- tools/ae.c | 2 +- 13 files changed, 716 insertions(+), 21 deletions(-) create mode 100644 new_changelogs/20261024T153948Z-added-brotli-decoder.md create mode 100644 std/brotli/test_brotli.ae diff --git a/Makefile b/Makefile index 0451c3e39..83fdbf0de 100644 --- a/Makefile +++ b/Makefile @@ -583,14 +583,13 @@ ifeq ($(BROTLI),0) BROTLI_CFLAGS := BROTLI_LDFLAGS := else - BROTLI_CFLAGS := $(shell pkg-config --cflags libbrotlienc 2>/dev/null) + BROTLI_CFLAGS := $(shell pkg-config --cflags libbrotlienc libbrotlidec 2>/dev/null) # --static so libbrotlicommon comes with it. It is a Requires.private of - # libbrotlienc, so a plain --libs omits it -- fine when the loader resolves + # libbrotlienc/libbrotlidec, so a plain --libs omits it -- fine when the loader resolves # a shared object, fatal on a static link: MinGW's ld then reports a wall of # undefined BrotliGetDictionary / BrotliDefaultAllocFunc / _kBrotli* - # references from inside libbrotlienc.a itself. Found on a real MSYS2 box; - # a Linux shared link hides it entirely. - BROTLI_LDFLAGS := $(shell pkg-config --static --libs libbrotlienc 2>/dev/null) + # references from inside libbrotlienc.a/libbrotlidec.a itself. + BROTLI_LDFLAGS := $(shell pkg-config --static --libs libbrotlienc libbrotlidec 2>/dev/null | tr ' ' '\n' | awk '!seen[$$0]++' | tr '\n' ' ') endif ifneq ($(BROTLI_LDFLAGS),) BROTLI_CFLAGS += -DAETHER_HAS_BROTLI diff --git a/compiler/codegen/codegen.c b/compiler/codegen/codegen.c index 7965f09da..682070df2 100644 --- a/compiler/codegen/codegen.c +++ b/compiler/codegen/codegen.c @@ -6673,7 +6673,7 @@ static const AetherLinkReq g_link_reqs[] = { /* Base64 entry points share aether_cryptography.o with OpenSSL users. */ { "std.encoding", "-lssl -lcrypto" }, { "std.zlib", "-lz" }, - { "std.brotli", "-lbrotlienc -lbrotlicommon" }, + { "std.brotli", "-lbrotlienc -lbrotlidec -lbrotlicommon" }, { "std.zstd", "-lzstd" }, { "std.yaml", "-lfyaml" }, /* These veneers reference the HTTP server/client objects from C, not diff --git a/docs/api/brotli.html b/docs/api/brotli.html index 3606ec8c4..5c3ff6823 100644 --- a/docs/api/brotli.html +++ b/docs/api/brotli.html @@ -24,13 +24,26 @@

Exports

  • get_compress_bytes
  • get_compress_length
  • release_compress
    • +
  • get_decompress_bytes
    • +
  • get_decompress_length
    • +
  • release_decompress
    • +
  • try_decompress_stream_new
    • +
  • get_decompress_stream_bytes
    • +
  • get_decompress_stream_length
    • +
  • release_decompress_stream
  • available
  • compress
    • +
  • decompress
    • +
  • decompress_max
  • stream_new
  • stream_write
  • stream_flush
  • stream_finish
  • stream_free
    • +
  • decompress_stream_new
    • +
  • decompress_stream_write
    • +
  • decompress_stream_finish
    • +
  • decompress_stream_free
  • MIN_QUALITY
  • MAX_QUALITY
  • DEFAULT_QUALITY
    • @@ -178,6 +191,38 @@

    get_compress_length externrelease_compress extern

    release_compress()
    +
    +

    get_decompress_bytes extern

    + 
    get_decompress_bytes() -> ptr
    +
    +
    +

    get_decompress_length extern

    + 
    get_decompress_length() -> int
    +
    +
    +

    release_decompress extern

    + 
    release_decompress()
    +
    +
    +

    try_decompress_stream_new extern

    + 
    try_decompress_stream_new(max_bytes) -> ptr
    +

    max_bytes: int

    +
    +
    +

    get_decompress_stream_bytes extern

    + 
    get_decompress_stream_bytes(handle) -> ptr
    +

    handle: ptr

    +
    +
    +

    get_decompress_stream_length extern

    + 
    get_decompress_stream_length(handle) -> int
    +

    handle: ptr

    +
    +
    +

    release_decompress_stream extern

    + 
    release_decompress_stream(handle)
    +

    handle: ptr

    +

    available fn

    available() -> int
    @@ -189,6 +234,18 @@

    compress fn

    data: byte[], quality: int

    One-shot: compress a body already in memory. Returns (bytes, count, "") on success, ("", 0, error) otherwise. Quality out of range is clamped.

    +
    +

    decompress fn

    + 
    decompress(data)
    +

    data: byte[]

    +

    One-shot: decompress brotli-encoded data. Returns (bytes, count, "") on success, ("", 0, error) otherwise.

    +
    +
    +

    decompress_max fn

    + 
    decompress_max(data, max_bytes)
    +

    data: byte[], max_bytes: int

    +

    One-shot decompress with a strict output limit (`max_bytes`). If output would exceed `max_bytes`, decoding is aborted and returns an error mitigating decompression bomb attack vectors (e.g. broccoli2shell / CVE-2026-111497).

    +

    stream_new fn

    stream_new(quality, window) -> (ptr, string)
    @@ -219,6 +276,30 @@

    stream_free fn

    handle: ptr

    Release the handle. Safe on null.

    +
    +

    decompress_stream_new fn

    + 
    decompress_stream_new(max_bytes) -> (ptr, string)
    +

    max_bytes: int

    +

    Streaming decompression. `max_bytes` sets a strict cumulative output upper bound across stream lifetime (0 = default safety ceiling).

    +
    +
    +

    decompress_stream_write fn

    + 
    decompress_stream_write(handle, data)
    +

    handle: ptr, data: byte[]

    +

    Feed bytes into a decompression stream. Returns (decompressed_chunk, count, "").

    +
    +
    +

    decompress_stream_finish fn

    + 
    decompress_stream_finish(handle) -> (string, int, string)
    +

    handle: ptr

    +

    Finish a decompression stream and flush any final decompressed bytes.

    +
    +
    +

    decompress_stream_free fn

    + 
    decompress_stream_free(handle)
    +

    handle: ptr

    +

    Release a decompression stream handle. Safe on null.

    +

    MIN_QUALITY const

    MIN_QUALITY = 0
    diff --git a/docs/api/index.html b/docs/api/index.html index 19fea2194..b14735ed0 100644 --- a/docs/api/index.html +++ b/docs/api/index.html @@ -149,7 +149,7 @@

    bits

    brotli

    std.brotli — Brotli compression (RFC 7932), streaming and one-shot.

    - 20 exports + 33 exports

    bytes

    @@ -679,13 +679,26 @@

    Search Results

  • get_compress_bytesbrotli
  • get_compress_lengthbrotli
  • release_compressbrotli
    • +
  • get_decompress_bytesbrotli
    • +
  • get_decompress_lengthbrotli
    • +
  • release_decompressbrotli
    • +
  • try_decompress_stream_newbrotli
    • +
  • get_decompress_stream_bytesbrotli
    • +
  • get_decompress_stream_lengthbrotli
    • +
  • release_decompress_streambrotli
  • availablebrotli
  • compressbrotli
    • +
  • decompressbrotli
    • +
  • decompress_maxbrotli
  • stream_newbrotli
  • stream_writebrotli
  • stream_flushbrotli
  • stream_finishbrotli
  • stream_freebrotli
    • +
  • decompress_stream_newbrotli
    • +
  • decompress_stream_writebrotli
    • +
  • decompress_stream_finishbrotli
    • +
  • decompress_stream_freebrotli
  • MIN_QUALITYbrotli
  • MAX_QUALITYbrotli
  • DEFAULT_QUALITYbrotli
    • diff --git a/docs/stdlib-reference.md b/docs/stdlib-reference.md index fa32982fc..50d8ad92a 100644 --- a/docs/stdlib-reference.md +++ b/docs/stdlib-reference.md @@ -18,7 +18,7 @@ header comment is the authoritative description. | `std.audit` | Query the sandbox audit trail. | 4 | [guide](../std/audit/README.md) · [source](../std/audit/module.ae) | | `std.bignum` | Arbitrary-precision integers. | 29 | [guide](../std/bignum/README.md) · [source](../std/bignum/module.ae) | | `std.bits` | Unsigned bit operations: rotates, shifts, popcount, leading zeros, unsigned divide. | 17 | [guide](../std/bits/README.md) · [source](../std/bits/module.ae) | -| `std.brotli` | Brotli compression, streaming and one-shot, for `Content-Encoding: br`. | 20 | [guide](../std/brotli/README.md) · [source](../std/brotli/module.ae) | +| `std.brotli` | Brotli compression, streaming and one-shot, for `Content-Encoding: br`. | 33 | [guide](../std/brotli/README.md) · [source](../std/brotli/module.ae) | | `std.bytes` | Mutable byte buffer with random access and overlap-safe copies. | 32 | [guide](../std/bytes/README.md) · [source](../std/bytes/module.ae) | | `std.capsicum` | FreeBSD Capsicum capability-mode bindings. | 28 | [guide](../std/capsicum/README.md) · [source](../std/capsicum/module.ae) | | `std.cas` | Content-addressed store keyed by the sha256 of file contents. | 7 | [guide](../std/cas/README.md) · [source](../std/cas/module.ae) | diff --git a/new_changelogs/20261024T153948Z-added-brotli-decoder.md b/new_changelogs/20261024T153948Z-added-brotli-decoder.md new file mode 100644 index 000000000..280842596 --- /dev/null +++ b/new_changelogs/20261024T153948Z-added-brotli-decoder.md @@ -0,0 +1 @@ +- `std.brotli`: Added one-shot (`decompress`, `decompress_max`) and streaming (`decompress_stream_new`, `decompress_stream_write`, `decompress_stream_finish`, `decompress_stream_free`) Brotli decoder APIs, with output size limit enforcement to defend against decompression bomb / zip bomb attack vectors. diff --git a/std/brotli/README.md b/std/brotli/README.md index 1473b614f..743ae0170 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -7,10 +7,30 @@ ahead of `gzip` in `Accept-Encoding`, and it typically beats gzip by 15-25% on text at comparable speed. This module wraps the system **libbrotlienc** (`apt install libbrotli-dev`, `brew install brotli`). -Compression only. An HTTP server compresses responses; nothing in-tree needs to -*decode* `br`, and a decoder is separate work. +Decompression and compression supported via `libbrotli`. -## Streaming +## Decompression and Security + +`decompress(data)` decompresses Brotli-encoded byte slices into their original string representation. + +To defend against decompression bombs / zip bombs (such as the "broccoli2shell" attack vector / CVE-2026-111497) where a small compressed payload expands into huge allocations, use `decompress_max`: + +```aether,fragment +plain, n, err = brotli.decompress_max(string.bytes(compressed), 1048576) // cap at 1MB +``` + +If the decompressed output exceeds `max_bytes`, `decompress_max` immediately aborts and returns an error (`"brotli decompress: output limit exceeded"`). + +Streaming decompression is also available: + +```aether,fragment +ds, err = brotli.decompress_stream_new(max_bytes) +chunk1, n1, err = brotli.decompress_stream_write(ds, data1) +tail, tn, err = brotli.decompress_stream_finish(ds) +brotli.decompress_stream_free(ds) +``` + +## Streaming Encoding The surface deliberately mirrors `std.zlib`'s streaming deflate, so a server negotiating `br` against `gzip` writes the same shape either way: @@ -55,6 +75,8 @@ fall back to gzip. ## Exports -`available`, `compress`, `stream_new`, `stream_write`, `stream_flush`, -`stream_finish`, `stream_free` (and the `MIN_QUALITY` / `MAX_QUALITY` / +`available`, `compress`, `decompress`, `decompress_max`, `stream_new`, +`stream_write`, `stream_flush`, `stream_finish`, `stream_free`, +`decompress_stream_new`, `decompress_stream_write`, `decompress_stream_finish`, +`decompress_stream_free` (and the `MIN_QUALITY` / `MAX_QUALITY` / `DEFAULT_QUALITY` / `FAST_QUALITY` constants), plus the `brotli_*` raw forms. diff --git a/std/brotli/aether_brotli.c b/std/brotli/aether_brotli.c index 03bd1ce16..ef64379cf 100644 --- a/std/brotli/aether_brotli.c +++ b/std/brotli/aether_brotli.c @@ -7,6 +7,7 @@ #ifdef AETHER_HAS_BROTLI #include +#include #endif /* The input bytes and their count. `data` is always raw: the std module @@ -201,6 +202,243 @@ int brotli_try_compress(const char* data, int length, int quality) { return 1; } +/* One-shot decompression output slot, thread-local */ +static _Thread_local unsigned char* tls_decomp_buf = NULL; +static _Thread_local size_t tls_decomp_cap = 0; +static _Thread_local int tls_decomp_len = 0; + +static void free_decomp_tls(void) { + if (tls_decomp_buf) { + aether_caps_free(tls_decomp_buf, tls_decomp_cap); + tls_decomp_buf = NULL; + } + tls_decomp_cap = 0; + tls_decomp_len = 0; +} + +int brotli_try_decompress(const char* data, int length, int max_bytes) { + free_decomp_tls(); + if (length < 0 || max_bytes < 0) return 0; + + size_t in_len; + const unsigned char* in = brotli_unwrap_bytes(data, length, &in_len); + if (in_len > 0 && !in) return 0; + + BrotliDecoderState* dec = BrotliDecoderCreateInstance(NULL, NULL, NULL); + if (!dec) return 0; + + size_t limit = (max_bytes > 0) ? (size_t)max_bytes : ((size_t)1 << 30); + size_t initial_cap = (in_len > 0 && in_len * 4 > 256) ? (in_len * 4) : 256; + if (initial_cap > limit) initial_cap = limit; + if (initial_cap == 0) initial_cap = 1; + + size_t cap = initial_cap; + unsigned char* out = (unsigned char*)aether_caps_malloc(cap); + if (!out) { + BrotliDecoderDestroyInstance(dec); + return 0; + } + + size_t avail_in = in_len; + const uint8_t* next_in = (const uint8_t*)in; + size_t total_out = 0; + int status = 0; + + for (;;) { + size_t avail_out = cap - total_out; + uint8_t* next_out = (uint8_t*)(out + total_out); + + BrotliDecoderResult res = BrotliDecoderDecompressStream( + dec, &avail_in, &next_in, &avail_out, &next_out, NULL); + + total_out = cap - avail_out; + + if (res == BROTLI_DECODER_RESULT_SUCCESS) { + if (avail_in > 0) { + status = 0; /* Trailing garbage after valid stream */ + } else { + status = 1; + } + break; + } else if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + if (total_out >= limit) { + status = -2; + break; + } + size_t ncap = cap * 2; + if (ncap > limit) ncap = limit; + if (ncap <= cap) { + status = -2; + break; + } + unsigned char* nb = (unsigned char*)aether_caps_malloc(ncap); + if (!nb) { + status = 0; + break; + } + memcpy(nb, out, total_out); + aether_caps_free(out, cap); + out = nb; + cap = ncap; + } else if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + status = 0; + break; + } else { + status = 0; + break; + } + } + + BrotliDecoderDestroyInstance(dec); + + if (status == 1) { + tls_decomp_buf = out; + tls_decomp_cap = cap; + tls_decomp_len = (int)total_out; + return 1; + } else { + aether_caps_free(out, cap); + return status; + } +} + +const char* brotli_get_decompress_bytes(void) { + return (const char*)(tls_decomp_buf ? tls_decomp_buf : (unsigned char*)""); +} + +int brotli_get_decompress_length(void) { + return tls_decomp_len; +} + +void brotli_release_decompress(void) { + free_decomp_tls(); +} + +typedef struct { + BrotliDecoderState* dec; + unsigned char* out; + size_t out_cap; + int out_len; + size_t total_out; + size_t limit; + int finished; + int broken; + int limit_exceeded; +} BrotliDecompressStream; + +void* brotli_try_decompress_stream_new(int max_bytes) { + if (max_bytes < 0) return NULL; + BrotliDecompressStream* z = (BrotliDecompressStream*)aether_caps_calloc(1, sizeof(BrotliDecompressStream)); + if (!z) return NULL; + z->dec = BrotliDecoderCreateInstance(NULL, NULL, NULL); + if (!z->dec) { + aether_caps_free(z, sizeof(BrotliDecompressStream)); + return NULL; + } + z->limit = (max_bytes > 0) ? (size_t)max_bytes : ((size_t)1 << 30); + return z; +} + +int brotli_try_decompress_stream_write(void* handle, const char* data, int length) { + BrotliDecompressStream* z = (BrotliDecompressStream*)handle; + if (!z || z->finished || z->broken || length < 0) return 0; + + size_t in_len; + const unsigned char* in = brotli_unwrap_bytes(data, length, &in_len); + if (in_len > 0 && !in) return 0; + + z->out_len = 0; + size_t used = 0; + + if (z->out_cap < 256) { + size_t cap = 256; + unsigned char* nb = (unsigned char*)aether_caps_malloc(cap); + if (!nb) { z->broken = 1; return 0; } + if (z->out) aether_caps_free(z->out, z->out_cap); + z->out = nb; + z->out_cap = cap; + } + + size_t avail_in = in_len; + const uint8_t* next_in = (const uint8_t*)in; + + for (;;) { + size_t avail_out = z->out_cap - used; + uint8_t* next_out = (uint8_t*)(z->out + used); + + BrotliDecoderResult res = BrotliDecoderDecompressStream( + z->dec, &avail_in, &next_in, &avail_out, &next_out, NULL); + + used = z->out_cap - avail_out; + + if (z->total_out + used > z->limit) { + z->broken = 1; + z->limit_exceeded = 1; + return -2; + } + + if (res == BROTLI_DECODER_RESULT_SUCCESS) { + z->finished = 1; + break; + } else if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + if (avail_in == 0) break; + } else if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + size_t remaining_limit = z->limit - z->total_out; + if (used >= remaining_limit) { + z->broken = 1; + z->limit_exceeded = 1; + return -2; + } + size_t ncap = z->out_cap * 2; + if (ncap > remaining_limit) ncap = remaining_limit; + if (ncap <= z->out_cap) { + z->broken = 1; + z->limit_exceeded = 1; + return -2; + } + unsigned char* nb = (unsigned char*)aether_caps_malloc(ncap); + if (!nb) { z->broken = 1; return 0; } + memcpy(nb, z->out, used); + aether_caps_free(z->out, z->out_cap); + z->out = nb; + z->out_cap = ncap; + } else { + z->broken = 1; + return 0; + } + } + + z->total_out += used; + z->out_len = (int)used; + return 1; +} + +int brotli_try_decompress_stream_finish(void* handle) { + BrotliDecompressStream* z = (BrotliDecompressStream*)handle; + if (!z || z->broken) return 0; + if (z->finished) { z->out_len = 0; return 1; } + return brotli_try_decompress_stream_write(handle, NULL, 0); +} + +const char* brotli_get_decompress_stream_bytes(void* handle) { + BrotliDecompressStream* z = (BrotliDecompressStream*)handle; + if (!z || !z->out) return ""; + return (const char*)z->out; +} + +int brotli_get_decompress_stream_length(void* handle) { + BrotliDecompressStream* z = (BrotliDecompressStream*)handle; + return z ? z->out_len : 0; +} + +void brotli_release_decompress_stream(void* handle) { + BrotliDecompressStream* z = (BrotliDecompressStream*)handle; + if (!z) return; + if (z->dec) BrotliDecoderDestroyInstance(z->dec); + if (z->out) aether_caps_free(z->out, z->out_cap); + aether_caps_free(z, sizeof(BrotliDecompressStream)); +} + #else /* !AETHER_HAS_BROTLI */ /* stream_new returning NULL is the one signal a caller cannot ignore: every @@ -224,6 +462,30 @@ int brotli_try_compress(const char* data, int length, int quality) { (void)data; (void)length; (void)quality; return 0; } +int brotli_try_decompress(const char* data, int length, int max_bytes) { + (void)data; (void)length; (void)max_bytes; return 0; +} + +const char* brotli_get_decompress_bytes(void) { return ""; } +int brotli_get_decompress_length(void) { return 0; } +void brotli_release_decompress(void) {} + +void* brotli_try_decompress_stream_new(int max_bytes) { + (void)max_bytes; return NULL; +} + +int brotli_try_decompress_stream_write(void* handle, const char* data, int length) { + (void)handle; (void)data; (void)length; return 0; +} + +int brotli_try_decompress_stream_finish(void* handle) { + (void)handle; return 0; +} + +const char* brotli_get_decompress_stream_bytes(void* handle) { (void)handle; return ""; } +int brotli_get_decompress_stream_length(void* handle) { (void)handle; return 0; } +void brotli_release_decompress_stream(void* handle) { (void)handle; } + #endif /* AETHER_HAS_BROTLI */ const char* brotli_get_compress_bytes(void) { diff --git a/std/brotli/aether_brotli.h b/std/brotli/aether_brotli.h index bb5810e4d..5e5991d1f 100644 --- a/std/brotli/aether_brotli.h +++ b/std/brotli/aether_brotli.h @@ -58,6 +58,26 @@ const char* brotli_get_compress_bytes(void); int brotli_get_compress_length(void); void brotli_release_compress(void); +/* One-shot decompression. `max_bytes` sets a strict upper bound on decompressed + * output size (0 = default safety ceiling). Returns 1 on success, 0 on decode + * error / corrupted bitstream, and -2 when `max_bytes` output limit is + * exceeded (mitigating decompression bombs / DoS attack vectors like + * broccoli2shell / CVE-2026-111497). */ +int brotli_try_decompress(const char* data, int length, int max_bytes); +const char* brotli_get_decompress_bytes(void); +int brotli_get_decompress_length(void); +void brotli_release_decompress(void); + +/* Streaming decompression. `max_bytes` sets a strict upper bound on cumulative + * decompressed output across all stream writes (0 = default safety ceiling). */ +void* brotli_try_decompress_stream_new(int max_bytes); +int brotli_try_decompress_stream_write(void* handle, const char* data, int length); +int brotli_try_decompress_stream_finish(void* handle); + +const char* brotli_get_decompress_stream_bytes(void* handle); +int brotli_get_decompress_stream_length(void* handle); +void brotli_release_decompress_stream(void* handle); + /* Why a system library rather than a vendored implementation: * libbrotlienc ships in every mainstream distro (apt libbrotli-dev, brew * brotli) and exposes exactly the streaming encoder this needs diff --git a/std/brotli/module.ae b/std/brotli/module.ae index 34991400a..bb1d8c54b 100644 --- a/std/brotli/module.ae +++ b/std/brotli/module.ae @@ -14,8 +14,7 @@ // tail, n, err = brotli.stream_finish(s) // at connection close // brotli.stream_free(s) // -// Compression only. An HTTP server compresses responses; nothing in-tree -// needs to DECODE br, and a decoder is a separate piece of work. +// One-shot and streaming compression and decompression. exports( brotli_try_stream_new, brotli_try_stream_flush, @@ -23,8 +22,14 @@ exports( brotli_release_stream, brotli_get_compress_bytes, brotli_get_compress_length, brotli_release_compress, - available, compress, + brotli_get_decompress_bytes, brotli_get_decompress_length, + brotli_release_decompress, + brotli_try_decompress_stream_new, brotli_get_decompress_stream_bytes, + brotli_get_decompress_stream_length, brotli_release_decompress_stream, + available, compress, decompress, decompress_max, stream_new, stream_write, stream_flush, stream_finish, stream_free, + decompress_stream_new, decompress_stream_write, decompress_stream_finish, + decompress_stream_free, // Quality levels for compress / stream_new MIN_QUALITY, MAX_QUALITY, DEFAULT_QUALITY, FAST_QUALITY ) @@ -49,6 +54,18 @@ extern brotli_get_compress_bytes() -> ptr extern brotli_get_compress_length() -> int extern brotli_release_compress() +extern brotli_try_decompress(data: ptr, length: int, max_bytes: int) -> int +extern brotli_get_decompress_bytes() -> ptr +extern brotli_get_decompress_length() -> int +extern brotli_release_decompress() + +extern brotli_try_decompress_stream_new(max_bytes: int) -> ptr +extern brotli_try_decompress_stream_write(handle: ptr, data: ptr, length: int) -> int +extern brotli_try_decompress_stream_finish(handle: ptr) -> int +extern brotli_get_decompress_stream_bytes(handle: ptr) -> ptr +extern brotli_get_decompress_stream_length(handle: ptr) -> int +extern brotli_release_decompress_stream(handle: ptr) + // Length-preserving string constructor from std.string — takes an explicit // byte count so binary payloads with embedded NULs survive the copy out of // the borrowed C buffer into an owned AetherString. @@ -146,3 +163,81 @@ stream_take(handle: ptr) -> { stream_free(handle: ptr) { if handle != null { brotli_release_stream(handle) } } + +// One-shot: decompress brotli-encoded data. Returns (bytes, count, "") on +// success, ("", 0, error) otherwise. +decompress(data: byte[]) -> { + return decompress_max(data, 0) +} + +// One-shot decompress with a strict output limit (`max_bytes`). If output +// would exceed `max_bytes`, decoding is aborted and returns an error +// mitigating decompression bomb attack vectors (e.g. broccoli2shell / CVE-2026-111497). +decompress_max(data: byte[], max_bytes: int) -> { + if data.len < 0 { unbounded("decompress") } + if max_bytes < 0 { return "", 0, "brotli decompress: max_bytes must be >= 0" } + if brotli_backend_available() == 0 { + return "", 0, "brotli unavailable" + } + rc = brotli_try_decompress(data, data.len, max_bytes) + if rc == -2 { + return "", 0, "brotli decompress: output limit exceeded" + } + if rc <= 0 { + return "", 0, "brotli decompress failed" + } + raw = brotli_get_decompress_bytes() + n = brotli_get_decompress_length() + owned = string_new_with_length(raw, n) + brotli_release_decompress() + return owned, n, "" +} + +// Streaming decompression. `max_bytes` sets a strict cumulative output upper +// bound across stream lifetime (0 = default safety ceiling). +decompress_stream_new(max_bytes: int) -> { + if brotli_backend_available() == 0 { + return null, "brotli unavailable" + } + if max_bytes < 0 { + return null, "brotli decompress_stream_new: max_bytes must be >= 0" + } + h = brotli_try_decompress_stream_new(max_bytes) + if h == null { + return null, "brotli decompress_stream_new failed" + } + return h, "" +} + +// Feed bytes into a decompression stream. Returns (decompressed_chunk, count, ""). +decompress_stream_write(handle: ptr, data: byte[]) -> { + if data.len < 0 { unbounded("decompress_stream_write") } + if handle == null { return "", 0, "brotli stream: null handle" } + rc = brotli_try_decompress_stream_write(handle, data, data.len) + if rc == -2 { return "", 0, "brotli decompress: output limit exceeded" } + if rc <= 0 { return "", 0, "brotli decompress_stream_write failed" } + return decompress_stream_take(handle) +} + +// Finish a decompression stream and flush any final decompressed bytes. +decompress_stream_finish(handle: ptr) -> { + if handle == null { return "", 0, "brotli stream: null handle" } + rc = brotli_try_decompress_stream_finish(handle) + if rc == -2 { return "", 0, "brotli decompress: output limit exceeded" } + if rc <= 0 { return "", 0, "brotli decompress_stream_finish failed" } + return decompress_stream_take(handle) +} + +// Copy the borrowed C buffer into an owned AetherString for decompression stream. +decompress_stream_take(handle: ptr) -> { + raw = brotli_get_decompress_stream_bytes(handle) + n = brotli_get_decompress_stream_length(handle) + if n == 0 { return "", 0, "" } + owned = string_new_with_length(raw, n) + return owned, n, "" +} + +// Release a decompression stream handle. Safe on null. +decompress_stream_free(handle: ptr) { + if handle != null { brotli_release_decompress_stream(handle) } +} diff --git a/std/brotli/test_brotli.ae b/std/brotli/test_brotli.ae new file mode 100644 index 000000000..f067395b3 --- /dev/null +++ b/std/brotli/test_brotli.ae @@ -0,0 +1,202 @@ +// std.brotli unit & security regression tests — co-located with std/brotli/ module. +// +// Includes test vector patterns and edge cases adapted from Google Brotli +// (https://github.com/google/brotli), subject to the MIT License: +// Copyright 2016 The Brotli Authors. All rights reserved. + +import std.spec +import std.brotli +import std.string +import std.bytes + +const SAMPLE = "the quick brown fox jumps over the lazy dog, and then does it again and again" + +main() { + fw = spec.init() + + unavailable = spec.skip_all_if(fw, brotli.available() == 0, + "brotli backend not built in") + + if unavailable == 0 { + spec.describe(fw, "std.brotli one-shot compress/decompress") { + spec.it("round-trips a string byte for byte") callback { + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + spec.assert_true(plen > 0, "compressed length is positive") + + back, blen, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_ok(derr, "decompress succeeds") + spec.assert_eq(blen, n, "length restored") + spec.assert_str_eq(back, SAMPLE, "content restored") + } + + spec.it("shrinks repetitive input") callback { + original = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + n = string.length(original) + packed, plen, err = brotli.compress(string.bytes(original)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + spec.assert_true(plen < n, "compressed ${plen} < original ${n}") + + back, blen, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_ok(derr, "decompress succeeds") + spec.assert_eq(blen, n, "length restored") + spec.assert_str_eq(back, original, "content restored") + } + + spec.it("round-trips at various quality levels") callback { + n = string.length(SAMPLE) + q = brotli.MIN_QUALITY + while q <= 9 { + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], q) + spec.assert_ok(err, "compress at quality ${q}") + back, blen, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_ok(derr, "decompress at quality ${q}") + spec.assert_str_eq(back, SAMPLE, "content restored at quality ${q}") + q = q + 3 + } + } + } + + spec.describe(fw, "std.brotli decompress_max security limits") { + spec.it("succeeds when output fits within max_bytes limit") callback { + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + + back, blen, derr = brotli.decompress_max(string.bytes(packed)[0..plen], n) + spec.assert_ok(derr, "decompress_max succeeds with exact limit") + spec.assert_eq(blen, n, "length matches") + spec.assert_str_eq(back, SAMPLE, "content restored") + } + + spec.it("refuses decompression bomb exceeding max_bytes limit") callback { + // Highly compressible payload (decompression bomb pattern) + b = bytes.new(2000) + i = 0 + while i < 2000 { + bytes.set(b, i, 120) // 'x' + i = i + 1 + } + bomb = bytes.finish(b, 2000) + + packed, plen, err = brotli.compress(string.bytes(bomb)[0..2000], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress bomb payload") + + _out, _olen, derr = brotli.decompress_max(string.bytes(packed)[0..plen], 500) + spec.assert_err_contains(derr, "output limit exceeded", "strictly halts decompression bomb when limit exceeded") + } + + spec.it("rejects negative max_bytes parameter") callback { + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + + _out, _olen, derr = brotli.decompress_max(string.bytes(packed)[0..plen], 0 - 5) + spec.assert_err_contains(derr, "max_bytes must be >= 0", "rejects negative limit") + } + } + + spec.describe(fw, "std.brotli edge cases & invalid input") { + spec.it("rejects garbage input on decompress") callback { + _out, _olen, derr = brotli.decompress(string.bytes("this is not brotli data")) + spec.assert_err(derr, "garbage input is rejected") + } + + spec.it("rejects empty buffer on decompress") callback { + _out, _olen, derr = brotli.decompress(string.bytes("")) + spec.assert_err(derr, "empty input buffer is rejected") + } + + spec.it("rejects trailing garbage appended to valid stream") callback { + // Adapted from Google Brotli decompress_test.py (test_garbage_appended) + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + + buf = bytes.new(plen + 5) + bytes.copy_from_string(buf, 0, packed, plen) + bytes.copy_from_string(buf, plen, "extra", 5) + corrupt = bytes.finish(buf, plen + 5) + + _out, _olen, derr = brotli.decompress(string.bytes(corrupt)[0..plen + 5]) + spec.assert_err(derr, "trailing garbage appended to stream is rejected") + } + + spec.it("rejects truncated bitstream") callback { + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + + // Truncate stream midway + trunc_len = plen / 2 + _out, _olen, derr = brotli.decompress(string.bytes(packed)[0..trunc_len]) + spec.assert_err(derr, "truncated bitstream is rejected") + } + } + + spec.describe(fw, "std.brotli streaming decompression") { + spec.it("decompresses streamed chunks accurately") callback { + s, err = brotli.stream_new(brotli.FAST_QUALITY, 0) + spec.assert_ok(err, "stream_new encoder") + + n = string.length(SAMPLE) + c1, n1, e1 = brotli.stream_write(s, string.bytes(SAMPLE)[0..n]) + spec.assert_ok(e1, "stream_write") + c2, n2, e2 = brotli.stream_finish(s) + spec.assert_ok(e2, "stream_finish") + + // Concatenate encoded output chunks + tot = n1 + n2 + buf = bytes.new(tot) + if n1 > 0 { bytes.copy_from_string(buf, 0, c1, n1) } + if n2 > 0 { bytes.copy_from_string(buf, n1, c2, n2) } + compressed = bytes.finish(buf, tot) + brotli.stream_free(s) + + // Decompress via streaming decoder + ds, derr = brotli.decompress_stream_new(0) + spec.assert_ok(derr, "decompress_stream_new") + + d1, dn1, de1 = brotli.decompress_stream_write(ds, string.bytes(compressed)[0..tot]) + spec.assert_ok(de1, "decompress_stream_write") + + d2, dn2, de2 = brotli.decompress_stream_finish(ds) + spec.assert_ok(de2, "decompress_stream_finish") + + full_decomp = "${d1}${d2}" + spec.assert_eq(dn1 + dn2, n, "decompressed total length matches original") + spec.assert_str_eq(full_decomp, SAMPLE, "decompressed stream content restored") + + brotli.decompress_stream_free(ds) + } + + spec.it("enforces max_bytes limit during streaming decompression") callback { + b = bytes.new(2000) + i = 0 + while i < 2000 { + bytes.set(b, i, 121) // 'y' + i = i + 1 + } + bomb = bytes.finish(b, 2000) + + packed, plen, err = brotli.compress(string.bytes(bomb)[0..2000], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress bomb") + + ds, derr = brotli.decompress_stream_new(300) + spec.assert_ok(derr, "decompress_stream_new with max_bytes=300") + + _d1, _dn1, de1 = brotli.decompress_stream_write(ds, string.bytes(packed)[0..plen]) + spec.assert_err_contains(de1, "output limit exceeded", "streaming decompress halts bomb exceeding limit") + + // Subsequent write on broken handle fails safely + _d2, _dn2, de2 = brotli.decompress_stream_write(ds, string.bytes("more")[0..4]) + spec.assert_err(de2, "broken stream handle rejects further writes") + + brotli.decompress_stream_free(ds) + } + } + } + + return spec.run_summary(fw) +} diff --git a/tests/integration/optional_library_linking/test_optional_library_linking.sh b/tests/integration/optional_library_linking/test_optional_library_linking.sh index 3f8b17e74..becc988ce 100755 --- a/tests/integration/optional_library_linking/test_optional_library_linking.sh +++ b/tests/integration/optional_library_linking/test_optional_library_linking.sh @@ -21,7 +21,7 @@ cat > cc-probe <<'EOF' for arg do printf '%s\n' "$arg" >> "$LINK_ARGS" case "$arg" in - -lssl|-lcrypto|-lz|-lnghttp2|-lpcre2-8|-lbrotlienc|-lbrotlicommon|-lzstd|-lfyaml) + -lssl|-lcrypto|-lz|-lnghttp2|-lpcre2-8|-lbrotlienc|-lbrotlidec|-lbrotlicommon|-lzstd|-lfyaml) case " $ALLOWED_LIBS " in *" $arg "*) ;; *) echo "unexpected optional dependency: $arg" >&2; exit 1 ;; @@ -43,7 +43,7 @@ check_args() { # Windows uses the native compiler; inspect its verbose command instead. if [ "$WINDOWS" = 1 ]; then tr ' ' '\n' < "$1" > "$LINK_ARGS"; fi [ -s "$LINK_ARGS" ] || fail 'C compiler command missing' - for lib in -lssl -lcrypto -lz -lnghttp2 -lpcre2-8 -lbrotlienc -lbrotlicommon -lzstd -lfyaml; do + for lib in -lssl -lcrypto -lz -lnghttp2 -lpcre2-8 -lbrotlienc -lbrotlidec -lbrotlicommon -lzstd -lfyaml; do if grep -Fxq -- "$lib" "$LINK_ARGS"; then case " $ALLOWED_LIBS " in *" $lib "*) ;; @@ -81,7 +81,7 @@ esac for entry in \ 'regex|-lpcre2-8' \ 'zlib|-lz' \ - 'brotli|-lbrotlienc -lbrotlicommon' \ + 'brotli|-lbrotlienc -lbrotlidec -lbrotlicommon' \ 'zstd|-lzstd' \ 'yaml|-lfyaml' \ 'cryptography|-lssl -lcrypto' \ @@ -150,7 +150,7 @@ main() { else { println("excluded") } } EOF -ALLOWED_LIBS='-lbrotlienc -lbrotlicommon' +ALLOWED_LIBS='-lbrotlienc -lbrotlidec -lbrotlicommon' export ALLOWED_LIBS : > "$LINK_ARGS" "$AE" build conditional.ae -D WITH_CODEC -o conditional-on --verbose > conditional-on.log 2>&1 \ diff --git a/tools/ae.c b/tools/ae.c index 9ea44f23b..30f9b8591 100644 --- a/tools/ae.c +++ b/tools/ae.c @@ -3019,7 +3019,7 @@ static unsigned optional_link_requirement(const char* tok, size_t len) { { "-lssl", LINK_OPENSSL }, { "-lcrypto", LINK_OPENSSL }, { "-lz", LINK_ZLIB }, { "-lnghttp2", LINK_NGHTTP2 }, { "-lpcre2-8", LINK_PCRE2 }, - { "-lbrotlienc", LINK_BROTLI }, { "-lbrotlicommon", LINK_BROTLI }, + { "-lbrotlienc", LINK_BROTLI }, { "-lbrotlidec", LINK_BROTLI }, { "-lbrotlicommon", LINK_BROTLI }, { "-lzstd", LINK_ZSTD }, { "-lfyaml", LINK_YAML }, }; for (size_t i = 0; i < sizeof(libs) / sizeof(libs[0]); i++) { From 336ae3d55fcfa104290337f1bd5d5d74d33f5086 Mon Sep 17 00:00:00 2001 From: Paul Hammant Date: Sat, 24 Oct 2026 17:46:43 +0100 Subject: [PATCH 2/2] std.brotli decoder: refuse an unpatched libbrotlidec, report truncation, pin the CVE classes The decoder from the previous commit, reviewed against the brotli CVEs other implementations have shipped: - CVE-2026-111497 (broccoli2shell): libbrotlidec 1.0.3-1.2.0 writes out of bounds on a malformed block switch. Decoding now refuses to run on an unfixed library. brotli_decoder_patched() asks the library to name error code -17, which the fix added in the same commit, so distro backports that keep the old version number (Debian 1.0.9-2+deb12u1) pass. Checked against upstream v1.2.0 (refused) and v1.2.1 (decodes). decoder_available() exports it. - decompress_stream_finish accepted a truncated stream as success; it now returns "truncated stream". The one-shot path names truncation too. - The streaming decoder ignored bytes after the end of the stream; they are now an error, as on the one-shot path. - Errors name their cause (output limit, truncated, corrupt, refused). - BROTLI_LDFLAGS kept the FIRST of each repeated library, putting -lbrotlicommon before -lbrotlidec, which a static link cannot resolve; it keeps the last now. - The README called broccoli2shell a decompression bomb; it is a memory-safety bug the output cap cannot stop. The README now has a table of which CVE class each defence answers. Tests added: a thousandfold bomb (8 MiB in a few hundred bytes) on both paths, a 300 KB binary payload with NULs at and one under its exact limit, a stream fed one byte at a time, truncation and trailing bytes on the stream, an empty body, and a large-window stream (fixture from the brotli 1.2.1 CLI). The truncation and trailing-byte tests failed against the previous commit; the large-window test fails with large windows enabled. Co-Authored-By: Claude Opus 5.5 --- Makefile | 16 +- docs/api/brotli.html | 22 +- docs/api/index.html | 5 +- docs/stdlib-reference.md | 2 +- .../20261024T153948Z-added-brotli-decoder.md | 13 +- std/brotli/README.md | 63 ++++-- std/brotli/aether_brotli.c | 33 ++- std/brotli/aether_brotli.h | 23 +- std/brotli/module.ae | 82 +++++--- std/brotli/test_brotli.ae | 196 +++++++++++++++++- tools/ae.c | 15 +- 11 files changed, 389 insertions(+), 81 deletions(-) diff --git a/Makefile b/Makefile index 83fdbf0de..1806728e8 100644 --- a/Makefile +++ b/Makefile @@ -585,11 +585,17 @@ ifeq ($(BROTLI),0) else BROTLI_CFLAGS := $(shell pkg-config --cflags libbrotlienc libbrotlidec 2>/dev/null) # --static so libbrotlicommon comes with it. It is a Requires.private of - # libbrotlienc/libbrotlidec, so a plain --libs omits it -- fine when the loader resolves - # a shared object, fatal on a static link: MinGW's ld then reports a wall of - # undefined BrotliGetDictionary / BrotliDefaultAllocFunc / _kBrotli* - # references from inside libbrotlienc.a/libbrotlidec.a itself. - BROTLI_LDFLAGS := $(shell pkg-config --static --libs libbrotlienc libbrotlidec 2>/dev/null | tr ' ' '\n' | awk '!seen[$$0]++' | tr '\n' ' ') + # libbrotlienc and libbrotlidec, so a plain --libs omits it -- fine when the + # loader resolves a shared object, fatal on a static link: MinGW's ld then + # reports a wall of undefined BrotliGetDictionary / BrotliDefaultAllocFunc / + # _kBrotli* references from inside libbrotlienc.a itself. Found on a real + # MSYS2 box; a Linux shared link hides it entirely. Asking for both + # libraries names -lbrotlicommon twice, and the link line must name each + # library once (optional_library_linking checks), so keep the LAST of each: + # a static link resolves left to right, and -lbrotlicommon has to follow + # libbrotlidec as well as libbrotlienc. + BROTLI_LDFLAGS := $(shell pkg-config --static --libs libbrotlienc libbrotlidec 2>/dev/null \ + | awk '{ o = ""; for (i = NF; i > 0; i--) if (!seen[$$i]++) o = $$i " " o; print o }') endif ifneq ($(BROTLI_LDFLAGS),) BROTLI_CFLAGS += -DAETHER_HAS_BROTLI diff --git a/docs/api/brotli.html b/docs/api/brotli.html index 5c3ff6823..570134da1 100644 --- a/docs/api/brotli.html +++ b/docs/api/brotli.html @@ -32,6 +32,7 @@

    Exports

  • get_decompress_stream_length
  • release_decompress_stream
  • available
    • +
  • decoder_available
  • compress
  • decompress
  • decompress_max
    • @@ -145,7 +146,7 @@

    Modules

    brotli

    -

    std.brotli — Brotli compression (RFC 7932), streaming and one-shot.

    +

    std.brotli — Brotli (RFC 7932) encoding and decoding, streaming and one-shot.

    import std.brotli

    @@ -226,7 +227,12 @@

    release_decompress_stream extern

    available fn

    available() -> int
    -

    1 when the toolchain was built with libbrotlienc detected. Use it to skip a test or pick a fallback encoding rather than threading an error everywhere.

    +

    1 when the toolchain was built with libbrotli detected. Use it to skip a test or pick a fallback encoding rather than threading an error everywhere.

    +

    +
    +

    decoder_available fn

    + 
    decoder_available() -> int
    +

    1 when decoding works here: brotli is built in AND the system libbrotlidec carries the CVE-2026-111497 fix (an out-of-bounds write on malformed input, libbrotli 1.0.3 to 1.2.0). Without the fix every decode refuses with an error rather than run a decoder that untrusted input can exploit; encoding is unaffected. The check asks the library itself, so a distro backport that keeps the old version number passes.

    compress fn

    @@ -238,13 +244,13 @@

    compress fn

    decompress fn

    decompress(data)

    data: byte[]

    -

    One-shot: decompress brotli-encoded data. Returns (bytes, count, "") on success, ("", 0, error) otherwise.

    +

    One-shot: decompress with the default 1 GiB ceiling. Returns (bytes, count, "") on success, ("", 0, error) otherwise. For input you did not produce, use decompress_max with a cap that fits the job.

    decompress_max fn

    decompress_max(data, max_bytes)

    data: byte[], max_bytes: int

    -

    One-shot decompress with a strict output limit (`max_bytes`). If output would exceed `max_bytes`, decoding is aborted and returns an error mitigating decompression bomb attack vectors (e.g. broccoli2shell / CVE-2026-111497).

    +

    One-shot with a cap: decoding stops, and returns an "output limit exceeded" error, as soon as the output would pass `max_bytes` (0 = the 1 GiB ceiling). Truncated input, corrupt input and bytes after the end of the stream are errors too.

    stream_new fn

    @@ -280,25 +286,25 @@

    stream_free fn

    decompress_stream_new fn

    decompress_stream_new(max_bytes) -> (ptr, string)

    max_bytes: int

    -

    Streaming decompression. `max_bytes` sets a strict cumulative output upper bound across stream lifetime (0 = default safety ceiling).

    +

    Open a decoding stream whose total output is capped at `max_bytes` (0 = the 1 GiB ceiling). Returns (handle, "") or (null, error).

    decompress_stream_write fn

    decompress_stream_write(handle, data)

    handle: ptr, data: byte[]

    -

    Feed bytes into a decompression stream. Returns (decompressed_chunk, count, "").

    +

    Feed compressed bytes in any size of chunk. Returns (bytes, count, "") with whatever they decoded to. After an error the handle refuses further writes.

    decompress_stream_finish fn

    decompress_stream_finish(handle) -> (string, int, string)

    handle: ptr

    -

    Finish a decompression stream and flush any final decompressed bytes.

    +

    Confirm the stream ended. Returns ("", 0, "") when it did, and a "truncated stream" error when the input stopped short of the end, so a cut connection is not mistaken for a shorter body. Still needs decompress_stream_free.

    decompress_stream_free fn

    decompress_stream_free(handle)

    handle: ptr

    -

    Release a decompression stream handle. Safe on null.

    +

    Release a decoding stream. Safe on null.

    MIN_QUALITY const

    diff --git a/docs/api/index.html b/docs/api/index.html index b14735ed0..4469d85b4 100644 --- a/docs/api/index.html +++ b/docs/api/index.html @@ -148,8 +148,8 @@

    bits

    brotli

    -

    std.brotli — Brotli compression (RFC 7932), streaming and one-shot.

    - 33 exports +

    std.brotli — Brotli (RFC 7932) encoding and decoding, streaming and one-shot.

    + 34 exports

    bytes

    @@ -687,6 +687,7 @@

    Search Results

  • get_decompress_stream_lengthbrotli
  • release_decompress_streambrotli
  • availablebrotli
    • +
  • decoder_availablebrotli
  • compressbrotli
  • decompressbrotli
  • decompress_maxbrotli
    • diff --git a/docs/stdlib-reference.md b/docs/stdlib-reference.md index 50d8ad92a..511ab1906 100644 --- a/docs/stdlib-reference.md +++ b/docs/stdlib-reference.md @@ -18,7 +18,7 @@ header comment is the authoritative description. | `std.audit` | Query the sandbox audit trail. | 4 | [guide](../std/audit/README.md) · [source](../std/audit/module.ae) | | `std.bignum` | Arbitrary-precision integers. | 29 | [guide](../std/bignum/README.md) · [source](../std/bignum/module.ae) | | `std.bits` | Unsigned bit operations: rotates, shifts, popcount, leading zeros, unsigned divide. | 17 | [guide](../std/bits/README.md) · [source](../std/bits/module.ae) | -| `std.brotli` | Brotli compression, streaming and one-shot, for `Content-Encoding: br`. | 33 | [guide](../std/brotli/README.md) · [source](../std/brotli/module.ae) | +| `std.brotli` | Brotli encoding and capped decoding, streaming and one-shot, for `Content-Encoding: br`. | 34 | [guide](../std/brotli/README.md) · [source](../std/brotli/module.ae) | | `std.bytes` | Mutable byte buffer with random access and overlap-safe copies. | 32 | [guide](../std/bytes/README.md) · [source](../std/bytes/module.ae) | | `std.capsicum` | FreeBSD Capsicum capability-mode bindings. | 28 | [guide](../std/capsicum/README.md) · [source](../std/capsicum/module.ae) | | `std.cas` | Content-addressed store keyed by the sha256 of file contents. | 7 | [guide](../std/cas/README.md) · [source](../std/cas/module.ae) | diff --git a/new_changelogs/20261024T153948Z-added-brotli-decoder.md b/new_changelogs/20261024T153948Z-added-brotli-decoder.md index 280842596..6ace03bab 100644 --- a/new_changelogs/20261024T153948Z-added-brotli-decoder.md +++ b/new_changelogs/20261024T153948Z-added-brotli-decoder.md @@ -1 +1,12 @@ -- `std.brotli`: Added one-shot (`decompress`, `decompress_max`) and streaming (`decompress_stream_new`, `decompress_stream_write`, `decompress_stream_finish`, `decompress_stream_free`) Brotli decoder APIs, with output size limit enforcement to defend against decompression bomb / zip bomb attack vectors. +- **`std.brotli` decodes.** `decompress` / `decompress_max` (one-shot) and + `decompress_stream_new` / `_write` / `_finish` / `_free` (streaming) wrap the + system libbrotlidec. Every decode takes a cap on what it will produce + (`max_bytes`, 0 = a 1 GiB ceiling), so a few hundred bytes of `br` cannot + expand into gigabytes (CVE-2025-6176, urllib3's CVE-2025-66471). Truncated + input, corrupt input and bytes after the end of the stream are errors, and + `decompress_stream_finish` reports a stream that was cut short. The decoder + refuses to run on a libbrotlidec without the CVE-2026-111497 + ("broccoli2shell") fix for its out-of-bounds write: `decoder_available()` + asks the library itself, so distro backports that keep the old version number + pass. Large-window streams stay refused. Building with brotli now needs + libbrotlidec beside libbrotlienc; both come in the same distro package. diff --git a/std/brotli/README.md b/std/brotli/README.md index 743ae0170..cba332bc8 100644 --- a/std/brotli/README.md +++ b/std/brotli/README.md @@ -1,35 +1,62 @@ # std.brotli -Brotli compression (RFC 7932): streaming and one-shot. +Brotli (RFC 7932) encoding and decoding, streaming and one-shot. `br` is what current browsers actually prefer: every modern browser lists it ahead of `gzip` in `Accept-Encoding`, and it typically beats gzip by 15-25% on -text at comparable speed. This module wraps the system **libbrotlienc** -(`apt install libbrotli-dev`, `brew install brotli`). +text at comparable speed. This module wraps the system **libbrotlienc** and +**libbrotlidec** (`apt install libbrotli-dev`, `brew install brotli`). -Decompression and compression supported via `libbrotli`. +## Decoding -## Decompression and Security - -`decompress(data)` decompresses Brotli-encoded byte slices into their original string representation. - -To defend against decompression bombs / zip bombs (such as the "broccoli2shell" attack vector / CVE-2026-111497) where a small compressed payload expands into huge allocations, use `decompress_max`: +Decoding is where brotli's CVEs are, because the input is whatever a peer +sent. Every decode takes a cap on what it will produce: ```aether,fragment -plain, n, err = brotli.decompress_max(string.bytes(compressed), 1048576) // cap at 1MB +body, n, err = brotli.decompress_max(string.bytes(req_body), 1048576) // 1 MiB ``` -If the decompressed output exceeds `max_bytes`, `decompress_max` immediately aborts and returns an error (`"brotli decompress: output limit exceeded"`). +Past the cap the decoder stops and returns `"... output limit exceeded"`. +`decompress(data)` is `decompress_max(data, 0)`, and 0 means a 1 GiB ceiling: +fine for data you produced, too generous for a request body. -Streaming decompression is also available: +The streaming decoder takes chunks of any size, and its cap covers the total: ```aether,fragment -ds, err = brotli.decompress_stream_new(max_bytes) -chunk1, n1, err = brotli.decompress_stream_write(ds, data1) -tail, tn, err = brotli.decompress_stream_finish(ds) +ds, err = brotli.decompress_stream_new(1048576) +chunk, n, err = brotli.decompress_stream_write(ds, string.bytes(part)) +_, _, err = brotli.decompress_stream_finish(ds) // "truncated stream" if cut short brotli.decompress_stream_free(ds) ``` +Truncated input, corrupt input and bytes after the end of the stream are all +errors, on both paths. A failed stream refuses further writes. + +### What it defends against + +| Issue | Where | What this module does | +|---|---|---| +| CVE-2026-111497 ("broccoli2shell" / Node2Shell): out-of-bounds write on a malformed block switch, RCE | libbrotlidec 1.0.3-1.2.0; Node.js, Bun, Python `brotli` | Refuses to decode on an unfixed libbrotlidec (below). | +| CVE-2025-6176: decompression bomb, no output limit | Python `brotli` < 1.2.0, Scrapy | `max_bytes` on every decode; tested with 8 MiB packed into a few hundred bytes. | +| CVE-2025-66471: a small streaming read decodes a whole highly-compressed chunk | urllib3 < 2.6.0 | The stream cap counts every byte a write produces, so one write cannot exceed it. | +| CVE-2020-8927: overflow when one-shot decoding > 2 GiB | libbrotli < 1.0.8 | The one-shot path uses the streaming decoder, lengths are `int`, and the fix check below rules out pre-1.0.8 libraries. | +| Large-window streams: the header can ask for a 1 GiB window outside any output cap | non-RFC 7932 extension | Left off; a test pins that such a stream is refused. | +| Truncation read as a complete body | a common decoder-wrapper bug | `decompress_stream_finish` reports `"truncated stream"`. | + +`std.http`'s client does not decode `Content-Encoding` itself, so the chained +encodings of urllib3's CVE-2025-66418 (`br, br, br, ...`) do not arise: a +caller decodes once, explicitly. + +### An unfixed system libbrotlidec is refused + +CVE-2026-111497 is in the system library, so linking against an old one +would make every decode exploitable. `decoder_available()` is 1 only when the +library carries the fix (upstream 1.0.10, 1.1.1, 1.2.1, or a distro backport +such as Debian's `1.0.9-2+deb12u1`); otherwise every decode returns an error +naming the CVE and encoding keeps working. The check asks the library to name +error code -17, which the fix added. A backport that keeps the old version +number still passes, where a version comparison would refuse it. + ## Streaming Encoding The surface deliberately mirrors `std.zlib`'s streaming deflate, so a server @@ -67,7 +94,7 @@ packed, n, err = brotli.compress(string.bytes(body), brotli.FAST_QUALITY) ## When the backend is absent -Built without libbrotlienc, `available()` returns 0 and `stream_new` returns +Built without libbrotli, `available()` returns 0 and `stream_new` returns `(null, "brotli unavailable")`. Every other stream call needs the handle, so a program fails where it *asks for* a stream rather than silently emitting nothing. A server negotiating content-encoding should check `available()` and @@ -75,8 +102,8 @@ fall back to gzip. ## Exports -`available`, `compress`, `decompress`, `decompress_max`, `stream_new`, -`stream_write`, `stream_flush`, `stream_finish`, `stream_free`, +`available`, `decoder_available`, `compress`, `decompress`, `decompress_max`, +`stream_new`, `stream_write`, `stream_flush`, `stream_finish`, `stream_free`, `decompress_stream_new`, `decompress_stream_write`, `decompress_stream_finish`, `decompress_stream_free` (and the `MIN_QUALITY` / `MAX_QUALITY` / `DEFAULT_QUALITY` / `FAST_QUALITY` constants), plus the `brotli_*` raw forms. diff --git a/std/brotli/aether_brotli.c b/std/brotli/aether_brotli.c index ef64379cf..210f08fbd 100644 --- a/std/brotli/aether_brotli.c +++ b/std/brotli/aether_brotli.c @@ -202,6 +202,20 @@ int brotli_try_compress(const char* data, int length, int quality) { return 1; } +/* CVE-2026-111497 ("broccoli2shell"): libbrotlidec 1.0.3 to 1.2.0 can write + * out of bounds on a malformed block switch, which is remote code execution + * for anything decoding untrusted `br`. The fix (1.0.10, 1.1.1, 1.2.1, and + * distro backports that keep the old version number, Debian's + * 1.0.9-2+deb12u1 among them) gave the reserved error code -17 a name, + * BLOCK_SWITCH, in the same commit. The library's own error-string table is + * compiled from that list, so asking it to name -17 tells a fixed decoder + * from an unfixed one where the version number cannot. An unfixed library + * is refused: decoding fails closed, encoding is unaffected. */ +int brotli_decoder_patched(void) { + const char* name = BrotliDecoderErrorString((BrotliDecoderErrorCode)-17); + return name && strstr(name, "BLOCK_SWITCH") ? 1 : 0; +} + /* One-shot decompression output slot, thread-local */ static _Thread_local unsigned char* tls_decomp_buf = NULL; static _Thread_local size_t tls_decomp_cap = 0; @@ -219,6 +233,7 @@ static void free_decomp_tls(void) { int brotli_try_decompress(const char* data, int length, int max_bytes) { free_decomp_tls(); if (length < 0 || max_bytes < 0) return 0; + if (!brotli_decoder_patched()) return -4; size_t in_len; const unsigned char* in = brotli_unwrap_bytes(data, length, &in_len); @@ -281,7 +296,7 @@ int brotli_try_decompress(const char* data, int length, int max_bytes) { out = nb; cap = ncap; } else if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { - status = 0; + status = -3; /* the input ends before the stream does */ break; } else { status = 0; @@ -327,7 +342,7 @@ typedef struct { } BrotliDecompressStream; void* brotli_try_decompress_stream_new(int max_bytes) { - if (max_bytes < 0) return NULL; + if (max_bytes < 0 || !brotli_decoder_patched()) return NULL; BrotliDecompressStream* z = (BrotliDecompressStream*)aether_caps_calloc(1, sizeof(BrotliDecompressStream)); if (!z) return NULL; z->dec = BrotliDecoderCreateInstance(NULL, NULL, NULL); @@ -378,6 +393,9 @@ int brotli_try_decompress_stream_write(void* handle, const char* data, int lengt } if (res == BROTLI_DECODER_RESULT_SUCCESS) { + /* Bytes after the end of the stream are corruption, not a + * second stream: the one-shot path refuses them too. */ + if (avail_in > 0) { z->broken = 1; return 0; } z->finished = 1; break; } else if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { @@ -413,11 +431,16 @@ int brotli_try_decompress_stream_write(void* handle, const char* data, int lengt return 1; } +/* The stream must have reached its end by now: a decoder still waiting for + * input means the body was cut short, and a truncated body is an error + * rather than the shorter plaintext decoded so far. Returns -3 for that. */ int brotli_try_decompress_stream_finish(void* handle) { BrotliDecompressStream* z = (BrotliDecompressStream*)handle; if (!z || z->broken) return 0; - if (z->finished) { z->out_len = 0; return 1; } - return brotli_try_decompress_stream_write(handle, NULL, 0); + z->out_len = 0; + if (z->finished) return 1; + z->broken = 1; + return -3; } const char* brotli_get_decompress_stream_bytes(void* handle) { @@ -462,6 +485,8 @@ int brotli_try_compress(const char* data, int length, int quality) { (void)data; (void)length; (void)quality; return 0; } +int brotli_decoder_patched(void) { return 0; } + int brotli_try_decompress(const char* data, int length, int max_bytes) { (void)data; (void)length; (void)max_bytes; return 0; } diff --git a/std/brotli/aether_brotli.h b/std/brotli/aether_brotli.h index 5e5991d1f..a4c11a838 100644 --- a/std/brotli/aether_brotli.h +++ b/std/brotli/aether_brotli.h @@ -58,18 +58,27 @@ const char* brotli_get_compress_bytes(void); int brotli_get_compress_length(void); void brotli_release_compress(void); -/* One-shot decompression. `max_bytes` sets a strict upper bound on decompressed - * output size (0 = default safety ceiling). Returns 1 on success, 0 on decode - * error / corrupted bitstream, and -2 when `max_bytes` output limit is - * exceeded (mitigating decompression bombs / DoS attack vectors like - * broccoli2shell / CVE-2026-111497). */ +/* 1 when the linked libbrotlidec carries the CVE-2026-111497 fix, which + * every decode below requires. 0 when it does not, or brotli is absent. */ +int brotli_decoder_patched(void); + +/* One-shot decompression. `max_bytes` caps the decoded size (0 = a 1 GiB + * ceiling). Returns 1 on success, -2 when the output would pass `max_bytes`, + * -3 when the input ends before the stream does, -4 when the decoder is + * refused (see brotli_decoder_patched), and 0 for anything else + * (corrupt input, bytes after the end, out of memory). The cap is what lets + * a server decode a request body it did not write: a few hundred bytes of + * brotli can describe gigabytes. */ int brotli_try_decompress(const char* data, int length, int max_bytes); const char* brotli_get_decompress_bytes(void); int brotli_get_decompress_length(void); void brotli_release_decompress(void); -/* Streaming decompression. `max_bytes` sets a strict upper bound on cumulative - * decompressed output across all stream writes (0 = default safety ceiling). */ +/* Streaming decompression. `max_bytes` caps the total decoded across every + * write (0 = a 1 GiB ceiling). write returns 1, -2 past the cap, or 0; + * finish returns 1 once the stream has ended and -3 when it has not. Any + * failure leaves the handle refusing further writes. Each call's output is + * held on the handle until the next call, as for the encoder. */ void* brotli_try_decompress_stream_new(int max_bytes); int brotli_try_decompress_stream_write(void* handle, const char* data, int length); int brotli_try_decompress_stream_finish(void* handle); diff --git a/std/brotli/module.ae b/std/brotli/module.ae index bb1d8c54b..5bf3a1ef7 100644 --- a/std/brotli/module.ae +++ b/std/brotli/module.ae @@ -1,9 +1,9 @@ -// std.brotli — Brotli compression (RFC 7932), streaming and one-shot. +// std.brotli — Brotli (RFC 7932) encoding and decoding, streaming and one-shot. // // `br` is what current browsers actually prefer over gzip: every modern // browser lists it ahead of gzip in Accept-Encoding, and it typically beats // gzip by 15-25% on text at comparable speed. This module wraps the system -// libbrotlienc (apt: libbrotli-dev, brew: brotli). +// libbrotlienc and libbrotlidec (apt: libbrotli-dev, brew: brotli). // // The streaming surface deliberately mirrors std.zlib's, so a server // negotiating `br` vs `gzip` writes the same shape either way: @@ -14,7 +14,11 @@ // tail, n, err = brotli.stream_finish(s) // at connection close // brotli.stream_free(s) // -// One-shot and streaming compression and decompression. +// Decoding has the same two shapes, and every decode takes a cap on how much +// it will produce, because a few hundred bytes of brotli can describe +// gigabytes: +// +// body, n, err = brotli.decompress_max(string.bytes(req_body), 1048576) exports( brotli_try_stream_new, brotli_try_stream_flush, @@ -26,7 +30,7 @@ exports( brotli_release_decompress, brotli_try_decompress_stream_new, brotli_get_decompress_stream_bytes, brotli_get_decompress_stream_length, brotli_release_decompress_stream, - available, compress, decompress, decompress_max, + available, decoder_available, compress, decompress, decompress_max, stream_new, stream_write, stream_flush, stream_finish, stream_free, decompress_stream_new, decompress_stream_write, decompress_stream_finish, decompress_stream_free, @@ -54,6 +58,7 @@ extern brotli_get_compress_bytes() -> ptr extern brotli_get_compress_length() -> int extern brotli_release_compress() +extern brotli_decoder_patched() -> int extern brotli_try_decompress(data: ptr, length: int, max_bytes: int) -> int extern brotli_get_decompress_bytes() -> ptr extern brotli_get_decompress_length() -> int @@ -83,7 +88,7 @@ fn unbounded(fname: string) { panic("brotli.${fname}: the slice has no bound (.len -1); give it one, as `(p as byte[])[0..n]`") } -// 1 when the toolchain was built with libbrotlienc detected. Use it to skip a +// 1 when the toolchain was built with libbrotli detected. Use it to skip a // test or pick a fallback encoding rather than threading an error everywhere. available() -> int { return brotli_backend_available() } @@ -164,28 +169,48 @@ stream_free(handle: ptr) { if handle != null { brotli_release_stream(handle) } } -// One-shot: decompress brotli-encoded data. Returns (bytes, count, "") on -// success, ("", 0, error) otherwise. +const UNPATCHED = "brotli decoder refused: the system libbrotlidec lacks the CVE-2026-111497 fix (1.0.10, 1.1.1, 1.2.1 or a distro backport)" + +// 1 when decoding works here: brotli is built in AND the system libbrotlidec +// carries the CVE-2026-111497 fix (an out-of-bounds write on malformed input, +// libbrotli 1.0.3 to 1.2.0). Without the fix every decode refuses with an +// error rather than run a decoder that untrusted input can exploit; encoding +// is unaffected. The check asks the library itself, so a distro backport +// that keeps the old version number passes. +decoder_available() -> int { + if brotli_backend_available() == 0 { return 0 } + return brotli_decoder_patched() +} + +// The message for a decoder status: -2 is the cap, -3 a stream cut short, +// -4 an unpatched decoder. +fn decode_error(fname: string, rc: int) -> string { + if rc == -2 { return "brotli ${fname}: output limit exceeded" } + if rc == -3 { return "brotli ${fname}: truncated stream" } + if rc == -4 { return UNPATCHED } + return "brotli ${fname}: corrupt input" +} + +// One-shot: decompress with the default 1 GiB ceiling. Returns +// (bytes, count, "") on success, ("", 0, error) otherwise. For input you did +// not produce, use decompress_max with a cap that fits the job. decompress(data: byte[]) -> { + if data.len < 0 { unbounded("decompress") } return decompress_max(data, 0) } -// One-shot decompress with a strict output limit (`max_bytes`). If output -// would exceed `max_bytes`, decoding is aborted and returns an error -// mitigating decompression bomb attack vectors (e.g. broccoli2shell / CVE-2026-111497). +// One-shot with a cap: decoding stops, and returns an "output limit +// exceeded" error, as soon as the output would pass `max_bytes` (0 = the +// 1 GiB ceiling). Truncated input, corrupt input and bytes after the end of +// the stream are errors too. decompress_max(data: byte[], max_bytes: int) -> { - if data.len < 0 { unbounded("decompress") } + if data.len < 0 { unbounded("decompress_max") } if max_bytes < 0 { return "", 0, "brotli decompress: max_bytes must be >= 0" } if brotli_backend_available() == 0 { return "", 0, "brotli unavailable" } rc = brotli_try_decompress(data, data.len, max_bytes) - if rc == -2 { - return "", 0, "brotli decompress: output limit exceeded" - } - if rc <= 0 { - return "", 0, "brotli decompress failed" - } + if rc != 1 { return "", 0, decode_error("decompress", rc) } raw = brotli_get_decompress_bytes() n = brotli_get_decompress_length() owned = string_new_with_length(raw, n) @@ -193,8 +218,8 @@ decompress_max(data: byte[], max_bytes: int) -> { return owned, n, "" } -// Streaming decompression. `max_bytes` sets a strict cumulative output upper -// bound across stream lifetime (0 = default safety ceiling). +// Open a decoding stream whose total output is capped at `max_bytes` +// (0 = the 1 GiB ceiling). Returns (handle, "") or (null, error). decompress_stream_new(max_bytes: int) -> { if brotli_backend_available() == 0 { return null, "brotli unavailable" @@ -202,6 +227,7 @@ decompress_stream_new(max_bytes: int) -> { if max_bytes < 0 { return null, "brotli decompress_stream_new: max_bytes must be >= 0" } + if brotli_decoder_patched() == 0 { return null, UNPATCHED } h = brotli_try_decompress_stream_new(max_bytes) if h == null { return null, "brotli decompress_stream_new failed" @@ -209,26 +235,28 @@ decompress_stream_new(max_bytes: int) -> { return h, "" } -// Feed bytes into a decompression stream. Returns (decompressed_chunk, count, ""). +// Feed compressed bytes in any size of chunk. Returns (bytes, count, "") with +// whatever they decoded to. After an error the handle refuses further writes. decompress_stream_write(handle: ptr, data: byte[]) -> { if data.len < 0 { unbounded("decompress_stream_write") } if handle == null { return "", 0, "brotli stream: null handle" } rc = brotli_try_decompress_stream_write(handle, data, data.len) - if rc == -2 { return "", 0, "brotli decompress: output limit exceeded" } - if rc <= 0 { return "", 0, "brotli decompress_stream_write failed" } + if rc != 1 { return "", 0, decode_error("decompress_stream_write", rc) } return decompress_stream_take(handle) } -// Finish a decompression stream and flush any final decompressed bytes. +// Confirm the stream ended. Returns ("", 0, "") when it did, and a +// "truncated stream" error when the input stopped short of the end, so a cut +// connection is not mistaken for a shorter body. Still needs +// decompress_stream_free. decompress_stream_finish(handle: ptr) -> { if handle == null { return "", 0, "brotli stream: null handle" } rc = brotli_try_decompress_stream_finish(handle) - if rc == -2 { return "", 0, "brotli decompress: output limit exceeded" } - if rc <= 0 { return "", 0, "brotli decompress_stream_finish failed" } + if rc != 1 { return "", 0, decode_error("decompress_stream_finish", rc) } return decompress_stream_take(handle) } -// Copy the borrowed C buffer into an owned AetherString for decompression stream. +// Copy the borrowed C buffer into an owned AetherString, as stream_take does. decompress_stream_take(handle: ptr) -> { raw = brotli_get_decompress_stream_bytes(handle) n = brotli_get_decompress_stream_length(handle) @@ -237,7 +265,7 @@ decompress_stream_take(handle: ptr) -> { return owned, n, "" } -// Release a decompression stream handle. Safe on null. +// Release a decoding stream. Safe on null. decompress_stream_free(handle: ptr) { if handle != null { brotli_release_decompress_stream(handle) } } diff --git a/std/brotli/test_brotli.ae b/std/brotli/test_brotli.ae index f067395b3..ee8a4d1f7 100644 --- a/std/brotli/test_brotli.ae +++ b/std/brotli/test_brotli.ae @@ -9,12 +9,74 @@ import std.brotli import std.string import std.bytes +// "hello large window" encoded by the brotli 1.2.1 CLI with +// --large_window=30: the non-RFC 7932 large-window format, whose header may ask +// the decoder for a window of up to 1 GiB before a byte of output exists. +const LARGE_WINDOW_BR: int[] = [0x11, 0x5e, 0x44, 0x00, 0xe0, 0x97, 0x2e, 0x43, 0x63, 0x7b, + 0x53, 0x84, 0xac, 0x52, 0x66, 0x38, 0x47, 0xa9, 0xeb, 0x00] + const SAMPLE = "the quick brown fox jumps over the lazy dog, and then does it again and again" +// `n` bytes cycling through 0..250, NULs included, so a decoder that stops +// at a NUL or mixes up chunk order is caught. +binary_payload(n: int) -> string { + b = bytes.new(n) + i = 0 + while i < n { + bytes.set(b, i, i % 251) + i = i + 1 + } + return bytes.finish(b, n) +} + +// `mb` megabytes of one byte, compressed through the streaming encoder so the +// plaintext never sits in memory at once: a small input that expands a +// thousandfold, the shape a decompression bomb takes. +bomb(mb: int) -> (string, int) { + chunk = bytes.new(65536) + i = 0 + while i < 65536 { + bytes.set(chunk, i, 0) + i = i + 1 + } + block = bytes.finish(chunk, 65536) + s, _e = brotli.stream_new(brotli.FAST_QUALITY, 0) + out = bytes.new(64) + total = 0 + k = 0 + while k < mb * 16 { + c, n, _e2 = brotli.stream_write(s, string.bytes(block)[0..65536]) + if n > 0 { + bytes.copy_from_string(out, total, c, n) + total = total + n + } + k = k + 1 + } + c, n, _e3 = brotli.stream_finish(s) + brotli.stream_free(s) + if n > 0 { bytes.copy_from_string(out, total, c, n) } + total = total + n + return bytes.finish(out, total), total +} + main() { fw = spec.init() - unavailable = spec.skip_all_if(fw, brotli.available() == 0, + if brotli.available() == 1 && brotli.decoder_available() == 0 { + // An unpatched system decoder: every decode must refuse (CVE-2026-111497). + spec.describe(fw, "std.brotli on an unpatched libbrotlidec") { + spec.it("refuses to decode") callback { + packed, plen, _e = brotli.compress(string.bytes(SAMPLE)[0..string.length(SAMPLE)], 5) + _o, _n, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_err_contains(derr, "CVE-2026-111497", "one-shot refuses") + _h, serr = brotli.decompress_stream_new(0) + spec.assert_err_contains(serr, "CVE-2026-111497", "streaming refuses") + } + } + return spec.run_summary(fw) + } + + unavailable = spec.skip_all_if(fw, brotli.decoder_available() == 0, "brotli backend not built in") if unavailable == 0 { @@ -97,7 +159,66 @@ main() { } } + spec.describe(fw, "std.brotli large and hostile input") { + spec.it("round-trips a binary payload larger than its first buffer") callback { + n = 300000 + plain = binary_payload(n) + packed, plen, err = brotli.compress(string.bytes(plain)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + back, blen, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_ok(derr, "decompress succeeds") + spec.assert_eq(blen, n, "length restored, NULs and all") + spec.assert_true(string.string_equals(back, plain) == 1, "content restored byte for byte") + _exact, elen, eerr = brotli.decompress_max(string.bytes(packed)[0..plen], n) + spec.assert_ok(eerr, "a limit equal to the output size is enough") + spec.assert_eq(elen, n, "length at the exact limit") + _o, _l, oerr = brotli.decompress_max(string.bytes(packed)[0..plen], n - 1) + spec.assert_err_contains(oerr, "output limit exceeded", "one byte under the size is refused") + } + + spec.it("stops a thousandfold bomb at the limit") callback { + packed, plen = bomb(8) + spec.assert_true(plen < 65536, "8 MiB packs into ${plen} bytes") + _o, _l, derr = brotli.decompress_max(string.bytes(packed)[0..plen], 1048576) + spec.assert_err_contains(derr, "output limit exceeded", "one-shot refuses past 1 MiB") + ds, _e = brotli.decompress_stream_new(1048576) + _d, _dn, serr = brotli.decompress_stream_write(ds, string.bytes(packed)[0..plen]) + spec.assert_err_contains(serr, "output limit exceeded", "a stream refuses past 1 MiB") + brotli.decompress_stream_free(ds) + } + } + spec.describe(fw, "std.brotli edge cases & invalid input") { + spec.it("round-trips an empty body") callback { + packed, plen, err = brotli.compress(string.bytes("")[0..0], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress of nothing") + spec.assert_true(plen > 0, "an empty stream still has a header") + back, blen, derr = brotli.decompress(string.bytes(packed)[0..plen]) + spec.assert_ok(derr, "decompress of an empty stream") + spec.assert_eq(blen, 0, "nothing comes back") + spec.assert_str_eq(back, "", "as an empty string") + } + + spec.it("refuses the large-window format") callback { + // Large windows are off in libbrotlidec unless a caller turns + // them on; this pins that nobody does, since the window is + // allocated from the header, outside max_bytes. + n = LARGE_WINDOW_BR.len + b = bytes.new(n) + i = 0 + while i < n { + bytes.set(b, i, LARGE_WINDOW_BR[i]) + i = i + 1 + } + lw = bytes.finish(b, n) + _o, _l, derr = brotli.decompress(string.bytes(lw)[0..n]) + spec.assert_err(derr, "one-shot refuses a large-window stream") + ds, _e = brotli.decompress_stream_new(0) + _d, _dn, serr = brotli.decompress_stream_write(ds, string.bytes(lw)[0..n]) + spec.assert_err(serr, "a stream refuses it too") + brotli.decompress_stream_free(ds) + } + spec.it("rejects garbage input on decompress") callback { _out, _olen, derr = brotli.decompress(string.bytes("this is not brotli data")) spec.assert_err(derr, "garbage input is rejected") @@ -171,6 +292,79 @@ main() { brotli.decompress_stream_free(ds) } + spec.it("decodes a stream fed one byte at a time") callback { + n = 5000 + plain = binary_payload(n) + packed, plen, err = brotli.compress(string.bytes(plain)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + ds, _e = brotli.decompress_stream_new(0) + out = bytes.new(n) + got = 0 + i = 0 + while i < plen { + c, cn, werr = brotli.decompress_stream_write(ds, string.bytes(packed)[i..i + 1]) + spec.assert_ok(werr, "write of byte ${i}") + if cn > 0 { + bytes.copy_from_string(out, got, c, cn) + got = got + cn + } + i = i + 1 + } + _t, tn, ferr = brotli.decompress_stream_finish(ds) + spec.assert_ok(ferr, "finish after the last byte") + spec.assert_eq(got + tn, n, "every byte came out") + spec.assert_true(string.string_equals(bytes.finish(out, got), plain) == 1, "in order") + brotli.decompress_stream_free(ds) + } + + spec.it("finish refuses a truncated stream") callback { + n = 5000 + plain = binary_payload(n) + packed, plen, err = brotli.compress(string.bytes(plain)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + ds, _e = brotli.decompress_stream_new(0) + _d, _dn, werr = brotli.decompress_stream_write(ds, string.bytes(packed)[0..plen - 3]) + spec.assert_ok(werr, "a prefix decodes so far") + _t, _tn, ferr = brotli.decompress_stream_finish(ds) + spec.assert_err_contains(ferr, "truncated", "the missing tail is reported") + brotli.decompress_stream_free(ds) + } + + spec.it("refuses bytes after the end of the stream") callback { + n = string.length(SAMPLE) + packed, plen, err = brotli.compress(string.bytes(SAMPLE)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + buf = bytes.new(plen + 5) + bytes.copy_from_string(buf, 0, packed, plen) + bytes.copy_from_string(buf, plen, "extra", 5) + padded = bytes.finish(buf, plen + 5) + ds, _e = brotli.decompress_stream_new(0) + _d, _dn, werr = brotli.decompress_stream_write(ds, string.bytes(padded)[0..plen + 5]) + spec.assert_err(werr, "trailing bytes in the same write are refused") + brotli.decompress_stream_free(ds) + + ds2, _e2 = brotli.decompress_stream_new(0) + _d2, _dn2, w2 = brotli.decompress_stream_write(ds2, string.bytes(packed)[0..plen]) + spec.assert_ok(w2, "the whole stream decodes") + _d3, _dn3, w3 = brotli.decompress_stream_write(ds2, string.bytes("extra")[0..5]) + spec.assert_err(w3, "a later write past the end is refused") + brotli.decompress_stream_free(ds2) + } + + spec.it("allows output exactly at a stream's limit") callback { + n = 5000 + plain = binary_payload(n) + packed, plen, err = brotli.compress(string.bytes(plain)[0..n], brotli.FAST_QUALITY) + spec.assert_ok(err, "compress succeeds") + ds, _e = brotli.decompress_stream_new(n) + _d, dn, werr = brotli.decompress_stream_write(ds, string.bytes(packed)[0..plen]) + spec.assert_ok(werr, "output equal to the limit") + _t, tn, ferr = brotli.decompress_stream_finish(ds) + spec.assert_ok(ferr, "finish") + spec.assert_eq(dn + tn, n, "all of it") + brotli.decompress_stream_free(ds) + } + spec.it("enforces max_bytes limit during streaming decompression") callback { b = bytes.new(2000) i = 0 diff --git a/tools/ae.c b/tools/ae.c index 30f9b8591..86aa25d60 100644 --- a/tools/ae.c +++ b/tools/ae.c @@ -3019,7 +3019,8 @@ static unsigned optional_link_requirement(const char* tok, size_t len) { { "-lssl", LINK_OPENSSL }, { "-lcrypto", LINK_OPENSSL }, { "-lz", LINK_ZLIB }, { "-lnghttp2", LINK_NGHTTP2 }, { "-lpcre2-8", LINK_PCRE2 }, - { "-lbrotlienc", LINK_BROTLI }, { "-lbrotlidec", LINK_BROTLI }, { "-lbrotlicommon", LINK_BROTLI }, + { "-lbrotlienc", LINK_BROTLI }, { "-lbrotlidec", LINK_BROTLI }, + { "-lbrotlicommon", LINK_BROTLI }, { "-lzstd", LINK_ZSTD }, { "-lfyaml", LINK_YAML }, }; for (size_t i = 0; i < sizeof(libs) / sizeof(libs[0]); i++) { @@ -4615,9 +4616,9 @@ void build_gcc_cmd(char* cmd, size_t size, const char* pcre2_libs = ""; #endif #ifdef AETHER_BROTLI_LIBS - /* libbrotlienc powers std.brotli. Empty when not detected, and that - * emptiness is load-bearing: a box without the library must not get a - * -lbrotlienc it cannot resolve. The std.brotli surface stays valid and + /* libbrotlienc and libbrotlidec power std.brotli. Empty when not + * detected, and that emptiness is load-bearing: a box without the + * libraries must not get a -lbrotlienc it cannot resolve. The std.brotli surface stays valid and * reports "brotli unavailable". */ const char* brotli_libs = (required_libs & LINK_BROTLI) ? AETHER_BROTLI_LIBS : ""; #else @@ -4935,9 +4936,9 @@ void build_gcc_cmd(char* cmd, size_t size, #else const char* pcre2_libs = ""; #endif - // libbrotlienc powers std.brotli. Empty when not detected, and that - // emptiness is load-bearing: a box without the library must not be handed - // a -lbrotlienc it cannot resolve. + // libbrotlienc and libbrotlidec power std.brotli. Empty when not detected, + // and that emptiness is load-bearing: a box without the libraries must not + // be handed a -lbrotlienc it cannot resolve. #ifdef AETHER_BROTLI_LIBS const char* brotli_libs = (required_libs & LINK_BROTLI) ? AETHER_BROTLI_LIBS : ""; #else