encode fn
diff --git a/new_changelogs/20261029T182754Z-security-cbor-decode-limits.md b/new_changelogs/20261029T182754Z-security-cbor-decode-limits.md
index 6deefba1f..591e59528 100644
--- a/new_changelogs/20261029T182754Z-security-cbor-decode-limits.md
+++ b/new_changelogs/20261029T182754Z-security-cbor-decode-limits.md
@@ -1 +1,13 @@
-- **`std.cbor`**: Added `CborDecodeOptions` with safe defaults (`parse_with`), duplicate map key rejection by default, declared length and container element bounds enforcement, and `encode_deterministic` (RFC 8949 §4.2 Core Deterministic Encoding).
+- **`std.cbor` decodes untrusted input safely by default.** `parse` now
+ refuses duplicate map keys (`cbor: duplicate map key "grant"`), where two
+ keys are duplicates when they are the same value however they were encoded,
+ and limits each array or map to 131072 elements or pairs. It also refuses a
+ declared length or count that cannot fit in the bytes left before
+ allocating for it. `parse_with(data, opts)` takes a `CborDecodeOptions`
+ (`default_decode_options()` gives the defaults) to change the limits, allow
+ duplicates, or refuse indefinite-length items. `encode_deterministic`
+ writes RFC 8949 §4.2.1 core deterministic encoding: shortest forms, definite
+ lengths and sorted map keys. Duplicate detection and key sorting are
+ linear-logarithmic, so a map at the limit cannot be used to spend minutes
+ of CPU. Refusing duplicate keys is a change to `parse`'s behaviour; use
+ `parse_with` with `allow_duplicate_keys = 1` for the old one.
diff --git a/std/cbor/module.ae b/std/cbor/module.ae
index 59364326f..6d8c268e9 100644
--- a/std/cbor/module.ae
+++ b/std/cbor/module.ae
@@ -10,6 +10,7 @@ import std.strbuilder
import std.bytes
import std.mem
import std.encoding
+import std.map(map_new, map_put_raw, map_has, map_free)
exports(
CBOR_INT, CBOR_BYTES, CBOR_TEXT, CBOR_ARRAY, CBOR_MAP, CBOR_TAG, CBOR_SIMPLE, CBOR_FLOAT,
@@ -30,9 +31,14 @@ const CBOR_TAG = 5
const CBOR_SIMPLE = 6
const CBOR_FLOAT = 7
-// Deepest nesting parse accepts by default, the same bound as std.json.
+// Deepest nesting parse accepts by default, the same bound as std.json. The
+// decoder recurses once per level, so without a bound a few hundred kilobytes
+// of one-element arrays overflow the stack.
const CBOR_MAX_DEPTH = 256
+// What parse_with accepts. default_decode_options() is what parse uses: safe
+// for input from an untrusted peer. The element and pair defaults are
+// fxamacker/cbor's.
struct CborDecodeOptions {
max_depth: int
max_array_elements: int
@@ -400,17 +406,29 @@ fn rd_byte(b: ptr, off: int) -> int {
return bytes.get(b, off) & 255
}
-fn map_has_duplicate_key(cv: *CborValue, key_cv: ptr) -> bool {
- n = list.size(cv.map_keys)
- i = 0
- while i < n {
- k = list.get_raw(cv.map_keys, i)
- if equal_val(k, key_cv) {
- return true
- }
- i = i + 1
- }
- return false
+// Duplicate map keys (RFC 8949 §5.6) are found through a set of the keys
+// seen so far in this map, keyed by the hex of each key's deterministic
+// encoding: two keys are the same value exactly when those encodings match,
+// whether either was written definite or indefinite. Hex, because a set key
+// is a C string and an encoding can hold a zero byte. A pairwise scan was
+// quadratic: 131072 small keys in about 400 KB cost billions of comparisons.
+// Returns "" when key_cv is new (and records it), else the error.
+fn seen_key_check(seen: ptr, key_cv: ptr) -> string {
+ enc, eerr = encode_deterministic(key_cv)
+ if eerr != "" { return eerr }
+ hx = encoding.hex_encode(string.bytes(enc))
+ string.release(enc)
+ if map_has(seen, hx) == 1 {
+ string.release(hx)
+ return duplicate_key_error(key_cv)
+ }
+ _ = map_put_raw(seen, hx, hx)
+ string.release(hx)
+ return ""
+}
+
+fn free_seen(seen: ptr) {
+ if seen != null { map_free(seen) }
}
fn duplicate_key_error(key_cv: ptr) -> string {
@@ -566,6 +584,8 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
len_val, err = read_value(r, ai)
if err != "" { return null, err }
string.release(err)
+ // Compared in long against what is left: narrowing a length of 2^31
+ // or more to int first made it negative and passed it as empty.
rem = (reader.len - reader.offset) as long
if len_val < 0 || len_val > rem { return null, "cbor: unexpected EOF" }
if mt == 3 && !valid_utf8(reader.buf, reader.offset, len_val as int) {
@@ -625,6 +645,9 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
}
string.release(err)
cv.ind = ind_of(ai)
+ // Every element takes at least one byte, so a count beyond what is
+ // left is truncated input, refused before the loop rather than
+ // narrowed to int (2^32 became 0 elements).
rem = (reader.len - reader.offset) as long
if len_val < 0 || len_val > rem {
free_value(cv)
@@ -655,6 +678,9 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
cv.type = CBOR_MAP
cv.map_keys = list.new()
cv.map_values = list.new()
+ seen = null
+ if reader.allow_duplicate_keys == 0 { seen = map_new() }
+ defer free_seen(seen)
if ai == 31 {
if reader.allow_indefinite == 0 {
free_value(cv)
@@ -683,14 +709,16 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
}
string.release(err)
- if reader.allow_duplicate_keys == 0 {
- if map_has_duplicate_key(cv as *CborValue, key_cv) {
- dup_err = duplicate_key_error(key_cv)
+ if seen != null {
+ dup_err = seen_key_check(seen, key_cv)
+ if dup_err != "" {
free_value(key_cv)
free_value(cv)
return null, dup_err
}
}
+ // A break here, where the value belongs, is not the end of
+ // the map: parse_item refuses it as an unexpected break.
val_cv, err2 = parse_item(r, depth + 1)
if err2 != "" {
@@ -712,6 +740,7 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
}
string.release(err)
cv.ind = ind_of(ai)
+ // Every entry is a key and a value, at least two bytes.
rem = (reader.len - reader.offset) as long
if len_val < 0 || len_val > rem / 2 {
free_value(cv)
@@ -731,9 +760,9 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
}
string.release(err)
- if reader.allow_duplicate_keys == 0 {
- if map_has_duplicate_key(cv as *CborValue, key_cv) {
- dup_err = duplicate_key_error(key_cv)
+ if seen != null {
+ dup_err = seen_key_check(seen, key_cv)
+ if dup_err != "" {
free_value(key_cv)
free_value(cv)
return null, dup_err
@@ -806,7 +835,10 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
return null, "cbor: invalid additional information"
}
-// Parse one CBOR data item that is the whole of `cbor_data` with options.
+// Parse one CBOR data item that is the whole of `cbor_data`, under `opts`.
+// Returns (value_ptr, "") on success, (null, error) on failure. Caller owns
+// the returned value and must free with cbor.free. Bytes left after the item
+// are an error (#2470): `0102` used to parse as 1.
parse_with(cbor_data: string, opts: CborDecodeOptions) -> ptr! {
ok = ""
dlen = string.length(cbor_data)
@@ -844,7 +876,8 @@ parse_with(cbor_data: string, opts: CborDecodeOptions) -> ptr! {
return res, ok
}
-// Parse one CBOR data item using safe default options.
+// parse_with under default_decode_options(): duplicate map keys refused,
+// 131072 elements or pairs per container, nesting to 256.
parse(cbor_data: string) -> ptr! {
return parse_with(cbor_data, default_decode_options())
}
@@ -1196,6 +1229,66 @@ fn bytes_lex_cmp(s1: string, s2: string) -> int {
return 0
}
+// Sort *MapPair entries by the bytes of their encoded keys (RFC 8949
+// §4.2.1). A merge sort: an insertion sort was quadratic in the pair count.
+// Encodings are prefix-free, so bytewise order needs no tie-break.
+fn sort_pairs(pairs: ptr, n: int) {
+ tmp = list.new()
+ i = 0
+ while i < n {
+ _ = list.add(tmp, list.get_raw(pairs, i))
+ i = i + 1
+ }
+ src = pairs
+ dst = tmp
+ width = 1
+ while width < n {
+ lo = 0
+ while lo < n {
+ mid = lo + width
+ if mid > n { mid = n }
+ hi = lo + 2 * width
+ if hi > n { hi = n }
+ a = lo
+ b = mid
+ k = lo
+ while k < hi {
+ take_a = 0
+ if a < mid {
+ if b >= hi {
+ take_a = 1
+ } else {
+ pa = list.get_raw(src, a) as *MapPair
+ pb = list.get_raw(src, b) as *MapPair
+ if bytes_lex_cmp(pa.enc_k, pb.enc_k) <= 0 { take_a = 1 }
+ }
+ }
+ if take_a == 1 {
+ list.set(dst, k, list.get_raw(src, a))
+ a = a + 1
+ } else {
+ list.set(dst, k, list.get_raw(src, b))
+ b = b + 1
+ }
+ k = k + 1
+ }
+ lo = hi
+ }
+ t = src
+ src = dst
+ dst = t
+ width = width * 2
+ }
+ if src != pairs {
+ i = 0
+ while i < n {
+ list.set(pairs, i, list.get_raw(src, i))
+ i = i + 1
+ }
+ }
+ list.free(tmp)
+}
+
encode_deterministic_item(w: ptr, cv: ptr) -> string {
if cv == null {
write_byte(w, 246)
@@ -1288,26 +1381,7 @@ encode_deterministic_item(w: ptr, cv: ptr) -> string {
return map_err
}
- // Insertion sort on pairs
- var sort_i = 1
- while sort_i < n {
- p_i_ptr, _ = list.get(pairs, sort_i)
- p_i = p_i_ptr as *MapPair
- var sort_j = sort_i - 1
- while sort_j >= 0 {
- p_j_ptr, _ = list.get(pairs, sort_j)
- p_j = p_j_ptr as *MapPair
- cmp = bytes_lex_cmp(p_j.enc_k, p_i.enc_k)
- if cmp > 0 {
- list.set(pairs, sort_j + 1, p_j as ptr)
- sort_j = sort_j - 1
- } else {
- break
- }
- }
- list.set(pairs, sort_j + 1, p_i as ptr)
- sort_i = sort_i + 1
- }
+ sort_pairs(pairs, n)
// Check for duplicate key encodings
var check_i = 0
diff --git a/std/cbor/test_cbor_hardening.ae b/std/cbor/test_cbor_hardening.ae
index 70cd995b9..5e2febba4 100644
--- a/std/cbor/test_cbor_hardening.ae
+++ b/std/cbor/test_cbor_hardening.ae
@@ -307,6 +307,54 @@ fn test_deterministic_encoding() -> int {
return f
}
+// Duplicate detection and key sorting at the default pair limit. A pairwise
+// duplicate scan or an insertion sort is quadratic here: 131072 keys made
+// billions of comparisons, so either regression shows as this test hanging.
+fn test_large_maps() -> int {
+ f = 0
+ n = 131072
+ m = cbor.obj()
+ i = 0
+ while i < n {
+ // descending, so a sort has work to do
+ _ = cbor.map_push(m, cbor.from_int((n - i) as long), cbor.from_int(0 as long))
+ i = i + 1
+ }
+ enc, eerr = cbor.encode(m)
+ cbor.free(m)
+ if eerr != "" {
+ println(" FAIL large map encode: ${eerr}")
+ return 1
+ }
+ back, perr = cbor.parse(enc)
+ if perr != "" {
+ println(" FAIL large map with ${n} distinct keys refused: ${perr}")
+ return 1
+ }
+ det, derr = cbor.encode_deterministic(back)
+ cbor.free(back)
+ if derr != "" {
+ println(" FAIL large map deterministic encode: ${derr}")
+ return 1
+ }
+ sorted, serr = cbor.parse(det)
+ if serr != "" {
+ println(" FAIL large map re-parse: ${serr}")
+ return 1
+ }
+ k0, _v0, _e0 = cbor.object_entry(sorted, 0)
+ kl, _vl, _el = cbor.object_entry(sorted, n - 1)
+ first = cbor.get_long(k0)
+ last = cbor.get_long(kl)
+ if first != 1 || last != (n as long) {
+ println(" FAIL large map keys not sorted: first ${first}, last ${last}")
+ f = f + 1
+ }
+ cbor.free(sorted)
+ if f == 0 { println(" PASS: ${n}-key map parses and sorts") }
+ return f
+}
+
main() {
println("=== test_cbor_hardening ===")
fails = 0
@@ -315,6 +363,7 @@ main() {
fails = fails + test_length_bombs()
fails = fails + test_option_boundaries()
fails = fails + test_deterministic_encoding()
+ fails = fails + test_large_maps()
if fails != 0 {
println("CBOR HARDENING: ${fails} failure(s)")
From aca9a7e51dff92de504301bf5128e06cee2f1bd6 Mon Sep 17 00:00:00 2001
From: Paul Hammant
Date: Thu, 29 Oct 2026 19:11:39 +0000
Subject: [PATCH 3/4] cbor: the duplicate-key set uses std.map's public map_put
check-raw-calls refuses a std module calling another module's raw C
binding (map_put_raw).
Co-Authored-By: Claude Opus 5.5
---
std/cbor/module.ae | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/std/cbor/module.ae b/std/cbor/module.ae
index 6d8c268e9..eab5739db 100644
--- a/std/cbor/module.ae
+++ b/std/cbor/module.ae
@@ -10,7 +10,7 @@ import std.strbuilder
import std.bytes
import std.mem
import std.encoding
-import std.map(map_new, map_put_raw, map_has, map_free)
+import std.map(map_new, map_put, map_has, map_free)
exports(
CBOR_INT, CBOR_BYTES, CBOR_TEXT, CBOR_ARRAY, CBOR_MAP, CBOR_TAG, CBOR_SIMPLE, CBOR_FLOAT,
@@ -422,7 +422,7 @@ fn seen_key_check(seen: ptr, key_cv: ptr) -> string {
string.release(hx)
return duplicate_key_error(key_cv)
}
- _ = map_put_raw(seen, hx, hx)
+ _ = map_put(seen, hx, hx)
string.release(hx)
return ""
}
From a796bd842a8fa35ba2f6c79f0b437f37a9f0f370 Mon Sep 17 00:00:00 2001
From: Paul Hammant
Date: Thu, 29 Oct 2026 19:39:27 +0000
Subject: [PATCH 4/4] cbor: duplicate keys found by sorting key encodings, not
a std.map set
Importing std.map into std.cbor fails check-warnings: std.map's get
mangles to map_get, the symbol of cbor's own map_get. Once a map is
parsed, its keys' deterministic encodings are now merge-sorted and
neighbours compared, which is still n log n. test_large_maps still parses
and sorts 131072 keys in about a second; with the check disabled, the
four duplicate-key cases fail.
Co-Authored-By: Claude Opus 5.5
---
std/cbor/module.ae | 112 ++++++++++++++++++++++++++-------------------
1 file changed, 65 insertions(+), 47 deletions(-)
diff --git a/std/cbor/module.ae b/std/cbor/module.ae
index eab5739db..553cda8c9 100644
--- a/std/cbor/module.ae
+++ b/std/cbor/module.ae
@@ -10,7 +10,6 @@ import std.strbuilder
import std.bytes
import std.mem
import std.encoding
-import std.map(map_new, map_put, map_has, map_free)
exports(
CBOR_INT, CBOR_BYTES, CBOR_TEXT, CBOR_ARRAY, CBOR_MAP, CBOR_TAG, CBOR_SIMPLE, CBOR_FLOAT,
@@ -406,29 +405,57 @@ fn rd_byte(b: ptr, off: int) -> int {
return bytes.get(b, off) & 255
}
-// Duplicate map keys (RFC 8949 §5.6) are found through a set of the keys
-// seen so far in this map, keyed by the hex of each key's deterministic
-// encoding: two keys are the same value exactly when those encodings match,
-// whether either was written definite or indefinite. Hex, because a set key
-// is a C string and an encoding can hold a zero byte. A pairwise scan was
-// quadratic: 131072 small keys in about 400 KB cost billions of comparisons.
-// Returns "" when key_cv is new (and records it), else the error.
-fn seen_key_check(seen: ptr, key_cv: ptr) -> string {
- enc, eerr = encode_deterministic(key_cv)
- if eerr != "" { return eerr }
- hx = encoding.hex_encode(string.bytes(enc))
- string.release(enc)
- if map_has(seen, hx) == 1 {
- string.release(hx)
- return duplicate_key_error(key_cv)
- }
- _ = map_put(seen, hx, hx)
- string.release(hx)
- return ""
-}
-
-fn free_seen(seen: ptr) {
- if seen != null { map_free(seen) }
+// Duplicate map keys (RFC 8949 §5.6): two keys are the same value exactly
+// when their deterministic encodings match, whether either was written
+// definite or indefinite. Once a map is parsed its key encodings are
+// merge-sorted and neighbours compared. A pairwise scan was quadratic:
+// 131072 small keys in about 400 KB cost billions of comparisons. (A hash
+// set from std.map would do, but importing std.map collides with this
+// module's own map_get.) Returns "" when every key is distinct, else the error.
+fn duplicate_key_check(cv: *CborValue) -> string {
+ n = list.size(cv.map_keys)
+ if n < 2 { return "" }
+ pairs = list.new()
+ err = ""
+ i = 0
+ while i < n {
+ k = list.get_raw(cv.map_keys, i)
+ ek, kerr = encode_deterministic(k)
+ if kerr != "" {
+ err = kerr
+ break
+ }
+ p = heap.new(MapPair)
+ p.key = k
+ p.val = null
+ p.enc_k = ek
+ p.enc_v = ""
+ _ = list.add(pairs, p as ptr)
+ i = i + 1
+ }
+ m = list.size(pairs)
+ if err == "" {
+ sort_pairs(pairs, m)
+ j = 0
+ while j < m - 1 {
+ pa = list.get_raw(pairs, j) as *MapPair
+ pb = list.get_raw(pairs, j + 1) as *MapPair
+ if bytes_lex_cmp(pa.enc_k, pb.enc_k) == 0 {
+ err = duplicate_key_error(pa.key)
+ break
+ }
+ j = j + 1
+ }
+ }
+ j = 0
+ while j < m {
+ p = list.get_raw(pairs, j) as *MapPair
+ string.release(p.enc_k)
+ heap.free(p)
+ j = j + 1
+ }
+ list.free(pairs)
+ return err
}
fn duplicate_key_error(key_cv: ptr) -> string {
@@ -678,9 +705,6 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
cv.type = CBOR_MAP
cv.map_keys = list.new()
cv.map_values = list.new()
- seen = null
- if reader.allow_duplicate_keys == 0 { seen = map_new() }
- defer free_seen(seen)
if ai == 31 {
if reader.allow_indefinite == 0 {
free_value(cv)
@@ -708,18 +732,8 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
return null, err
}
string.release(err)
-
- if seen != null {
- dup_err = seen_key_check(seen, key_cv)
- if dup_err != "" {
- free_value(key_cv)
- free_value(cv)
- return null, dup_err
- }
- }
// A break here, where the value belongs, is not the end of
// the map: parse_item refuses it as an unexpected break.
-
val_cv, err2 = parse_item(r, depth + 1)
if err2 != "" {
free_value(key_cv)
@@ -731,6 +745,13 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
_ = list.add(cv.map_values, val_cv)
map_count = map_count + 1
}
+ if reader.allow_duplicate_keys == 0 {
+ dup_err = duplicate_key_check(cv as *CborValue)
+ if dup_err != "" {
+ free_value(cv)
+ return null, dup_err
+ }
+ }
return cv as ptr, ""
}
len_val, err = read_value(r, ai)
@@ -759,16 +780,6 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
return null, err
}
string.release(err)
-
- if seen != null {
- dup_err = seen_key_check(seen, key_cv)
- if dup_err != "" {
- free_value(key_cv)
- free_value(cv)
- return null, dup_err
- }
- }
-
val_cv, err2 = parse_item(r, depth + 1)
if err2 != "" {
free_value(key_cv)
@@ -780,6 +791,13 @@ parse_item(r: ptr, depth: int) -> (ptr, string) {
_ = list.add(cv.map_values, val_cv)
i = i + 1
}
+ if reader.allow_duplicate_keys == 0 {
+ dup_err = duplicate_key_check(cv as *CborValue)
+ if dup_err != "" {
+ free_value(cv)
+ return null, dup_err
+ }
+ }
return cv as ptr, ""
}
if mt == 6 {