diff --git a/conformance/fixtures/claims/numeric-name.json b/conformance/fixtures/claims/numeric-name.json new file mode 100644 index 0000000..43e99bf --- /dev/null +++ b/conformance/fixtures/claims/numeric-name.json @@ -0,0 +1,9 @@ +{ + "name": "numeric-name", + "description": "Decimal claim names remain valid names even when a host language stores them as integer map keys.", + "input": { + "1": "one", + "0": "zero" + }, + "expected": "0:zero\n1:one\n" +} diff --git a/conformance/fixtures/jcs/ecmascript-numbers.json b/conformance/fixtures/jcs/ecmascript-numbers.json index 4905a2b..9e2f87a 100644 --- a/conformance/fixtures/jcs/ecmascript-numbers.json +++ b/conformance/fixtures/jcs/ecmascript-numbers.json @@ -1,6 +1,6 @@ { "name": "ecmascript-numbers", "description": "Numbers use ECMAScript serialization at fixed and exponential notation boundaries.", - "input": "{\"z\":-0,\"a\":1e30,\"b\":4.50,\"c\":2e-3,\"d\":1e-7,\"e\":1e-6}", - "expected": "{\"a\":1e+30,\"b\":4.5,\"c\":0.002,\"d\":1e-7,\"e\":0.000001,\"z\":0}" + "input": "{\"a\":1e30,\"b\":4.50,\"c\":2e-3,\"d\":1e-7,\"e\":1e-6}", + "expected": "{\"a\":1e+30,\"b\":4.5,\"c\":0.002,\"d\":1e-7,\"e\":0.000001}" } diff --git a/conformance/fixtures/jcs/malformed-surrogate-json.json b/conformance/fixtures/jcs/malformed-surrogate-json.json new file mode 100644 index 0000000..44b4702 --- /dev/null +++ b/conformance/fixtures/jcs/malformed-surrogate-json.json @@ -0,0 +1,6 @@ +{ + "name": "malformed-surrogate-json", + "description": "Malformed JSON takes precedence over lone-surrogate classification.", + "input": "{\"value\":\"\\uD800", + "error": "jcs-invalid-json" +} diff --git a/conformance/fixtures/jcs/negative-underflow.json b/conformance/fixtures/jcs/negative-underflow.json new file mode 100644 index 0000000..6073497 --- /dev/null +++ b/conformance/fixtures/jcs/negative-underflow.json @@ -0,0 +1,6 @@ +{ + "name": "negative-underflow", + "description": "RFC 8785 erratum 7920 rejects a negative number that underflows to binary64 negative zero.", + "input": "{\"value\":-1e-400}", + "error": "jcs-number" +} diff --git a/conformance/fixtures/jcs/negative-zero.json b/conformance/fixtures/jcs/negative-zero.json new file mode 100644 index 0000000..0cb96c5 --- /dev/null +++ b/conformance/fixtures/jcs/negative-zero.json @@ -0,0 +1,6 @@ +{ + "name": "negative-zero", + "description": "RFC 8785 erratum 7920 rejects a literal negative zero token.", + "input": "{\"value\":-0}", + "error": "jcs-number" +} diff --git a/conformance/fixtures/jcs/oversized-malformed-json.json b/conformance/fixtures/jcs/oversized-malformed-json.json new file mode 100644 index 0000000..5ce8498 --- /dev/null +++ b/conformance/fixtures/jcs/oversized-malformed-json.json @@ -0,0 +1,7 @@ +{ + "name": "oversized-malformed-json", + "description": "The source ceiling takes precedence over malformed JSON classification.", + "input": "{", + "repeat": 1048577, + "error": "resource-limit-exceeded" +} diff --git a/conformance/fixtures/jcs/rfc8785-appendix-b-numbers.json b/conformance/fixtures/jcs/rfc8785-appendix-b-numbers.json index d93b008..1e0f6af 100644 --- a/conformance/fixtures/jcs/rfc8785-appendix-b-numbers.json +++ b/conformance/fixtures/jcs/rfc8785-appendix-b-numbers.json @@ -1,6 +1,6 @@ { "name": "rfc8785-appendix-b-numbers", "description": "Binary64 values use the ECMAScript spellings listed in RFC 8785 Appendix B.", - "input": "[0,-0,5e-324,-5e-324,1.7976931348623157e308,-1.7976931348623157e308,9007199254740992,-9007199254740992,295147905179352830000,9.999999999999997e22,1e23,1.0000000000000001e23,999999999999999700000,999999999999999900000,1e21,9.999999999999997e-7,0.000001,333333333.3333332,333333333.33333325,333333333.3333333,333333333.3333334,333333333.33333343,-0.0000033333333333333333,1424953923781206.25]", - "expected": "[0,0,5e-324,-5e-324,1.7976931348623157e+308,-1.7976931348623157e+308,9007199254740992,-9007199254740992,295147905179352830000,9.999999999999997e+22,1e+23,1.0000000000000001e+23,999999999999999700000,999999999999999900000,1e+21,9.999999999999997e-7,0.000001,333333333.3333332,333333333.33333325,333333333.3333333,333333333.3333334,333333333.33333343,-0.0000033333333333333333,1424953923781206.2]" + "input": "[0,5e-324,-5e-324,1.7976931348623157e308,-1.7976931348623157e308,9007199254740992,-9007199254740992,295147905179352830000,9.999999999999997e22,1e23,1.0000000000000001e23,999999999999999700000,999999999999999900000,1e21,9.999999999999997e-7,0.000001,333333333.3333332,333333333.33333325,333333333.3333333,333333333.3333334,333333333.33333343,-0.0000033333333333333333,1424953923781206.25]", + "expected": "[0,5e-324,-5e-324,1.7976931348623157e+308,-1.7976931348623157e+308,9007199254740992,-9007199254740992,295147905179352830000,9.999999999999997e+22,1e+23,1.0000000000000001e+23,999999999999999700000,999999999999999900000,1e+21,9.999999999999997e-7,0.000001,333333333.3333332,333333333.33333325,333333333.3333333,333333333.3333334,333333333.33333343,-0.0000033333333333333333,1424953923781206.2]" } diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs index 5023e6b..d025898 100644 --- a/ffi/src/lib.rs +++ b/ffi/src/lib.rs @@ -174,7 +174,7 @@ mod capi { Some(bytes) => match std::str::from_utf8(bytes) { Ok(base) => Some(base), Err(_) => { - let bytes = b"invalid-utf8".to_vec().into_boxed_slice(); + let bytes = b"parser-profile-unsupported".to_vec().into_boxed_slice(); *out_len = bytes.len(); *out = Box::into_raw(bytes) as *mut u8; return 1; @@ -190,7 +190,7 @@ mod capi { }, ) } - Err(_) => Err("invalid-utf8".to_string()), + Err(_) => Err("parser-profile-unsupported".to_string()), }; let (status, bytes) = match result { Ok(text) => (0, text.into_bytes()), @@ -261,7 +261,8 @@ mod capi { } /// Normalize a UTF-8 byte string with the profile-v1 size limits. - /// Status 0 returns normalized text, status 1 returns `invalid-utf8` or + /// Status 0 returns normalized text, status 1 returns + /// `parser-profile-unsupported` or /// `resource-limit-exceeded`, and status 2 indicates invalid pointers. /// Valid output pointers are initialized before input decoding. #[no_mangle] diff --git a/go/canonicalize.go b/go/canonicalize.go index b23e899..10d3636 100644 --- a/go/canonicalize.go +++ b/go/canonicalize.go @@ -105,7 +105,7 @@ func NormalizeTextChecked(text string, opts ...Options) (string, error) { return "", fmt.Errorf("resource-limit-exceeded") } if !utf8.ValidString(text) { - return "", fmt.Errorf("invalid-utf8") + return "", fmt.Errorf("parser-profile-unsupported") } result := normalizeText(text, opts...) if len(result) > maxResourceBytes { diff --git a/go/canonicalize_test.go b/go/canonicalize_test.go index b8f5f33..dae3b8b 100644 --- a/go/canonicalize_test.go +++ b/go/canonicalize_test.go @@ -721,6 +721,23 @@ func TestExtractClaimsFromSignedSectionRejectsNormalizedDuplicate(t *testing.T) } } +func TestExtractClaimsFromSignedSectionSelectsFirstNestedSection(t *testing.T) { + claims, err := ExtractClaimsFromSignedSection(`
`) + if err != nil { + t.Fatalf("ExtractClaimsFromSignedSection: %v", err) + } + if claims["author"] != "Nested first" { + t.Fatalf("selected claims = %#v, want first nested signed-section", claims) + } +} + +func TestExtractClaimsFromSignedSectionChecksFieldSizeBeforeDuplicate(t *testing.T) { + _, err := ExtractClaimsFromSignedSection(``) + if err == nil || !strings.Contains(err.Error(), "resource-limit-exceeded") { + t.Fatalf("oversized duplicate claim error = %v, want resource-limit-exceeded", err) + } +} + func TestResolveKeyChain(t *testing.T) { pemStr, _, _ := newEd25519PEM(t) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -826,6 +843,14 @@ func TestCanonicalizeEndorsementDocumentRejectsDuplicateMembers(t *testing.T) { } } +func TestCanonicalizeJSONDocumentRejectsNegativeZero(t *testing.T) { + for _, document := range []string{`{"value":-0}`, `{"value":-1e-400}`} { + if _, err := CanonicalizeJSONDocument([]byte(document)); err == nil || !strings.Contains(err.Error(), "jcs-number") { + t.Errorf("%s: expected jcs-number, got %v", document, err) + } + } +} + func TestVerifyEndorsementMissingFields(t *testing.T) { cases := []Endorsement{ {Endorser: "", Endorsement: "x", Signature: "x", Timestamp: "x"}, diff --git a/go/extract.go b/go/extract.go index 3d9e0d3..9d92494 100644 --- a/go/extract.go +++ b/go/extract.go @@ -60,19 +60,21 @@ var ( // to line feeds, decodes entities, and runs the full text normalization // pipeline. The returned string is trimmed. func ExtractCanonicalText(html string, opts ...Options) (string, error) { - if len(html) > maxResourceBytes || !utf8.ValidString(html) { + if len(html) > maxResourceBytes { return "", fmt.Errorf("resource-limit-exceeded") } + if !utf8.ValidString(html) { + return "", fmt.Errorf("parser-profile-unsupported") + } var o Options if len(opts) > 0 { o = opts[0] } - base, err := parseBaseURL(o.BaseURL) - if err != nil { + if err := validatePortableHTML(html); err != nil { return "", err } - - if err := validatePortableHTML(html); err != nil { + base, err := parseBaseURL(o.BaseURL) + if err != nil { return "", err } fragment, err := htmlpkg.ParseFragment(strings.NewReader(html), &htmlpkg.Node{Type: htmlpkg.ElementNode, Data: "div", DataAtom: atom.Div}) @@ -91,12 +93,15 @@ func ExtractCanonicalText(html string, opts ...Options) (string, error) { } // ExtractClaimsFromSignedSection returns claim metadata from direct child meta -// elements. If the fragment contains a top-level signed-section, that element -// supplies the children; otherwise the fragment is treated as section inner HTML. +// elements of the first signed-section anywhere in the fragment. If there is +// no signed-section, the fragment is treated as section inner HTML. func ExtractClaimsFromSignedSection(source string) (map[string]string, error) { - if len(source) > maxResourceBytes || !utf8.ValidString(source) { + if len(source) > maxResourceBytes { return nil, fmt.Errorf("resource-limit-exceeded") } + if !utf8.ValidString(source) { + return nil, fmt.Errorf("parser-profile-unsupported") + } if err := validatePortableHTML(source); err != nil { return nil, err } @@ -105,13 +110,10 @@ func ExtractClaimsFromSignedSection(source string) (map[string]string, error) { return nil, fmt.Errorf("parser-profile-unsupported: %v", err) } children := fragment - for _, node := range fragment { - if node.Type == htmlpkg.ElementNode && strings.EqualFold(node.Data, "signed-section") { - children = children[:0] - for child := node.FirstChild; child != nil; child = child.NextSibling { - children = append(children, child) - } - break + if section := firstSignedSection(fragment); section != nil { + children = children[:0] + for child := section.FirstChild; child != nil; child = child.NextSibling { + children = append(children, child) } } claims := make(map[string]string) @@ -155,6 +157,30 @@ func ExtractClaimsFromSignedSection(source string) (map[string]string, error) { return claims, nil } +// firstSignedSection walks the parsed tree in document order. A signed +// section nested inside another element is still the first candidate when it +// appears before a later top-level section. +func firstSignedSection(nodes []*htmlpkg.Node) *htmlpkg.Node { + for _, node := range nodes { + if section := firstSignedSectionNode(node); section != nil { + return section + } + } + return nil +} + +func firstSignedSectionNode(node *htmlpkg.Node) *htmlpkg.Node { + if node.Type == htmlpkg.ElementNode && strings.EqualFold(node.Data, "signed-section") { + return node + } + for child := node.FirstChild; child != nil; child = child.NextSibling { + if section := firstSignedSectionNode(child); section != nil { + return section + } + } + return nil +} + // x/net/html repairs malformed input and does not expose parser diagnostics. // Keep a strict source preflight so repaired trees cannot enter the portable // profile silently. @@ -413,7 +439,7 @@ func finalizeCanonicalParts(parts []string) string { for strings.Contains(text, "\n\n") { text = strings.ReplaceAll(text, "\n\n", "\n") } - return strings.Trim(text, " \n") + return strings.TrimSpace(text) } func appendAttributeRecords(parts *[]string, elementName string, attrs map[string]string, base *whatwgurl.Url) error { diff --git a/go/final_hardening_test.go b/go/final_hardening_test.go index 2bc2c24..c541cec 100644 --- a/go/final_hardening_test.go +++ b/go/final_hardening_test.go @@ -102,6 +102,33 @@ func TestParserPreflightRejectsCommentsAndDeclarations(t *testing.T) { } } +func TestExtractionRejectsInvalidUTF8(t *testing.T) { + if _, err := ExtractCanonicalText(string([]byte{'<', 'p', '>', 0xff, '<', '/', 'p', '>'})); err == nil || !strings.Contains(err.Error(), "parser-profile-unsupported") { + t.Fatalf("invalid UTF-8 extraction error = %v, want parser-profile-unsupported", err) + } + if _, err := ExtractClaimsFromSignedSection(string([]byte{'<', 'm', 'e', 't', 'a', ' ', 0xff, '>'})); err == nil || !strings.Contains(err.Error(), "parser-profile-unsupported") { + t.Fatalf("invalid UTF-8 claim extraction error = %v, want parser-profile-unsupported", err) + } +} + +func TestNormalizeTextCheckedRejectsInvalidUTF8WithParserProfileError(t *testing.T) { + if _, err := NormalizeTextChecked(string([]byte{0xff})); err == nil || !strings.Contains(err.Error(), "parser-profile-unsupported") { + t.Fatalf("NormalizeTextChecked error = %v, want parser-profile-unsupported", err) + } +} + +func TestSourceLimitPrecedesInvalidUTF8Classification(t *testing.T) { + if _, err := NormalizeTextChecked(strings.Repeat(string([]byte{0xff}), maxResourceBytes+1)); err == nil || !strings.Contains(err.Error(), "resource-limit-exceeded") { + t.Fatalf("oversized invalid UTF-8 normalization error = %v, want resource-limit-exceeded", err) + } +} + +func TestExtractionPreflightRunsBeforeInvalidBaseURL(t *testing.T) { + if _, err := ExtractCanonicalText("

unclosed", Options{BaseURL: "not a URL"}); err == nil || !strings.Contains(err.Error(), "parser-profile-unsupported") { + t.Fatalf("ExtractCanonicalText error = %v, want parser-profile-unsupported before base URL handling", err) + } +} + func TestRemoteKeyResolversBoundResponseBodies(t *testing.T) { body := strings.Repeat("x", maxRemoteKeyBytes+1) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -152,3 +179,21 @@ func TestJCSRejectsNestingBeyondLimit(t *testing.T) { t.Fatalf("JSON at nesting limit rejected: %v", err) } } + +func TestJCSMalformedJSONPrecedesSurrogateClassification(t *testing.T) { + if _, err := CanonicalizeJSONDocument([]byte(`{"value":"\uD800`)); err == nil || !strings.Contains(err.Error(), "jcs-invalid-json") { + t.Fatalf("malformed surrogate JSON error = %v, want jcs-invalid-json", err) + } +} + +func TestExtractionAppliesOutputLimitAfterFinalization(t *testing.T) { + unit := `

` + source := strings.Repeat(unit, 10000) + output, err := ExtractCanonicalText(source, Options{BaseURL: "https://example.com/"}) + if err != nil { + t.Fatalf("finalized output should be within the limit: %v", err) + } + if len(output) != 1039999 { + t.Fatalf("finalized output length = %d, want 1039999", len(output)) + } +} diff --git a/go/json.go b/go/json.go index f084ce0..d191c1e 100644 --- a/go/json.go +++ b/go/json.go @@ -169,6 +169,9 @@ func (p *strictJSONParser) string() (string, error) { p.pos += 4 if u >= 0xd800 && u <= 0xdbff { if p.pos+6 > len(p.data) || p.data[p.pos] != '\\' || p.data[p.pos+1] != 'u' { + if !hasUnescapedQuote(p.data, p.pos) { + return "", p.fail() + } return "", fmt.Errorf("jcs-invalid-surrogate") } p.pos += 2 @@ -186,6 +189,9 @@ func (p *strictJSONParser) string() (string, error) { } b.WriteRune(utf16.DecodeRune(rune(u), rune(lo))) } else if u >= 0xdc00 && u <= 0xdfff { + if !hasUnescapedQuote(p.data, p.pos) { + return "", p.fail() + } return "", fmt.Errorf("jcs-invalid-surrogate") } else { b.WriteRune(rune(u)) @@ -196,6 +202,25 @@ func (p *strictJSONParser) string() (string, error) { } return "", p.fail() } + +func hasUnescapedQuote(data []byte, start int) bool { + escaped := false + for _, c := range data[start:] { + if escaped { + escaped = false + continue + } + if c == '\\' { + escaped = true + continue + } + if c == '"' { + return true + } + } + return false +} + func (p *strictJSONParser) number() (float64, error) { start := p.pos if p.pos < len(p.data) && p.data[p.pos] == '-' { @@ -238,6 +263,11 @@ func (p *strictJSONParser) number() (float64, error) { if e != nil || n != n || n > 1.7976931348623157e308 || n < -1.7976931348623157e308 { return 0, fmt.Errorf("jcs-number") } + // RFC 8785 erratum 7920: reject negative zero, including negative values + // whose magnitude underflows to zero during binary64 parsing. + if p.data[start] == '-' && n == 0 { + return 0, fmt.Errorf("jcs-number") + } return n, nil } func (p *strictJSONParser) array() (*jsonValue, error) { diff --git a/javascript/index.js b/javascript/index.js index 3fbbf37..985b877 100644 --- a/javascript/index.js +++ b/javascript/index.js @@ -89,6 +89,10 @@ const ELLIPSIS_RE = /\u2026/g; export function normalizeText(text, options = {}) { if (typeof text !== "string") throw new TypeError("normalizeText expects a string"); checkResourceBytes(text, "source"); + // JavaScript strings can contain lone UTF-16 surrogates, but they have no + // UTF-8 source representation. Reject them before any normalization rather + // than allowing TextEncoder to silently replace them with U+FFFD. + assertUnicodeScalarString(text, "parser-profile-unsupported"); const { preserveWhitespace = false } = options; // Phase 1: Unicode NFKC normalization @@ -293,6 +297,7 @@ export function extractCanonicalText(html, options = {}) { } function parseHTMLFragment(html) { + assertUnicodeScalarString(html, "parser-profile-unsupported"); validatePortableSource(html); const errors = []; const fragment = parse5.parseFragment(html, { @@ -560,8 +565,8 @@ export function canonicalizeClaims(claims) { }) .map(([name, value]) => { if (!name) throw new Error("claim-malformed"); - if (seen.has(name)) throw new Error(`claim-duplicate: ${name}`); if (utf8Length(name) > MAX_CLAIM_FIELD_BYTES || utf8Length(value) > MAX_CLAIM_FIELD_BYTES) throw new Error("resource-limit-exceeded"); + if (seen.has(name)) throw new Error(`claim-duplicate: ${name}`); seen.add(name); return [name, value]; }) @@ -590,10 +595,7 @@ export function extractClaimsFromSignedSection(html) { checkResourceBytes(html, "source"); const fragment = parseHTMLFragment(html); - let section = fragment; - for (const node of fragment.childNodes || []) { - if (node.tagName?.toLowerCase() === "signed-section") { section = node; break; } - } + const section = findFirstSignedSection(fragment) ?? fragment; const claims = {}; const seen = new Set(); for (const child of section.childNodes || []) { @@ -603,8 +605,12 @@ export function extractClaimsFromSignedSection(html) { const claimName = normalizeText(attrs.get("name")).trim(); const content = normalizeText(attrs.get("content")).trim(); if (!claimName) throw new Error("claim-malformed"); + if (seen.size >= MAX_CLAIMS) throw new Error("resource-limit-exceeded"); + // Check field limits before duplicate detection. A duplicate oversized + // field is still a resource violation and must not be accepted as a + // duplicate-name failure. + if (utf8Length(claimName) > MAX_CLAIM_FIELD_BYTES || utf8Length(content) > MAX_CLAIM_FIELD_BYTES) throw new Error("resource-limit-exceeded"); if (seen.has(claimName)) throw new Error(`claim-duplicate: ${claimName}`); - if (seen.size >= MAX_CLAIMS || utf8Length(claimName) > MAX_CLAIM_FIELD_BYTES || utf8Length(content) > MAX_CLAIM_FIELD_BYTES) throw new Error("resource-limit-exceeded"); seen.add(claimName); claims[claimName] = content; } @@ -612,6 +618,15 @@ export function extractClaimsFromSignedSection(html) { return claims; } +function findFirstSignedSection(node) { + for (const child of node.childNodes || []) { + if (child.tagName?.toLowerCase() === "signed-section") return child; + const nested = findFirstSignedSection(child); + if (nested) return nested; + } + return null; +} + // === Signature binding (spec Β§2.1) === /** @@ -688,6 +703,9 @@ export function buildSigningPayloadV1({ throw new Error(`signing-object-invalid: ${name}`); } } + // Enforce the input side of the resource profile before URL parsing or any + // other derived-field work can shorten or otherwise transform the values. + jcsInputBytes({ contentHash, claimsHash, documentURL, scope, keyid, algorithm, signedAt }); validateSignedAtV1(signedAt); return canonicalizeJson({ algorithm, @@ -1265,15 +1283,15 @@ export function buildEndorsementBinding(e) { return canonicalizeJson(unsigned); } -function assertUnicodeScalarString(value) { +function assertUnicodeScalarString(value, errorCode = "jcs-invalid-surrogate") { for (let i = 0; i < value.length; i++) { const unit = value.charCodeAt(i); if (unit >= 0xd800 && unit <= 0xdbff) { const low = value.charCodeAt(i + 1); - if (!(low >= 0xdc00 && low <= 0xdfff)) throw new Error("jcs-invalid-surrogate"); + if (!(low >= 0xdc00 && low <= 0xdfff)) throw new Error(errorCode); i++; } else if (unit >= 0xdc00 && unit <= 0xdfff) { - throw new Error("jcs-invalid-surrogate"); + throw new Error(errorCode); } } } @@ -1287,10 +1305,15 @@ function serializeJcs(value, depth = 0) { if (typeof value === "boolean") return value ? "true" : "false"; if (typeof value === "number") { if (!Number.isFinite(value)) throw new Error("non-finite JSON number"); + if (Object.is(value, -0)) throw new Error("jcs-number"); return JSON.stringify(value); } if (Array.isArray(value)) { if (depth >= MAX_JCS_DEPTH) throw new Error("resource-limit-exceeded"); + if (value.length > MAX_RESOURCE_BYTES) throw new Error("resource-limit-exceeded"); + for (let index = 0; index < value.length; index++) { + if (!Object.hasOwn(value, index)) throw new Error("unsupported JSON value: sparse array"); + } return `[${value.map((item) => serializeJcs(item, depth + 1)).join(",")}]`; } if (value && typeof value === "object") { @@ -1305,8 +1328,67 @@ function serializeJcs(value, depth = 0) { throw new Error(`unsupported JSON value: ${typeof value}`); } +// Object APIs do not receive a raw JSON byte string, so account for the +// complete JSON-shaped value before serialization. This catches oversized +// aggregate inputs even when a later canonicalization step could shorten a +// string (for example, URL normalization in a signing payload). The final +// serialized result is checked separately for the output ceiling. +function jcsInputBytes(value, depth = 0, ancestors = new WeakSet()) { + const add = (left, right) => { + const total = left + right; + if (total > MAX_RESOURCE_BYTES) throw new Error("resource-limit-exceeded"); + return total; + }; + if (value === null) return 4; + if (typeof value === "string") { + assertUnicodeScalarString(value); + return add(2, utf8Length(value)); + } + if (typeof value === "boolean") return value ? 4 : 5; + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("non-finite JSON number"); + if (Object.is(value, -0)) throw new Error("jcs-number"); + return JSON.stringify(value).length; + } + if (Array.isArray(value)) { + if (depth >= MAX_JCS_DEPTH || value.length > MAX_RESOURCE_BYTES) throw new Error("resource-limit-exceeded"); + if (ancestors.has(value)) throw new Error("unsupported JSON value: cyclic object"); + ancestors.add(value); + let total = 2; + for (let index = 0; index < value.length; index++) { + if (!Object.hasOwn(value, index)) throw new Error("unsupported JSON value: sparse array"); + total = add(total, jcsInputBytes(value[index], depth + 1, ancestors)); + if (index + 1 < value.length) total = add(total, 1); + } + ancestors.delete(value); + return total; + } + if (value && typeof value === "object") { + if (depth >= MAX_JCS_DEPTH) throw new Error("resource-limit-exceeded"); + if (ancestors.has(value)) throw new Error("unsupported JSON value: cyclic object"); + ancestors.add(value); + let total = 2; + const keys = Object.keys(value); + for (let index = 0; index < keys.length; index++) { + const key = keys[index]; + assertUnicodeScalarString(key); + if (value[key] === undefined) throw new Error("unsupported JSON value: undefined"); + total = add(total, add(2, utf8Length(key))); + total = add(total, 1); + total = add(total, jcsInputBytes(value[key], depth + 1, ancestors)); + if (index + 1 < keys.length) total = add(total, 1); + } + ancestors.delete(value); + return total; + } + throw new Error(`unsupported JSON value: ${typeof value}`); +} + export function canonicalizeJson(value) { - return serializeJcs(value); + jcsInputBytes(value); + const result = serializeJcs(value); + checkResourceBytes(result, "output"); + return result; } // A small strict JSON parser is used for raw documents. JSON.parse is unable @@ -1330,6 +1412,9 @@ class StrictJsonParser { const raw = match[0]; const number = Number(raw); if (!Number.isFinite(number)) throw new Error("jcs-number"); + // RFC 8785 erratum 7920: reject every JSON token that decodes to IEEE-754 + // negative zero, including values whose magnitude underflows to zero. + if (raw[0] === "-" && Object.is(number, -0)) throw new Error("jcs-number"); this.index += raw.length; return number; } @@ -1349,11 +1434,22 @@ class StrictJsonParser { if (!/^[0-9a-fA-F]{4}$/.test(hex)) this.fail(); const unit = parseInt(hex, 16); this.index += 4; if (unit >= 0xd800 && unit <= 0xdbff) { - if (this.source.slice(this.index, this.index + 2) !== "\\u" || !/^[0-9a-fA-F]{4}$/.test(this.source.slice(this.index + 2, this.index + 6))) throw new Error("jcs-invalid-surrogate"); - const low = parseInt(this.source.slice(this.index + 2, this.index + 6), 16); + // Keep malformed JSON distinct from a valid JSON string containing + // an unpaired UTF-16 code unit. A truncated string or malformed + // low-surrogate escape must reach the generic JSON error path. + if (this.source.slice(this.index, this.index + 2) !== "\\u") { + if (!hasUnescapedQuote(this.source, this.index)) this.fail(); + throw new Error("jcs-invalid-surrogate"); + } + const lowHex = this.source.slice(this.index + 2, this.index + 6); + if (!/^[0-9a-fA-F]{4}$/.test(lowHex)) this.fail(); + const low = parseInt(lowHex, 16); if (low < 0xdc00 || low > 0xdfff) throw new Error("jcs-invalid-surrogate"); out += String.fromCodePoint(0x10000 + ((unit - 0xd800) << 10) + low - 0xdc00); this.index += 6; - } else if (unit >= 0xdc00 && unit <= 0xdfff) throw new Error("jcs-invalid-surrogate"); + } else if (unit >= 0xdc00 && unit <= 0xdfff) { + if (!hasUnescapedQuote(this.source, this.index)) this.fail(); + throw new Error("jcs-invalid-surrogate"); + } else out += String.fromCharCode(unit); continue; } @@ -1388,6 +1484,17 @@ class StrictJsonParser { parse() { const result = this.value(); this.ws(); if (this.index !== this.source.length) this.fail(); return result; } } +function hasUnescapedQuote(source, index) { + let escaped = false; + for (let i = index; i < source.length; i++) { + const c = source[i]; + if (escaped) { escaped = false; continue; } + if (c === "\\") { escaped = true; continue; } + if (c === '"') return true; + } + return false; +} + export function canonicalizeJsonDocument(document) { if (typeof document !== "string") throw new TypeError("canonicalizeJsonDocument expects a string"); checkResourceBytes(document, "source"); diff --git a/javascript/test.js b/javascript/test.js index c7f32bc..3910bff 100644 --- a/javascript/test.js +++ b/javascript/test.js @@ -16,6 +16,7 @@ import { resolveKey, verifyEndorsement, isKeyRevoked, + canonicalizeJson, canonicalizeJsonDocument, } from './index.js'; import * as nodeCrypto from 'node:crypto'; @@ -165,6 +166,14 @@ await check('extractClaimsFromSignedSection includes all direct child meta only' assert(!('Nested' in claims), 'nested meta must not be extracted'); }); +await check('extractClaimsFromSignedSection selects the first signed-section anywhere', () => { + const claims = extractClaimsFromSignedSection(` +
+ + `); + assertEq(claims.author, 'Nested first'); +}); + await check('extractClaimsFromSignedSection rejects duplicate normalized names', () => { let threw = false; try { @@ -175,6 +184,16 @@ await check('extractClaimsFromSignedSection rejects duplicate normalized names', assert(threw, 'expected duplicate direct child meta names to fail'); }); +await check('extractClaimsFromSignedSection checks oversized fields before duplicates', () => { + let threw = false; + try { + extractClaimsFromSignedSection(``); + } catch (error) { + threw = String(error).includes('resource-limit-exceeded'); + } + assert(threw, 'oversized duplicate claim must fail as a resource violation'); +}); + await check('extractCanonicalText signs href/src/alt/aria-label attributes', () => { const out = extractCanonicalText( '

linkHero β€” image

', @@ -205,6 +224,28 @@ await check('extractCanonicalText bounds malformed quoted tag scanning', () => { assert(performance.now() - started < 1000, 'malformed quoted tag scanning must stay bounded'); }); +await check('UTF-16 surrogates cannot enter the UTF-8 canonical profile', () => { + let normalizeThrew = false; + try { normalizeText('\ud800'); } catch (error) { + normalizeThrew = String(error).includes('parser-profile-unsupported'); + } + assert(normalizeThrew, 'normalizeText must reject lone surrogates'); + + let extractThrew = false; + try { extractCanonicalText(`

bad\ud800

`); } catch (error) { + extractThrew = String(error).includes('parser-profile-unsupported'); + } + assert(extractThrew, 'extractCanonicalText must reject non-UTF-8 source'); +}); + +await check('source limits precede malformed UTF-16 classification', () => { + let rejected = false; + try { normalizeText('\ud800'.repeat(1024 * 1024 + 1)); } catch (error) { + rejected = String(error).includes('resource-limit-exceeded'); + } + assert(rejected, 'oversized malformed source must report the resource limit'); +}); + await check('extractCanonicalText accepts qualified tag names and enforces element depth', () => { assertEq(extractCanonicalText('qualified'), 'qualified'); const withinLimit = ''.repeat(256) + 'deep' + ''.repeat(256); @@ -246,6 +287,79 @@ await check('canonicalizeJsonDocument rejects excessive nesting', () => { assert(threw, 'JCS nesting beyond 256 levels must be rejected'); }); +await check('canonicalizeJsonDocument rejects negative zero and underflow', () => { + for (const input of ['{"value":-0}', '{"value":-1e-400}']) { + let threw = false; + try { canonicalizeJsonDocument(input); } catch (error) { + threw = String(error).includes('jcs-number'); + } + assert(threw, `negative zero must be rejected: ${input}`); + } +}); + +await check('canonicalizeJson rejects negative zero in object input', () => { + for (const input of [-0, { value: -0 }, [-0]]) { + let rejected = false; + try { canonicalizeJson(input); } catch (error) { + rejected = String(error).includes('jcs-number'); + } + if (!rejected) throw new Error(`expected object input to reject negative zero: ${JSON.stringify(input)}`); + } +}); + +await check('canonicalizeJson rejects sparse and undefined values', () => { + const sparse = []; + sparse.length = 1; + let sparseRejected = false; + try { canonicalizeJson(sparse); } catch (error) { sparseRejected = String(error).includes('sparse array'); } + assert(sparseRejected, 'sparse arrays must be rejected'); + + let undefinedRejected = false; + try { canonicalizeJson({ value: undefined }); } catch (error) { undefinedRejected = String(error).includes('undefined'); } + assert(undefinedRejected, 'undefined object values must be rejected'); + let surrogateRejected = false; + try { canonicalizeJson('\ud800'); } catch (error) { surrogateRejected = String(error).includes('jcs-invalid-surrogate'); } + assert(surrogateRejected, 'lone surrogates must use the JCS failure vocabulary'); +}); + +await check('canonicalizeJson and buildSigningPayloadV1 enforce the input ceiling', () => { + let jsonRejected = false; + try { canonicalizeJson('x'.repeat(1024 * 1024)); } catch (error) { jsonRejected = String(error).includes('resource-limit-exceeded'); } + assert(jsonRejected, 'oversized object-call input must be rejected'); + + let outputRejected = false; + try { canonicalizeJson('\0'.repeat(1024 * 1024 - 2)); } catch (error) { outputRejected = String(error).includes('resource-limit-exceeded'); } + assert(outputRejected, 'expanded canonical JSON output must be rejected'); + + let payloadRejected = false; + try { + buildSigningPayloadV1({ + contentHash: 'x'.repeat(1024 * 1024), + claimsHash: 'sha256:claims', + documentURL: 'https://example.org/article', + scope: 'url', + keyid: 'did:web:example.org', + algorithm: 'ed25519', + signedAt: '2026-05-01T10:30:00Z', + }); + } catch (error) { payloadRejected = String(error).includes('resource-limit-exceeded'); } + assert(payloadRejected, 'oversized signing payload input must be rejected'); +}); + +await check('canonicalizeJsonDocument classifies malformed surrogate JSON as JSON syntax', () => { + let malformed = false; + try { canonicalizeJsonDocument('{"value":"\\uD800'); } catch (error) { + malformed = String(error).includes('jcs-invalid-json'); + } + assert(malformed, 'malformed JSON must precede surrogate classification'); +}); + +await check('extractCanonicalText applies output limit after finalization', () => { + const unit = '

'; + const output = extractCanonicalText(unit.repeat(10000), { baseUrl: 'https://example.com/' }); + assertEq(output.length, 1039999); +}); + await check('decodeCanonicalBase64 rejects padded and base64url forms', () => { assertEq(new TextDecoder().decode(decodeCanonicalBase64('Zm9v')), 'foo'); let padded = false; diff --git a/php/src/Canonicalize.php b/php/src/Canonicalize.php index bdac275..47cc2e8 100644 --- a/php/src/Canonicalize.php +++ b/php/src/Canonicalize.php @@ -93,6 +93,11 @@ public static function normalizeText(string $text, bool $preserveWhitespace = fa if (strlen($text) > self::MAX_RESOURCE_BYTES) { throw new \InvalidArgumentException('resource-limit-exceeded'); } + // PHP strings are byte sequences. Reject malformed UTF-8 before the + // Unicode normalizer or PCRE can replace it or return null. + if (preg_match('//u', $text) !== 1) { + throw new \InvalidArgumentException('parser-profile-unsupported'); + } // Phase 1: Unicode NFKC normalization. // Handles ligatures, fullwidth/halfwidth, presentation forms, @@ -643,9 +648,6 @@ private static function appendCanonicalPart(array &$parts, string $value): void private static function finalizeCanonicalParts(array $parts): string { $text = implode('', $parts); - if (strlen($text) > self::MAX_RESOURCE_BYTES) { - throw new \InvalidArgumentException('resource-limit-exceeded'); - } while (strpos($text, ' ') !== false) { $text = str_replace(' ', ' ', $text); } @@ -653,7 +655,11 @@ private static function finalizeCanonicalParts(array $parts): string while (strpos($text, "\n\n") !== false) { $text = str_replace("\n\n", "\n", $text); } - return trim($text, " \n"); + $text = trim($text); + if (strlen($text) > self::MAX_RESOURCE_BYTES) { + throw new \InvalidArgumentException('resource-limit-exceeded'); + } + return $text; } /** @@ -806,7 +812,9 @@ public static function canonicalizeClaims(array $claims): string $entries = []; $seen = []; foreach ($claims as $name => $value) { - if (!is_string($name) || !is_string($value)) { + // PHP converts decimal string keys such as "1" to integers when + // constructing an array. They still represent valid claim names. + if ((!is_string($name) && !is_int($name)) || !is_string($value)) { throw new \InvalidArgumentException('claim-malformed'); } $normName = trim(self::normalizeText((string) $name)); @@ -1072,6 +1080,11 @@ private static function scanJsonValue(string $s, int &$i, int $depth): void if (is_infinite((float) $token)) { throw new \InvalidArgumentException('jcs-number'); } + // RFC 8785 erratum 7920: reject negative zero, including negative + // values whose magnitude underflows to zero during binary64 parsing. + if ($token[0] === '-' && (float) $token == 0.0) { + throw new \InvalidArgumentException('jcs-number'); + } } /** @param mixed $value */ diff --git a/php/tests/CanonicalizeJcsTest.php b/php/tests/CanonicalizeJcsTest.php index d1ffcd5..e031d21 100644 --- a/php/tests/CanonicalizeJcsTest.php +++ b/php/tests/CanonicalizeJcsTest.php @@ -11,8 +11,8 @@ class CanonicalizeJcsTest extends TestCase public function testCanonicalizesRawDocument(): void { $this->assertSame( - '{"a":1e+30,"b":4.5,"z":0,"πŸ˜€":2,"ξ€€":1}', - Canonicalize::canonicalizeJsonDocument('{"z":-0,"a":1e30,"b":4.50,"πŸ˜€":2,"ξ€€":1}') + '{"a":1e+30,"b":4.5,"πŸ˜€":2,"ξ€€":1}', + Canonicalize::canonicalizeJsonDocument('{"a":1e30,"b":4.50,"πŸ˜€":2,"ξ€€":1}') ); } @@ -22,8 +22,8 @@ public function testNumberFormattingIgnoresSerializePrecision(): void ini_set('serialize_precision', '3'); try { $this->assertSame( - '[0,0,5e-324,1e+23,0.000001,333333333.33333325]', - Canonicalize::canonicalizeJsonDocument('[0,-0,5e-324,1e23,1e-6,333333333.33333325]') + '[0,5e-324,1e+23,0.000001,333333333.33333325]', + Canonicalize::canonicalizeJsonDocument('[0,5e-324,1e23,1e-6,333333333.33333325]') ); } finally { if ($previous !== false) ini_set('serialize_precision', $previous); @@ -54,6 +54,9 @@ public static function unsafeJsonProvider(): array ['{"a":1,"a":2}', 'jcs-duplicate-key'], ['"\\uD800"', 'jcs-invalid-surrogate'], ['{"n":1e400}', 'jcs-number'], + ['{"n":-0}', 'jcs-number'], + ['{"n":-1e-400}', 'jcs-number'], + ['{"value":"\\uD800', 'jcs-invalid-json'], ]; } } diff --git a/php/tests/CanonicalizeTest.php b/php/tests/CanonicalizeTest.php index 5eede7f..f857c7e 100644 --- a/php/tests/CanonicalizeTest.php +++ b/php/tests/CanonicalizeTest.php @@ -114,6 +114,20 @@ public function testPreserveWhitespace(): void $this->assertStringContainsString(' ', $result); } + public function testRejectsMalformedUtf8Source(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('parser-profile-unsupported'); + Canonicalize::normalizeText("bad\xFF"); + } + + public function testSourceLimitPrecedesMalformedUtf8Classification(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('resource-limit-exceeded'); + Canonicalize::normalizeText(str_repeat("\xFF", 1024 * 1024 + 1)); + } + /** @dataProvider malformedFragmentProvider */ public function testRejectsParserAmbiguities(string $html, string $reason): void { @@ -187,4 +201,23 @@ public function testInvalidBaseUrlIsRejectedWithoutUrlAttributes(): void $this->expectExceptionMessage('attribute-canonicalization-failed'); Canonicalize::extractCanonicalText('

text

', false, 'not a URL'); } + + public function testDecimalNumericClaimNamesRemainValid(): void + { + $this->assertSame( + "0:zero\n1:one\n", + Canonicalize::canonicalizeClaims(['1' => 'one', '0' => 'zero']) + ); + } + + public function testOutputLimitAppliesAfterFinalization(): void + { + $unit = '

'; + $output = Canonicalize::extractCanonicalText( + str_repeat($unit, 10000), + false, + 'https://example.com/' + ); + $this->assertSame(1039999, strlen($output)); + } } diff --git a/python/README.md b/python/README.md index f91428f..59ff275 100644 --- a/python/README.md +++ b/python/README.md @@ -74,7 +74,7 @@ content = extract_canonical_text( claims = canonicalize_claims({'License': 'CC-BY-4.0'}) assert claims == 'License:CC-BY-4.0\n' -payload = canonicalize_json_document('{"z":-0,"a":1e30}') +payload = canonicalize_json_document('{"z":0,"a":1e30}') assert payload == '{"a":1e+30,"z":0}' ``` diff --git a/python/htmltrust_canonicalization/_extract.py b/python/htmltrust_canonicalization/_extract.py index ec86b10..ae9796c 100644 --- a/python/htmltrust_canonicalization/_extract.py +++ b/python/htmltrust_canonicalization/_extract.py @@ -10,6 +10,8 @@ from __future__ import annotations +import re + from bs4 import BeautifulSoup, NavigableString, Tag from html5lib.html5parser import HTMLParser from pywhatwgurl import URL @@ -499,7 +501,11 @@ def _escape_text(value: str) -> str: def _finalize_parts(text: str) -> str: while " " in text: text = text.replace(" ", " ") - text = text.replace(" \n", "\n").replace("\n ", "\n") + # Finalization is shared with the other bindings, including for + # preserve_whitespace extraction. Spaces and tabs adjacent to a boundary + # are formatting introduced by the DOM walk and must not remain around + # the emitted line feed. + text = re.sub(r"[ \t]*\n[ \t]*", "\n", text) while "\n\n" in text: text = text.replace("\n\n", "\n") text = text.strip() diff --git a/python/htmltrust_canonicalization/_jcs.py b/python/htmltrust_canonicalization/_jcs.py index 817c01c..5ff741b 100644 --- a/python/htmltrust_canonicalization/_jcs.py +++ b/python/htmltrust_canonicalization/_jcs.py @@ -51,6 +51,10 @@ def _finite(value: str) -> float: raise ValueError("jcs-number") from exc if not math.isfinite(number): raise ValueError("jcs-number") + # RFC 8785 erratum 7920: reject negative zero, including negative values + # whose magnitude underflows to zero during binary64 parsing. + if value.startswith("-") and number == 0.0: + raise ValueError("jcs-number") return number @@ -77,11 +81,17 @@ def canonicalize_json_document(document: str | bytes) -> str: raise TypeError("canonicalize_json_document expects raw JSON text") try: document_bytes = document.encode("utf-8", "strict") if isinstance(document, str) else bytes(document) - document_bytes.decode("utf-8", "strict") - except (UnicodeEncodeError, UnicodeDecodeError) as exc: + except UnicodeEncodeError as exc: raise ValueError("jcs-invalid-surrogate") from exc if len(document_bytes) > _MAX_DOCUMENT_BYTES: raise ValueError("resource-limit-exceeded") + try: + document_bytes.decode("utf-8", "strict") + except UnicodeDecodeError as exc: + # A bytes API receives source octets, so malformed UTF-8 is malformed + # JSON. A Python str containing an actual surrogate is handled above + # as a JCS surrogate violation instead. + raise ValueError("jcs-invalid-json") from exc _enforce_nesting_limit(document_bytes) try: value = json.loads( diff --git a/python/htmltrust_canonicalization/_normalize.py b/python/htmltrust_canonicalization/_normalize.py index be7c054..105fb0e 100644 --- a/python/htmltrust_canonicalization/_normalize.py +++ b/python/htmltrust_canonicalization/_normalize.py @@ -164,7 +164,14 @@ def normalize_text(text: str, preserve_whitespace: bool = False) -> str: """ if not isinstance(text, str): raise TypeError("normalize_text expects a str") - if len(text.encode("utf-8")) > _MAX_RESOURCE_BYTES: + # A gross source-size breach takes precedence over malformed Unicode. + if len(text) > _MAX_RESOURCE_BYTES: + raise ValueError("resource-limit-exceeded") + try: + source_bytes = text.encode("utf-8") + except UnicodeEncodeError as exc: + raise ValueError("parser-profile-unsupported") from exc + if len(source_bytes) > _MAX_RESOURCE_BYTES: raise ValueError("resource-limit-exceeded") # Phase 1: NFKC -- ligatures, fullwidth/halfwidth, presentation forms, diff --git a/python/tests/test_extract.py b/python/tests/test_extract.py index 8d053b5..ae55189 100644 --- a/python/tests/test_extract.py +++ b/python/tests/test_extract.py @@ -81,6 +81,11 @@ def test_normalization_pipeline_applied(): ) +def test_preserve_whitespace_finalization_strips_runs_around_newlines(): + html = "
before \t \n \t after
" + assert extract_canonical_text(html, preserve_whitespace=True) == "before\nafter" + + def test_nested_blocks(): """Deeply nested block structure still collapses repeated line feeds.""" html = ( @@ -122,6 +127,11 @@ def test_parser_preflight_rejects_unclosed_and_foster_parented_text(html): extract_canonical_text(html) +def test_parser_preflight_rejects_invalid_utf8_surrogate(): + with pytest.raises(ValueError, match="parser-profile-unsupported"): + extract_canonical_text("

bad\ud800

") + + def test_colon_qualified_elements_count_toward_depth_limit(): nested = "".join("" for _ in range(257)) closing = "".join("" for _ in range(257)) @@ -152,6 +162,15 @@ def test_invalid_base_url_fails_without_url_attributes(): extract_canonical_text("

x

", base_url="not a URL") +def test_output_limit_applies_after_finalization(): + unit = '

' + output = extract_canonical_text( + unit * 10_000, + base_url="https://example.com/", + ) + assert len(output.encode("utf-8")) == 1_039_999 + + def test_signed_semantic_attributes_are_canonicalized(): html = ( '

link' diff --git a/python/tests/test_jcs.py b/python/tests/test_jcs.py index 704b5e0..49c78d6 100644 --- a/python/tests/test_jcs.py +++ b/python/tests/test_jcs.py @@ -5,8 +5,8 @@ def test_jcs_sorts_utf16_keys_and_numbers(): assert canonicalize_json_document( - '{"z":-0,"a":1e30,"b":4.50,"πŸ˜€":2,"ξ€€":1}' - ) == '{"a":1e+30,"b":4.5,"z":0,"πŸ˜€":2,"ξ€€":1}' + '{"a":1e30,"b":4.50,"πŸ˜€":2,"ξ€€":1}' + ) == '{"a":1e+30,"b":4.5,"πŸ˜€":2,"ξ€€":1}' def test_jcs_uses_binary64_for_large_integer_tokens(): @@ -21,6 +21,8 @@ def test_jcs_uses_binary64_for_large_integer_tokens(): ('{"a":1,"a":2}', "jcs-duplicate-key"), ('"\\uD800"', "jcs-invalid-surrogate"), ('{"n":1e400}', "jcs-number"), + ('{"n":-0}', "jcs-number"), + ('{"n":-1e-400}', "jcs-number"), ], ) def test_jcs_rejects_unsafe_raw_json(document, reason): @@ -32,3 +34,18 @@ def test_jcs_rejects_excessive_nesting(): document = "[" * 257 + "0" + "]" * 257 with pytest.raises(ValueError, match="resource-limit-exceeded"): canonicalize_json_document(document) + + +def test_jcs_malformed_json_precedes_surrogate_classification(): + with pytest.raises(ValueError, match="jcs-invalid-json"): + canonicalize_json_document('{"value":"\\uD800') + + +def test_jcs_source_limit_precedes_malformed_json(): + with pytest.raises(ValueError, match="resource-limit-exceeded"): + canonicalize_json_document("{" * (1024 * 1024 + 1)) + + +def test_jcs_bytes_reject_invalid_utf8_as_malformed_json(): + with pytest.raises(ValueError, match="jcs-invalid-json"): + canonicalize_json_document(b'{"value":"\xff"}') diff --git a/python/tests/test_normalize.py b/python/tests/test_normalize.py index 4863598..a838c88 100644 --- a/python/tests/test_normalize.py +++ b/python/tests/test_normalize.py @@ -69,6 +69,16 @@ def test_preserve_whitespace(): assert normalize_text(src, preserve_whitespace=True) == src +def test_rejects_invalid_utf8_surrogate(): + with pytest.raises(ValueError, match="parser-profile-unsupported"): + normalize_text("\ud800") + + +def test_source_limit_precedes_invalid_utf8_classification(): + with pytest.raises(ValueError, match="resource-limit-exceeded"): + normalize_text("\ud800" * (1024 * 1024 + 1)) + + def test_normalize_text_rejects_non_string(): with pytest.raises(TypeError): normalize_text(123) # type: ignore[arg-type] diff --git a/rust/README.md b/rust/README.md index cacd812..2e348c1 100644 --- a/rust/README.md +++ b/rust/README.md @@ -46,7 +46,8 @@ htmltrust-canonicalization = "0.3" - `try_normalize_text` normalizes a UTF-8 `str` and enforces the 1 MiB source and output limits. -- `try_normalize_text_v1` accepts bytes and also rejects invalid UTF-8. +- `try_normalize_text_v1` accepts bytes and rejects invalid UTF-8 as + `parser-profile-unsupported`. - `try_extract_canonical_text_with_options` parses HTML with explicit compatibility whitespace and base URL options. Profile-v1 callers use `preserve_whitespace: false`; the portable profile rejects nesting deeper diff --git a/rust/src/lib.rs b/rust/src/lib.rs index d78acde..d4b1478 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -25,7 +25,6 @@ use url::Url; /// Maximum size of a source document and its canonical output. pub const MAX_DOCUMENT_BYTES: usize = 1024 * 1024; const MAX_ELEMENT_DEPTH: usize = 256; -const INVALID_UTF8: &str = "invalid-utf8"; const PARSER_UNSUPPORTED: &str = "parser-profile-unsupported"; const RESOURCE_LIMIT: &str = "resource-limit-exceeded"; @@ -220,7 +219,7 @@ pub fn try_normalize_text_v1(text: &[u8], preserve_whitespace: bool) -> Result MAX_DOCUMENT_BYTES { return Err(RESOURCE_LIMIT.to_string()); } - let text = std::str::from_utf8(text).map_err(|_| INVALID_UTF8.to_string())?; + let text = std::str::from_utf8(text).map_err(|_| PARSER_UNSUPPORTED.to_string())?; try_normalize_text(text, preserve_whitespace) } @@ -332,9 +331,6 @@ pub fn try_extract_canonical_text_with_options( options.preserve_whitespace, )?; - if out.len() > MAX_DOCUMENT_BYTES { - return Err(RESOURCE_LIMIT.to_string()); - } let result = finalize_parts(&out, options.preserve_whitespace); if result.len() > MAX_DOCUMENT_BYTES { return Err(RESOURCE_LIMIT.to_string()); @@ -354,9 +350,11 @@ pub fn try_extract_canonical_text_v1( if base_url.is_some_and(|base| base.len() > MAX_DOCUMENT_BYTES) { return Err(RESOURCE_LIMIT.to_string()); } - let html = std::str::from_utf8(html).map_err(|_| INVALID_UTF8.to_string())?; + let html = std::str::from_utf8(html).map_err(|_| PARSER_UNSUPPORTED.to_string())?; let base = match base_url { - Some(bytes) => Some(std::str::from_utf8(bytes).map_err(|_| INVALID_UTF8.to_string())?), + Some(bytes) => { + Some(std::str::from_utf8(bytes).map_err(|_| PARSER_UNSUPPORTED.to_string())?) + } None => None, }; try_extract_canonical_text_with_base_url(html, base) @@ -795,7 +793,7 @@ fn finalize_parts(text: &str, _preserve_whitespace: bool) -> String { while text.contains("\n\n") { text = text.replace("\n\n", "\n"); } - text.trim_matches(&[' ', '\n'][..]).to_string() + text.trim().to_string() } /// Compute the canonical serialization of a claim map. @@ -892,12 +890,33 @@ pub fn canonicalize_json_document(raw: &[u8]) -> Result { return Err("resource-limit-exceeded".to_string()); } enforce_json_nesting_limit(raw)?; + let value = match parse_strict_json(raw) { + Ok(value) => value, + Err(_error) if has_lone_surrogate_escape(raw) => { + // serde_json rejects lone UTF-16 surrogate escapes while parsing, + // so validate the same bytes with surrogate escapes replaced by a + // scalar placeholder. This second parse only distinguishes a + // syntactically valid lone-surrogate document from malformed JSON; + // it never supplies the value used for canonicalization. + let sanitized = replace_surrogate_escapes(raw); + match parse_strict_json(&sanitized) { + Ok(_) => return Err("jcs-invalid-surrogate".to_string()), + Err(sanitized_error) => return Err(map_json_error(sanitized_error)), + } + } + Err(error) => return Err(map_json_error(error)), + }; + // Only classify surrogate escapes after the JSON parser has accepted the + // complete document. A malformed document such as an unterminated string + // containing `\uD800` is jcs-invalid-json, not jcs-invalid-surrogate. if has_lone_surrogate_escape(raw) { return Err("jcs-invalid-surrogate".to_string()); } - let mut de = serde_json::Deserializer::from_slice(raw); - let value = StrictJson::deserialize(&mut de).map_err(map_json_error)?; - de.end().map_err(map_json_error)?; + // RFC 8785 erratum 7920: reject negative zero, including negative values + // whose magnitude underflows to zero during binary64 parsing. + if has_negative_zero_number(raw) { + return Err("jcs-number".to_string()); + } let output = serde_json_canonicalizer::to_string(&value) .map_err(|e| format!("jcs-invalid-json: {e}"))?; if output.len() > MAX_DOCUMENT_BYTES { @@ -906,6 +925,63 @@ pub fn canonicalize_json_document(raw: &[u8]) -> Result { Ok(output) } +/// Detect negative JSON number tokens that parse as IEEE-754 negative zero. +/// This lexical pass is needed because serde_json presents an integer token +/// such as `-0` to the visitor as the signed integer zero, losing its sign. +fn has_negative_zero_number(raw: &[u8]) -> bool { + let mut in_string = false; + let mut escaped = false; + let mut index = 0; + while index < raw.len() { + let byte = raw[index]; + if in_string { + if escaped { + escaped = false; + } else if byte == b'\\' { + escaped = true; + } else if byte == b'"' { + in_string = false; + } + index += 1; + continue; + } + if byte == b'"' { + in_string = true; + index += 1; + continue; + } + if byte == b'-' { + let start = index; + index += 1; + while index < raw.len() + && !matches!( + raw[index], + b' ' | b'\t' | b'\r' | b'\n' | b',' | b']' | b'}' + ) + { + index += 1; + } + if let Ok(token) = std::str::from_utf8(&raw[start..index]) { + if let Ok(number) = token.parse::() { + if number == 0.0 && number.is_sign_negative() { + return true; + } + } + } + continue; + } + index += 1; + } + false +} + +fn parse_strict_json(raw: &[u8]) -> Result { + let mut de = serde_json::Deserializer::from_slice(raw); + let value = StrictJson::deserialize(&mut de)?; + de.end()?; + Ok(value) +} + fn enforce_json_nesting_limit(raw: &[u8]) -> Result<(), String> { const MAX_NESTING_DEPTH: usize = 256; let mut depth = 0usize; @@ -939,9 +1015,7 @@ fn enforce_json_nesting_limit(raw: &[u8]) -> Result<(), String> { fn map_json_error(error: serde_json::Error) -> String { let msg = error.to_string(); - if msg.contains("surrogate") { - "jcs-invalid-surrogate".to_string() - } else if msg.contains("number out of range") || msg.contains("invalid number") { + if msg.contains("number out of range") || msg.contains("invalid number") { "jcs-number".to_string() } else if msg.contains("duplicate object key") { "jcs-duplicate-key".to_string() @@ -950,6 +1024,37 @@ fn map_json_error(error: serde_json::Error) -> String { } } +fn replace_surrogate_escapes(raw: &[u8]) -> Vec { + let mut output = Vec::with_capacity(raw.len()); + let mut i = 0; + while i < raw.len() { + let slash_run = if raw[i] == b'\\' { + let mut count = 0; + let mut p = i; + while p > 0 && raw[p - 1] == b'\\' { + count += 1; + p -= 1; + } + count + } else { + 0 + }; + if raw[i] == b'\\' + && slash_run % 2 == 0 + && i + 5 < raw.len() + && raw[i + 1] == b'u' + && hex4(&raw[i + 2..i + 6]).is_some_and(|v| (0xD800..=0xDFFF).contains(&v)) + { + output.extend_from_slice(br"\uFFFD"); + i += 6; + } else { + output.push(raw[i]); + i += 1; + } + } + output +} + fn hex4(bytes: &[u8]) -> Option { if bytes.len() < 4 { return None; diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index 51cb2db..3166e87 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -21,6 +21,16 @@ fn jcs_rejects_excessive_nesting() { ); } +#[test] +fn jcs_rejects_negative_zero_and_underflow() { + for document in [&br#"{"value":-0}"#[..], &br#"{"value":-1e-400}"#[..]] { + assert_eq!( + canonicalize_json_document(document), + Err("jcs-number".to_string()) + ); + } +} + /// One conformance vector. `(input_a, input_b, should_match, description)`. type Case = (&'static str, &'static str, bool, &'static str); @@ -163,11 +173,64 @@ fn fallible_text_apis_enforce_source_and_output_limits() { fn fallible_text_apis_reject_invalid_utf8() { assert_eq!( htmltrust_canonicalization::try_normalize_text_v1(b"\xff", false), - Err("invalid-utf8".into()) + Err("parser-profile-unsupported".into()) ); assert_eq!( htmltrust_canonicalization::try_extract_canonical_text_v1(b"\xff", None), - Err("invalid-utf8".into()) + Err("parser-profile-unsupported".into()) + ); + assert_eq!( + htmltrust_canonicalization::try_normalize_text_v1( + &vec![0xff; MAX_DOCUMENT_BYTES + 1], + false, + ), + Err("resource-limit-exceeded".into()) + ); +} + +#[test] +fn checked_claim_map_enforces_normalized_field_limits() { + let mut claims = BTreeMap::new(); + claims.insert("x".repeat(4097), "value".to_string()); + assert_eq!( + htmltrust_canonicalization::canonicalize_claims_checked(&claims), + Err("resource-limit-exceeded".into()) + ); + + let mut claims = BTreeMap::new(); + claims.insert("name".to_string(), "x".repeat(4097)); + assert_eq!( + htmltrust_canonicalization::canonicalize_claims_checked(&claims), + Err("resource-limit-exceeded".into()) + ); +} + +#[test] +fn extraction_applies_output_limit_after_finalization() { + let unit = r#"

"#; + let source = unit.repeat(10_000); + let output = htmltrust_canonicalization::try_extract_canonical_text_with_base_url( + &source, + Some("https://example.com/"), + ) + .expect("finalized output is within the limit"); + assert_eq!(output.len(), 1_039_999); +} + +#[test] +fn malformed_json_precedes_surrogate_classification() { + let error = canonicalize_json_document(br#"{"value":"\uD800"#).unwrap_err(); + assert!( + error.starts_with("jcs-invalid-json:"), + "unexpected error: {error}" + ); +} + +#[test] +fn valid_json_lone_surrogate_is_classified_after_parsing() { + assert_eq!( + canonicalize_json_document(br#""\uD800""#), + Err("jcs-invalid-surrogate".into()) ); }