From 025e690f1baabb65933cc7e5f48c5ba764c69b24 Mon Sep 17 00:00:00 2001 From: jaideeppyne Date: Fri, 28 Aug 2026 06:42:32 +0530 Subject: [PATCH 1/4] Apply GB5 before GB9b in GraphemeCursor::provide_context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `provide_context` short-circuits to "no break" whenever the last codepoint of the supplied pre-context chunk is `Prepend`, applying GB9b directly. GB4 and GB5 are ordered before GB9b in UAX #29, so that shortcut is wrong when the codepoint after the cursor is `Control`, `CR` or `LF`: there is a break there regardless of the `Prepend`. GraphemeBreakTest-17.0.0.txt spells this out, attributing the break to rule 5.0: ÷ 06DD ÷ 000D ÷ # ÷ [0.2] ARABIC END OF AYAH (Prepend) ÷ [5.0] (CR) ÷ [0.3] `check_pair` already orders the rules correctly, so the `Graphemes` iterator and any cursor given the whole string are unaffected; only a cursor answering the query across a chunk boundary disagreed: let s = "\u{06dd}\r"; let mut c = GraphemeCursor::new(2, s.len(), true); c.is_boundary(&s[2..], 2); // Err(PreContext(2)) c.provide_context(&s[..2], 0); c.is_boundary(&s[2..], 2); // Ok(false), should be Ok(true) Skip the shortcut when the following category is one GB4/GB5 decides. The remaining `_` arm of the `match self.state` below already records `cat_before`, so the retried `is_boundary` reaches `check_pair` and applies the rules in the right order. The `InCbConsonant`, `Regional` and `Emoji` states are unreachable with a `Control`/`CR`/`LF` after the cursor, so their reliance on the shortcut is untouched. Found by running the cursor over the whole of GraphemeBreakTest.txt with the input split into one chunk per codepoint. --- src/grapheme.rs | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/grapheme.rs b/src/grapheme.rs index 0b39f0c..05e2a3a 100644 --- a/src/grapheme.rs +++ b/src/grapheme.rs @@ -441,7 +441,12 @@ impl GraphemeCursor { use crate::tables::grapheme as gr; assert!(chunk_start.saturating_add(chunk.len()) == self.pre_context_offset.unwrap()); self.pre_context_offset = None; - if self.is_extended && chunk_start + chunk.len() == self.offset { + // GB4 and GB5 come before GB9b, so a Control, CR or LF after the + // Prepend still breaks; only apply the GB9b shortcut when the + // following category cannot be decided by an earlier rule. + let after_breaks_first = + matches!(self.cat_after, Some(gr::GC_Control | gr::GC_CR | gr::GC_LF)); + if self.is_extended && !after_breaks_first && chunk_start + chunk.len() == self.offset { let ch = chunk.chars().next_back().unwrap(); if self.grapheme_category(ch) == gr::GC_Prepend { self.decide(false); // GB9b @@ -918,6 +923,29 @@ fn test_grapheme_cursor_chunk_start_require_precontext() { assert_eq!(c.is_boundary(&s[1..], 1), Ok(false)); } +#[test] +fn test_grapheme_cursor_prepend_before_control_on_chunk_start() { + // GB4 and GB5 take precedence over GB9b, so `Prepend` does not glue a + // following `Control`, `CR` or `LF` to itself. GraphemeBreakTest.txt: + // ÷ 06DD ÷ 000D ÷ # ÷ [0.2] ARABIC END OF AYAH (Prepend) ÷ [5.0] (CR) ÷ [0.3] + for s in ["\u{06dd}\r", "\u{06dd}\n", "\u{06dd}\0"] { + let prepend_len = "\u{06dd}".len(); + // The whole-string answer, which the `Graphemes` iterator already gets right. + assert_eq!( + GraphemeCursor::new(prepend_len, s.len(), true).is_boundary(s, 0), + Ok(true) + ); + // The same query answered from separate chunks must agree. + let mut c = GraphemeCursor::new(prepend_len, s.len(), true); + assert_eq!( + c.is_boundary(&s[prepend_len..], prepend_len), + Err(GraphemeIncomplete::PreContext(prepend_len)) + ); + c.provide_context(&s[..prepend_len], 0); + assert_eq!(c.is_boundary(&s[prepend_len..], prepend_len), Ok(true)); + } +} + #[test] fn test_grapheme_cursor_prev_boundary() { let s = "abcd"; From f3f031c4604b18acef56e60ee8a44f4796aa2aa5 Mon Sep 17 00:00:00 2001 From: jaideeppyne Date: Fri, 28 Aug 2026 07:58:26 +0530 Subject: [PATCH 2/4] Add manual test cases for Prepend before Control/CR/LF Adds the segmentation expectations to EXTRA_SAME so they can be read off and compared against other implementations, plus a chunked GraphemeCursor test, since tests/test.rs previously only exercised the graphemes iterator and so never reached provide_context. --- src/grapheme.rs | 30 +------------------------- tests/test.rs | 56 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 29 deletions(-) diff --git a/src/grapheme.rs b/src/grapheme.rs index 05e2a3a..0b39f0c 100644 --- a/src/grapheme.rs +++ b/src/grapheme.rs @@ -441,12 +441,7 @@ impl GraphemeCursor { use crate::tables::grapheme as gr; assert!(chunk_start.saturating_add(chunk.len()) == self.pre_context_offset.unwrap()); self.pre_context_offset = None; - // GB4 and GB5 come before GB9b, so a Control, CR or LF after the - // Prepend still breaks; only apply the GB9b shortcut when the - // following category cannot be decided by an earlier rule. - let after_breaks_first = - matches!(self.cat_after, Some(gr::GC_Control | gr::GC_CR | gr::GC_LF)); - if self.is_extended && !after_breaks_first && chunk_start + chunk.len() == self.offset { + if self.is_extended && chunk_start + chunk.len() == self.offset { let ch = chunk.chars().next_back().unwrap(); if self.grapheme_category(ch) == gr::GC_Prepend { self.decide(false); // GB9b @@ -923,29 +918,6 @@ fn test_grapheme_cursor_chunk_start_require_precontext() { assert_eq!(c.is_boundary(&s[1..], 1), Ok(false)); } -#[test] -fn test_grapheme_cursor_prepend_before_control_on_chunk_start() { - // GB4 and GB5 take precedence over GB9b, so `Prepend` does not glue a - // following `Control`, `CR` or `LF` to itself. GraphemeBreakTest.txt: - // ÷ 06DD ÷ 000D ÷ # ÷ [0.2] ARABIC END OF AYAH (Prepend) ÷ [5.0] (CR) ÷ [0.3] - for s in ["\u{06dd}\r", "\u{06dd}\n", "\u{06dd}\0"] { - let prepend_len = "\u{06dd}".len(); - // The whole-string answer, which the `Graphemes` iterator already gets right. - assert_eq!( - GraphemeCursor::new(prepend_len, s.len(), true).is_boundary(s, 0), - Ok(true) - ); - // The same query answered from separate chunks must agree. - let mut c = GraphemeCursor::new(prepend_len, s.len(), true); - assert_eq!( - c.is_boundary(&s[prepend_len..], prepend_len), - Err(GraphemeIncomplete::PreContext(prepend_len)) - ); - c.provide_context(&s[..prepend_len], 0); - assert_eq!(c.is_boundary(&s[prepend_len..], prepend_len), Ok(true)); - } -} - #[test] fn test_grapheme_cursor_prev_boundary() { let s = "abcd"; diff --git a/tests/test.rs b/tests/test.rs index 5a89234..4babc1a 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -45,6 +45,13 @@ fn test_graphemes() { "\u{1F938}\u{1F3FE}\u{1F3FE}", &["\u{1F938}\u{1F3FE}\u{1F3FE}"], ), + // GB4 and GB5 come before GB9b, so a Prepend does not join to a + // following Control, CR or LF. ARABIC END OF AYAH is Prepend. + ("\u{06DD}\r", &["\u{06DD}", "\r"]), + ("\u{06DD}\n", &["\u{06DD}", "\n"]), + ("\u{06DD}\u{0000}", &["\u{06DD}", "\u{0000}"]), + // ARABIC NUMBER SIGN, the other Prepend used in these tests + ("\u{0600}\r", &["\u{0600}", "\r"]), ]; for &(s, g) in TEST_SAME.iter().chain(EXTRA_SAME) { @@ -107,6 +114,55 @@ fn test_graphemes() { assert_eq!(gr, b); } +// The `graphemes` iterator sees the whole string at once, so it always had these +// right. `GraphemeCursor` answers the same question one chunk at a time, and +// `provide_context` applied GB9b directly on a trailing Prepend without checking +// whether GB4 or GB5 had already decided the boundary. Feeding the input one +// codepoint at a time is the smallest way to reach that path. +#[test] +fn test_grapheme_cursor_chunked_matches_iterator() { + use unicode_segmentation::{GraphemeCursor, GraphemeIncomplete}; + + const CASES: &[&str] = &[ + "\u{06DD}\r", + "\u{06DD}\n", + "\u{06DD}\u{0000}", + "\u{0600}\r", + "\u{0600}a", + "a\u{06DD}\r", + ]; + + for &s in CASES { + for is_extended in [true, false] { + // Prepend is only joined in extended mode, so the expected + // boundaries have to come from the matching iterator. + let expected: Vec = std::iter::once(0) + .chain( + UnicodeSegmentation::grapheme_indices(s, is_extended).map(|(i, g)| i + g.len()), + ) + .collect(); + + for (offset, _) in s.char_indices().skip(1) { + let mut cursor = GraphemeCursor::new(offset, s.len(), is_extended); + let boundary = loop { + match cursor.is_boundary(&s[offset..], offset) { + Ok(b) => break b, + Err(GraphemeIncomplete::PreContext(n)) => { + cursor.provide_context(&s[..n], 0); + } + Err(e) => panic!("{s:?} at {offset}: unexpected {e:?}"), + } + }; + assert_eq!( + boundary, + expected.contains(&offset), + "{s:?} extended={is_extended} offset={offset}" + ); + } + } + } +} + #[test] fn test_words() { use crate::testdata::TEST_WORD; From 23d592368106647e2362c2125179eb353fcea224 Mon Sep 17 00:00:00 2001 From: jaideeppyne Date: Fri, 28 Aug 2026 07:59:12 +0530 Subject: [PATCH 3/4] Restore the provide_context fix The previous commit accidentally reverted src/grapheme.rs while adding the integration tests. --- src/grapheme.rs | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/grapheme.rs b/src/grapheme.rs index 0b39f0c..05e2a3a 100644 --- a/src/grapheme.rs +++ b/src/grapheme.rs @@ -441,7 +441,12 @@ impl GraphemeCursor { use crate::tables::grapheme as gr; assert!(chunk_start.saturating_add(chunk.len()) == self.pre_context_offset.unwrap()); self.pre_context_offset = None; - if self.is_extended && chunk_start + chunk.len() == self.offset { + // GB4 and GB5 come before GB9b, so a Control, CR or LF after the + // Prepend still breaks; only apply the GB9b shortcut when the + // following category cannot be decided by an earlier rule. + let after_breaks_first = + matches!(self.cat_after, Some(gr::GC_Control | gr::GC_CR | gr::GC_LF)); + if self.is_extended && !after_breaks_first && chunk_start + chunk.len() == self.offset { let ch = chunk.chars().next_back().unwrap(); if self.grapheme_category(ch) == gr::GC_Prepend { self.decide(false); // GB9b @@ -918,6 +923,29 @@ fn test_grapheme_cursor_chunk_start_require_precontext() { assert_eq!(c.is_boundary(&s[1..], 1), Ok(false)); } +#[test] +fn test_grapheme_cursor_prepend_before_control_on_chunk_start() { + // GB4 and GB5 take precedence over GB9b, so `Prepend` does not glue a + // following `Control`, `CR` or `LF` to itself. GraphemeBreakTest.txt: + // ÷ 06DD ÷ 000D ÷ # ÷ [0.2] ARABIC END OF AYAH (Prepend) ÷ [5.0] (CR) ÷ [0.3] + for s in ["\u{06dd}\r", "\u{06dd}\n", "\u{06dd}\0"] { + let prepend_len = "\u{06dd}".len(); + // The whole-string answer, which the `Graphemes` iterator already gets right. + assert_eq!( + GraphemeCursor::new(prepend_len, s.len(), true).is_boundary(s, 0), + Ok(true) + ); + // The same query answered from separate chunks must agree. + let mut c = GraphemeCursor::new(prepend_len, s.len(), true); + assert_eq!( + c.is_boundary(&s[prepend_len..], prepend_len), + Err(GraphemeIncomplete::PreContext(prepend_len)) + ); + c.provide_context(&s[..prepend_len], 0); + assert_eq!(c.is_boundary(&s[prepend_len..], prepend_len), Ok(true)); + } +} + #[test] fn test_grapheme_cursor_prev_boundary() { let s = "abcd"; From 17ee61e4c4efe9863f08147d8d676893ca73cdc1 Mon Sep 17 00:00:00 2001 From: jaideeppyne Date: Wed, 2 Sep 2026 09:58:04 +0530 Subject: [PATCH 4/4] Fix test formatting for Rust 2018 --- tests/test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test.rs b/tests/test.rs index 4babc1a..a93ad37 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -150,7 +150,7 @@ fn test_grapheme_cursor_chunked_matches_iterator() { Err(GraphemeIncomplete::PreContext(n)) => { cursor.provide_context(&s[..n], 0); } - Err(e) => panic!("{s:?} at {offset}: unexpected {e:?}"), + Err(e) => panic!("{:?} at {}: unexpected {:?}", s, offset, e), } }; assert_eq!(