From 8a14ad6676f984c8d90bd60989bd655c60cde802 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 22:57:07 -0400 Subject: [PATCH] forward-serve: whitespace around a reasoning block is layout, not text (Refs #1786) 'Answer with one word' came back as "\n\nParis", and a model that thinks nothing sent a reasoning chunk of two newlines. The whitespace leading the reasoning, and the whitespace leading content that follows a block, are dropped: the rule app/think_split.h applies on the in-process route. Two existing arms pinned the newline after the opening tag as part of the reasoning; they are updated. Co-Authored-By: Claude Fable 5.2 --- app/npu_answer_parser.h | 22 ++++++++++++++-- docs/npu-chat-template.md | 7 +++++ tests/npu_answer_parser_test.cpp | 44 ++++++++++++++++++++++++++++++-- tests/npu_sampling_test.cpp | 6 +++-- 4 files changed, 73 insertions(+), 6 deletions(-) diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index ab1c95ee2..9d01d74e6 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -180,8 +180,22 @@ class AnswerStream { private: enum class Mode { content, reasoning, call }; - void emit(std::vector>& out, bool is_reasoning, const std::string& text) { - if (text.empty()) return; + // Whitespace around a reasoning block is the model's layout, not part of either text: `\n...\n\n\n` + // then the answer. Left in, the answer to "one word" comes back as "\n\nParis", and a model that thinks nothing + // (`\n\n`) sends a reasoning chunk that is two newlines. So the whitespace that LEADS the reasoning + // is dropped, and so is the whitespace that leads the content when that content follows a reasoning block -- + // the rule app/think_split.h has always applied on the in-process route ("whitespace around the block is + // dropped"), so the two routes return the same strings. Content with no reasoning before it is not touched. + void emit(std::vector>& out, bool is_reasoning, const std::string& raw) { + if (raw.empty()) return; + std::string text = raw; + bool& trim = is_reasoning ? trim_reasoning_ : trim_content_; + if (trim) { + const size_t b = text.find_first_not_of(" \t\r\n"); + if (b == std::string::npos) return; // all whitespace so far: nothing to send yet + text.erase(0, b); + trim = false; + } if (is_reasoning) { reasoning_ += text; saw_reasoning_ = true; @@ -221,6 +235,8 @@ class AnswerStream { } if (starts(tags_.think_close)) { flush_plain(); + // what follows a block is separated from it by layout whitespace -- unless content came first + if (mode_ == Mode::reasoning && !saw_text_) trim_content_ = true; mode_ = Mode::content; i += std::string(tags_.think_close).size(); continue; @@ -377,6 +393,8 @@ class AnswerStream { std::vector calls_; bool saw_reasoning_ = false; bool saw_text_ = false; + bool trim_reasoning_ = true; // until the reasoning's first non-whitespace character + bool trim_content_ = false; // set when a reasoning block closes before any content; cleared by the first text }; // Does the rendered prompt leave the model INSIDE a reasoning block? Some templates end their generation prompt diff --git a/docs/npu-chat-template.md b/docs/npu-chat-template.md index b8581feef..00d258ea4 100644 --- a/docs/npu-chat-template.md +++ b/docs/npu-chat-template.md @@ -207,6 +207,13 @@ it did not: the whole reasoning came back in `content`, `reasoning_content` was dropped, so nothing in the reply marked where the answer began. With thinking off the same template opens and closes the block in the prompt, and the answer is content from its first token. +**Whitespace around the block is layout.** A model writes `\n…\n\n\n` and then its answer. The +whitespace that leads the reasoning is not returned, and neither is the whitespace that leads the content when the +content follows a reasoning block, so "answer with one word" comes back as `Paris`, not `\n\nParis`, and a model +that thinks nothing (`\n\n`) sends no reasoning chunk at all. Whitespace inside either text is +kept, and content with no reasoning before it is returned exactly as written. This is the rule `app/think_split.h` +applies on the in-process route, so both routes return the same strings. + One exception: `tool_choice: "required"` or a named function installs a grammar that constrains the reply to the `qwen3` call markup for every container (`app/npu_tool_grammar.h`), so that reply is split as `qwen3`. A grammar in Gemma's own call syntax is not written yet. diff --git a/tests/npu_answer_parser_test.cpp b/tests/npu_answer_parser_test.cpp index e66c93951..428451076 100644 --- a/tests/npu_answer_parser_test.cpp +++ b/tests/npu_answer_parser_test.cpp @@ -151,7 +151,7 @@ int main() { "without the flag the reasoning is filed under content, with the closing tag dropped, so nothing marks where the answer starts (the measured fault)"); const auto a = parse_answer(reply, Q, false, /*starts_in_reasoning=*/true); check(a.reasoning == "Thinking Process:\n1. The capital of France is Paris.\n", "with it the reasoning is the reasoning"); - check(a.content == "\n\nParis" && !contains(a.content, "think"), "and the content is the answer, with no tag in it"); + check(a.content == "Paris" && !contains(a.content, "think"), "and the content is the answer, with no tag and no layout newlines in it"); // an answer cut off inside the block: all reasoning, nothing invented as content const auto cut = parse_answer("Thinking Process:\n1. Analyze", Q, false, true); @@ -177,10 +177,50 @@ int main() { // per-token mask: reasoning tokens get no logprob entry, content tokens do const std::vector toks = {"Thinking", " Process", "", "\n\n", "Paris"}; const std::vector m = content_mask(toks, Q, false, true); - check(m == std::vector({0, 0, 0, 1, 1}), "the content mask starts after the closing tag"); + check(m == std::vector({0, 0, 0, 0, 1}), "the content mask starts at the answer, after the closing tag and its layout"); check(content_mask(toks, Q, false) == std::vector({1, 1, 0, 1, 1}), "where without the flag the reasoning tokens counted as content"); } + // Whitespace around a reasoning block is layout (the rule app/think_split.h applies on the in-process route). + { + const AnswerFamily Q = AnswerFamily::qwen3; + const auto a = parse_answer("\nthe capital is Paris\n\n\nParis", Q, false); + check(a.reasoning == "the capital is Paris\n" && a.content == "Paris", + "the newline after and the blank line after are not part of either text"); + const auto none = parse_answer("\n\n\n\nItem 3 is the 3D printer.", Q, false); + check(none.reasoning.empty() && none.content == "Item 3 is the 3D printer.", + "a model that thinks nothing sends no reasoning at all, and the answer starts at its first character"); + const auto plain = parse_answer("\n indented answer", Q, false); + check(plain.content == "\n indented answer", "content with no reasoning before it is returned as written"); + const auto mid = parse_answer("First.x Second.", Q, false); + check(mid.content == "First. Second.", "content that came before a block keeps the space that follows the block"); + const auto inner = parse_answer("a\n\n bone\n\n two", Q, false); + check(inner.reasoning == "a\n\n b" && inner.content == "one\n\n two", "whitespace inside either text is kept"); + const auto ws = parse_answer("x\n\n", Q, false); + check(ws.content.empty() && ws.reasoning == "x", "an answer that is only layout after the block is empty content"); + + // streamed: the layout arrives in its own pieces and none of them is sent + AnswerStream st(Q, false); + std::vector> out; + for (const std::string& piece : {std::string(""), std::string("\n"), std::string("\n"), std::string(""), + std::string("\n\n"), std::string("Item"), std::string(" 3")}) + st.feed(piece, out); + st.finish(out); + check(out.size() == 2 && !out[0].first && out[0].second == "Item" && out[1].second == " 3", + "streamed, an empty block and its layout produce no chunk; the first chunk is the answer's first word"); + + // the per-token mask agrees: layout tokens are not content tokens + const std::vector m = content_mask({"", "\n", "why", "", "\n\n", "Paris", "\n"}, Q, false); + check(m == std::vector({0, 0, 0, 0, 0, 1, 1}), "layout tokens after the block get no content entry; the answer's own do"); + + // a tool call straight after the block is unaffected + const auto call = parse_answer("t\n{\"name\":\"ping\",\"arguments\":{}}", Q, true); + check(call.calls.size() == 1 && call.content_null, "a call after the block and a newline is a call with null content"); + // without tools the marker is text and is not eaten by the trim + const auto text = parse_answer("t\nx", Q, false); + check(text.content == "x", "the first text after the block is kept whole, whatever it starts with"); + } + std::printf("%s\n", failures ? "FAILED" : "all cases ok"); return failures ? 1 : 0; } diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index e791743f6..515695117 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -844,7 +844,8 @@ void test_gemma4_answer_parser() { } { const auto a = parse_answer("<|channel>thought\nlet me thinkDone.", G, false); - check(a.reasoning == "\nlet me think", "gemma4: the thought channel is reasoning"); + check(a.reasoning == "let me think" && a.content == "Done.", + "gemma4: the thought channel is reasoning, without the newline the template puts after the tag"); check(a.content == "Done.", "gemma4: and the reply after it is content"); } { @@ -920,7 +921,8 @@ void test_answer_parser() { const auto F = AnswerFamily::qwen3; { // reasoning, then content: the Qwen3 shape this route sees on every answer const auto a = parse_answer("\nOkay, the user asked.", F, false); - check(a.reasoning == "\nOkay, the user asked.", "reasoning_content is the inside of "); + check(a.reasoning == "Okay, the user asked.", + "reasoning_content is the inside of , without the newline the model puts after the tag"); check(a.content.empty() && a.calls.empty(), "a reasoning-only answer has no content and no calls"); check(!a.content_null, "content is empty string, not null, when there are no calls");