diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index b5da1fa66..4027f26e8 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -153,6 +153,10 @@ struct CallStats { double prompt_ms = 0; double predicted_ms = 0; npu::PrefixDecision prefix{-1, 0, "off"}; // "off" when --prefix-slots 0 + // engine#2111: each generated token's text, in order, and how many of them were reasoning. -1 = not counted (a + // /v1/completions reply is raw text with no split), so the usage object then carries no details at all. + std::vector pieces; + int reasoning_tokens = -1; }; // One generated token's log-probability entry, with the text that token produced. The two travel @@ -549,6 +553,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // pre-top_k/top_p/temperature and the grammar mask cannot narrow it. if (want_lp) held.emplace_back(Emitted{piece, npu::logprobs_of(logits, next, opts.sample.top_logprobs)}, piece.size()); + stats.pieces.push_back(piece); // engine#2111: reasoning_tokens is counted over these ++produced; const size_t stop_at = npu::find_stop(pending, opts.stop); if (stop_at != std::string::npos) { @@ -597,14 +602,19 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // OpenAI's usage object plus the two fields issue #650 reports: how many prompt tokens came // from a saved prefix, and what the cache decided (which slot, and why). auto usage_json = [](const CallStats& st) { - return json{{"prompt_tokens", st.prompt_tokens}, - {"completion_tokens", st.completion_tokens}, - {"total_tokens", st.prompt_tokens + st.completion_tokens}, - {"prompt_tokens_details", {{"cached_tokens", st.prefix.cached_tokens}}}, - {"flm_prefix_cache", {{"slot", st.prefix.slot}, - {"cached_tokens", st.prefix.cached_tokens}, - {"reason", st.prefix.reason}, - {"pinned", st.prefix.pinned}}}}; + json u{{"prompt_tokens", st.prompt_tokens}, + {"completion_tokens", st.completion_tokens}, + {"total_tokens", st.prompt_tokens + st.completion_tokens}, + {"prompt_tokens_details", {{"cached_tokens", st.prefix.cached_tokens}}}, + {"flm_prefix_cache", {{"slot", st.prefix.slot}, + {"cached_tokens", st.prefix.cached_tokens}, + {"reason", st.prefix.reason}, + {"pinned", st.prefix.pinned}}}}; + // engine#2111: OpenAI's field for "how much of the budget went into reasoning". With content "" and + // finish_reason "length", reasoning_tokens == completion_tokens is the sentence that says the budget ran out + // inside a thinking block, rather than that the model said nothing. Additive: absent where nothing was split. + if (st.reasoning_tokens >= 0) u["completion_tokens_details"] = {{"reasoning_tokens", st.reasoning_tokens}}; + return u; }; // OpenAI's logprobs.content entry: the token's text, its log-probability, and the most likely @@ -639,6 +649,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // parts; a reply that is only tool calls says content: null. const npu::ParsedAnswer ans = npu::parse_answer(content, family, opts.tools_sent, starts_in_reasoning); + stats.reasoning_tokens = npu::reasoning_token_count(stats.pieces, family, opts.tools_sent, starts_in_reasoning); json message{{"role", "assistant"}}; message["content"] = ans.content_null ? json(nullptr) : json(ans.content); if (!ans.reasoning.empty()) message["reasoning_content"] = ans.reasoning; @@ -767,6 +778,10 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib { json tail = chunk_tpl; tail["choices"] = json::array(); + // engine#2111: the same count as the non-streamed reply's, over the same pieces. + if (chat) + stats.reasoning_tokens = + npu::reasoning_token_count(stats.pieces, family, opts.tools_sent, starts_in_reasoning); tail["usage"] = usage_json(stats); tail["timings"] = timings_json(stats); send("data: " + tail.dump() + "\n\n"); diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index 9d01d74e6..f6453fcb0 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -172,6 +172,9 @@ class AnswerStream { // The calls seen so far, in order. A call needs tools_sent, per the issue's rule. const std::vector& calls() const { return calls_; } bool saw_reasoning() const { return saw_reasoning_; } + // Inside a reasoning block right now (after the opening tag, before the closing one). reasoning_token_count() + // reads it before and after each token, so the tags themselves and held-back partial tags count with the block. + bool in_reasoning() const { return mode_ == Mode::reasoning; } bool saw_text() const { return saw_text_; } const std::string& reasoning() const { return reasoning_; } @@ -442,6 +445,57 @@ inline std::vector content_mask(const std::vector& pieces, An return mask; } +// How many of `pieces` (one generated token's text each, in order) were reasoning: OpenAI's +// usage.completion_tokens_details.reasoning_tokens (engine#2111). An empty `content` with finish_reason "length" reads +// like "the model said nothing" when the whole budget went into a thinking block, and this is the number that says so. +// A token is reasoning when none of its text reached `content` and the block was open before it, after it, or it put +// text into the block: so the opening and closing tags, a partial tag the holdback keeps, and an unclosed block at the +// end of the budget all count, while call markup and the whitespace after a closed block do not. A token that +// straddles the boundary ("The") is content, as content_mask() says. Same splitter, so the count, the split +// and the logprob entries cannot disagree. +inline int reasoning_token_count(const std::vector& pieces, AnswerFamily family, bool tools_sent, + bool starts_in_reasoning = false) { + AnswerStream s(family, tools_sent, starts_in_reasoning); + std::vector> out; + int n = 0; + bool last_counted = false; + // Tokens that produced nothing before anything was produced (leading whitespace, the first half of an opening tag + // the holdback keeps) belong to whatever comes next: to the block if the next token opens it, to nothing otherwise. + int pending = 0; + bool started = false; + for (const auto& piece : pieces) { + const bool before = s.in_reasoning(); + out.clear(); + s.feed(piece, out); + bool content = false, reasoning = false; + for (const auto& [is_reasoning, text] : out) { + (void)text; + (is_reasoning ? reasoning : content) = true; + } + last_counted = !content && (reasoning || before || s.in_reasoning()); + if (last_counted) { + ++n; + if (!started) n += pending; + started = true; + } else if (content) { + started = true; + } else if (!started) { + ++pending; + } + } + // What the holdback still holds is flushed by finish(): if it is content, the last token's text reached `content`. + out.clear(); + s.finish(out); + for (const auto& [is_reasoning, text] : out) { + (void)text; + if (!is_reasoning && last_counted) { + --n; + last_counted = false; + } + } + return n; +} + // The whole answer at once: the same splitter, so the two paths cannot disagree. inline ParsedAnswer parse_answer(const std::string& text, AnswerFamily family, bool tools_sent, bool starts_in_reasoning = false) { diff --git a/app/think_split.h b/app/think_split.h index 5ba670400..efb06201f 100644 --- a/app/think_split.h +++ b/app/think_split.h @@ -16,6 +16,7 @@ #include #include +#include namespace onebit { @@ -73,4 +74,35 @@ struct ThinkSplit { } }; +// How many of `pieces` (one generated token's text each, in order) were reasoning, for usage.completion_tokens_details +// .reasoning_tokens (engine#2111): the twin of npu::reasoning_token_count() over this route's splitter. A token is +// reasoning when none of its text reached content and the block was open before it, after it, or it put text into +// it -- so both tags, a held-back partial tag and an unclosed block at the end of the budget count. +inline int think_reasoning_tokens(const std::vector& pieces, bool opened) { + ThinkSplit s(opened); + int n = 0; + bool last_counted = false; + int pending = 0; // leading tokens that produced nothing yet: they join the block if the next one opens it + bool started = false; + for (const auto& piece : pieces) { + const bool before = s.in_think; + bool content = false, reasoning = false; + s.feed(piece, [&](bool r, const std::string&) { (r ? reasoning : content) = true; }); + last_counted = !content && (reasoning || before || s.in_think); + if (last_counted) { + ++n; + if (!started) n += pending; + started = true; + } else if (content) { + started = true; + } else if (!started) { + ++pending; + } + } + bool flushed_content = false; + s.flush([&](bool r, const std::string&) { if (!r) flushed_content = true; }); + if (flushed_content && last_counted) --n; + return n; +} + } // namespace onebit diff --git a/app/unified.cpp b/app/unified.cpp index 102ab340d..c44f2a436 100644 --- a/app/unified.cpp +++ b/app/unified.cpp @@ -237,12 +237,27 @@ void handle_generate(Engine& e, const httplib::Request& req, httplib::Response& return "data: " + c.dump() + "\n\n"; }; - auto usage = [](const npu::GenerateResult& r, size_t n_prompt, size_t n_cached) { + // engine#2111: a chat reply also says how many completion tokens were reasoning (OpenAI's + // completion_tokens_details.reasoning_tokens), so content "" with finish_reason "length" can be told apart from a + // model that said nothing: the budget ran out inside the thinking block. Counted over the generated ids (the end + // token excluded, as completion_tokens excludes it) with this route's own splitter; 0 when nothing is split. + auto usage = [&e, chat, split_reasoning, think_opened](const npu::GenerateResult& r, size_t n_prompt, size_t n_cached) { const size_t completion = r.tokens.size() - (r.stopped_at_eos ? 1 : 0); - return json{{"prompt_tokens", n_prompt}, - {"completion_tokens", completion}, - {"total_tokens", n_prompt + completion}, - {"prompt_tokens_details", {{"cached_tokens", n_cached}}}}; + json u{{"prompt_tokens", n_prompt}, + {"completion_tokens", completion}, + {"total_tokens", n_prompt + completion}, + {"prompt_tokens_details", {{"cached_tokens", n_cached}}}}; + if (chat) { + int reasoning = 0; + if (split_reasoning) { + std::vector pieces; + pieces.reserve(completion); + for (size_t i = 0; i < completion; ++i) pieces.push_back(e.tok->decode(r.tokens[i])); + reasoning = think_reasoning_tokens(pieces, think_opened); + } + u["completion_tokens_details"] = {{"reasoning_tokens", reasoning}}; + } + return u; }; auto timings = [](const npu::GenerateResult& r, const std::string& cache) { json t{{"prompt_ms", r.prefill_ms}, diff --git a/docs/serve.md b/docs/serve.md index b10b4fce7..226cb3d72 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -522,6 +522,13 @@ part of this build. `--npu-opt KEY=VALUE` passes an option through to an add-on `stream_options.include_usage` gets a last chunk with `usage` and `timings`, as OpenAI's API sends it. +A chat reply on the NPU also says how many of its `completion_tokens` were reasoning, in OpenAI's +field `usage.completion_tokens_details.reasoning_tokens` (engine#2111). `"content": ""` with +`"finish_reason": "length"` and `reasoning_tokens` equal to `completion_tokens` means the budget ran out +inside the thinking block, not that the model said nothing: the text is in `reasoning_content`; raise +`max_tokens` or send `enable_thinking: false`. Both tags of the block count as reasoning; +the answer and tool-call markup do not. A `/v1/completions` reply is raw text and carries no details. + `chat_template_kwargs.enable_thinking: false` works on every device. The GPU backends apply the model's own chat template. The NPU route emits what Qwen3's template does: an empty think block after the assistant prefix. For Qwen3.6-35B-A3B it diff --git a/tests/npu_answer_parser_test.cpp b/tests/npu_answer_parser_test.cpp index 428451076..0fe0262eb 100644 --- a/tests/npu_answer_parser_test.cpp +++ b/tests/npu_answer_parser_test.cpp @@ -221,6 +221,44 @@ int main() { check(text.content == "x", "the first text after the block is kept whole, whatever it starts with"); } + // engine#2111: usage.completion_tokens_details.reasoning_tokens -- how much of the budget went into reasoning. + { + const AnswerFamily Q = AnswerFamily::qwen3; + // the issue's row: max_tokens 8, the template opened the block, and every token is still inside it + const std::vector eight = {"Okay", ",", " the", " user", " asks", " about", " the", " capital"}; + check(reasoning_token_count(eight, Q, false, true) == 8, + "an unclosed block that took the whole budget: every token is reasoning (content \"\", 8 of 8)"); + const auto empty = parse_answer("Okay, the user asks about the capital", Q, false, true); + check(empty.content.empty() && empty.reasoning == "Okay, the user asks about the capital", + "... and that reply's content is empty while its reasoning carries the text"); + // the model opens the block itself: both tags count with it, the layout and the answer do not + check(reasoning_token_count({"", "\n", "why", "", "\n\n", "Paris", "\n"}, Q, false) == 4, + "a closed block: , its text, are reasoning; the layout after it and the answer are not"); + // a tag split across tokens counts with the block on both sides + check(reasoning_token_count({"", "plan", "", "Paris"}, Q, false) == 5, + "a tag split over two tokens is reasoning on both halves"); + // a token that straddles the boundary reached content, as content_mask says + check(reasoning_token_count({"", "plan", "Paris"}, Q, false) == 2, + "a token carrying the closing tag and the first answer text is content"); + // a block this splitter opens after the answer started: its opening tag counts with it + check(reasoning_token_count({"Hi.", "", "x", "", "Bye"}, Q, false) == 3, + "a block opened after content: , x, are reasoning; the prose around it is not"); + // no reasoning at all + check(reasoning_token_count({"Par", "is", "."}, Q, false) == 0, "a reply with no block has 0 reasoning tokens"); + check(reasoning_token_count({}, Q, false, true) == 0, "no tokens, no reasoning tokens"); + // call markup is neither content nor reasoning, and is not counted as reasoning + check(reasoning_token_count({"", "t", "", "", "{\"name\":\"ping\",\"arguments\":{}}", + ""}, Q, true) == 3, + "a tool call after the block is not reasoning"); + // the count agrees with the mask: a reasoning token never has a content entry + const std::vector mixed = {"", "a", "b", "", "\n", "X", "Y"}; + const std::vector m = content_mask(mixed, Q, false); + int content_entries = 0; + for (char c : m) content_entries += c ? 1 : 0; + check(reasoning_token_count(mixed, Q, false) + content_entries <= int(mixed.size()), + "reasoning tokens and content entries never overlap"); + } + std::printf("%s\n", failures ? "FAILED" : "all cases ok"); return failures ? 1 : 0; } diff --git a/tests/think_split_test.cpp b/tests/think_split_test.cpp index d93af2573..909aada57 100644 --- a/tests/think_split_test.cpp +++ b/tests/think_split_test.cpp @@ -39,6 +39,18 @@ int main(){ check("no think", run(false,{"Par","is"}), "Paris", ""); check("literal tag after content", run(false,{"use tags"}), "use tags", ""); check("unfinished think", run(false,{"still going"}), "", "still going"); + + // engine#2111: reasoning_tokens on this route, the same rule as npu::reasoning_token_count(). + auto count = [](const char* name, int got, int want) { + const bool ok = got == want; fails += !ok; + std::printf("%s %s: reasoning_tokens=%d (want %d)\n", ok ? "PASS" : "FAIL", name, got, want); + }; + count("budget spent in an opened block", onebit::think_reasoning_tokens({"Okay", ",", " the", " user"}, true), 4); + count("closed block then answer", onebit::think_reasoning_tokens({"", "\n", "plan", "", "\n\n", "Paris"}, false), 4); + count("tag split over tokens", onebit::think_reasoning_tokens({"", "pl", "an", "Paris"}, false), 5); + count("closing tag and answer in one token", onebit::think_reasoning_tokens({"", "plan", "Paris"}, false), 2); + count("no block", onebit::think_reasoning_tokens({"Par", "is"}, false), 0); + count("no tokens", onebit::think_reasoning_tokens({}, true), 0); return fails; }