diff --git a/CMakeLists.txt b/CMakeLists.txt index d6823978d..35a85bc99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1857,6 +1857,9 @@ endif() # same sources against the runtime stub (engine#1946) cannot drift from this one. onebit_npu_forward_library(onebit_npu_forward onebit_npu_rt) target_sources(onebit PRIVATE app/forward_serve.cpp) + # engine#2111: the usage object's pure function, its own TU so tests/serve_usage_test.cpp can link it without + # dragging in npu/rt.h (see cmake/tests-serve-usage.cmake). + target_sources(onebit PRIVATE app/serve_usage.cpp) # app/embed_serve.cpp's device path includes npu/embed/embed_npu.h (engine#1311): the embedding route's # ternary linears on the array, through the same int8 GEMM the forward links. target_include_directories(onebit PRIVATE ${CMAKE_SOURCE_DIR}/npu/embed ${CMAKE_SOURCE_DIR}/app diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index dfdb4d0c6..27dede566 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -76,6 +76,7 @@ namespace onebit { + namespace { // A prompt that leaves no room for an answer inside the context (the model's, or --ctx-size). @@ -605,20 +606,19 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // OpenAI's usage object plus the two fields issue #650 reports: how many prompt tokens came // from a saved prefix, and what the cache decided (which slot, and why). + // engine#2111: the usage object is `onebit::usage_json_text()` now (app/forward_serve.h) - a pure function, so + // the rule "reasoning_tokens appears iff the route could count it" has known answers without an NPU + // (tests/serve_usage_test.cpp). The JSON stays here, where the rest of the reply is built. auto usage_json = [](const CallStats& st) { - json u{{"prompt_tokens", st.prompt_tokens}, - {"completion_tokens", st.completion_tokens}, - {"total_tokens", st.prompt_tokens + st.completion_tokens}, - {"prompt_tokens_details", {{"cached_tokens", st.prefix.cached_tokens}}}, - {"flm_prefix_cache", {{"slot", st.prefix.slot}, - {"cached_tokens", st.prefix.cached_tokens}, - {"reason", st.prefix.reason}, - {"pinned", st.prefix.pinned}}}}; - // engine#2111: OpenAI's field for "how much of the budget went into reasoning". With content "" and - // finish_reason "length", reasoning_tokens == completion_tokens is the sentence that says the budget ran out - // inside a thinking block, rather than that the model said nothing. Additive: absent where nothing was split. - if (st.reasoning_tokens >= 0) u["completion_tokens_details"] = {{"reasoning_tokens", st.reasoning_tokens}}; - return u; + UsageCounters u; + u.prompt_tokens = st.prompt_tokens; + u.completion_tokens = st.completion_tokens; + u.cached_tokens = st.prefix.cached_tokens; + u.prefix_slot = st.prefix.slot; + u.prefix_reason = st.prefix.reason; + u.prefix_pinned = st.prefix.pinned; + u.reasoning_tokens = st.reasoning_tokens; + return json::parse(usage_json_text(u)); }; // OpenAI's logprobs.content entry: the token's text, its log-probability, and the most likely diff --git a/app/forward_serve.h b/app/forward_serve.h index 7b4093646..813951ec1 100644 --- a/app/forward_serve.h +++ b/app/forward_serve.h @@ -26,6 +26,29 @@ namespace onebit { int run_forward_serve(int argc, char** argv); +// engine#2111: the `usage` object a chat reply carries, as a pure function of the counters a call produced. +// +// It lived as a lambda inside the reply builder, which meant the one thing the issue is about - *is +// completion_tokens_details.reasoning_tokens present, and when is it absent?* - could only be observed by +// running a model on the NPU. As a function it is a decision table with known answers (tests/serve_usage_test.cpp), +// and the rule is the same one this fleet applies everywhere: a field that says "the budget went into reasoning" +// must be distinguishable from a field that says nothing, and `-1` ("the route could not count it") is neither. +struct UsageCounters { + int prompt_tokens = 0; + int completion_tokens = 0; + int cached_tokens = 0; // prefix cache: how many prompt tokens came from a saved prefix + int prefix_slot = -1; // -1 = "off" (no prefix slots configured) + std::string prefix_reason; // why the cache decided what it decided + bool prefix_pinned = false; + int reasoning_tokens = -1; // -1 = the route could not count it; 0 = counted, none; >0 = that many +}; + +// Declared here, defined in forward_serve.cpp beside the reply it belongs to (nlohmann/json stays out of this +// header, so a test that only wants the decision table does not take the JSON dependency). +std::string usage_json_text(const UsageCounters& c); + + + // engine#1862: `1bit npu-run --ids` on a pack whose kernel directory holds the SEALED set (SEALED-*/full_i8_*), which is // every pack since that route shipped. Runs the ids through ONE multi-row pass of the same forward `forward-serve` // drives -- the pass the k-row identity check needs -- and prints each row's argmax. `host_gemm` (NPU_INFER_HOST_GEMM) diff --git a/app/serve_usage.cpp b/app/serve_usage.cpp new file mode 100644 index 000000000..38fc02788 --- /dev/null +++ b/app/serve_usage.cpp @@ -0,0 +1,49 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#2111: the chat reply's `usage` object, as a pure function. +// +// Its own translation unit for one reason: tests/serve_usage_test.cpp has to link it, and linking +// app/forward_serve.cpp would drag in the NPU runtime (npu/rt.h refuses a build without ONEBIT_NPU_RT_LIB), so +// the field this issue is about could only ever be checked by running a model on the device. Here it is a decision +// table with known answers - see the header comment for the three cases that must stay distinct. + +#include "forward_serve.h" + +#include + +namespace onebit { + +using json = nlohmann::json; + +// The usage object, verbatim what the reply carries. Two fields are not OpenAI's and are here because a lane +// needed them: how many prompt tokens came from a saved prefix, and what the cache decided. +// `reasoning_tokens` is ABSENT when the route could not count it (-1), which is not the same as 0 (counted, none). +std::string usage_json_text(const UsageCounters& c) { + json u{{"prompt_tokens", c.prompt_tokens}, + {"completion_tokens", c.completion_tokens}, + {"total_tokens", c.prompt_tokens + c.completion_tokens}, + {"prompt_tokens_details", {{"cached_tokens", c.cached_tokens}}}, + {"flm_prefix_cache", {{"slot", c.prefix_slot}, + {"cached_tokens", c.cached_tokens}, + {"reason", c.prefix_reason}, + {"pinned", c.prefix_pinned}}}}; + // With content "" and finish_reason "length", reasoning_tokens == completion_tokens is the sentence that says the + // budget ran out inside a thinking block, rather than that the model said nothing. + if (c.reasoning_tokens >= 0) u["completion_tokens_details"] = {{"reasoning_tokens", c.reasoning_tokens}}; + return u.dump(); +} + +} // namespace onebit diff --git a/cmake/tests-serve-usage.cmake b/cmake/tests-serve-usage.cmake new file mode 100644 index 000000000..cd0963b06 --- /dev/null +++ b/cmake/tests-serve-usage.cmake @@ -0,0 +1,23 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# engine#2111: the chat reply's `usage` object as a decision table. It links app/serve_usage.cpp and NOT +# app/forward_serve.cpp, because the latter pulls `npu/rt.h`, which refuses a build with no NPU runtime - and that is +# exactly why this field could only ever be observed on the device before. +add_executable(serve_usage_test tests/serve_usage_test.cpp app/serve_usage.cpp) +target_include_directories(serve_usage_test PRIVATE app) +target_link_libraries(serve_usage_test PRIVATE nlohmann_json::nlohmann_json) +add_test(NAME serve_usage COMMAND serve_usage_test) +add_dependencies(onebit serve_usage_test) diff --git a/tests/serve_usage_test.cpp b/tests/serve_usage_test.cpp new file mode 100644 index 000000000..280857548 --- /dev/null +++ b/tests/serve_usage_test.cpp @@ -0,0 +1,107 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#2111: the reply's `usage` object, checked as a decision table instead of on a device. +// +// The issue is that an empty `content` with `finish_reason: "length"` reads like "the model said nothing" when the +// budget went into a thinking block the engine strips. The field that tells them apart is +// `completion_tokens_details.reasoning_tokens`, and the three cases below must stay three different answers: +// +// reasoning_tokens > 0 the route counted them; the field is PRESENT with that number +// reasoning_tokens == 0 the route counted and found none; the field is PRESENT with 0 +// reasoning_tokens == -1 the route could not count (e.g. /v1/completions); the field is ABSENT, and must never +// be read as zero +// +// No model, no NPU, no HTTP: app/serve_usage.cpp is the only thing this links. + +#include "forward_serve.h" + +#include + +#include +#include + +namespace { + +int bad = 0; + +void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++bad; +} + +nlohmann::json usage(const onebit::UsageCounters& c) { + return nlohmann::json::parse(onebit::usage_json_text(c)); +} + +bool has_reasoning(const nlohmann::json& u) { + return u.contains("completion_tokens_details") && u["completion_tokens_details"].contains("reasoning_tokens"); +} + +} // namespace + +int main() { + { + onebit::UsageCounters c; + c.prompt_tokens = 24; + c.completion_tokens = 8; + c.reasoning_tokens = 8; + auto u = usage(c); + check(has_reasoning(u) && u["completion_tokens_details"]["reasoning_tokens"] == 8, + "an unclosed block that took the whole budget: the field is present with 8 of 8"); + check(u["completion_tokens"] == 8 && u["total_tokens"] == 32, + "and the counters are untouched by it (completion 8, total 24+8)"); + } + { + onebit::UsageCounters c; + c.prompt_tokens = 10; + c.completion_tokens = 5; + c.reasoning_tokens = 0; + auto u = usage(c); + check(has_reasoning(u) && u["completion_tokens_details"]["reasoning_tokens"] == 0, + "a route that counted and found none reports 0, which is not the same as absent"); + } + { + onebit::UsageCounters c; + c.prompt_tokens = 10; + c.completion_tokens = 5; + c.reasoning_tokens = -1; + auto u = usage(c); + check(!has_reasoning(u), "a route that could not count it omits the field entirely (never a zero)"); + } + { + onebit::UsageCounters c; + c.prompt_tokens = 7; + c.completion_tokens = 3; + c.cached_tokens = 4; + c.prefix_slot = 2; + c.prefix_reason = "hit"; + c.prefix_pinned = true; + auto u = usage(c); + check(u["prompt_tokens_details"]["cached_tokens"] == 4, "the prefix cache's cached_tokens is still reported"); + check(u["flm_prefix_cache"]["slot"] == 2 && u["flm_prefix_cache"]["reason"] == "hit" && + u["flm_prefix_cache"]["pinned"] == true, + "and the cache decision (slot, reason, pinned) is unchanged by the extraction"); + } + { + // the default-constructed struct is the shape a /v1/completions reply has: no counting was possible + onebit::UsageCounters c; + check(!has_reasoning(usage(c)), "a default UsageCounters omits the field, so a caller that never set it is safe"); + check(usage(c)["flm_prefix_cache"]["slot"] == -1, "and 'off' is -1, as the prefix cache already spells it"); + } + + std::printf("serve_usage: %s\n", bad ? "FAIL" : "ok"); + return bad ? 1 : 0; +}