From d47afeb281b0690e9ceb723ea45fe8c6df1c73d4 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 08:03:05 -0400 Subject: [PATCH] forward-serve: a request's tools reach the chat template, and the function/parameter call form is read (Refs #1788) The chat route passed only chat_template_kwargs and enable_thinking to the container's template, so a template's tools block never rendered: MiniCPM-V 4.7 got 25 prompt tokens where its template renders 292 and answered in prose. Three changes: - npu::request_template_vars: tools are a template variable, in the caller's key order. - the qwen3 answer markup also reads VALUE, the form Qwen3.5's and MiniCPM-V 4.7's templates teach. - npu::messages_for_template: an earlier call's arguments string is handed to the template as the object. Co-Authored-By: Claude Fable 5.2 --- app/forward_serve.cpp | 13 +++---- app/npu_answer_parser.h | 58 ++++++++++++++++++++++++++++++++ app/npu_template.h | 37 ++++++++++++++++++++ docs/npu-chat-template.md | 22 +++++++++++- tests/npu_answer_parser_test.cpp | 46 +++++++++++++++++++++++++ tests/npu_template_test.cpp | 57 +++++++++++++++++++++++++++++++ 6 files changed, 226 insertions(+), 7 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index db836fc45..dfdb4d0c6 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -239,12 +239,13 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // template that serializes its input (a tools block through `tojson`) renders the // request's key order into the prompt, so it has to be the caller's order, not ours. const nlohmann::ordered_json ordered = nlohmann::ordered_json::parse(req.body); - nlohmann::ordered_json extra = nlohmann::ordered_json::object(); - if (ordered.contains("chat_template_kwargs") && ordered["chat_template_kwargs"].is_object()) - extra = ordered["chat_template_kwargs"]; - extra["enable_thinking"] = npu::thinking_from_body(body); - prompt = npu::render_chat_template(e.tmpl, ordered.at("messages"), /*add_generation_prompt=*/true, - extra, e.bos_text, e.eos_text); + // The functions the request offers go the same way (npu::request_template_vars): the + // template is where a model learns what it may call. + const nlohmann::ordered_json extra = npu::request_template_vars(ordered, npu::thinking_from_body(body)); + // An earlier call's arguments arrive as a JSON string; the template is written against the + // object (npu::messages_for_template). + prompt = npu::render_chat_template(e.tmpl, npu::messages_for_template(ordered.at("messages")), + /*add_generation_prompt=*/true, extra, e.bos_text, e.eos_text); if (e.prefix_pin) { // A template that cannot render the probe is no reason to refuse the request: it only means // there is no early pin, and the log says so. diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index f6453fcb0..4bd0b210a 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -273,6 +273,11 @@ class AnswerStream { mode_ = Mode::content; return; } + if (fam_ == AnswerFamily::qwen3 && close_function_call()) { + call_text_.clear(); + mode_ = Mode::content; + return; + } try { const nlohmann::json j = nlohmann::json::parse(call_text_); ToolCall c; @@ -367,6 +372,59 @@ class AnswerStream { return out; } + // The second call form a block carries (Qwen3.5's template, and MiniCPM-V 4.7's, teach it): + // + // + // VALUE + // + // + // The template writes a string argument bare and everything else through `tojson`, so a value is read back + // the same way: an object, an array, a number, true/false/null are JSON; anything else is the string itself, + // less the one newline the form puts on each side. False on any other shape (the caller then tries the JSON + // form and, failing that, puts the markup back into content). + // shortcut: without the request's schema a string argument whose text is a JSON literal ("17", "true") comes + // back as that literal; type by the tool's parameter schema here if a caller needs it kept a string. + bool close_function_call() { + const std::string body = answer_detail::trim(call_text_); + static const std::string fopen_ = "', fopen_.size()); + if (name_end == std::string::npos) return false; + ToolCall c; + c.id = "call_" + std::to_string(calls_.size()); + c.name = answer_detail::trim(body.substr(fopen_.size(), name_end - fopen_.size())); + if (c.name.empty() || c.name.find('<') != std::string::npos) return false; + nlohmann::ordered_json args = nlohmann::ordered_json::object(); + size_t i = name_end + 1; + const size_t end = body.size() - fclose_.size(); + auto space = [&] { while (i < end && (body[i] == ' ' || body[i] == '\n' || body[i] == '\r' || body[i] == '\t')) ++i; }; + for (space(); i < end; space()) { + if (body.compare(i, popen_.size(), popen_) != 0) return false; + const size_t key_end = body.find('>', i + popen_.size()); + if (key_end == std::string::npos || key_end >= end) return false; + const std::string key = answer_detail::trim(body.substr(i + popen_.size(), key_end - i - popen_.size())); + const size_t close = body.find(pclose_, key_end + 1); + if (key.empty() || close == std::string::npos || close > end) return false; + std::string value = body.substr(key_end + 1, close - key_end - 1); + if (!value.empty() && value.front() == '\n') value.erase(0, 1); + if (!value.empty() && value.back() == '\n') value.pop_back(); + const std::string t = answer_detail::trim(value); + nlohmann::ordered_json v = value; + const bool literal = !t.empty() && (t[0] == '{' || t[0] == '[' || t[0] == '-' || std::isdigit((unsigned char)t[0]) || + t == "true" || t == "false" || t == "null"); + if (literal) { + const nlohmann::ordered_json parsed = nlohmann::ordered_json::parse(t, nullptr, /*allow_exceptions=*/false); + if (!parsed.is_discarded() && !parsed.is_string()) v = parsed; + } + args[key] = std::move(v); + i = close + pclose_.size(); + } + c.arguments = args.dump(); + calls_.push_back(std::move(c)); + return true; + } + bool close_gemma_call() { const std::string body = answer_detail::trim(call_text_); if (body.compare(0, 5, "call:") != 0) return false; diff --git a/app/npu_template.h b/app/npu_template.h index 6abd5485d..0137e85cc 100644 --- a/app/npu_template.h +++ b/app/npu_template.h @@ -105,6 +105,43 @@ inline bool renderer_built() { #endif } +// The template variables one chat request contributes, besides its messages: `chat_template_kwargs` +// as sent, the functions the request offers (`tools`), and `enable_thinking`. `tools` is here because +// a template is the only place a model is told what it may call: a request whose tools never reach +// the template is answered in prose (MiniCPM-V 4.7 rendered 25 tokens where its template, given the +// same request, renders 292 -- engine#1788). An empty or non-array `tools` is left out, so a +// template's `{% if tools %}` reads it as "none offered". `body` is the request as parsed in the +// caller's key order (see render_chat_template below for why). +inline nlohmann::ordered_json request_template_vars(const nlohmann::ordered_json& body, bool enable_thinking) { + nlohmann::ordered_json extra = nlohmann::ordered_json::object(); + if (body.contains("chat_template_kwargs") && body["chat_template_kwargs"].is_object()) + extra = body["chat_template_kwargs"]; + if (body.contains("tools") && body["tools"].is_array() && !body["tools"].empty()) extra["tools"] = body["tools"]; + extra["enable_thinking"] = enable_thinking; + return extra; +} + +// The request's messages as a template expects them. The OpenAI API spells an earlier call's +// `function.arguments` as a JSON *string*; a chat template is written against the object (transformers +// hands it a dict): MiniCPM-V 4.7's walks `tool_call.arguments|items`, which does not exist for a string, +// and Qwen3's `arguments | tojson` would quote the string a second time. So a string that parses to a +// JSON object becomes that object, in the string's own key order; anything else is left as sent. +inline nlohmann::ordered_json messages_for_template(nlohmann::ordered_json messages) { + if (!messages.is_array()) return messages; + for (auto& m : messages) { + if (!m.is_object() || !m.contains("tool_calls") || !m["tool_calls"].is_array()) continue; + for (auto& call : m["tool_calls"]) { + if (!call.is_object() || !call.contains("function") || !call["function"].is_object()) continue; + auto& fn = call["function"]; + if (!fn.contains("arguments") || !fn["arguments"].is_string()) continue; + nlohmann::ordered_json parsed = + nlohmann::ordered_json::parse(fn["arguments"].get(), nullptr, /*allow_exceptions=*/false); + if (parsed.is_object()) fn["arguments"] = std::move(parsed); + } + } + return messages; +} + // Render `messages` with the container's template. // // messages the request's messages array, as the OpenAI API spells it diff --git a/docs/npu-chat-template.md b/docs/npu-chat-template.md index 00d258ea4..6e3dc147e 100644 --- a/docs/npu-chat-template.md +++ b/docs/npu-chat-template.md @@ -192,12 +192,32 @@ reads the answer markup from the template it renders and prints it at load: | the rendered template contains | markup | reasoning | a tool call | |---|---|---|---| | `<|tool_call>` or `<|channel>thought` | `gemma4` | `<|channel>thought … ` | `<|tool_call>call:NAME{key:value}`, strings as `<|"|>…<|"|>` | -| neither, or no template (ChatML) | `qwen3` | ` … ` | `{"name": …, "arguments": {…}}` | +| neither, or no template (ChatML) | `qwen3` | ` … ` | `{"name": …, "arguments": {…}}`, or `VALUE` | Both give the client the same OpenAI shape: `reasoning_content`, `content`, and `tool_calls` whose `arguments` is a JSON string. A Gemma call body that is not well formed (a string that never closes, a key with no value, an unclosed object or array, an empty value) is not a call: the markup is returned in `content` unchanged. +**Tools go through the template (engine#1788).** A request's `tools` are a template variable, next to +`chat_template_kwargs` and `enable_thinking` (`npu::request_template_vars`), in the caller's key order; an empty +list is left out. The template is the only place a model is told what it may call: before this the route kept +`tools` for the grammar and the answer parser only, so MiniCPM-V 4.7's 292-token tools prompt was rendered as 25 +tokens and the model answered in prose. Two things follow from rendering the pack's own template: + +- The call form is the template's. Qwen3.5's and MiniCPM-V 4.7's teach `` with one + `` block per argument; the `qwen3` markup reads both that and the JSON object. A parameter's + value is a JSON value when it is an object, an array, a number, `true`, `false` or `null`, and the text itself + otherwise (the template writes strings bare), so a string argument whose text is such a literal (`17`) comes + back as the literal. A block that is neither form is returned in `content` unchanged. +- An earlier call in `messages` carries `function.arguments` as a JSON string, as the API spells it. A template + is written against the object, so a string that parses to a JSON object is handed over as that object + (`npu::messages_for_template`); before this the second turn of a tool conversation was a 400 on a template + that walks the arguments. + +Measured on the NPU with `MiniCPM-V-4.7-1B-NPU2` (`get_weather(city)`, temperature 0): the first reply is +`finish_reason: tool_calls` with `get_weather {"city":"Paris"}`; with the tool result appended the reply is "The +weather in Paris is 17 degrees Celsius with light rain." and the prefix cache reuses 291 of the 376 prompt tokens. + **A reasoning block the prompt opened.** Some templates end their generation prompt with the opening tag itself: Qwen3.5's ends `<|im_start|>assistant\n\n` when thinking is on. The model then never writes that tag; its answer starts with the reasoning and the first tag in it is the closing one. `forward-serve` checks whether the diff --git a/tests/npu_answer_parser_test.cpp b/tests/npu_answer_parser_test.cpp index 0fe0262eb..bf546b082 100644 --- a/tests/npu_answer_parser_test.cpp +++ b/tests/npu_answer_parser_test.cpp @@ -97,6 +97,52 @@ int main() { check(contains(a.content, "to emit a call."), "without tools, the rest of the text is kept"); } + // The function/parameter call form (engine#1788): Qwen3.5's template and MiniCPM-V 4.7's teach + // VALUE + // and the reply below is what MiniCPM-V 4.7 1B wrote on the NPU. It used to come back as content. + { + const auto a = parse_answer( + "I need to get the current weather for Paris, so I will call the get_weather function.\n\n\n" + "\n\nParis\n\n\n", + AnswerFamily::qwen3, true); + check(a.calls.size() == 1 && a.calls[0].name == "get_weather", "function form: one call, named"); + check(a.calls.size() == 1 && a.calls[0].arguments == "{\"city\":\"Paris\"}", "function form: a bare value is a string argument"); + check(!contains(a.content, "") && !contains(a.content, "\n\n\n3\n\n\n[\"a\", \"b\"]\n\n" + "\nline one\nline two\n\n\n2026-11-11\n\n\n", + AnswerFamily::qwen3, true); + check(a.calls.size() == 1 && + a.calls[0].arguments == "{\"nights\":3,\"guests\":[\"a\",\"b\"],\"note\":\"line one\\nline two\",\"when\":\"2026-11-11\"}", + "function form: JSON values are values, text is a string (a date is not a number), in the model's order"); + check(a.content_null, "function form: a call-only reply has null content"); + } + { + const auto a = parse_answer("\n\n\n", AnswerFamily::qwen3, true); + check(a.calls.size() == 1 && a.calls[0].name == "ping" && a.calls[0].arguments == "{}", "function form: no parameters is {}"); + } + { + // Malformed: a parameter that never closes, and text between parameters. Neither is a call. + const auto a = parse_answer("1", AnswerFamily::qwen3, true); + check(a.calls.empty() && contains(a.content, ""), "function form: an unclosed parameter goes back to content"); + const auto b = parse_answer("stray1", AnswerFamily::qwen3, true); + check(b.calls.empty() && contains(b.content, "stray"), "function form: text between parameters goes back to content"); + } + { + // Streaming: the same reply in three-byte pieces gives the same call and no markup in the deltas. + const std::string text = "ok \n\n\nParis\n\n\n"; + onebit::npu::AnswerStream st(AnswerFamily::qwen3, true); + std::vector> out; + for (size_t at = 0; at < text.size(); at += 3) st.feed(text.substr(at, 3), out); + st.finish(out); + std::string deltas; + for (const auto& d : out) deltas += d.second; + check(st.calls().size() == 1 && st.calls()[0].arguments == "{\"city\":\"Paris\"}", "function form: streamed in pieces, the same call"); + check(!contains(deltas, "function"), "function form: streamed, no markup in the deltas"); + } // 5. Malformed call body: never dropped, goes back into content with its markup. { const auto a = parse_answer("xnot json at all", AnswerFamily::qwen3, true); diff --git a/tests/npu_template_test.cpp b/tests/npu_template_test.cpp index 1c70d0ab2..9ba43d17c 100644 --- a/tests/npu_template_test.cpp +++ b/tests/npu_template_test.cpp @@ -121,6 +121,63 @@ int main(int argc, char** argv) { run_fixture(a); } + // The chat route's own variables (engine#1788): a request's `tools` reach the template, in the + // caller's key order, and an empty list is "none offered". The route used to pass only + // chat_template_kwargs and enable_thinking, so a template's tools block never rendered. + { + onebit::npu::ChatTemplate t; + t.text = "{% if tools %}TOOLS {{ tools | tojson }} {% endif %}{% if enable_thinking %}think {% endif %}" + "{{ k }}{{ messages[0].content }}"; + t.source = "test"; + const nlohmann::ordered_json body = nlohmann::ordered_json::parse( + R"({"messages":[{"role":"user","content":"hi"}],"chat_template_kwargs":{"k":"K "},)" + R"("tools":[{"type":"function","function":{"name":"get_weather","description":"d","parameters":{"type":"object","properties":{"zeta":{"type":"string"},"alpha":{"type":"string"}}}}}]})"); + try { + const nlohmann::ordered_json vars = onebit::npu::request_template_vars(body, true); + const std::string out = onebit::npu::render_chat_template(t, body.at("messages"), true, vars); + check(out.find("get_weather") != std::string::npos, "request tools: the template sees the request's tools"); + const size_t z = out.find("zeta"), a = out.find("alpha"); + check(z != std::string::npos && a != std::string::npos && z < a, "request tools: in the caller's key order"); + check(out.find("think K hi") != std::string::npos, "request tools: kwargs and enable_thinking still arrive"); + nlohmann::ordered_json none = body; + none["tools"] = nlohmann::ordered_json::array(); + const std::string plain = onebit::npu::render_chat_template( + t, none.at("messages"), true, onebit::npu::request_template_vars(none, false)); + check(plain == "K hi", "request tools: an empty list renders no tools block"); + } catch (const std::exception& e) { + check(false, std::string("request tools: rendered (") + e.what() + ")"); + } + } + + // An earlier call in the conversation (engine#1788): the API's arguments string reaches the template + // as the object, so a template that walks it renders; the route used to answer 400 on the second turn. + { + onebit::npu::ChatTemplate t; + t.text = "{% for m in messages %}{% if m.tool_calls %}{% for c in m.tool_calls %}{{ c.function.name }}(" + "{% for k, v in c.function.arguments|items %}{{ k }}={{ v }};{% endfor %}){% endfor %}{% endif %}{% endfor %}"; + t.source = "test"; + const nlohmann::ordered_json msgs = nlohmann::ordered_json::parse( + R"([{"role":"user","content":"q"},{"role":"assistant","content":"","tool_calls":[{"id":"call_0","type":"function",)" + R"("function":{"name":"get_weather","arguments":"{\"zeta\":\"Paris\",\"alpha\":2}"}}]},)" + R"({"role":"tool","tool_call_id":"call_0","content":"17"}])"); + bool raw_threw = false; + try { + (void)onebit::npu::render_chat_template(t, msgs, true); + } catch (const onebit::npu::TemplateError&) { + raw_threw = true; + } + check(raw_threw, "call arguments: as the API sends them (a string) this template cannot walk them"); + try { + const std::string out = onebit::npu::render_chat_template(t, onebit::npu::messages_for_template(msgs), true); + check(out == "get_weather(zeta=Paris;alpha=2;)", "call arguments: the template walks the object, in the string's key order"); + } catch (const std::exception& e) { + check(false, std::string("call arguments: rendered (") + e.what() + ")"); + } + nlohmann::ordered_json odd = msgs; + odd[1]["tool_calls"][0]["function"]["arguments"] = "not json"; + check(onebit::npu::messages_for_template(odd) == odd, "call arguments: a string that is not a JSON object is left as sent"); + } + // A container that declares no template is this layer's "no", not a ChatML render: the route decides // what to do about that (app/forward_serve.cpp: ChatML with anchors, else the #987 400). {