Skip to content

forward-serve: a request's tools reach the chat template — MiniCPM-V 4.7 answered a tool request in prose, now a call and a round trip (Refs #1788) #2172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bong-water-water-bong merged 1 commit into from
Nov 11, 2026
Merged

forward-serve: a request's tools reach the chat template — MiniCPM-V 4.7 answered a tool request in prose, now a call and a round trip (Refs #1788) #2172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions app/forward_serve.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,12 +239,13 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib
// template that serializes its input (a tools block through `tojson`) renders the
// request's key order into the prompt, so it has to be the caller's order, not ours.
const nlohmann::ordered_json ordered = nlohmann::ordered_json::parse(req.body);
nlohmann::ordered_json extra = nlohmann::ordered_json::object();
if (ordered.contains("chat_template_kwargs") && ordered["chat_template_kwargs"].is_object())
extra = ordered["chat_template_kwargs"];
extra["enable_thinking"] = npu::thinking_from_body(body);
prompt = npu::render_chat_template(e.tmpl, ordered.at("messages"), /*add_generation_prompt=*/true,
extra, e.bos_text, e.eos_text);
// The functions the request offers go the same way (npu::request_template_vars): the
// template is where a model learns what it may call.
const nlohmann::ordered_json extra = npu::request_template_vars(ordered, npu::thinking_from_body(body));
// An earlier call's arguments arrive as a JSON string; the template is written against the
// object (npu::messages_for_template).
prompt = npu::render_chat_template(e.tmpl, npu::messages_for_template(ordered.at("messages")),
/*add_generation_prompt=*/true, extra, e.bos_text, e.eos_text);
if (e.prefix_pin) {
// A template that cannot render the probe is no reason to refuse the request: it only means
// there is no early pin, and the log says so.
Expand Down
58 changes: 58 additions & 0 deletions app/npu_answer_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,11 @@ class AnswerStream {
mode_ = Mode::content;
return;
}
if (fam_ == AnswerFamily::qwen3 && close_function_call()) {
call_text_.clear();
mode_ = Mode::content;
return;
}
try {
const nlohmann::json j = nlohmann::json::parse(call_text_);
ToolCall c;
Expand Down Expand Up @@ -367,6 +372,59 @@ class AnswerStream {
return out;
}

// The second call form a <tool_call> block carries (Qwen3.5's template, and MiniCPM-V 4.7's, teach it):
// <function=NAME>
// <parameter=KEY>
// VALUE
// </parameter>
// </function>
// The template writes a string argument bare and everything else through `tojson`, so a value is read back
// the same way: an object, an array, a number, true/false/null are JSON; anything else is the string itself,
// less the one newline the form puts on each side. False on any other shape (the caller then tries the JSON
// form and, failing that, puts the markup back into content).
// shortcut: without the request's schema a string argument whose text is a JSON literal ("17", "true") comes
// back as that literal; type by the tool's parameter schema here if a caller needs it kept a string.
bool close_function_call() {
const std::string body = answer_detail::trim(call_text_);
static const std::string fopen_ = "<function=", fclose_ = "</function>", popen_ = "<parameter=", pclose_ = "</parameter>";
if (body.compare(0, fopen_.size(), fopen_) != 0) return false;
if (body.size() < fclose_.size() || body.compare(body.size() - fclose_.size(), fclose_.size(), fclose_) != 0) return false;
const size_t name_end = body.find('>', fopen_.size());
if (name_end == std::string::npos) return false;
ToolCall c;
c.id = "call_" + std::to_string(calls_.size());
c.name = answer_detail::trim(body.substr(fopen_.size(), name_end - fopen_.size()));
if (c.name.empty() || c.name.find('<') != std::string::npos) return false;
nlohmann::ordered_json args = nlohmann::ordered_json::object();
size_t i = name_end + 1;
const size_t end = body.size() - fclose_.size();
auto space = [&] { while (i < end && (body[i] == ' ' || body[i] == '\n' || body[i] == '\r' || body[i] == '\t')) ++i; };
for (space(); i < end; space()) {
if (body.compare(i, popen_.size(), popen_) != 0) return false;
const size_t key_end = body.find('>', i + popen_.size());
if (key_end == std::string::npos || key_end >= end) return false;
const std::string key = answer_detail::trim(body.substr(i + popen_.size(), key_end - i - popen_.size()));
const size_t close = body.find(pclose_, key_end + 1);
if (key.empty() || close == std::string::npos || close > end) return false;
std::string value = body.substr(key_end + 1, close - key_end - 1);
if (!value.empty() && value.front() == '\n') value.erase(0, 1);
if (!value.empty() && value.back() == '\n') value.pop_back();
const std::string t = answer_detail::trim(value);
nlohmann::ordered_json v = value;
const bool literal = !t.empty() && (t[0] == '{' || t[0] == '[' || t[0] == '-' || std::isdigit((unsigned char)t[0]) ||
t == "true" || t == "false" || t == "null");
if (literal) {
const nlohmann::ordered_json parsed = nlohmann::ordered_json::parse(t, nullptr, /*allow_exceptions=*/false);
if (!parsed.is_discarded() && !parsed.is_string()) v = parsed;
}
args[key] = std::move(v);
i = close + pclose_.size();
}
c.arguments = args.dump();
calls_.push_back(std::move(c));
return true;
}

bool close_gemma_call() {
const std::string body = answer_detail::trim(call_text_);
if (body.compare(0, 5, "call:") != 0) return false;
Expand Down
37 changes: 37 additions & 0 deletions app/npu_template.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,43 @@ inline bool renderer_built() {
#endif
}

// The template variables one chat request contributes, besides its messages: `chat_template_kwargs`
// as sent, the functions the request offers (`tools`), and `enable_thinking`. `tools` is here because
// a template is the only place a model is told what it may call: a request whose tools never reach
// the template is answered in prose (MiniCPM-V 4.7 rendered 25 tokens where its template, given the
// same request, renders 292 -- engine#1788). An empty or non-array `tools` is left out, so a
// template's `{% if tools %}` reads it as "none offered". `body` is the request as parsed in the
// caller's key order (see render_chat_template below for why).
inline nlohmann::ordered_json request_template_vars(const nlohmann::ordered_json& body, bool enable_thinking) {
nlohmann::ordered_json extra = nlohmann::ordered_json::object();
if (body.contains("chat_template_kwargs") && body["chat_template_kwargs"].is_object())
extra = body["chat_template_kwargs"];
if (body.contains("tools") && body["tools"].is_array() && !body["tools"].empty()) extra["tools"] = body["tools"];
extra["enable_thinking"] = enable_thinking;
return extra;
}

// The request's messages as a template expects them. The OpenAI API spells an earlier call's
// `function.arguments` as a JSON *string*; a chat template is written against the object (transformers
// hands it a dict): MiniCPM-V 4.7's walks `tool_call.arguments|items`, which does not exist for a string,
// and Qwen3's `arguments | tojson` would quote the string a second time. So a string that parses to a
// JSON object becomes that object, in the string's own key order; anything else is left as sent.
inline nlohmann::ordered_json messages_for_template(nlohmann::ordered_json messages) {
if (!messages.is_array()) return messages;
for (auto& m : messages) {
if (!m.is_object() || !m.contains("tool_calls") || !m["tool_calls"].is_array()) continue;
for (auto& call : m["tool_calls"]) {
if (!call.is_object() || !call.contains("function") || !call["function"].is_object()) continue;
auto& fn = call["function"];
if (!fn.contains("arguments") || !fn["arguments"].is_string()) continue;
nlohmann::ordered_json parsed =
nlohmann::ordered_json::parse(fn["arguments"].get<std::string>(), nullptr, /*allow_exceptions=*/false);
if (parsed.is_object()) fn["arguments"] = std::move(parsed);
}
}
return messages;
}

// Render `messages` with the container's template.
//
// messages the request's messages array, as the OpenAI API spells it
Expand Down
22 changes: 21 additions & 1 deletion docs/npu-chat-template.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,12 +192,32 @@ reads the answer markup from the template it renders and prints it at load:
| the rendered template contains | markup | reasoning | a tool call |
|---|---|---|---|
| `<|tool_call>` or `<|channel>thought` | `gemma4` | `<|channel>thought … <channel|>` | `<|tool_call>call:NAME{key:value}<tool_call|>`, strings as `<|"|>…<|"|>` |
| neither, or no template (ChatML) | `qwen3` | `<think> … </think>` | `<tool_call>{"name": …, "arguments": {…}}</tool_call>` |
| neither, or no template (ChatML) | `qwen3` | `<think> … </think>` | `<tool_call>{"name": …, "arguments": {…}}</tool_call>`, or `<tool_call><function=NAME><parameter=KEY>VALUE</parameter></function></tool_call>` |

Both give the client the same OpenAI shape: `reasoning_content`, `content`, and `tool_calls` whose `arguments`
is a JSON string. A Gemma call body that is not well formed (a string that never closes, a key with no value,
an unclosed object or array, an empty value) is not a call: the markup is returned in `content` unchanged.

**Tools go through the template (engine#1788).** A request's `tools` are a template variable, next to
`chat_template_kwargs` and `enable_thinking` (`npu::request_template_vars`), in the caller's key order; an empty
list is left out. The template is the only place a model is told what it may call: before this the route kept
`tools` for the grammar and the answer parser only, so MiniCPM-V 4.7's 292-token tools prompt was rendered as 25
tokens and the model answered in prose. Two things follow from rendering the pack's own template:

- The call form is the template's. Qwen3.5's and MiniCPM-V 4.7's teach `<function=NAME>` with one
`<parameter=KEY>` block per argument; the `qwen3` markup reads both that and the JSON object. A parameter's
value is a JSON value when it is an object, an array, a number, `true`, `false` or `null`, and the text itself
otherwise (the template writes strings bare), so a string argument whose text is such a literal (`17`) comes
back as the literal. A block that is neither form is returned in `content` unchanged.
- An earlier call in `messages` carries `function.arguments` as a JSON string, as the API spells it. A template
is written against the object, so a string that parses to a JSON object is handed over as that object
(`npu::messages_for_template`); before this the second turn of a tool conversation was a 400 on a template
that walks the arguments.

Measured on the NPU with `MiniCPM-V-4.7-1B-NPU2` (`get_weather(city)`, temperature 0): the first reply is
`finish_reason: tool_calls` with `get_weather {"city":"Paris"}`; with the tool result appended the reply is "The
weather in Paris is 17 degrees Celsius with light rain." and the prefix cache reuses 291 of the 376 prompt tokens.

**A reasoning block the prompt opened.** Some templates end their generation prompt with the opening tag itself:
Qwen3.5's ends `<|im_start|>assistant\n<think>\n` when thinking is on. The model then never writes that tag; its
answer starts with the reasoning and the first tag in it is the closing one. `forward-serve` checks whether the
Expand Down
46 changes: 46 additions & 0 deletions tests/npu_answer_parser_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,52 @@ int main() {
check(contains(a.content, "to emit a call."), "without tools, the rest of the text is kept");
}

// The function/parameter call form (engine#1788): Qwen3.5's template and MiniCPM-V 4.7's teach
// <tool_call><function=NAME><parameter=KEY>VALUE</parameter></function></tool_call>
// and the reply below is what MiniCPM-V 4.7 1B wrote on the NPU. It used to come back as content.
{
const auto a = parse_answer(
"I need to get the current weather for Paris, so I will call the get_weather function.\n\n<tool_call>\n"
"<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n</function>\n</tool_call>",
AnswerFamily::qwen3, true);
check(a.calls.size() == 1 && a.calls[0].name == "get_weather", "function form: one call, named");
check(a.calls.size() == 1 && a.calls[0].arguments == "{\"city\":\"Paris\"}", "function form: a bare value is a string argument");
check(!contains(a.content, "<tool_call>") && !contains(a.content, "<function="), "function form: the markup leaves content");
check(contains(a.content, "I will call the get_weather function"), "function form: the prose before it stays");
}
{
const auto a = parse_answer(
"<tool_call>\n<function=book>\n<parameter=nights>\n3\n</parameter>\n<parameter=guests>\n[\"a\", \"b\"]\n</parameter>\n"
"<parameter=note>\nline one\nline two\n</parameter>\n<parameter=when>\n2026-11-11\n</parameter>\n</function>\n</tool_call>",
AnswerFamily::qwen3, true);
check(a.calls.size() == 1 &&
a.calls[0].arguments == "{\"nights\":3,\"guests\":[\"a\",\"b\"],\"note\":\"line one\\nline two\",\"when\":\"2026-11-11\"}",
"function form: JSON values are values, text is a string (a date is not a number), in the model's order");
check(a.content_null, "function form: a call-only reply has null content");
}
{
const auto a = parse_answer("<tool_call>\n<function=ping>\n</function>\n</tool_call>", AnswerFamily::qwen3, true);
check(a.calls.size() == 1 && a.calls[0].name == "ping" && a.calls[0].arguments == "{}", "function form: no parameters is {}");
}
{
// Malformed: a parameter that never closes, and text between parameters. Neither is a call.
const auto a = parse_answer("<tool_call><function=f><parameter=x>1</function></tool_call>", AnswerFamily::qwen3, true);
check(a.calls.empty() && contains(a.content, "<function=f>"), "function form: an unclosed parameter goes back to content");
const auto b = parse_answer("<tool_call><function=f>stray<parameter=x>1</parameter></function></tool_call>", AnswerFamily::qwen3, true);
check(b.calls.empty() && contains(b.content, "stray"), "function form: text between parameters goes back to content");
}
{
// Streaming: the same reply in three-byte pieces gives the same call and no markup in the deltas.
const std::string text = "ok <tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n</function>\n</tool_call>";
onebit::npu::AnswerStream st(AnswerFamily::qwen3, true);
std::vector<std::pair<bool, std::string>> out;
for (size_t at = 0; at < text.size(); at += 3) st.feed(text.substr(at, 3), out);
st.finish(out);
std::string deltas;
for (const auto& d : out) deltas += d.second;
check(st.calls().size() == 1 && st.calls()[0].arguments == "{\"city\":\"Paris\"}", "function form: streamed in pieces, the same call");
check(!contains(deltas, "function"), "function form: streamed, no markup in the deltas");
}
// 5. Malformed call body: never dropped, goes back into content with its markup.
{
const auto a = parse_answer("x<tool_call>not json at all</tool_call>", AnswerFamily::qwen3, true);
Expand Down
57 changes: 57 additions & 0 deletions tests/npu_template_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,63 @@ int main(int argc, char** argv) {
run_fixture(a);
}

// The chat route's own variables (engine#1788): a request's `tools` reach the template, in the
// caller's key order, and an empty list is "none offered". The route used to pass only
// chat_template_kwargs and enable_thinking, so a template's tools block never rendered.
{
onebit::npu::ChatTemplate t;
t.text = "{% if tools %}TOOLS {{ tools | tojson }} {% endif %}{% if enable_thinking %}think {% endif %}"
"{{ k }}{{ messages[0].content }}";
t.source = "test";
const nlohmann::ordered_json body = nlohmann::ordered_json::parse(
R"({"messages":[{"role":"user","content":"hi"}],"chat_template_kwargs":{"k":"K "},)"
R"("tools":[{"type":"function","function":{"name":"get_weather","description":"d","parameters":{"type":"object","properties":{"zeta":{"type":"string"},"alpha":{"type":"string"}}}}}]})");
try {
const nlohmann::ordered_json vars = onebit::npu::request_template_vars(body, true);
const std::string out = onebit::npu::render_chat_template(t, body.at("messages"), true, vars);
check(out.find("get_weather") != std::string::npos, "request tools: the template sees the request's tools");
const size_t z = out.find("zeta"), a = out.find("alpha");
check(z != std::string::npos && a != std::string::npos && z < a, "request tools: in the caller's key order");
check(out.find("think K hi") != std::string::npos, "request tools: kwargs and enable_thinking still arrive");
nlohmann::ordered_json none = body;
none["tools"] = nlohmann::ordered_json::array();
const std::string plain = onebit::npu::render_chat_template(
t, none.at("messages"), true, onebit::npu::request_template_vars(none, false));
check(plain == "K hi", "request tools: an empty list renders no tools block");
} catch (const std::exception& e) {
check(false, std::string("request tools: rendered (") + e.what() + ")");
}
}

// An earlier call in the conversation (engine#1788): the API's arguments string reaches the template
// as the object, so a template that walks it renders; the route used to answer 400 on the second turn.
{
onebit::npu::ChatTemplate t;
t.text = "{% for m in messages %}{% if m.tool_calls %}{% for c in m.tool_calls %}{{ c.function.name }}("
"{% for k, v in c.function.arguments|items %}{{ k }}={{ v }};{% endfor %}){% endfor %}{% endif %}{% endfor %}";
t.source = "test";
const nlohmann::ordered_json msgs = nlohmann::ordered_json::parse(
R"([{"role":"user","content":"q"},{"role":"assistant","content":"","tool_calls":[{"id":"call_0","type":"function",)"
R"("function":{"name":"get_weather","arguments":"{\"zeta\":\"Paris\",\"alpha\":2}"}}]},)"
R"({"role":"tool","tool_call_id":"call_0","content":"17"}])");
bool raw_threw = false;
try {
(void)onebit::npu::render_chat_template(t, msgs, true);
} catch (const onebit::npu::TemplateError&) {
raw_threw = true;
}
check(raw_threw, "call arguments: as the API sends them (a string) this template cannot walk them");
try {
const std::string out = onebit::npu::render_chat_template(t, onebit::npu::messages_for_template(msgs), true);
check(out == "get_weather(zeta=Paris;alpha=2;)", "call arguments: the template walks the object, in the string's key order");
} catch (const std::exception& e) {
check(false, std::string("call arguments: rendered (") + e.what() + ")");
}
nlohmann::ordered_json odd = msgs;
odd[1]["tool_calls"][0]["function"]["arguments"] = "not json";
check(onebit::npu::messages_for_template(odd) == odd, "call arguments: a string that is not a JSON object is left as sent");
}

// A container that declares no template is this layer's "no", not a ChatML render: the route decides
// what to do about that (app/forward_serve.cpp: ChatML with anchors, else the #987 400).
{
Expand Down
Loading