diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 69212f868..ec7c17046 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -1301,8 +1301,7 @@ int run_forward_serve(int argc, char** argv) { e.tmpl.present() ? "" : " -- the chat route uses ChatML with anchors, else refuses"); const bool can_render = e.tmpl.present() && npu::renderer_built(); if (can_render) e.answer_family = npu::answer_family_of_template(e.tmpl.text); - std::fprintf(stderr, "1bit forward-serve: answer markup=%s\n", - e.answer_family == npu::AnswerFamily::gemma4 ? "gemma4" : "qwen3"); + std::fprintf(stderr, "1bit forward-serve: answer markup=%s\n", npu::answer_detail::family_name(e.answer_family)); if (e.tmpl.present() && !npu::renderer_built()) std::fprintf(stderr, "1bit forward-serve: this build has no chat-template renderer (ONEBIT_JINJA=OFF), so the " diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index 4bd0b210a..3ccb513b1 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -28,7 +28,8 @@ // // Families: Qwen3's markup -- `…` for reasoning and // `{"name": …, "arguments": {…}}` for calls, which is what this route's ChatML -// models emit -- and Gemma 4's (`<|channel>thought…`, `<|tool_call>call:f{…}`). +// models emit -- Gemma 4's (`<|channel>thought…`, `<|tool_call>call:f{…}`) and GPT-OSS's +// harmony (`<|channel|>analysis<|message|>…<|end|>`, `to=functions.f…<|message|>{…}<|call|>`). // answer_family_of_template() picks one from the container's own chat template. AnswerFamily exists so // LFM2 can be added without another signature change. #pragma once @@ -53,6 +54,12 @@ enum class AnswerFamily { // A call body is `call:NAME{key:value,...}` with string values escaped as <|"|>v<|"|> -- NOT JSON, which is // why a JSON-only reader fails on it (the vendor's response_schema ships the same two regexes this follows). gemma4, + // GPT-OSS's harmony format (engine#1786): the reply is a run of messages, each `<|channel|>NAME<|message|>TEXT` + // ended by <|end|>, the next headed by `<|start|>assistant`. The analysis channel is reasoning, final is content, + // and a call is `to=functions.NAME<|channel|>commentary json<|message|>ARGS<|call|>` (the template's own order; + // the model may also write `<|channel|>commentary to=functions.NAME <|constrain|>json<|message|>`). Measured on + // FastFlowLM's GPT-OSS-20B-NPU2: parsed as qwen3, every reply came back with the raw headers in content. + harmony, }; // One tool call, in OpenAI's shape: function.arguments is a JSON *string*, so the parsed object is @@ -83,6 +90,7 @@ inline const char* family_name(AnswerFamily f) { switch (f) { case AnswerFamily::qwen3: return "qwen3"; case AnswerFamily::gemma4: return "gemma4"; + case AnswerFamily::harmony: return "harmony"; } return "?"; } @@ -103,14 +111,29 @@ inline FamilyTags tags_of(AnswerFamily f) { // The reasoning channel opens with <|channel> immediately followed by the literal word `thought`; the // template emits exactly `<|channel>thought\n…\n`, so the pair is the tag. return FamilyTags{"<|channel>thought", "", "<|tool_call>", ""}; + case AnswerFamily::harmony: + // <|end|> closes any message; after the analysis one that is the end of the reasoning. The call opens at + // the recipient (`to=functions.`), which both header orders contain, and closes at <|call|>. + return FamilyTags{"<|channel|>analysis<|message|>", "<|end|>", "to=functions.", "<|call|>"}; } throw std::runtime_error(std::string("answer parser: unsupported family ") + family_name(f)); } +// Header markup that is neither reasoning, content nor a call: dropped where it appears outside a call body. Only +// harmony has any -- the next message's `<|start|>assistant`, the final channel's header (whose text is content), +// a commentary preamble's header, and the stop token should a detokenizer render it. +inline std::vector drop_tags_of(AnswerFamily f) { + if (f == AnswerFamily::harmony) + return {"<|start|>assistant", "<|channel|>final<|message|>", "<|channel|>commentary", "<|message|>", + "<|return|>"}; + return {}; +} + // How many trailing bytes of `buf` could still become a tag: the longest suffix that is a proper // prefix of one of the family's tags. Those are held back until the next chunk decides them. -inline size_t holdback(const std::string& buf, const FamilyTags& t) { - const char* tags[4] = {t.think_open, t.think_close, t.call_open, t.call_close}; +inline size_t holdback(const std::string& buf, const FamilyTags& t, const std::vector& drops = {}) { + std::vector tags = {t.think_open, t.think_close, t.call_open, t.call_close}; + tags.insert(tags.end(), drops.begin(), drops.end()); size_t hold = 0; for (const char* tag : tags) { const size_t n = std::string(tag).size(); @@ -120,6 +143,13 @@ inline size_t holdback(const std::string& buf, const FamilyTags& t) { break; } } + // A whole tag that ends the buffer and reaches back past the held tail is that tag, not the start of another: + // `<|start|>assistant` ends in the `t` that opens `to=functions.`, and holding the `t` back split the tag, so + // neither half matched and the header went out as text (byte-at-a-time stream, harmony). + for (const char* tag : tags) { + const size_t n = std::char_traits::length(tag); + if (n > hold && buf.size() >= n && buf.compare(buf.size() - n, n, tag) == 0) return 0; + } return hold; } @@ -131,6 +161,9 @@ inline size_t holdback(const std::string& buf, const FamilyTags& t) { // `` and neither of those. No template, or one with neither marker, is qwen3 -- what every // container was parsed as before this function existed. inline AnswerFamily answer_family_of_template(const std::string& template_text) { + // GPT-OSS's template writes `<|channel|>analysis<|message|>` for a past turn's thinking (openai/gpt-oss-20b and + // FastFlowLM's GPT-OSS-20B-NPU2: five each); no other family's template carries `<|channel|>`. + if (template_text.find("<|channel|>analysis<|message|>") != std::string::npos) return AnswerFamily::harmony; const answer_detail::FamilyTags g = answer_detail::tags_of(AnswerFamily::gemma4); if (template_text.find(g.call_open) != std::string::npos || template_text.find(g.think_open) != std::string::npos) @@ -145,12 +178,13 @@ class AnswerStream { // `starts_in_reasoning`: the PROMPT already opened the reasoning block (prompt_opens_reasoning below), so the // answer begins inside it and the first thing to look for is the closing tag. AnswerStream(AnswerFamily family, bool tools_sent, bool starts_in_reasoning = false) - : fam_(family), tags_(answer_detail::tags_of(family)), tools_sent_(tools_sent), + : fam_(family), tags_(answer_detail::tags_of(family)), drops_(answer_detail::drop_tags_of(family)), + tools_sent_(tools_sent), mode_(starts_in_reasoning ? Mode::reasoning : Mode::content) {} void feed(const std::string& chunk, std::vector>& out) { buf_ += chunk; - const size_t hold = answer_detail::holdback(buf_, tags_); + const size_t hold = answer_detail::holdback(buf_, tags_, drops_); const std::string safe = buf_.substr(0, buf_.size() - hold); buf_.erase(0, buf_.size() - hold); scan(safe, out); @@ -222,7 +256,8 @@ class AnswerStream { plain.clear(); }; while (i < safe.size()) { - if (safe[i] == '<') { + // Every tag opens with '<' except harmony's recipient, `to=functions.`, so its own first byte is tested too. + if (safe[i] == '<' || safe[i] == tags_.call_open[0]) { // Compared at the tag's own length, never through a fixed window: this read `safe.substr(i, 16)` // once, and gemma4's thinking tag `<|channel>thought` is 17 bytes, so it could never match -- every // tag as long as the window read as text (engine#1779). No width is left here to outgrow. @@ -261,6 +296,23 @@ class AnswerStream { i += std::string(tags_.call_close).size(); continue; } + if (mode_ != Mode::call) { + bool dropped = false; + for (const char* d : drops_) + if (starts(d)) { + flush_plain(); + // The final channel follows the reasoning without an <|end|> when the model skips one: + // its header ends the reasoning all the same. + if (mode_ == Mode::reasoning) { + if (!saw_text_) trim_content_ = true; + mode_ = Mode::content; + } + i += std::char_traits::length(d); + dropped = true; + break; + } + if (dropped) continue; + } } plain += safe[i++]; } @@ -273,6 +325,13 @@ class AnswerStream { mode_ = Mode::content; return; } + if (fam_ == AnswerFamily::harmony) { + if (!close_harmony_call()) + emit(out, false, std::string(tags_.call_open) + call_text_ + tags_.call_close); + call_text_.clear(); + mode_ = Mode::content; + return; + } if (fam_ == AnswerFamily::qwen3 && close_function_call()) { call_text_.clear(); mode_ = Mode::content; @@ -384,6 +443,31 @@ class AnswerStream { // form and, failing that, puts the markup back into content). // shortcut: without the request's schema a string argument whose text is a JSON literal ("17", "true") comes // back as that literal; type by the tool's parameter schema here if a caller needs it kept a string. + // `NAME<|channel|>commentary json<|message|>ARGS` or `NAME <|constrain|>json<|message|>ARGS` (what follows + // `to=functions.`): the name runs to the first space or `<`, the arguments are the JSON after the last + // <|message|>. False on anything else, and the caller puts the markup back into content. + bool close_harmony_call() { + static const std::string msg = "<|message|>"; + const size_t end = call_text_.find_first_of(" <"); + const size_t m = call_text_.rfind(msg); + if (end == 0 || end == std::string::npos || m == std::string::npos || m < end) return false; + const std::string name = call_text_.substr(0, end); + for (const char c : name) + if (!(std::isalnum(static_cast(c)) || c == '_' || c == '-' || c == '.')) return false; + try { + const nlohmann::json j = nlohmann::json::parse(call_text_.substr(m + msg.size())); + if (!j.is_object()) return false; + ToolCall c; + c.id = "call_" + std::to_string(calls_.size()); + c.name = name; + c.arguments = j.dump(); + calls_.push_back(std::move(c)); + return true; + } catch (const std::exception&) { + return false; + } + } + bool close_function_call() { const std::string body = answer_detail::trim(call_text_); static const std::string fopen_ = " drops_; // header markup dropped outside a call (harmony only) bool tools_sent_ = false; Mode mode_ = Mode::content; std::string buf_; // held back: may still become a tag diff --git a/docs/npu-chat-template.md b/docs/npu-chat-template.md index 6e3dc147e..ef3a45645 100644 --- a/docs/npu-chat-template.md +++ b/docs/npu-chat-template.md @@ -192,12 +192,19 @@ reads the answer markup from the template it renders and prints it at load: | the rendered template contains | markup | reasoning | a tool call | |---|---|---|---| | `<|tool_call>` or `<|channel>thought` | `gemma4` | `<|channel>thought … ` | `<|tool_call>call:NAME{key:value}`, strings as `<|"|>…<|"|>` | +| `<|channel|>analysis<|message|>` (GPT-OSS) | `harmony` | `<|channel|>analysis<|message|> … <|end|>`; `<|channel|>final<|message|>` heads the content | `to=functions.NAME<|channel|>commentary json<|message|>{…}<|call|>`, either header order | | neither, or no template (ChatML) | `qwen3` | ` … ` | `{"name": …, "arguments": {…}}`, or `VALUE` | Both give the client the same OpenAI shape: `reasoning_content`, `content`, and `tool_calls` whose `arguments` is a JSON string. A Gemma call body that is not well formed (a string that never closes, a key with no value, an unclosed object or array, an empty value) is not a call: the markup is returned in `content` unchanged. +`harmony` drops the headers between messages (`<|start|>assistant`, `<|channel|>final<|message|>`, a commentary +preamble's `<|channel|>commentary` and `<|message|>`) outside a call. Measured on the NPU with +`GPT-OSS-20B-NPU2` before this markup existed: the reply was parsed as `qwen3`, so "What is the capital of +France?" came back with `content` beginning `<|channel|>analysis<|message|>The user asks…` and no +`reasoning_content`. + **Tools go through the template (engine#1788).** A request's `tools` are a template variable, next to `chat_template_kwargs` and `enable_thinking` (`npu::request_template_vars`), in the caller's key order; an empty list is left out. The template is the only place a model is told what it may call: before this the route kept diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index a92f3d434..603bbba95 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -951,6 +951,54 @@ void test_gemma4_answer_parser() { } } +// GPT-OSS's harmony markup (engine#1786). The first reply is the one FastFlowLM's GPT-OSS-20B-NPU2 gave on the NPU +// (byte for byte, then the final message the 32-token budget cut off); the call forms are the pack's template's own +// order and the order the model card shows, asserted on name and JSON arguments. +void test_harmony_answer_parser() { + const auto H = AnswerFamily::harmony; + { + const std::string reply = + "<|channel|>analysis<|message|>The user asks: \"What is the capital of France? Answer with one word.\" " + "The answer is \"Paris\". So we answer \"Paris\".<|end|><|start|>assistant<|channel|>final<|message|>Paris"; + const auto a = parse_answer(reply, H, false); + check(a.reasoning.rfind("The user asks:", 0) == 0 && a.reasoning.find("<|") == std::string::npos, + "harmony: the analysis channel is reasoning, with no header left in it"); + check(a.content == "Paris", "harmony: the final channel is content, headers dropped"); + // Streamed one byte at a time: the same split. + AnswerStream s(H, false); + std::vector> out; + for (const char c : reply) s.feed(std::string(1, c), out); + s.finish(out); + check(s.reasoning() == a.reasoning && s.content() == "Paris", "harmony: byte-at-a-time stream agrees"); + } + { + const auto a = parse_answer("<|channel|>analysis<|message|>think<|channel|>final<|message|>Done.", H, false); + check(a.reasoning == "think" && a.content == "Done.", "harmony: a final header without <|end|> ends reasoning"); + const auto b = parse_answer("<|channel|>final<|message|>Only this.<|return|>", H, false); + check(b.reasoning.empty() && b.content == "Only this.", "harmony: no analysis, and a rendered stop token"); + } + { + const std::string tmpl_form = + "<|channel|>analysis<|message|>Need weather.<|end|><|start|>assistant to=functions.get_weather" + "<|channel|>commentary json<|message|>{\"city\":\"Paris\"}<|call|>"; + const auto a = parse_answer(tmpl_form, H, true); + check(a.calls.size() == 1 && a.calls[0].name == "get_weather" && a.calls[0].arguments == R"({"city":"Paris"})", + "harmony: the template's call order is a call"); + check(a.reasoning == "Need weather." && a.content.empty() && a.content_null, + "harmony: a call-only reply has content: null"); + const auto b = parse_answer( + "<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>{\"city\":\"Oslo\"}<|call|>", H, true); + check(b.calls.size() == 1 && b.calls[0].name == "get_weather" && b.calls[0].arguments == R"({"city":"Oslo"})", + "harmony: the card's call order is a call too"); + const auto c = parse_answer(tmpl_form, H, false); + check(c.calls.empty() && c.content.find("to=functions.get_weather") != std::string::npos, + "harmony: without tools the call markup stays in content"); + const std::string bad = "to=functions.f<|channel|>commentary json<|message|>{not json<|call|>"; + const auto d = parse_answer(bad, H, true); + check(d.calls.empty() && d.content == bad, "harmony: arguments that are not JSON are back in content, whole"); + } +} + // The family comes from the container's chat template. The Gemma 4 lines are the two literals the pack's own // chat_template carries (amd/gemma-4-e4b-it-mobius-int4_rai_1.9.0_npu: `<|tool_call>` x1, `<|channel>thought` // x1, `` x0, `` x0), not a guess at them. @@ -964,6 +1012,9 @@ void test_answer_family_of_template() { "a template that writes <|tool_call>call: is gemma4"); check(answer_family_of_template("{{- '<|channel>thought\\n' -}}") == AnswerFamily::gemma4, "a template with the thought channel and no tools is gemma4 too"); + check(answer_family_of_template("{{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.thinking -}}") == + AnswerFamily::harmony, + "a template that writes the analysis channel is harmony"); check(answer_family_of_template("<|im_start|>user\n{{ content }}<|im_end|>") == AnswerFamily::qwen3, "a template with neither marker is qwen3"); } @@ -972,7 +1023,7 @@ void test_answer_family_of_template() { // every byte. The scanner once read tags through a 16-byte window, so a 17-byte tag read as text (engine#1779); // this arm fails for any family whose tags the scanner cannot match, including one added later. void test_every_family_tag_is_matched() { - for (const AnswerFamily f : {AnswerFamily::qwen3, AnswerFamily::gemma4}) { + for (const AnswerFamily f : {AnswerFamily::qwen3, AnswerFamily::gemma4, AnswerFamily::harmony}) { const auto t = onebit::npu::answer_detail::tags_of(f); const std::string name = onebit::npu::answer_detail::family_name(f); const std::string think = std::string(t.think_open) + "R" + t.think_close + "C"; @@ -1359,6 +1410,7 @@ int main() { test_answer_stream(); test_tool_grammar(); test_gemma4_answer_parser(); + test_harmony_answer_parser(); test_every_family_tag_is_matched(); test_answer_family_of_template(); test_tool_request_parsing();