diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp
index 69212f868..ec7c17046 100644
--- a/app/forward_serve.cpp
+++ b/app/forward_serve.cpp
@@ -1301,8 +1301,7 @@ int run_forward_serve(int argc, char** argv) {
e.tmpl.present() ? "" : " -- the chat route uses ChatML with anchors, else refuses");
const bool can_render = e.tmpl.present() && npu::renderer_built();
if (can_render) e.answer_family = npu::answer_family_of_template(e.tmpl.text);
- std::fprintf(stderr, "1bit forward-serve: answer markup=%s\n",
- e.answer_family == npu::AnswerFamily::gemma4 ? "gemma4" : "qwen3");
+ std::fprintf(stderr, "1bit forward-serve: answer markup=%s\n", npu::answer_detail::family_name(e.answer_family));
if (e.tmpl.present() && !npu::renderer_built())
std::fprintf(stderr,
"1bit forward-serve: this build has no chat-template renderer (ONEBIT_JINJA=OFF), so the "
diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h
index 4bd0b210a..3ccb513b1 100644
--- a/app/npu_answer_parser.h
+++ b/app/npu_answer_parser.h
@@ -28,7 +28,8 @@
//
// Families: Qwen3's markup -- `…` for reasoning and
// `{"name": …, "arguments": {…}}` for calls, which is what this route's ChatML
-// models emit -- and Gemma 4's (`<|channel>thought…`, `<|tool_call>call:f{…}`).
+// models emit -- Gemma 4's (`<|channel>thought…`, `<|tool_call>call:f{…}`) and GPT-OSS's
+// harmony (`<|channel|>analysis<|message|>…<|end|>`, `to=functions.f…<|message|>{…}<|call|>`).
// answer_family_of_template() picks one from the container's own chat template. AnswerFamily exists so
// LFM2 can be added without another signature change.
#pragma once
@@ -53,6 +54,12 @@ enum class AnswerFamily {
// A call body is `call:NAME{key:value,...}` with string values escaped as <|"|>v<|"|> -- NOT JSON, which is
// why a JSON-only reader fails on it (the vendor's response_schema ships the same two regexes this follows).
gemma4,
+ // GPT-OSS's harmony format (engine#1786): the reply is a run of messages, each `<|channel|>NAME<|message|>TEXT`
+ // ended by <|end|>, the next headed by `<|start|>assistant`. The analysis channel is reasoning, final is content,
+ // and a call is `to=functions.NAME<|channel|>commentary json<|message|>ARGS<|call|>` (the template's own order;
+ // the model may also write `<|channel|>commentary to=functions.NAME <|constrain|>json<|message|>`). Measured on
+ // FastFlowLM's GPT-OSS-20B-NPU2: parsed as qwen3, every reply came back with the raw headers in content.
+ harmony,
};
// One tool call, in OpenAI's shape: function.arguments is a JSON *string*, so the parsed object is
@@ -83,6 +90,7 @@ inline const char* family_name(AnswerFamily f) {
switch (f) {
case AnswerFamily::qwen3: return "qwen3";
case AnswerFamily::gemma4: return "gemma4";
+ case AnswerFamily::harmony: return "harmony";
}
return "?";
}
@@ -103,14 +111,29 @@ inline FamilyTags tags_of(AnswerFamily f) {
// The reasoning channel opens with <|channel> immediately followed by the literal word `thought`; the
// template emits exactly `<|channel>thought\n…\n`, so the pair is the tag.
return FamilyTags{"<|channel>thought", "", "<|tool_call>", ""};
+ case AnswerFamily::harmony:
+ // <|end|> closes any message; after the analysis one that is the end of the reasoning. The call opens at
+ // the recipient (`to=functions.`), which both header orders contain, and closes at <|call|>.
+ return FamilyTags{"<|channel|>analysis<|message|>", "<|end|>", "to=functions.", "<|call|>"};
}
throw std::runtime_error(std::string("answer parser: unsupported family ") + family_name(f));
}
+// Header markup that is neither reasoning, content nor a call: dropped where it appears outside a call body. Only
+// harmony has any -- the next message's `<|start|>assistant`, the final channel's header (whose text is content),
+// a commentary preamble's header, and the stop token should a detokenizer render it.
+inline std::vector drop_tags_of(AnswerFamily f) {
+ if (f == AnswerFamily::harmony)
+ return {"<|start|>assistant", "<|channel|>final<|message|>", "<|channel|>commentary", "<|message|>",
+ "<|return|>"};
+ return {};
+}
+
// How many trailing bytes of `buf` could still become a tag: the longest suffix that is a proper
// prefix of one of the family's tags. Those are held back until the next chunk decides them.
-inline size_t holdback(const std::string& buf, const FamilyTags& t) {
- const char* tags[4] = {t.think_open, t.think_close, t.call_open, t.call_close};
+inline size_t holdback(const std::string& buf, const FamilyTags& t, const std::vector& drops = {}) {
+ std::vector tags = {t.think_open, t.think_close, t.call_open, t.call_close};
+ tags.insert(tags.end(), drops.begin(), drops.end());
size_t hold = 0;
for (const char* tag : tags) {
const size_t n = std::string(tag).size();
@@ -120,6 +143,13 @@ inline size_t holdback(const std::string& buf, const FamilyTags& t) {
break;
}
}
+ // A whole tag that ends the buffer and reaches back past the held tail is that tag, not the start of another:
+ // `<|start|>assistant` ends in the `t` that opens `to=functions.`, and holding the `t` back split the tag, so
+ // neither half matched and the header went out as text (byte-at-a-time stream, harmony).
+ for (const char* tag : tags) {
+ const size_t n = std::char_traits::length(tag);
+ if (n > hold && buf.size() >= n && buf.compare(buf.size() - n, n, tag) == 0) return 0;
+ }
return hold;
}
@@ -131,6 +161,9 @@ inline size_t holdback(const std::string& buf, const FamilyTags& t) {
// `` and neither of those. No template, or one with neither marker, is qwen3 -- what every
// container was parsed as before this function existed.
inline AnswerFamily answer_family_of_template(const std::string& template_text) {
+ // GPT-OSS's template writes `<|channel|>analysis<|message|>` for a past turn's thinking (openai/gpt-oss-20b and
+ // FastFlowLM's GPT-OSS-20B-NPU2: five each); no other family's template carries `<|channel|>`.
+ if (template_text.find("<|channel|>analysis<|message|>") != std::string::npos) return AnswerFamily::harmony;
const answer_detail::FamilyTags g = answer_detail::tags_of(AnswerFamily::gemma4);
if (template_text.find(g.call_open) != std::string::npos ||
template_text.find(g.think_open) != std::string::npos)
@@ -145,12 +178,13 @@ class AnswerStream {
// `starts_in_reasoning`: the PROMPT already opened the reasoning block (prompt_opens_reasoning below), so the
// answer begins inside it and the first thing to look for is the closing tag.
AnswerStream(AnswerFamily family, bool tools_sent, bool starts_in_reasoning = false)
- : fam_(family), tags_(answer_detail::tags_of(family)), tools_sent_(tools_sent),
+ : fam_(family), tags_(answer_detail::tags_of(family)), drops_(answer_detail::drop_tags_of(family)),
+ tools_sent_(tools_sent),
mode_(starts_in_reasoning ? Mode::reasoning : Mode::content) {}
void feed(const std::string& chunk, std::vector>& out) {
buf_ += chunk;
- const size_t hold = answer_detail::holdback(buf_, tags_);
+ const size_t hold = answer_detail::holdback(buf_, tags_, drops_);
const std::string safe = buf_.substr(0, buf_.size() - hold);
buf_.erase(0, buf_.size() - hold);
scan(safe, out);
@@ -222,7 +256,8 @@ class AnswerStream {
plain.clear();
};
while (i < safe.size()) {
- if (safe[i] == '<') {
+ // Every tag opens with '<' except harmony's recipient, `to=functions.`, so its own first byte is tested too.
+ if (safe[i] == '<' || safe[i] == tags_.call_open[0]) {
// Compared at the tag's own length, never through a fixed window: this read `safe.substr(i, 16)`
// once, and gemma4's thinking tag `<|channel>thought` is 17 bytes, so it could never match -- every
// tag as long as the window read as text (engine#1779). No width is left here to outgrow.
@@ -261,6 +296,23 @@ class AnswerStream {
i += std::string(tags_.call_close).size();
continue;
}
+ if (mode_ != Mode::call) {
+ bool dropped = false;
+ for (const char* d : drops_)
+ if (starts(d)) {
+ flush_plain();
+ // The final channel follows the reasoning without an <|end|> when the model skips one:
+ // its header ends the reasoning all the same.
+ if (mode_ == Mode::reasoning) {
+ if (!saw_text_) trim_content_ = true;
+ mode_ = Mode::content;
+ }
+ i += std::char_traits::length(d);
+ dropped = true;
+ break;
+ }
+ if (dropped) continue;
+ }
}
plain += safe[i++];
}
@@ -273,6 +325,13 @@ class AnswerStream {
mode_ = Mode::content;
return;
}
+ if (fam_ == AnswerFamily::harmony) {
+ if (!close_harmony_call())
+ emit(out, false, std::string(tags_.call_open) + call_text_ + tags_.call_close);
+ call_text_.clear();
+ mode_ = Mode::content;
+ return;
+ }
if (fam_ == AnswerFamily::qwen3 && close_function_call()) {
call_text_.clear();
mode_ = Mode::content;
@@ -384,6 +443,31 @@ class AnswerStream {
// form and, failing that, puts the markup back into content).
// shortcut: without the request's schema a string argument whose text is a JSON literal ("17", "true") comes
// back as that literal; type by the tool's parameter schema here if a caller needs it kept a string.
+ // `NAME<|channel|>commentary json<|message|>ARGS` or `NAME <|constrain|>json<|message|>ARGS` (what follows
+ // `to=functions.`): the name runs to the first space or `<`, the arguments are the JSON after the last
+ // <|message|>. False on anything else, and the caller puts the markup back into content.
+ bool close_harmony_call() {
+ static const std::string msg = "<|message|>";
+ const size_t end = call_text_.find_first_of(" <");
+ const size_t m = call_text_.rfind(msg);
+ if (end == 0 || end == std::string::npos || m == std::string::npos || m < end) return false;
+ const std::string name = call_text_.substr(0, end);
+ for (const char c : name)
+ if (!(std::isalnum(static_cast(c)) || c == '_' || c == '-' || c == '.')) return false;
+ try {
+ const nlohmann::json j = nlohmann::json::parse(call_text_.substr(m + msg.size()));
+ if (!j.is_object()) return false;
+ ToolCall c;
+ c.id = "call_" + std::to_string(calls_.size());
+ c.name = name;
+ c.arguments = j.dump();
+ calls_.push_back(std::move(c));
+ return true;
+ } catch (const std::exception&) {
+ return false;
+ }
+ }
+
bool close_function_call() {
const std::string body = answer_detail::trim(call_text_);
static const std::string fopen_ = " drops_; // header markup dropped outside a call (harmony only)
bool tools_sent_ = false;
Mode mode_ = Mode::content;
std::string buf_; // held back: may still become a tag
diff --git a/docs/npu-chat-template.md b/docs/npu-chat-template.md
index 6e3dc147e..ef3a45645 100644
--- a/docs/npu-chat-template.md
+++ b/docs/npu-chat-template.md
@@ -192,12 +192,19 @@ reads the answer markup from the template it renders and prints it at load:
| the rendered template contains | markup | reasoning | a tool call |
|---|---|---|---|
| `<|tool_call>` or `<|channel>thought` | `gemma4` | `<|channel>thought … ` | `<|tool_call>call:NAME{key:value}`, strings as `<|"|>…<|"|>` |
+| `<|channel|>analysis<|message|>` (GPT-OSS) | `harmony` | `<|channel|>analysis<|message|> … <|end|>`; `<|channel|>final<|message|>` heads the content | `to=functions.NAME<|channel|>commentary json<|message|>{…}<|call|>`, either header order |
| neither, or no template (ChatML) | `qwen3` | ` … ` | `{"name": …, "arguments": {…}}`, or `VALUE` |
Both give the client the same OpenAI shape: `reasoning_content`, `content`, and `tool_calls` whose `arguments`
is a JSON string. A Gemma call body that is not well formed (a string that never closes, a key with no value,
an unclosed object or array, an empty value) is not a call: the markup is returned in `content` unchanged.
+`harmony` drops the headers between messages (`<|start|>assistant`, `<|channel|>final<|message|>`, a commentary
+preamble's `<|channel|>commentary` and `<|message|>`) outside a call. Measured on the NPU with
+`GPT-OSS-20B-NPU2` before this markup existed: the reply was parsed as `qwen3`, so "What is the capital of
+France?" came back with `content` beginning `<|channel|>analysis<|message|>The user asks…` and no
+`reasoning_content`.
+
**Tools go through the template (engine#1788).** A request's `tools` are a template variable, next to
`chat_template_kwargs` and `enable_thinking` (`npu::request_template_vars`), in the caller's key order; an empty
list is left out. The template is the only place a model is told what it may call: before this the route kept
diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp
index a92f3d434..603bbba95 100644
--- a/tests/npu_sampling_test.cpp
+++ b/tests/npu_sampling_test.cpp
@@ -951,6 +951,54 @@ void test_gemma4_answer_parser() {
}
}
+// GPT-OSS's harmony markup (engine#1786). The first reply is the one FastFlowLM's GPT-OSS-20B-NPU2 gave on the NPU
+// (byte for byte, then the final message the 32-token budget cut off); the call forms are the pack's template's own
+// order and the order the model card shows, asserted on name and JSON arguments.
+void test_harmony_answer_parser() {
+ const auto H = AnswerFamily::harmony;
+ {
+ const std::string reply =
+ "<|channel|>analysis<|message|>The user asks: \"What is the capital of France? Answer with one word.\" "
+ "The answer is \"Paris\". So we answer \"Paris\".<|end|><|start|>assistant<|channel|>final<|message|>Paris";
+ const auto a = parse_answer(reply, H, false);
+ check(a.reasoning.rfind("The user asks:", 0) == 0 && a.reasoning.find("<|") == std::string::npos,
+ "harmony: the analysis channel is reasoning, with no header left in it");
+ check(a.content == "Paris", "harmony: the final channel is content, headers dropped");
+ // Streamed one byte at a time: the same split.
+ AnswerStream s(H, false);
+ std::vector> out;
+ for (const char c : reply) s.feed(std::string(1, c), out);
+ s.finish(out);
+ check(s.reasoning() == a.reasoning && s.content() == "Paris", "harmony: byte-at-a-time stream agrees");
+ }
+ {
+ const auto a = parse_answer("<|channel|>analysis<|message|>think<|channel|>final<|message|>Done.", H, false);
+ check(a.reasoning == "think" && a.content == "Done.", "harmony: a final header without <|end|> ends reasoning");
+ const auto b = parse_answer("<|channel|>final<|message|>Only this.<|return|>", H, false);
+ check(b.reasoning.empty() && b.content == "Only this.", "harmony: no analysis, and a rendered stop token");
+ }
+ {
+ const std::string tmpl_form =
+ "<|channel|>analysis<|message|>Need weather.<|end|><|start|>assistant to=functions.get_weather"
+ "<|channel|>commentary json<|message|>{\"city\":\"Paris\"}<|call|>";
+ const auto a = parse_answer(tmpl_form, H, true);
+ check(a.calls.size() == 1 && a.calls[0].name == "get_weather" && a.calls[0].arguments == R"({"city":"Paris"})",
+ "harmony: the template's call order is a call");
+ check(a.reasoning == "Need weather." && a.content.empty() && a.content_null,
+ "harmony: a call-only reply has content: null");
+ const auto b = parse_answer(
+ "<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>{\"city\":\"Oslo\"}<|call|>", H, true);
+ check(b.calls.size() == 1 && b.calls[0].name == "get_weather" && b.calls[0].arguments == R"({"city":"Oslo"})",
+ "harmony: the card's call order is a call too");
+ const auto c = parse_answer(tmpl_form, H, false);
+ check(c.calls.empty() && c.content.find("to=functions.get_weather") != std::string::npos,
+ "harmony: without tools the call markup stays in content");
+ const std::string bad = "to=functions.f<|channel|>commentary json<|message|>{not json<|call|>";
+ const auto d = parse_answer(bad, H, true);
+ check(d.calls.empty() && d.content == bad, "harmony: arguments that are not JSON are back in content, whole");
+ }
+}
+
// The family comes from the container's chat template. The Gemma 4 lines are the two literals the pack's own
// chat_template carries (amd/gemma-4-e4b-it-mobius-int4_rai_1.9.0_npu: `<|tool_call>` x1, `<|channel>thought`
// x1, `` x0, `` x0), not a guess at them.
@@ -964,6 +1012,9 @@ void test_answer_family_of_template() {
"a template that writes <|tool_call>call: is gemma4");
check(answer_family_of_template("{{- '<|channel>thought\\n' -}}") == AnswerFamily::gemma4,
"a template with the thought channel and no tools is gemma4 too");
+ check(answer_family_of_template("{{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.thinking -}}") ==
+ AnswerFamily::harmony,
+ "a template that writes the analysis channel is harmony");
check(answer_family_of_template("<|im_start|>user\n{{ content }}<|im_end|>") == AnswerFamily::qwen3,
"a template with neither marker is qwen3");
}
@@ -972,7 +1023,7 @@ void test_answer_family_of_template() {
// every byte. The scanner once read tags through a 16-byte window, so a 17-byte tag read as text (engine#1779);
// this arm fails for any family whose tags the scanner cannot match, including one added later.
void test_every_family_tag_is_matched() {
- for (const AnswerFamily f : {AnswerFamily::qwen3, AnswerFamily::gemma4}) {
+ for (const AnswerFamily f : {AnswerFamily::qwen3, AnswerFamily::gemma4, AnswerFamily::harmony}) {
const auto t = onebit::npu::answer_detail::tags_of(f);
const std::string name = onebit::npu::answer_detail::family_name(f);
const std::string think = std::string(t.think_open) + "R" + t.think_close + "C";
@@ -1359,6 +1410,7 @@ int main() {
test_answer_stream();
test_tool_grammar();
test_gemma4_answer_parser();
+ test_harmony_answer_parser();
test_every_family_tag_is_matched();
test_answer_family_of_template();
test_tool_request_parsing();