Skip to content

forward-serve: GPT-OSS replies are split by harmony markup, not qwen3 (refs #1786) #2298

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bong-water-water-bong merged 1 commit into from
Nov 12, 2026
Merged

forward-serve: GPT-OSS replies are split by harmony markup, not qwen3 (refs #1786) #2298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions app/forward_serve.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1301,8 +1301,7 @@ int run_forward_serve(int argc, char** argv) {
e.tmpl.present() ? "" : " -- the chat route uses ChatML with anchors, else refuses");
const bool can_render = e.tmpl.present() && npu::renderer_built();
if (can_render) e.answer_family = npu::answer_family_of_template(e.tmpl.text);
std::fprintf(stderr, "1bit forward-serve: answer markup=%s\n",
e.answer_family == npu::AnswerFamily::gemma4 ? "gemma4" : "qwen3");
std::fprintf(stderr, "1bit forward-serve: answer markup=%s\n", npu::answer_detail::family_name(e.answer_family));
if (e.tmpl.present() && !npu::renderer_built())
std::fprintf(stderr,
"1bit forward-serve: this build has no chat-template renderer (ONEBIT_JINJA=OFF), so the "
Expand Down
97 changes: 91 additions & 6 deletions app/npu_answer_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
//
// Families: Qwen3's markup -- `<think>…</think>` for reasoning and
// `<tool_call>{"name": …, "arguments": {…}}</tool_call>` for calls, which is what this route's ChatML
// models emit -- and Gemma 4's (`<|channel>thought…<channel|>`, `<|tool_call>call:f{…}<tool_call|>`).
// models emit -- Gemma 4's (`<|channel>thought…<channel|>`, `<|tool_call>call:f{…}<tool_call|>`) and GPT-OSS's
// harmony (`<|channel|>analysis<|message|>…<|end|>`, `to=functions.f…<|message|>{…}<|call|>`).
// answer_family_of_template() picks one from the container's own chat template. AnswerFamily exists so
// LFM2 can be added without another signature change.
#pragma once
Expand All @@ -53,6 +54,12 @@ enum class AnswerFamily {
// A call body is `call:NAME{key:value,...}` with string values escaped as <|"|>v<|"|> -- NOT JSON, which is
// why a JSON-only reader fails on it (the vendor's response_schema ships the same two regexes this follows).
gemma4,
// GPT-OSS's harmony format (engine#1786): the reply is a run of messages, each `<|channel|>NAME<|message|>TEXT`
// ended by <|end|>, the next headed by `<|start|>assistant`. The analysis channel is reasoning, final is content,
// and a call is `to=functions.NAME<|channel|>commentary json<|message|>ARGS<|call|>` (the template's own order;
// the model may also write `<|channel|>commentary to=functions.NAME <|constrain|>json<|message|>`). Measured on
// FastFlowLM's GPT-OSS-20B-NPU2: parsed as qwen3, every reply came back with the raw headers in content.
harmony,
};

// One tool call, in OpenAI's shape: function.arguments is a JSON *string*, so the parsed object is
Expand Down Expand Up @@ -83,6 +90,7 @@ inline const char* family_name(AnswerFamily f) {
switch (f) {
case AnswerFamily::qwen3: return "qwen3";
case AnswerFamily::gemma4: return "gemma4";
case AnswerFamily::harmony: return "harmony";
}
return "?";
}
Expand All @@ -103,14 +111,29 @@ inline FamilyTags tags_of(AnswerFamily f) {
// The reasoning channel opens with <|channel> immediately followed by the literal word `thought`; the
// template emits exactly `<|channel>thought\n…\n<channel|>`, so the pair is the tag.
return FamilyTags{"<|channel>thought", "<channel|>", "<|tool_call>", "<tool_call|>"};
case AnswerFamily::harmony:
// <|end|> closes any message; after the analysis one that is the end of the reasoning. The call opens at
// the recipient (`to=functions.`), which both header orders contain, and closes at <|call|>.
return FamilyTags{"<|channel|>analysis<|message|>", "<|end|>", "to=functions.", "<|call|>"};
}
throw std::runtime_error(std::string("answer parser: unsupported family ") + family_name(f));
}

// Header markup that is neither reasoning, content nor a call: dropped where it appears outside a call body. Only
// harmony has any -- the next message's `<|start|>assistant`, the final channel's header (whose text is content),
// a commentary preamble's header, and the stop token should a detokenizer render it.
inline std::vector<const char*> drop_tags_of(AnswerFamily f) {
if (f == AnswerFamily::harmony)
return {"<|start|>assistant", "<|channel|>final<|message|>", "<|channel|>commentary", "<|message|>",
"<|return|>"};
return {};
}

// How many trailing bytes of `buf` could still become a tag: the longest suffix that is a proper
// prefix of one of the family's tags. Those are held back until the next chunk decides them.
inline size_t holdback(const std::string& buf, const FamilyTags& t) {
const char* tags[4] = {t.think_open, t.think_close, t.call_open, t.call_close};
inline size_t holdback(const std::string& buf, const FamilyTags& t, const std::vector<const char*>& drops = {}) {
std::vector<const char*> tags = {t.think_open, t.think_close, t.call_open, t.call_close};
tags.insert(tags.end(), drops.begin(), drops.end());
size_t hold = 0;
for (const char* tag : tags) {
const size_t n = std::string(tag).size();
Expand All @@ -120,6 +143,13 @@ inline size_t holdback(const std::string& buf, const FamilyTags& t) {
break;
}
}
// A whole tag that ends the buffer and reaches back past the held tail is that tag, not the start of another:
// `<|start|>assistant` ends in the `t` that opens `to=functions.`, and holding the `t` back split the tag, so
// neither half matched and the header went out as text (byte-at-a-time stream, harmony).
for (const char* tag : tags) {
const size_t n = std::char_traits<char>::length(tag);
if (n > hold && buf.size() >= n && buf.compare(buf.size() - n, n, tag) == 0) return 0;
}
return hold;
}

Expand All @@ -131,6 +161,9 @@ inline size_t holdback(const std::string& buf, const FamilyTags& t) {
// `<tool_call>` and neither of those. No template, or one with neither marker, is qwen3 -- what every
// container was parsed as before this function existed.
inline AnswerFamily answer_family_of_template(const std::string& template_text) {
// GPT-OSS's template writes `<|channel|>analysis<|message|>` for a past turn's thinking (openai/gpt-oss-20b and
// FastFlowLM's GPT-OSS-20B-NPU2: five each); no other family's template carries `<|channel|>`.
if (template_text.find("<|channel|>analysis<|message|>") != std::string::npos) return AnswerFamily::harmony;
const answer_detail::FamilyTags g = answer_detail::tags_of(AnswerFamily::gemma4);
if (template_text.find(g.call_open) != std::string::npos ||
template_text.find(g.think_open) != std::string::npos)
Expand All @@ -145,12 +178,13 @@ class AnswerStream {
// `starts_in_reasoning`: the PROMPT already opened the reasoning block (prompt_opens_reasoning below), so the
// answer begins inside it and the first thing to look for is the closing tag.
AnswerStream(AnswerFamily family, bool tools_sent, bool starts_in_reasoning = false)
: fam_(family), tags_(answer_detail::tags_of(family)), tools_sent_(tools_sent),
: fam_(family), tags_(answer_detail::tags_of(family)), drops_(answer_detail::drop_tags_of(family)),
tools_sent_(tools_sent),
mode_(starts_in_reasoning ? Mode::reasoning : Mode::content) {}

void feed(const std::string& chunk, std::vector<std::pair<bool, std::string>>& out) {
buf_ += chunk;
const size_t hold = answer_detail::holdback(buf_, tags_);
const size_t hold = answer_detail::holdback(buf_, tags_, drops_);
const std::string safe = buf_.substr(0, buf_.size() - hold);
buf_.erase(0, buf_.size() - hold);
scan(safe, out);
Expand Down Expand Up @@ -222,7 +256,8 @@ class AnswerStream {
plain.clear();
};
while (i < safe.size()) {
if (safe[i] == '<') {
// Every tag opens with '<' except harmony's recipient, `to=functions.`, so its own first byte is tested too.
if (safe[i] == '<' || safe[i] == tags_.call_open[0]) {
// Compared at the tag's own length, never through a fixed window: this read `safe.substr(i, 16)`
// once, and gemma4's thinking tag `<|channel>thought` is 17 bytes, so it could never match -- every
// tag as long as the window read as text (engine#1779). No width is left here to outgrow.
Expand Down Expand Up @@ -261,6 +296,23 @@ class AnswerStream {
i += std::string(tags_.call_close).size();
continue;
}
if (mode_ != Mode::call) {
bool dropped = false;
for (const char* d : drops_)
if (starts(d)) {
flush_plain();
// The final channel follows the reasoning without an <|end|> when the model skips one:
// its header ends the reasoning all the same.
if (mode_ == Mode::reasoning) {
if (!saw_text_) trim_content_ = true;
mode_ = Mode::content;
}
i += std::char_traits<char>::length(d);
dropped = true;
break;
}
if (dropped) continue;
}
}
plain += safe[i++];
}
Expand All @@ -273,6 +325,13 @@ class AnswerStream {
mode_ = Mode::content;
return;
}
if (fam_ == AnswerFamily::harmony) {
if (!close_harmony_call())
emit(out, false, std::string(tags_.call_open) + call_text_ + tags_.call_close);
call_text_.clear();
mode_ = Mode::content;
return;
}
if (fam_ == AnswerFamily::qwen3 && close_function_call()) {
call_text_.clear();
mode_ = Mode::content;
Expand Down Expand Up @@ -384,6 +443,31 @@ class AnswerStream {
// form and, failing that, puts the markup back into content).
// shortcut: without the request's schema a string argument whose text is a JSON literal ("17", "true") comes
// back as that literal; type by the tool's parameter schema here if a caller needs it kept a string.
// `NAME<|channel|>commentary json<|message|>ARGS` or `NAME <|constrain|>json<|message|>ARGS` (what follows
// `to=functions.`): the name runs to the first space or `<`, the arguments are the JSON after the last
// <|message|>. False on anything else, and the caller puts the markup back into content.
bool close_harmony_call() {
static const std::string msg = "<|message|>";
const size_t end = call_text_.find_first_of(" <");
const size_t m = call_text_.rfind(msg);
if (end == 0 || end == std::string::npos || m == std::string::npos || m < end) return false;
const std::string name = call_text_.substr(0, end);
for (const char c : name)
if (!(std::isalnum(static_cast<unsigned char>(c)) || c == '_' || c == '-' || c == '.')) return false;
try {
const nlohmann::json j = nlohmann::json::parse(call_text_.substr(m + msg.size()));
if (!j.is_object()) return false;
ToolCall c;
c.id = "call_" + std::to_string(calls_.size());
c.name = name;
c.arguments = j.dump();
calls_.push_back(std::move(c));
return true;
} catch (const std::exception&) {
return false;
}
}

bool close_function_call() {
const std::string body = answer_detail::trim(call_text_);
static const std::string fopen_ = "<function=", fclose_ = "</function>", popen_ = "<parameter=", pclose_ = "</parameter>";
Expand Down Expand Up @@ -445,6 +529,7 @@ class AnswerStream {

AnswerFamily fam_;
answer_detail::FamilyTags tags_;
std::vector<const char*> drops_; // header markup dropped outside a call (harmony only)
bool tools_sent_ = false;
Mode mode_ = Mode::content;
std::string buf_; // held back: may still become a tag
Expand Down
7 changes: 7 additions & 0 deletions docs/npu-chat-template.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,12 +192,19 @@ reads the answer markup from the template it renders and prints it at load:
| the rendered template contains | markup | reasoning | a tool call |
|---|---|---|---|
| `<|tool_call>` or `<|channel>thought` | `gemma4` | `<|channel>thought … <channel|>` | `<|tool_call>call:NAME{key:value}<tool_call|>`, strings as `<|"|>…<|"|>` |
| `<|channel|>analysis<|message|>` (GPT-OSS) | `harmony` | `<|channel|>analysis<|message|> … <|end|>`; `<|channel|>final<|message|>` heads the content | `to=functions.NAME<|channel|>commentary json<|message|>{…}<|call|>`, either header order |
| neither, or no template (ChatML) | `qwen3` | `<think> … </think>` | `<tool_call>{"name": …, "arguments": {…}}</tool_call>`, or `<tool_call><function=NAME><parameter=KEY>VALUE</parameter></function></tool_call>` |

Both give the client the same OpenAI shape: `reasoning_content`, `content`, and `tool_calls` whose `arguments`
is a JSON string. A Gemma call body that is not well formed (a string that never closes, a key with no value,
an unclosed object or array, an empty value) is not a call: the markup is returned in `content` unchanged.

`harmony` drops the headers between messages (`<|start|>assistant`, `<|channel|>final<|message|>`, a commentary
preamble's `<|channel|>commentary` and `<|message|>`) outside a call. Measured on the NPU with
`GPT-OSS-20B-NPU2` before this markup existed: the reply was parsed as `qwen3`, so "What is the capital of
France?" came back with `content` beginning `<|channel|>analysis<|message|>The user asks…` and no
`reasoning_content`.

**Tools go through the template (engine#1788).** A request's `tools` are a template variable, next to
`chat_template_kwargs` and `enable_thinking` (`npu::request_template_vars`), in the caller's key order; an empty
list is left out. The template is the only place a model is told what it may call: before this the route kept
Expand Down
54 changes: 53 additions & 1 deletion tests/npu_sampling_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -951,6 +951,54 @@ void test_gemma4_answer_parser() {
}
}

// GPT-OSS's harmony markup (engine#1786). The first reply is the one FastFlowLM's GPT-OSS-20B-NPU2 gave on the NPU
// (byte for byte, then the final message the 32-token budget cut off); the call forms are the pack's template's own
// order and the order the model card shows, asserted on name and JSON arguments.
void test_harmony_answer_parser() {
const auto H = AnswerFamily::harmony;
{
const std::string reply =
"<|channel|>analysis<|message|>The user asks: \"What is the capital of France? Answer with one word.\" "
"The answer is \"Paris\". So we answer \"Paris\".<|end|><|start|>assistant<|channel|>final<|message|>Paris";
const auto a = parse_answer(reply, H, false);
check(a.reasoning.rfind("The user asks:", 0) == 0 && a.reasoning.find("<|") == std::string::npos,
"harmony: the analysis channel is reasoning, with no header left in it");
check(a.content == "Paris", "harmony: the final channel is content, headers dropped");
// Streamed one byte at a time: the same split.
AnswerStream s(H, false);
std::vector<std::pair<bool, std::string>> out;
for (const char c : reply) s.feed(std::string(1, c), out);
s.finish(out);
check(s.reasoning() == a.reasoning && s.content() == "Paris", "harmony: byte-at-a-time stream agrees");
}
{
const auto a = parse_answer("<|channel|>analysis<|message|>think<|channel|>final<|message|>Done.", H, false);
check(a.reasoning == "think" && a.content == "Done.", "harmony: a final header without <|end|> ends reasoning");
const auto b = parse_answer("<|channel|>final<|message|>Only this.<|return|>", H, false);
check(b.reasoning.empty() && b.content == "Only this.", "harmony: no analysis, and a rendered stop token");
}
{
const std::string tmpl_form =
"<|channel|>analysis<|message|>Need weather.<|end|><|start|>assistant to=functions.get_weather"
"<|channel|>commentary json<|message|>{\"city\":\"Paris\"}<|call|>";
const auto a = parse_answer(tmpl_form, H, true);
check(a.calls.size() == 1 && a.calls[0].name == "get_weather" && a.calls[0].arguments == R"({"city":"Paris"})",
"harmony: the template's call order is a call");
check(a.reasoning == "Need weather." && a.content.empty() && a.content_null,
"harmony: a call-only reply has content: null");
const auto b = parse_answer(
"<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>{\"city\":\"Oslo\"}<|call|>", H, true);
check(b.calls.size() == 1 && b.calls[0].name == "get_weather" && b.calls[0].arguments == R"({"city":"Oslo"})",
"harmony: the card's call order is a call too");
const auto c = parse_answer(tmpl_form, H, false);
check(c.calls.empty() && c.content.find("to=functions.get_weather") != std::string::npos,
"harmony: without tools the call markup stays in content");
const std::string bad = "to=functions.f<|channel|>commentary json<|message|>{not json<|call|>";
const auto d = parse_answer(bad, H, true);
check(d.calls.empty() && d.content == bad, "harmony: arguments that are not JSON are back in content, whole");
}
}

// The family comes from the container's chat template. The Gemma 4 lines are the two literals the pack's own
// chat_template carries (amd/gemma-4-e4b-it-mobius-int4_rai_1.9.0_npu: `<|tool_call>` x1, `<|channel>thought`
// x1, `<tool_call>` x0, `<think>` x0), not a guess at them.
Expand All @@ -964,6 +1012,9 @@ void test_answer_family_of_template() {
"a template that writes <|tool_call>call: is gemma4");
check(answer_family_of_template("{{- '<|channel>thought\\n<channel|>' -}}") == AnswerFamily::gemma4,
"a template with the thought channel and no tools is gemma4 too");
check(answer_family_of_template("{{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.thinking -}}") ==
AnswerFamily::harmony,
"a template that writes the analysis channel is harmony");
check(answer_family_of_template("<|im_start|>user\n{{ content }}<|im_end|>") == AnswerFamily::qwen3,
"a template with neither marker is qwen3");
}
Expand All @@ -972,7 +1023,7 @@ void test_answer_family_of_template() {
// every byte. The scanner once read tags through a 16-byte window, so a 17-byte tag read as text (engine#1779);
// this arm fails for any family whose tags the scanner cannot match, including one added later.
void test_every_family_tag_is_matched() {
for (const AnswerFamily f : {AnswerFamily::qwen3, AnswerFamily::gemma4}) {
for (const AnswerFamily f : {AnswerFamily::qwen3, AnswerFamily::gemma4, AnswerFamily::harmony}) {
const auto t = onebit::npu::answer_detail::tags_of(f);
const std::string name = onebit::npu::answer_detail::family_name(f);
const std::string think = std::string(t.think_open) + "R" + t.think_close + "C";
Expand Down Expand Up @@ -1359,6 +1410,7 @@ int main() {
test_answer_stream();
test_tool_grammar();
test_gemma4_answer_parser();
test_harmony_answer_parser();
test_every_family_tag_is_matched();
test_answer_family_of_template();
test_tool_request_parsing();
Expand Down
Loading