forward-serve: the turn end is read off the container's own template — Phi4-mini ran on past 'Paris<|end|>' to the token budget (Refs #1786)

app/forward_serve.cpp

@@ -1175,9 +1175,27 @@ int run_forward_serve(int argc, char** argv) {
     // after the template, because whether the route renders ChatML is known only now.
     {
         const bool chatml_route = !can_render && e.chat_ok;
+        // The marker this route's prompts close an assistant turn with.  On the ChatML fallback it is <|im_end|>.
+        // With the container's own template it is read off the template: render a two-message exchange and take
+        // what follows the assistant's text (npu::turn_end_marker).  A template that cannot render that exchange
+        // contributes nothing, and the load line below shows what the serve stops on either way.
+        std::string turn_end = chatml_route ? std::string("<|im_end|>") : std::string();
+        if (can_render) {
+            static const char* const kProbe = "\xC2\xA4turn-end-probe\xC2\xA4";
+            try {
+                nlohmann::ordered_json exchange = nlohmann::ordered_json::array();
+                exchange.push_back({{"role", "user"}, {"content", "u"}});
+                exchange.push_back({{"role", "assistant"}, {"content", kProbe}});
+                const std::string rendered = npu::render_chat_template(e.tmpl, exchange, /*add_generation_prompt=*/false,
+                                                                       nlohmann::ordered_json::object(), e.bos_text, e.eos_text);
+                const size_t at = rendered.find(kProbe);
+                if (at != std::string::npos) turn_end = npu::turn_end_marker(rendered.substr(at + std::strlen(kProbe)));
+            } catch (const std::exception& ex) {
+                std::fprintf(stderr, "1bit forward-serve: the template did not render a two-message exchange (%s); no turn-end marker from it\n", ex.what());
+            }
+        }
         e.eos = npu::stop_token_ids(npu::read_json_or_null(cfg), npu::read_json_or_null(dir + "/tokenizer_config.json"),
-                                    chatml_route ? std::string("<|im_end|>") : std::string(),
-                                    [&](const std::string& text) { return e.tok->token_id(text); });
+                                    turn_end, [&](const std::string& text) { return e.tok->token_id(text); });
         std::string named;
         for (size_t i = 0; i < e.eos.size(); i++)
             named += std::string(i ? ", " : "") + std::to_string(e.eos[i]) + " " + e.tok->decode(e.eos[i]) +

app/npu_stop_tokens.h

@@ -66,6 +66,34 @@ inline std::string eos_token_text(const nlohmann::json& tokenizer_config) {
     return {};
 }
 
+/// The marker a chat template closes an assistant turn with, read off the template's own output.  `after` is the
+/// text a template rendered immediately after an assistant message's content; the marker is the `<...>` token it
+/// starts with (leading whitespace skipped), or empty when it starts with anything else.
+///
+/// Why this exists: a model ends its turn with whatever its template ends an assistant turn with, and neither
+/// config.json nor tokenizer_config.json has to name that token.  FastFlowLM's Phi4-mini-Instruct-NPU2 names
+/// <|endoftext|> in both, and its template closes every turn with <|end|> (200020); the upstream checkpoint
+/// lists it in generation_config.json, which the pack does not ship.  Served with only the two files' ids, the
+/// answer was "Paris<|end|><|assistant|>**Follow-up Question 1:** ..." until the token budget ran out
+/// (engine#1786).  The template is the one file every chat pack has, and it cannot disagree with itself.
+///
+///   "<|end|>"            -> "<|end|>"          (Phi)
+///   "<|im_end|>\n"       -> "<|im_end|>"       (ChatML)
+///   "<|eot_id|>"         -> "<|eot_id|>"       (Llama 3)
+///   "<end_of_turn>\n"    -> "<end_of_turn>"    (Gemma)
+///   "\n", "", "</s>x"... -> "" unless it starts with a whole <...> token: plain text after a turn is not a stop.
+inline std::string turn_end_marker(const std::string& after) {
+    size_t i = 0;
+    while (i < after.size() && (after[i] == ' ' || after[i] == '\n' || after[i] == '\r' || after[i] == '\t')) ++i;
+    if (i >= after.size() || after[i] != '<') return {};
+    const size_t close = after.find('>', i);
+    if (close == std::string::npos || close - i < 2) return {};
+    const std::string marker = after.substr(i, close - i + 1);
+    for (char c : marker)
+        if (c == ' ' || c == '\n' || c == '\r' || c == '\t') return {};
+    return marker;
+}
+
 /// Every id that ends a generation on a container, in the order they were found and without repeats:
 ///
 ///   1. config.json's `eos_token_id` (one integer or a list): what eos_token_ids() reads;
@@ -75,9 +103,10 @@ inline std::string eos_token_text(const nlohmann::json& tokenizer_config) {
 ///      model ends a chat turn with the second.  With only the first, the answer came back as
 ///      "Item 3 is the 3D printer.<|im_end|>\n": the turn end served as text and one more token generated
 ///      after it (engine#1785's row on that container);
-///   3. `chat_turn_end`, when the caller's chat route renders a prompt whose turns end with a marker of its
-///      own (ChatML's <|im_end|>): a prompt we close with that marker is answered by a turn that ends with it,
-///      whatever the two files say.  Pass an empty string when the route has no such marker.
+///   3. `chat_turn_end`, the marker the chat route's prompt closes an assistant turn with: ChatML's <|im_end|>
+///      on the fallback route, or turn_end_marker() of the container's own template when that is rendered.  A
+///      prompt whose turns end with that marker is answered by a turn that ends with it, whatever the two files
+///      say.  Pass an empty string when the route has no such marker.
 ///
 /// `id_of` maps a token's text to its id, or a negative number when the tokenizer has no such token; a name it
 /// does not know adds nothing.

docs/npu.md

@@ -70,9 +70,15 @@ curl -s http://127.0.0.1:8000/v1/chat/completions -H 'Content-Type: application/
 
 **What ends an answer.** The serve stops on every id in `config.json`'s `eos_token_id` (one integer or a list), on
 `<|endoftext|>` when the tokenizer has it, on `tokenizer_config.json`'s `eos_token` when that file is beside the
-tokenizer, and, when the chat route falls back to ChatML, on `<|im_end|>`. The two files need not agree: FastFlowLM's
+tokenizer, and on the marker the chat route's prompts close an assistant turn with: `<|im_end|>` on the ChatML
+fallback, or, when the container's own template is rendered, whatever that template puts right after an assistant
+message (`<|end|>` for Phi, `<|eot_id|>` for Llama 3, `<end_of_turn>` for Gemma), read off a rendered two-message
+exchange at load. The two files need not agree: FastFlowLM's
 `Qwen3.5-2B-NPU2` container names `<|endoftext|>` in the first and `<|im_end|>` in the second, and the model ends a
-chat turn with the second. The load prints the list (`forward-serve: stops on …`).
+chat turn with the second. And neither file has to name the turn end at all: FastFlowLM's
+`Phi4-mini-Instruct-NPU2` names `<|endoftext|>` in both, its template closes every turn with `<|end|>`, and before
+the template was consulted its answer ran on past `Paris<|end|><|assistant|>` to the token budget. The load prints
+the list (`forward-serve: stops on …`).
 
 `1bit serve -m <model dir>` serves such a directory on the NPU behind the OpenAI-compatible API
 ([serve.md](serve.md)); inside Lemonade, that is what its `onebit` backend runs (`--onebit npu`).

tests/npu_stop_tokens_test.cpp

@@ -121,6 +121,37 @@ int main() {
         std::remove(bad.c_str());
     }
 
+    // 7. the turn end read off the template (engine#1786, FastFlowLM's Phi4-mini-Instruct-NPU2): neither file names
+    //    <|end|>, the template closes every turn with it, and the answer ran on past it.
+    {
+        using onebit::npu::turn_end_marker;
+        check(turn_end_marker("<|end|>") == "<|end|>", "Phi: the marker right after the assistant's text");
+        check(turn_end_marker("<|im_end|>\n") == "<|im_end|>", "ChatML: the marker, without the newline after it");
+        check(turn_end_marker("<|eot_id|>") == "<|eot_id|>" && turn_end_marker("<end_of_turn>\n") == "<end_of_turn>",
+              "Llama 3 and Gemma");
+        check(turn_end_marker("\n<|im_end|>\n<|im_start|>user") == "<|im_end|>", "whitespace before it is skipped, what follows is ignored");
+        check(turn_end_marker("").empty() && turn_end_marker("\n").empty() && turn_end_marker("  \n ").empty(),
+              "nothing, or only whitespace, after the turn: no marker");
+        check(turn_end_marker("Human: next").empty() && turn_end_marker("x<|end|>").empty(),
+              "plain text after the turn is not a stop, even with a marker later in it");
+        check(turn_end_marker("<").empty() && turn_end_marker("<>").empty() && turn_end_marker("<|end").empty() &&
+                  turn_end_marker("< not a token >").empty(),
+              "a lone bracket, an empty pair, an unclosed one and one with spaces in it are not markers");
+        // into the list: Phi's two files name only <|endoftext|>
+        const auto id_of = [](const std::string& t) {
+            if (t == "<|endoftext|>") return 199999;
+            if (t == "<|end|>") return 200020;
+            return -1;
+        };
+        const json cfg = json::parse(R"({"eos_token_id": 199999})");
+        const json tok = json::parse(R"({"eos_token": "<|endoftext|>"})");
+        check_ids(onebit::npu::stop_token_ids(cfg, tok, "", id_of), {199999}, "from the two files alone the serve stops on <|endoftext|> only (the fault)");
+        check_ids(onebit::npu::stop_token_ids(cfg, tok, turn_end_marker("<|end|>"), id_of), {199999, 200020},
+                  "with the template's turn end it also stops on <|end|>");
+        check_ids(onebit::npu::stop_token_ids(cfg, tok, turn_end_marker("<|eot|>"), id_of), {199999},
+                  "a marker the tokenizer does not have as one token adds nothing");
+    }
+
     if (failures == 0) std::printf("all stop-token checks passed\n");
     return failures == 0 ? 0 : 1;
 }