diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 27472a966..911e95d5a 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -1175,9 +1175,27 @@ int run_forward_serve(int argc, char** argv) { // after the template, because whether the route renders ChatML is known only now. { const bool chatml_route = !can_render && e.chat_ok; + // The marker this route's prompts close an assistant turn with. On the ChatML fallback it is <|im_end|>. + // With the container's own template it is read off the template: render a two-message exchange and take + // what follows the assistant's text (npu::turn_end_marker). A template that cannot render that exchange + // contributes nothing, and the load line below shows what the serve stops on either way. + std::string turn_end = chatml_route ? std::string("<|im_end|>") : std::string(); + if (can_render) { + static const char* const kProbe = "\xC2\xA4turn-end-probe\xC2\xA4"; + try { + nlohmann::ordered_json exchange = nlohmann::ordered_json::array(); + exchange.push_back({{"role", "user"}, {"content", "u"}}); + exchange.push_back({{"role", "assistant"}, {"content", kProbe}}); + const std::string rendered = npu::render_chat_template(e.tmpl, exchange, /*add_generation_prompt=*/false, + nlohmann::ordered_json::object(), e.bos_text, e.eos_text); + const size_t at = rendered.find(kProbe); + if (at != std::string::npos) turn_end = npu::turn_end_marker(rendered.substr(at + std::strlen(kProbe))); + } catch (const std::exception& ex) { + std::fprintf(stderr, "1bit forward-serve: the template did not render a two-message exchange (%s); no turn-end marker from it\n", ex.what()); + } + } e.eos = npu::stop_token_ids(npu::read_json_or_null(cfg), npu::read_json_or_null(dir + "/tokenizer_config.json"), - chatml_route ? std::string("<|im_end|>") : std::string(), - [&](const std::string& text) { return e.tok->token_id(text); }); + turn_end, [&](const std::string& text) { return e.tok->token_id(text); }); std::string named; for (size_t i = 0; i < e.eos.size(); i++) named += std::string(i ? ", " : "") + std::to_string(e.eos[i]) + " " + e.tok->decode(e.eos[i]) + diff --git a/app/npu_stop_tokens.h b/app/npu_stop_tokens.h index d3fe3c3f7..281adf045 100644 --- a/app/npu_stop_tokens.h +++ b/app/npu_stop_tokens.h @@ -66,6 +66,34 @@ inline std::string eos_token_text(const nlohmann::json& tokenizer_config) { return {}; } +/// The marker a chat template closes an assistant turn with, read off the template's own output. `after` is the +/// text a template rendered immediately after an assistant message's content; the marker is the `<...>` token it +/// starts with (leading whitespace skipped), or empty when it starts with anything else. +/// +/// Why this exists: a model ends its turn with whatever its template ends an assistant turn with, and neither +/// config.json nor tokenizer_config.json has to name that token. FastFlowLM's Phi4-mini-Instruct-NPU2 names +/// <|endoftext|> in both, and its template closes every turn with <|end|> (200020); the upstream checkpoint +/// lists it in generation_config.json, which the pack does not ship. Served with only the two files' ids, the +/// answer was "Paris<|end|><|assistant|>**Follow-up Question 1:** ..." until the token budget ran out +/// (engine#1786). The template is the one file every chat pack has, and it cannot disagree with itself. +/// +/// "<|end|>" -> "<|end|>" (Phi) +/// "<|im_end|>\n" -> "<|im_end|>" (ChatML) +/// "<|eot_id|>" -> "<|eot_id|>" (Llama 3) +/// "\n" -> "" (Gemma) +/// "\n", "", "x"... -> "" unless it starts with a whole <...> token: plain text after a turn is not a stop. +inline std::string turn_end_marker(const std::string& after) { + size_t i = 0; + while (i < after.size() && (after[i] == ' ' || after[i] == '\n' || after[i] == '\r' || after[i] == '\t')) ++i; + if (i >= after.size() || after[i] != '<') return {}; + const size_t close = after.find('>', i); + if (close == std::string::npos || close - i < 2) return {}; + const std::string marker = after.substr(i, close - i + 1); + for (char c : marker) + if (c == ' ' || c == '\n' || c == '\r' || c == '\t') return {}; + return marker; +} + /// Every id that ends a generation on a container, in the order they were found and without repeats: /// /// 1. config.json's `eos_token_id` (one integer or a list): what eos_token_ids() reads; @@ -75,9 +103,10 @@ inline std::string eos_token_text(const nlohmann::json& tokenizer_config) { /// model ends a chat turn with the second. With only the first, the answer came back as /// "Item 3 is the 3D printer.<|im_end|>\n": the turn end served as text and one more token generated /// after it (engine#1785's row on that container); -/// 3. `chat_turn_end`, when the caller's chat route renders a prompt whose turns end with a marker of its -/// own (ChatML's <|im_end|>): a prompt we close with that marker is answered by a turn that ends with it, -/// whatever the two files say. Pass an empty string when the route has no such marker. +/// 3. `chat_turn_end`, the marker the chat route's prompt closes an assistant turn with: ChatML's <|im_end|> +/// on the fallback route, or turn_end_marker() of the container's own template when that is rendered. A +/// prompt whose turns end with that marker is answered by a turn that ends with it, whatever the two files +/// say. Pass an empty string when the route has no such marker. /// /// `id_of` maps a token's text to its id, or a negative number when the tokenizer has no such token; a name it /// does not know adds nothing. diff --git a/docs/npu.md b/docs/npu.md index a54b87124..dd2d49702 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -70,9 +70,15 @@ curl -s http://127.0.0.1:8000/v1/chat/completions -H 'Content-Type: application/ **What ends an answer.** The serve stops on every id in `config.json`'s `eos_token_id` (one integer or a list), on `<|endoftext|>` when the tokenizer has it, on `tokenizer_config.json`'s `eos_token` when that file is beside the -tokenizer, and, when the chat route falls back to ChatML, on `<|im_end|>`. The two files need not agree: FastFlowLM's +tokenizer, and on the marker the chat route's prompts close an assistant turn with: `<|im_end|>` on the ChatML +fallback, or, when the container's own template is rendered, whatever that template puts right after an assistant +message (`<|end|>` for Phi, `<|eot_id|>` for Llama 3, `` for Gemma), read off a rendered two-message +exchange at load. The two files need not agree: FastFlowLM's `Qwen3.5-2B-NPU2` container names `<|endoftext|>` in the first and `<|im_end|>` in the second, and the model ends a -chat turn with the second. The load prints the list (`forward-serve: stops on …`). +chat turn with the second. And neither file has to name the turn end at all: FastFlowLM's +`Phi4-mini-Instruct-NPU2` names `<|endoftext|>` in both, its template closes every turn with `<|end|>`, and before +the template was consulted its answer ran on past `Paris<|end|><|assistant|>` to the token budget. The load prints +the list (`forward-serve: stops on …`). `1bit serve -m ` serves such a directory on the NPU behind the OpenAI-compatible API ([serve.md](serve.md)); inside Lemonade, that is what its `onebit` backend runs (`--onebit npu`). diff --git a/tests/npu_stop_tokens_test.cpp b/tests/npu_stop_tokens_test.cpp index 6ff5c20ae..9ebb686f6 100644 --- a/tests/npu_stop_tokens_test.cpp +++ b/tests/npu_stop_tokens_test.cpp @@ -121,6 +121,37 @@ int main() { std::remove(bad.c_str()); } + // 7. the turn end read off the template (engine#1786, FastFlowLM's Phi4-mini-Instruct-NPU2): neither file names + // <|end|>, the template closes every turn with it, and the answer ran on past it. + { + using onebit::npu::turn_end_marker; + check(turn_end_marker("<|end|>") == "<|end|>", "Phi: the marker right after the assistant's text"); + check(turn_end_marker("<|im_end|>\n") == "<|im_end|>", "ChatML: the marker, without the newline after it"); + check(turn_end_marker("<|eot_id|>") == "<|eot_id|>" && turn_end_marker("\n") == "", + "Llama 3 and Gemma"); + check(turn_end_marker("\n<|im_end|>\n<|im_start|>user") == "<|im_end|>", "whitespace before it is skipped, what follows is ignored"); + check(turn_end_marker("").empty() && turn_end_marker("\n").empty() && turn_end_marker(" \n ").empty(), + "nothing, or only whitespace, after the turn: no marker"); + check(turn_end_marker("Human: next").empty() && turn_end_marker("x<|end|>").empty(), + "plain text after the turn is not a stop, even with a marker later in it"); + check(turn_end_marker("<").empty() && turn_end_marker("<>").empty() && turn_end_marker("<|end").empty() && + turn_end_marker("< not a token >").empty(), + "a lone bracket, an empty pair, an unclosed one and one with spaces in it are not markers"); + // into the list: Phi's two files name only <|endoftext|> + const auto id_of = [](const std::string& t) { + if (t == "<|endoftext|>") return 199999; + if (t == "<|end|>") return 200020; + return -1; + }; + const json cfg = json::parse(R"({"eos_token_id": 199999})"); + const json tok = json::parse(R"({"eos_token": "<|endoftext|>"})"); + check_ids(onebit::npu::stop_token_ids(cfg, tok, "", id_of), {199999}, "from the two files alone the serve stops on <|endoftext|> only (the fault)"); + check_ids(onebit::npu::stop_token_ids(cfg, tok, turn_end_marker("<|end|>"), id_of), {199999, 200020}, + "with the template's turn end it also stops on <|end|>"); + check_ids(onebit::npu::stop_token_ids(cfg, tok, turn_end_marker("<|eot|>"), id_of), {199999}, + "a marker the tokenizer does not have as one token adds nothing"); + } + if (failures == 0) std::printf("all stop-token checks passed\n"); return failures == 0 ? 0 : 1; }