From 259bd8f61848f6b63c11f7878e9ee7534f6d3831 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:44:26 -0400 Subject: [PATCH 1/2] serve: ONEBIT_WARMUP selects which warm-up request is sent (engine#1539's bisect) serve sends an HRX backend two warm-ups once it is ready: a ~485-token sampling chat request (HRX's per-shape program build paid at load instead of on the client's first request) and a 16-token /completion prompt (the 13-20 token small-batch kernels). They are not interchangeable -- the first samples, the second does not -- so for engine#1539 (flint-40b-a2.5b returns all-NaN logits through 1bit serve and not through its child) the question "which warm-up can this model not survive" cannot be asked with ONEBIT_NO_WARMUP, which turns off both. ONEBIT_WARMUP=long|short|none now selects them, and reaches the --print-backend plan as well, since warmup_requests() feeds both. Unset or both is the shipped pair; an unrecognised value keeps it rather than silently disabling a warm-up (a typo must not quietly change a load). tests/print_backend.sh asserts each selection against the plan's warmups array (script-based add_test, no device, no model) and that "warmup" still mirrors warmups[0]. docs/serve.md documents it. --- app/serve.cpp | 24 ++++++++++++++++++++---- docs/serve.md | 7 ++++++- tests/print_backend.sh | 18 ++++++++++++++++++ 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/app/serve.cpp b/app/serve.cpp index d59d687fa..da1b29eab 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -1649,13 +1649,24 @@ int serve_only(const std::string& why) { return kServeOnly; } -// The warm-up requests (warm_backend, below): HRX only, and ONEBIT_NO_WARMUP=1 turns them off. serve sends +// The warm-up requests (warm_backend, below): HRX only, and ONEBIT_NO_WARMUP=1 (or ONEBIT_WARMUP=none) +// turns them off; ONEBIT_WARMUP picks which of the two, which is how engine#1539's bisect runs. serve sends // them itself; --print-backend hands the same requests to Lemonade as the plan's "warmups" ([{path, body}], // in order; "warmup" is the first one, for a lemond that reads only that), and Lemonade's onebit recipe // sends them once the backend is ready, before it marks the model loaded. +// ONEBIT_WARMUP selects which warm-up requests are sent (engine#1539's bisect): "" or "both" = the shipped pair, +// "long" = only the sampling chat request, "short" = only the 16-token /completion prompt, "none" = neither (the +// spelling ONEBIT_NO_WARMUP already had). The two requests are not interchangeable -- the first samples and the +// second does not -- so which one a model cannot survive is a question one load can answer instead of a rebuild. +static std::string warmup_selection() { + const char* which = getenv("ONEBIT_WARMUP"); + return which ? std::string(which) : std::string(); +} + static bool warmup_wanted(const std::string& device) { const std::string dev = device.substr(0, device.find(' ')); - return getenv("ONEBIT_NO_WARMUP") == nullptr && (dev == "hrx" || dev == "hybrid"); + return getenv("ONEBIT_NO_WARMUP") == nullptr && warmup_selection() != "none" && + (dev == "hrx" || dev == "hybrid"); } static const char* const kWarmupPath = "/v1/chat/completions"; @@ -1689,8 +1700,13 @@ static json warmup_short_body() { } static json warmup_requests() { - return json::array({ json{ { "path", kWarmupPath }, { "body", warmup_body() } }, - json{ { "path", "/completion" }, { "body", warmup_short_body() } } }); + const std::string sel = warmup_selection(); + const bool want_chat = sel.empty() || sel == "both" || sel == "long"; + const bool want_short = sel.empty() || sel == "both" || sel == "short"; + json out = json::array(); + if (want_chat) out.push_back(json{ { "path", kWarmupPath }, { "body", warmup_body() } }); + if (want_short) out.push_back(json{ { "path", "/completion" }, { "body", warmup_short_body() } }); + return out; } // --print-backend: the llama-server serve would start for this model, as one JSON line on stdout, diff --git a/docs/serve.md b/docs/serve.md index f7e241301..b10b4fce7 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -333,7 +333,12 @@ What serve did on the request path is the backend's own on this route: `"warmups": [{"path", "body"}, ...]` (and `"warmup"`, the first one, for an older lemond), on `hrx` chat plans only; Lemonade's onebit recipe sends them in order after `/health` answers and before it marks the model loaded. `ONEBIT_NO_WARMUP=1` in lemond's environment leaves them out of the plan, as - it turns serve's own warm-up off. + it turns serve's own warm-up off. `ONEBIT_WARMUP=long|short|none` selects *which* of the two, in + serve's own environment and in the plan alike (`long` = the chat request, `short` = the 16-token + completion, `none` = neither; unset or `both` is the shipped pair, and an unrecognised value keeps + it). The two are not interchangeable — the first samples, the second does not — so when a model + cannot survive one of them, this is the difference between one load per hypothesis and a rebuild + (engine#1539). **Why the 16-token request.** The first batch of 13 to 20 tokens after a load takes ggml-hrx's small-batch matmul kernels, which read the weights in a layout ggml-hrx builds on the host the first diff --git a/tests/print_backend.sh b/tests/print_backend.sh index bf0f56481..66d997899 100755 --- a/tests/print_backend.sh +++ b/tests/print_backend.sh @@ -29,6 +29,7 @@ # --device cpu, --parallel 2, --embedding and a file without the key leave it out, # - an hrx chat plan carries "warmups" (serve's warm-up requests, in order: the chat request, then a # 16-token /completion prompt (" a" x 16, n_predict 0)) and "warmup" (the first one, for an older lemond), +# - ONEBIT_WARMUP=long|short|none picks which warm-up request(s) the plan carries (engine#1539's bisect) # - an hrx chat plan carries "warmup" (serve's warm-up request, for Lemonade to send before the # model counts as loaded); ONEBIT_NO_WARMUP=1, cpu and role servers have none. # @@ -180,6 +181,23 @@ check "hrx plan: warmups = [the chat request, a 16-token /completion], warmup = out=$(ONEBIT_NO_WARMUP=1 "$bin" serve -m "$scratch/small.gguf" --device hrx --llama-server "$here/fake_backend.py" \ --no-mem-check --port 43132 --print-backend 2>"$scratch/err"); rc=$? check "hrx plan: ONEBIT_NO_WARMUP=1, no warmup (exit $rc)" '[ "$rc" = 0 ] && [ "$(get "\"warmup\" in d or \"warmups\" in d")" = False ]' +# engine#1539: the two warm-ups are not interchangeable (the first samples, the second does not), so which one a +# model cannot survive is now a selector rather than a rebuild. ONEBIT_WARMUP picks them; "none" is the spelling +# ONEBIT_NO_WARMUP already had, and an unknown value keeps the shipped pair rather than silently sending none. +for _sel in long short; do + out=$(ONEBIT_WARMUP=$_sel "$bin" serve -m "$scratch/small.gguf" --device hrx --llama-server "$here/fake_backend.py" \ + --no-mem-check --port 43133 --print-backend 2>"$scratch/err"); rc=$? + if [ "$_sel" = long ]; then _want=/v1/chat/completions; else _want=/completion; fi + check "hrx plan: ONEBIT_WARMUP=$_sel sends only $_want (exit $rc)" \ + '[ "$rc" = 0 ] && [ "$(get "[w[\"path\"] for w in d[\"warmups\"]]")" = "[$_want]" ] && [ "$(get "d[\"warmup\"] == d[\"warmups\"][0]")" = True ]' +done +out=$(ONEBIT_WARMUP=none "$bin" serve -m "$scratch/small.gguf" --device hrx --llama-server "$here/fake_backend.py" \ + --no-mem-check --port 43134 --print-backend 2>"$scratch/err"); rc=$? +check "hrx plan: ONEBIT_WARMUP=none, no warmup (exit $rc)" '[ "$rc" = 0 ] && [ "$(get "\"warmup\" in d or \"warmups\" in d")" = False ]' +out=$(ONEBIT_WARMUP=bogus "$bin" serve -m "$scratch/small.gguf" --device hrx --llama-server "$here/fake_backend.py" \ + --no-mem-check --port 43135 --print-backend 2>"$scratch/err"); rc=$? +check "hrx plan: an unknown ONEBIT_WARMUP keeps the shipped pair (exit $rc)" \ + '[ "$rc" = 0 ] && [ "$(get "len(d[\"warmups\"])")" = 2 ]' out=$(mplan small cpu); rc=$? check "cpu plan: no warmup (exit $rc)" '[ "$rc" = 0 ] && [ "$(get "\"warmup\" in d")" = False ]' From 53263c4d6e138f8863c11583457176a5bec9ec40 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:07:59 -0400 Subject: [PATCH 2/2] serve: an unknown ONEBIT_WARMUP keeps the shipped pair, and the plan never mirrors a null warmup Two defects the test found once the binary existed, neither visible in the diff: - the selection fell through to ZERO warm-ups for an unrecognised value, contradicting what the comment and docs/serve.md say (a typo must not quietly disable a warm-up that moves a measured 2 s off the client's first request). warmup_requests() now normalises anything that is not long/short/none to the shipped pair. - --print-backend then did out["warmup"] = out["warmups"][0], and nlohmann resizes an empty array on non-const operator[] and inserts null, so a plan could carry "warmup": null -- worse than absent, because a consumer reading it gets a value it cannot parse. Guarded with !empty(). The test's own assertion was wrong too (python renders a one-element list with quotes), which is why running it mattered more than reading it. --- app/serve.cpp | 11 +++++++---- tests/print_backend.sh | 6 +++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/app/serve.cpp b/app/serve.cpp index da1b29eab..789dbda5c 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -1700,9 +1700,12 @@ static json warmup_short_body() { } static json warmup_requests() { - const std::string sel = warmup_selection(); - const bool want_chat = sel.empty() || sel == "both" || sel == "long"; - const bool want_short = sel.empty() || sel == "both" || sel == "short"; + // An unrecognised value is the shipped pair, not "none": a typo must not quietly disable a warm-up that moves a + // measured 2 s off the client's first request. Normalised here so a caller cannot see the two states differently. + std::string sel = warmup_selection(); + if (sel != "long" && sel != "short" && sel != "none") sel = "both"; + const bool want_chat = sel == "both" || sel == "long"; + const bool want_short = sel == "both" || sel == "short"; json out = json::array(); if (want_chat) out.push_back(json{ { "path", kWarmupPath }, { "body", warmup_body() } }); if (want_short) out.push_back(json{ { "path", "/completion" }, { "body", warmup_short_body() } }); @@ -1748,7 +1751,7 @@ int print_backend(const Options& o, const Launch& l) { // a chat backend on HRX: the request Lemonade sends before it marks the model loaded (warmup_wanted) if (o.role.empty() && warmup_wanted(l.device)) { out["warmups"] = warmup_requests(); - out["warmup"] = out["warmups"][0]; + if (!out["warmups"].empty()) out["warmup"] = out["warmups"][0]; } std::printf("%s\n", out.dump().c_str()); return 0; diff --git a/tests/print_backend.sh b/tests/print_backend.sh index 66d997899..338402a5e 100755 --- a/tests/print_backend.sh +++ b/tests/print_backend.sh @@ -188,8 +188,12 @@ for _sel in long short; do out=$(ONEBIT_WARMUP=$_sel "$bin" serve -m "$scratch/small.gguf" --device hrx --llama-server "$here/fake_backend.py" \ --no-mem-check --port 43133 --print-backend 2>"$scratch/err"); rc=$? if [ "$_sel" = long ]; then _want=/v1/chat/completions; else _want=/completion; fi + # A join, not print(list): python renders a one-element list as ['/completion'], quotes included, and comparing + # against a bare [$_want] is a test that fails on a correct plan (found by running it). + _got=$(get "' '.join(w[\"path\"] for w in d[\"warmups\"])") + _mirrors=$(get "d[\"warmup\"] == d[\"warmups\"][0]") check "hrx plan: ONEBIT_WARMUP=$_sel sends only $_want (exit $rc)" \ - '[ "$rc" = 0 ] && [ "$(get "[w[\"path\"] for w in d[\"warmups\"]]")" = "[$_want]" ] && [ "$(get "d[\"warmup\"] == d[\"warmups\"][0]")" = True ]' + '[ "$rc" = 0 ] && [ "$_got" = "$_want" ] && [ "$_mirrors" = True ]' done out=$(ONEBIT_WARMUP=none "$bin" serve -m "$scratch/small.gguf" --device hrx --llama-server "$here/fake_backend.py" \ --no-mem-check --port 43134 --print-backend 2>"$scratch/err"); rc=$?