From 72d2dccf2c16418a5fb00b2aa3a6baa553ed5a69 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 16:42:09 -0400 Subject: [PATCH] npu: the decode lm head runs on the host by default; NPU_HEAD_HOST=0 keeps it on the device (Refs #1785) Measured on two models with byte-identical logits (#2191), and window11 --compare accepts the HEAD M=1 profile row against a device-head run (#2209), which was the condition for the default. Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 21 ++++++++++++++------- npu/forward/src/q4nx_forward.cpp | 14 ++++++++------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index d16c579a4..8493d4f3d 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -470,15 +470,16 @@ paragraphs up — which is the reason it exists: the target was self-consistent, own scalar reference, and decoding a different matrix than the container meant. -**The decode head can run on the host (engine#1785), `NPU_HEAD_HOST=1`.** A decode step sends one row +**The decode head runs on the host (engine#1785); `NPU_HEAD_HOST=0` puts it back on the device.** A decode step sends one row through every tile of the int8 head, and a launch costs the same weight pass for one row as for a full -batch, so those launches are the most expensive use of the device in a token. With `NPU_HEAD_HOST=1` the -single-row head is computed on the host instead, from the same image the device reads (the resident weight +batch, so those launches are the most expensive use of the device in a token. The +single-row head is therefore computed on the host, from the same image the device reads (the resident weight buffer's host mapping, no second copy), with the device path's own activation quantiser and the same epilogue. The sums are int32, so the logits are the device head's bit for bit, not an approximation of them. `NPU_HEAD_HOST_THREADS` sets the threads the tiles are split over (default 4, at most 16). It applies to the int8 tiled head only: the ternary (`pb16i`) head, the gemv head and batched head rows (prefill, -verify) stay on the device. Off by default. +verify) stay on the device. The serve log says which it is: `lm head: on the host (default; NPU_HEAD_HOST=0 for the +device), N tile(s), T thread(s)`. If the image cannot be read on the host the head goes to the device, with one line. Measured on the NPU with FastFlowLM's `Qwen3.5-2B-NPU2` container (vocabulary 248,320, width 2,048, 21 head tiles), same binary, device head and host head interleaved, 24 greedy tokens per request, on a busy @@ -503,9 +504,15 @@ serve (96 of 96): | head plus following layers | 386 ms (2.6 tok/s) | 334 ms (3.0 tok/s) | So the host head takes 68 ms off the head and the next token's layers give 15 ms of it back, a net 52 ms -(13 %) per token on this model. It stays off by default because it changes the profile table's row set (one -`HEAD` row at M=1 in place of the M=16 launches), and `scripts/npu-window11.sh --compare` requires the two -binaries it compares to print the same rows. The layers are the other four fifths of a token and are untouched. +(13 %) per token on this model. On `MiniCPM-V-4.7-1B-NPU2` (width 1,024, 35 head tiles) the same alternation, 100 +tokens at load 23, read 89.2 ms against 5.3 ms for the head and 320.4 against 321.2 ms for the following layers, so +nothing is given back there: about 84 ms of a 410 ms token. Its 96 logits dumps are byte-identical to a device-only +serve's. + +It is the default because the logits are the device's on both models and the gain is measured inside one serve on +both. The profile table then has one `HEAD` row at M=1 in place of the M=16 launches; +`scripts/npu-window11.sh --compare` accepts exactly that difference against a device-head run +([npu-plan.md](npu-plan.md)). The layers are the other four fifths of a token and are untouched. ### The t2s stream (what the XDNA decode kernel reads) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 56352f526..0b6dd9e9c 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -6021,11 +6021,13 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector // NPU_HEAD_HOST=alt is the measuring form: calls alternate host, device, host, ... in ONE serve, and each // call logs the previous one's head time and the time from its end to this call (the layers of the token in // between). Two serves on a shared box differ by more than the head costs; adjacent tokens of one serve do not. + // On unless NPU_HEAD_HOST=0: measured on two models with byte-identical logits (docs/npu.md), and the window11 + // row comparison accepts its `HEAD M=1` profile row. Any value other than 0 or alt reads as on. static const int head_host_mode = [] { const char* e = getenv("NPU_HEAD_HOST"); - if (!e) return 0; - if (e[0] == '1' && e[1] == 0) return 1; - return std::strcmp(e, "alt") == 0 ? 2 : 0; + if (!e) return 1; + if (e[0] == '0' && e[1] == 0) return 0; + return std::strcmp(e, "alt") == 0 ? 2 : 1; }(); static long head_calls = 0; const bool head_host = head_host_mode == 1 || (head_host_mode == 2 && (head_calls % 2) == 0); @@ -6093,14 +6095,14 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector static bool said = false; if (!said) { said = true; - fprintf(stderr, " [forward] lm head: on the host (NPU_HEAD_HOST=%s), %d tile(s), %d thread(s), first pass %.1f ms\n", - head_host_mode == 2 ? "alt" : "1", ntiles, nthreads, ms); + fprintf(stderr, " [forward] lm head: on the host (%s), %d tile(s), %d thread(s), first pass %.1f ms\n", + head_host_mode == 2 ? "NPU_HEAD_HOST=alt" : "default; NPU_HEAD_HOST=0 for the device", ntiles, nthreads, ms); } if (xd_->profiling()) xd_->note_head(1, ms, 0.0, 0.0, 0.0); if (ph) { g_ph.h_pack += ms; g_ph.h_total = ph_ms(hp0); g_in_head = false; } return true; } - fprintf(stderr, " [forward] lm head: NPU_HEAD_HOST is set but the image cannot be read on the host; using the device\n"); + fprintf(stderr, " [forward] lm head: the image cannot be read on the host; using the device\n"); } for (int t = 0; t < ntiles; t++) { const int n0 = t * T, n1 = std::min(NV, n0 + T);