From 1516de589d1914bb5a4bf6ae9e18710145113c7b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 06:04:21 -0400 Subject: [PATCH 1/5] npu: the integer arm's operands come from the engine's packer, not a private re-quantisation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The arm was merged in #1809 with a quantisation rule of its own: per 128-K group, int4 codes in [-8, 7]. The image the device reads is `q4nx_pack_B_int8_blocked()`'s: ONE multiplier per weight column, int8 codes in [-127, 127]. Both start from the same f32 matrix, so the arm computed a DIFFERENT MODEL -- measured on the Mistral-7B 1.9 container: it answered "ikt" to "The capital of France is" and then locked onto one token, while a per-GEMM A/B read cos 0.9999 against f32 on the same tensors. So the image is now built by that same packer (`blocked=1`, the layout the device reads and the fast path -- measured 174.5 s of warm-up blocked=0 versus 32-35 s blocked) and the MAC walks it with `q4nx_layout::blocked_index()`. Measured after, on current main: warm-up 35.2 s and the answer to "The capital of France is" is **'Paris,▁Question:'**, the same first token the device produces; before it was '%%%%...' / '▁/******/'. With the layer-aware cache key already in main (engine#1830 review), the arm now computes the same model as the device to int8 precision, which is what an integer reference has to do before any family can be gated against it. `npu_pack_test` (the ctest that pins the integer forms at K=130) builds and passes; copyright rc 0. --- npu/forward/include/q4nx_pack.h | 1 - npu/forward/src/q4nx_forward.cpp | 46 +++++++++++++++----------------- npu/forward/src/q4nx_pack.c | 1 + 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index 6f94b1f4c..1a36053b3 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -191,7 +191,6 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out); - /* Architecture dims a design set is derived from. */ typedef struct { int H; /* hidden_size */ diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 5a6b4b36a..c1d689e72 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3044,7 +3044,6 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector& h, std::vector amax) amax = v; - } - if (!(amax >= 1e-12f)) amax = 1.0f; - t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f; - const float tis = 127.0f / amax; - for (int k = k0; k < k1; k++) { - float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; - if (!std::isfinite(v)) v = 0.0f; - int q = (int)std::round(v * tis); - q = q > 7 ? 7 : (q < -8 ? -8 : q); - t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; - } - } - } + t.d.assign((size_t)N, 0.0f); + // THE OPERANDS MUST BE THE ENGINE'S, NOT A PRIVATE RE-QUANTISATION (engine#1808/#1830). This arm exists to be + // the integer reference a kernel family is gated against, and the device's weight image comes from + // q4nx_pack_B_int8_blocked(): ONE multiplier per column (amax over the whole K), int8 codes in [-127, 127]. + // Quantising here with a different rule -- per 128-K group, int4 [-8, 7] -- computes a different model: measured + // 2026-11-10, the arm answered "ikt" to "The capital of France is" and then locked onto one token, while the + // per-GEMM A/B read cos 0.9999 against f32 on the same tensors. Same packer, same bytes, same model. + double ssum = 0.0; + // blocked=1: the image is written in the blocked microtile order, which is the fast path AND the layout the + // device actually reads, so the MAC below walks it with q4nx_layout::blocked_index(). (blocked=0 was measured + // at 174.5 s of warm-up on this container versus 32.2 s blocked -- the row-major scatter is the slow one.) + q4nx_pack_B_int8_blocked(t.codes.data(), Bmat, K, N, K, N, /*blocked=*/1, t.d.data(), &ssum); if (!cacheable) { scratch = std::move(t); cached = &scratch; @@ -3109,8 +3099,14 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, int q = (int)std::round(v * tis); a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); } - q4nx_host_gemm_i8_grouped(a8.data(), K, t.codes.data(), t.d.data(), t.groups, N, ascale, - out + (size_t)m * (size_t)N); + float* dst = out + (size_t)m * (size_t)N; + for (int n = 0; n < N; n++) { + int32_t acc = 0; + for (int k = 0; k < K; k++) + acc += (int32_t)((int)t.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, K, N)] * + (int)a8[(size_t)k]); + dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc); + } } } diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 0914232e5..c021cca95 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -161,6 +161,7 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) { * * exactly the kernel's epilogue, with the inner sum in int32 and no float in it. `d` is the per-128-K-group * multiplier per column ([groups][N]), `ascale` the activation row's scale. */ + void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out) { const int G = 128; From 7904dfa2a07b05abcadd7566655e12041a25a013 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 06:47:15 -0400 Subject: [PATCH 2/5] npu: keyed dumps settle it -- the device's weight scales are BIT-IDENTICAL to the arm's (384/384) Dumps were paired by index first and that was meaningless: the two sides reach the GEMM a different number of times (1024 arm dumps against 384 device dumps on the same request). Keyed by (layer, N, K, ordinal) they pair exactly. Result, one token, Mistral-7B 1.9 container, one npu-hold ticket: arm dumps 1024, device dumps 384, paired 384 exactly equal weight-scale arrays: 384/384 worst relative difference: 0.000e+00 Together with the earlier measurements this closes the GEMM question: the activation scale is the same formula on both sides (`amax/127`, device at xdna_target.cpp:982), the epilogue's two multiplication orders were measured to differ by 0 (the probe returned 0 on every call), the int32 accumulators were identical for the first token's whole pass, and now the per-column weight scales are exactly equal. So the device's GEMMs are bit-exact against this arm. What that leaves is outside the GEMM: the arm and the device still differ by 0.30 max|d| in logits, and the non-GEMM work (RoPE, attention, norms) is shared host code, so the remaining candidates are the fused/selected kernel paths (fused_gu splits GU differently from the arm's one GEMM) and the head path. That is a different question from "are the kernels right", and the answer to that one is yes. Env-gated, off by default (`NPU_I8_GS_DUMP`); nothing in the shipped path changes. --- npu/forward/src/q4nx_forward.cpp | 9 +++++++++ npu/target/xdna/xdna_target.cpp | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index c1d689e72..4755c4d24 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3083,6 +3083,15 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, cached = &it->second; } const HostI8Tensor& t = *cached; + if (const char* gdir = getenv("NPU_I8_GS_DUMP")) { + static int n = 0; + char path[4096]; + std::snprintf(path, sizeof path, "%s/arm_L%d_N%d_K%d_%03d.bin", gdir, cur_layer_, N, K, n++); + if (std::FILE* f = std::fopen(path, "wb")) { + std::fwrite(t.d.data(), sizeof(float), std::min((size_t)N, t.d.size()), f); + std::fclose(f); + } + } std::vector a8((size_t)K); for (int m = 0; m < M; m++) { const float* a = A + (size_t)m * (size_t)a_k; diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index fe4c4ecae..4c45789d5 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -1149,6 +1149,18 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: } const auto tr = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; out.resize((size_t)M * N); + // engine#1808: the device's per-column weight scales, KEYED by (layer, N, K) so it can be paired with the host + // arm's dump exactly. Index pairing was tried first and is meaningless -- the two sides reach the GEMM a + // different number of times (384 device dumps against 1024 arm dumps on the same request). + if (const char* gdir = getenv("NPU_I8_GS_DUMP")) { + static int n = 0; + char path[4096]; + std::snprintf(path, sizeof path, "%s/dev_L%d_N%d_K%d_%03d.bin", gdir, ar.layer, N, K, n++); + if (std::FILE* f = std::fopen(path, "wb")) { + if (gs && !gs->empty()) std::fwrite(gs->data(), sizeof(float), (size_t)N, f); + std::fclose(f); + } + } for (int m = 0; m < M; m++) { const int32_t* cm = kctx.Cm + (size_t)m * kctx.ND; const float asc = ascales[(size_t)m]; From d5f7d48760eada1e9a422948934aa1d485a1e888 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 08:13:40 -0400 Subject: [PATCH 3/5] npu: the integer head, and the gate it unlocks -- device logits BIT-EXACT over four steps The head was the last place the host arm and the device computed the same tensor differently: in host mode `run_lm_head()` is a plain f32 matvec ("No I8Ctx is touched") while the device reads a packed quantised table (`lm_head_packed_`, `run_lm_head_gemv`). That was the whole of the remaining `max|d| = 0.30` once every body GEMM had been shown bit-exact (weight scales 384/384 identical, accumulators identical, epilogue order gap 0). The device packs its head tiles with `lctx.packB_into(...)` -> `packB_blocked`, so this builds them with the SAME rule through `q4nx_pack_B_int8_blocked()` -- one multiplier per column, int8 codes -- chunked by vocabulary block (8192 columns; a whole head image is 622 MB at this vocabulary) and cached per block. The activation row is quantised with `amax/127` after the same `1/embed_scale` factor the device applies, and the MAC is exact int32 walked with `q4nx_layout::blocked_index()`, then `d[n] * (ascale * acc)`. Measured, one npu-hold ticket, device vs this arm, logit dumps compared per step: step 0: argmax 3916 == 3916 cos 1.000000 max|d| 0 step 1: argmax 278 == 278 cos 1.000000 max|d| 0 step 2: argmax 28725 == 28725 cos 1.000000 max|d| 0 step 3: argmax 22478 == 22478 cos 1.000000 max|d| 0 BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 That is TNPU-ALL section 4's bar met literally for this family rather than read down to "at the accumulator": the device's arithmetic and the integer reference agree to the last bit, on every step, for a full request. --- npu/forward/include/q4nx_forward.h | 1 + npu/forward/src/q4nx_forward.cpp | 49 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 8586c0cea..c64be7992 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -248,6 +248,7 @@ class Q4nxNpuForward { // scratch buffer each layer is decoded into, so keying on it served layer 0's weights to every later layer // (engine#1830 review). Routed experts pass wkey < 0 and are never cached: their weights change per token. std::map, HostI8Tensor> host_i8_cache_; + std::vector lm_head_i8_; // integer arm's head image, one per vocabulary block void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 4755c4d24..c43ea283c 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -5215,6 +5215,55 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector if (npu::model::run_lm_head(*xd_, h.data(), via_target.data())) { logits = std::move(via_target); return true; } } if (!host_gemm_ && xd_ && xd_->head_ctx() && xd_->head_ctx()->gemv) return run_lm_head_gemv(h, logits); + if (host_gemm_ && host_gemm_i8_) { + // engine#1808: the head's integer twin. Without this the arm's head is a plain f32 matvec ("No I8Ctx is + // touched" below) while the device's head reads a packed quantised table, which is exactly the 0.30 max|d| + // that was left once every body GEMM had been shown bit-exact. The device packs its head tiles with + // lctx.packB_into(...) -> packB_blocked, so the arm builds them with the same rule through + // q4nx_pack_B_int8_blocked(): one multiplier per column, int8 codes. Chunked by vocabulary block, because a + // whole head image is H x V bytes (622 MB at this vocabulary). + const float inv_es = (sem_.tie_embeddings && sem_.embed_scale > 0) ? 1.0f / sem_.embed_scale : 1.0f; + const int TB = 8192; // vocabulary block; a tile's image is H x TB bytes + if ((int)lm_head_i8_.size() != (NV + TB - 1) / TB) lm_head_i8_.assign((NV + TB - 1) / TB, HostI8Tensor()); + logits.assign((size_t)NV, 0.0f); + std::vector a8v((size_t)H); + float amax = 0.0f; + for (int k = 0; k < H; k++) { + const float v = std::fabs(h[(size_t)k] * inv_es); + if (v > amax) amax = v; + } + const float ascale = amax > 0 ? amax / 127.0f : 1.0f; + for (int k = 0; k < H; k++) { + int q = (int)std::round(h[(size_t)k] * inv_es / ascale); + a8v[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); + } + std::vector bt((size_t)H * (size_t)TB); + std::vector a8i((size_t)H); + for (size_t i = 0; i < a8v.size(); i++) a8i[i] = a8v[i]; + for (int t = 0; t < (int)lm_head_i8_.size(); t++) { + const int n0 = t * TB, n1 = std::min(NV, n0 + TB), Tn = n1 - n0; + HostI8Tensor& c = lm_head_i8_[(size_t)t]; + if (c.codes.empty()) { + for (int n = n0; n < n1; n++) + for (int k = 0; k < H; k++) bt[(size_t)k * Tn + (n - n0)] = lm_at(n, k); + c.N = Tn; + c.codes.assign((size_t)H * (size_t)Tn, 0); + c.d.assign((size_t)Tn, 0.0f); + double ssum = 0.0; + q4nx_pack_B_int8_blocked(c.codes.data(), bt.data(), H, Tn, H, Tn, /*blocked=*/1, c.d.data(), &ssum); + } + for (int n = 0; n < Tn; n++) { + int32_t acc = 0; + for (int k = 0; k < H; k++) + acc += (int32_t)((int)c.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, H, Tn)] * + (int)a8i[(size_t)k]); + float v = c.d[(size_t)n] * (ascale * (float)acc); + if (sem_.logit_softcap > 0) v = sem_.logit_softcap * std::tanh(v / sem_.logit_softcap); + logits[(size_t)(n0 + n)] = v; + } + } + return true; + } if (host_gemm_) { // Offline mode: plain float matvec against the (possibly dequantized) // tied table, matching the Python reference. No I8Ctx is touched. From 7e6481e1cdc6e9930a3b9bd862382cd8cca7b5e5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 12:40:22 -0400 Subject: [PATCH 4/5] docs/npu: say what NPU_INFER_HOST_GEMM=i8 is for, and what it measured The integer arm was only discoverable from the code. It exists because an f32 host arm cannot gate a kernel family: the device's weights are int8 codes with one multiplier per column and int32 accumulators, so f32 differs by quantisation alone (cos 0.9960-0.9974, an argmax flip at step 3 on the h4096 family, neither of them a kernel fault). Documented: what it builds (the engine's own packer and blocked layout, so the reference shares the image rather than re-deriving one), that the LM head is included in vocabulary blocks, the measured BIT-EXACT result on kernels-h4096-b8 with its repeats and edge case, that it is a reference rather than a speed path, and what it does not claim. --- docs/npu.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index 98d287a95..99f7ca874 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1712,6 +1712,45 @@ usable beside another NPU engine, and what its limits are: not wait for other shared holders"; `LOCK_EX` = "I want the device to myself". A holder keeps its fd open for the life of the process; closing or exiting releases it. `LOCK_EX` and `LOCK_SH` never overlap, in either order, on one file. +### `NPU_INFER_HOST_GEMM=i8`: the integer arm, and why a float one cannot gate a kernel + +`NPU_INFER_HOST_GEMM=1` runs the forward on the host with **f32** GEMMs. That is enough to tell a structural +break (garbage output) from a working route, and it is *not* enough to check a kernel family: the device's +weights are int8 codes with one multiplier per column and its accumulators are int32, so an f32 arm differs from +it by quantisation alone -- measured on the h4096 family, cosine 0.9960-0.9974 and an argmax flip at step 3, +none of which is a kernel fault. A family gated against f32 can therefore never be *bit-exact*, which is the bar +`TISSUES-ALL` §4 asks for. + +`NPU_INFER_HOST_GEMM=i8` is the reference that can. It builds each body GEMM's weight image with the engine's +own packer (`q4nx_pack_B_int8_blocked()`: one multiplier per column, int8 codes, blocked microtile order), walks +it with `q4nx_layout::blocked_index()`, quantises the activation row to int8 with `amax/127`, accumulates in +int32 and applies `d[n] * (ascale * acc)`. **Same packer, same image, same order** -- that is the whole point: a +reference that re-quantises by a different rule computes a different model, and will disagree with a correct +device (measured: per-128-K int4 codes answer `ikt` to a question four other packs answer correctly on). + +The LM head is included, in vocabulary blocks (a whole head image is H x V bytes, 622 MB at 150k vocabulary). +Without it the body is exact and the head is not, and the residual shows up as `max|d| ~ 0.3` on logits that no +amount of body-side agreement can remove. + +**What it measures.** Run the device arm and the `=i8` arm on the same request with `--dump-logits`, then +`scripts/families/compare-logits.py` (npu-kernels). On `kernels-h4096-b8` with Mistral-7B-Instruct-v0.2 1.9: + +``` +step 0: argmax 3916 == 3916 cos 1.000000 max|d| 0 +step 1: argmax 278 == 278 cos 1.000000 max|d| 0 +step 2: argmax 28725 == 28725 cos 1.000000 max|d| 0 +step 3: argmax 22478 == 22478 cos 1.000000 max|d| 0 +BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 +``` +three repeats of the device arm, an edge case (`Hi`), and run-to-run determinism all `max|d| = 0`. + +**What it costs.** The arm warms up slowly (it builds one image per tensor, ~35 s on a 7B container) and is not +a speed path: it exists to be a reference. It is off by default and `=1` (f32) keeps its old meaning. + +**What it does not claim.** Nothing about throughput, and nothing about a family it has not been run on. The +same gate on the other four-h4096 sets needs a pack with *their* geometry, which is a download, not a code +change. + - **Who may take what.** Anything that honours the file interlocks with us: `forward-serve` (`--npu-streams` decides which mode), the in-process route (`LOCK_EX`, always -- `app/unified.cpp`), the ELF runner (`LOCK_EX`, always). **A run that never opens the device takes no lock**: with From c7c504309f3ba6910b05978a198122be0a1e4415 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 12:45:27 -0400 Subject: [PATCH 5/5] docs/npu: the two-arm command sequence for the bit-exact gate A documented reference that nobody can run is a claim, not an instrument: the section now carries the two forward-serve invocations, the dump naming scheme, and the three-step discipline (compare per position, repeat the device arm for determinism, add an edge case). --- docs/npu.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index 99f7ca874..0eaeb0620 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1744,6 +1744,18 @@ BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 ``` three repeats of the device arm, an edge case (`Hi`), and run-to-run determinism all `max|d| = 0`. +**How to run the two arms.** One ticket each, dumps compared per step: + +``` +1bit forward-serve -m /model.q4nx --kernels /npu -p 20081 --dump-logits /tmp/g/i8 \ + & # with NPU_INFER_HOST_GEMM=i8 in the environment +1bit forward-serve -m /model.q4nx --kernels /npu -p 20082 --dump-logits /tmp/g/dev +scripts/families/compare-logits.py /tmp/g/i8 /tmp/g/dev # BIT-EXACT at tolerance 0 +``` +`--dump-logits` writes `step__.bin` per generated step, so the comparison is per position rather than +per run. Repeat the device arm three times for determinism, and add an edge case (a two-token prompt), which is +what the table above did. + **What it costs.** The arm warms up slowly (it builds one image per tensor, ~35 s on a 7B container) and is not a speed path: it exists to be a reference. It is off by default and `=1` (f32) keeps its old meaning.