From 1516de589d1914bb5a4bf6ae9e18710145113c7b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 06:04:21 -0400 Subject: [PATCH 01/12] npu: the integer arm's operands come from the engine's packer, not a private re-quantisation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The arm was merged in #1809 with a quantisation rule of its own: per 128-K group, int4 codes in [-8, 7]. The image the device reads is `q4nx_pack_B_int8_blocked()`'s: ONE multiplier per weight column, int8 codes in [-127, 127]. Both start from the same f32 matrix, so the arm computed a DIFFERENT MODEL -- measured on the Mistral-7B 1.9 container: it answered "ikt" to "The capital of France is" and then locked onto one token, while a per-GEMM A/B read cos 0.9999 against f32 on the same tensors. So the image is now built by that same packer (`blocked=1`, the layout the device reads and the fast path -- measured 174.5 s of warm-up blocked=0 versus 32-35 s blocked) and the MAC walks it with `q4nx_layout::blocked_index()`. Measured after, on current main: warm-up 35.2 s and the answer to "The capital of France is" is **'Paris,▁Question:'**, the same first token the device produces; before it was '%%%%...' / '▁/******/'. With the layer-aware cache key already in main (engine#1830 review), the arm now computes the same model as the device to int8 precision, which is what an integer reference has to do before any family can be gated against it. `npu_pack_test` (the ctest that pins the integer forms at K=130) builds and passes; copyright rc 0. --- npu/forward/include/q4nx_pack.h | 1 - npu/forward/src/q4nx_forward.cpp | 46 +++++++++++++++----------------- npu/forward/src/q4nx_pack.c | 1 + 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index 6f94b1f4c..1a36053b3 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -191,7 +191,6 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out); - /* Architecture dims a design set is derived from. */ typedef struct { int H; /* hidden_size */ diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 5a6b4b36a..c1d689e72 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3044,7 +3044,6 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector& h, std::vector amax) amax = v; - } - if (!(amax >= 1e-12f)) amax = 1.0f; - t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f; - const float tis = 127.0f / amax; - for (int k = k0; k < k1; k++) { - float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; - if (!std::isfinite(v)) v = 0.0f; - int q = (int)std::round(v * tis); - q = q > 7 ? 7 : (q < -8 ? -8 : q); - t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; - } - } - } + t.d.assign((size_t)N, 0.0f); + // THE OPERANDS MUST BE THE ENGINE'S, NOT A PRIVATE RE-QUANTISATION (engine#1808/#1830). This arm exists to be + // the integer reference a kernel family is gated against, and the device's weight image comes from + // q4nx_pack_B_int8_blocked(): ONE multiplier per column (amax over the whole K), int8 codes in [-127, 127]. + // Quantising here with a different rule -- per 128-K group, int4 [-8, 7] -- computes a different model: measured + // 2026-11-10, the arm answered "ikt" to "The capital of France is" and then locked onto one token, while the + // per-GEMM A/B read cos 0.9999 against f32 on the same tensors. Same packer, same bytes, same model. + double ssum = 0.0; + // blocked=1: the image is written in the blocked microtile order, which is the fast path AND the layout the + // device actually reads, so the MAC below walks it with q4nx_layout::blocked_index(). (blocked=0 was measured + // at 174.5 s of warm-up on this container versus 32.2 s blocked -- the row-major scatter is the slow one.) + q4nx_pack_B_int8_blocked(t.codes.data(), Bmat, K, N, K, N, /*blocked=*/1, t.d.data(), &ssum); if (!cacheable) { scratch = std::move(t); cached = &scratch; @@ -3109,8 +3099,14 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, int q = (int)std::round(v * tis); a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); } - q4nx_host_gemm_i8_grouped(a8.data(), K, t.codes.data(), t.d.data(), t.groups, N, ascale, - out + (size_t)m * (size_t)N); + float* dst = out + (size_t)m * (size_t)N; + for (int n = 0; n < N; n++) { + int32_t acc = 0; + for (int k = 0; k < K; k++) + acc += (int32_t)((int)t.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, K, N)] * + (int)a8[(size_t)k]); + dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc); + } } } diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 0914232e5..c021cca95 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -161,6 +161,7 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) { * * exactly the kernel's epilogue, with the inner sum in int32 and no float in it. `d` is the per-128-K-group * multiplier per column ([groups][N]), `ascale` the activation row's scale. */ + void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out) { const int G = 128; From 7904dfa2a07b05abcadd7566655e12041a25a013 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 06:47:15 -0400 Subject: [PATCH 02/12] npu: keyed dumps settle it -- the device's weight scales are BIT-IDENTICAL to the arm's (384/384) Dumps were paired by index first and that was meaningless: the two sides reach the GEMM a different number of times (1024 arm dumps against 384 device dumps on the same request). Keyed by (layer, N, K, ordinal) they pair exactly. Result, one token, Mistral-7B 1.9 container, one npu-hold ticket: arm dumps 1024, device dumps 384, paired 384 exactly equal weight-scale arrays: 384/384 worst relative difference: 0.000e+00 Together with the earlier measurements this closes the GEMM question: the activation scale is the same formula on both sides (`amax/127`, device at xdna_target.cpp:982), the epilogue's two multiplication orders were measured to differ by 0 (the probe returned 0 on every call), the int32 accumulators were identical for the first token's whole pass, and now the per-column weight scales are exactly equal. So the device's GEMMs are bit-exact against this arm. What that leaves is outside the GEMM: the arm and the device still differ by 0.30 max|d| in logits, and the non-GEMM work (RoPE, attention, norms) is shared host code, so the remaining candidates are the fused/selected kernel paths (fused_gu splits GU differently from the arm's one GEMM) and the head path. That is a different question from "are the kernels right", and the answer to that one is yes. Env-gated, off by default (`NPU_I8_GS_DUMP`); nothing in the shipped path changes. --- npu/forward/src/q4nx_forward.cpp | 9 +++++++++ npu/target/xdna/xdna_target.cpp | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index c1d689e72..4755c4d24 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3083,6 +3083,15 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, cached = &it->second; } const HostI8Tensor& t = *cached; + if (const char* gdir = getenv("NPU_I8_GS_DUMP")) { + static int n = 0; + char path[4096]; + std::snprintf(path, sizeof path, "%s/arm_L%d_N%d_K%d_%03d.bin", gdir, cur_layer_, N, K, n++); + if (std::FILE* f = std::fopen(path, "wb")) { + std::fwrite(t.d.data(), sizeof(float), std::min((size_t)N, t.d.size()), f); + std::fclose(f); + } + } std::vector a8((size_t)K); for (int m = 0; m < M; m++) { const float* a = A + (size_t)m * (size_t)a_k; diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index fe4c4ecae..4c45789d5 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -1149,6 +1149,18 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: } const auto tr = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; out.resize((size_t)M * N); + // engine#1808: the device's per-column weight scales, KEYED by (layer, N, K) so it can be paired with the host + // arm's dump exactly. Index pairing was tried first and is meaningless -- the two sides reach the GEMM a + // different number of times (384 device dumps against 1024 arm dumps on the same request). + if (const char* gdir = getenv("NPU_I8_GS_DUMP")) { + static int n = 0; + char path[4096]; + std::snprintf(path, sizeof path, "%s/dev_L%d_N%d_K%d_%03d.bin", gdir, ar.layer, N, K, n++); + if (std::FILE* f = std::fopen(path, "wb")) { + if (gs && !gs->empty()) std::fwrite(gs->data(), sizeof(float), (size_t)N, f); + std::fclose(f); + } + } for (int m = 0; m < M; m++) { const int32_t* cm = kctx.Cm + (size_t)m * kctx.ND; const float asc = ascales[(size_t)m]; From d5f7d48760eada1e9a422948934aa1d485a1e888 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 08:13:40 -0400 Subject: [PATCH 03/12] npu: the integer head, and the gate it unlocks -- device logits BIT-EXACT over four steps The head was the last place the host arm and the device computed the same tensor differently: in host mode `run_lm_head()` is a plain f32 matvec ("No I8Ctx is touched") while the device reads a packed quantised table (`lm_head_packed_`, `run_lm_head_gemv`). That was the whole of the remaining `max|d| = 0.30` once every body GEMM had been shown bit-exact (weight scales 384/384 identical, accumulators identical, epilogue order gap 0). The device packs its head tiles with `lctx.packB_into(...)` -> `packB_blocked`, so this builds them with the SAME rule through `q4nx_pack_B_int8_blocked()` -- one multiplier per column, int8 codes -- chunked by vocabulary block (8192 columns; a whole head image is 622 MB at this vocabulary) and cached per block. The activation row is quantised with `amax/127` after the same `1/embed_scale` factor the device applies, and the MAC is exact int32 walked with `q4nx_layout::blocked_index()`, then `d[n] * (ascale * acc)`. Measured, one npu-hold ticket, device vs this arm, logit dumps compared per step: step 0: argmax 3916 == 3916 cos 1.000000 max|d| 0 step 1: argmax 278 == 278 cos 1.000000 max|d| 0 step 2: argmax 28725 == 28725 cos 1.000000 max|d| 0 step 3: argmax 22478 == 22478 cos 1.000000 max|d| 0 BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 That is TNPU-ALL section 4's bar met literally for this family rather than read down to "at the accumulator": the device's arithmetic and the integer reference agree to the last bit, on every step, for a full request. --- npu/forward/include/q4nx_forward.h | 1 + npu/forward/src/q4nx_forward.cpp | 49 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 8586c0cea..c64be7992 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -248,6 +248,7 @@ class Q4nxNpuForward { // scratch buffer each layer is decoded into, so keying on it served layer 0's weights to every later layer // (engine#1830 review). Routed experts pass wkey < 0 and are never cached: their weights change per token. std::map, HostI8Tensor> host_i8_cache_; + std::vector lm_head_i8_; // integer arm's head image, one per vocabulary block void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 4755c4d24..c43ea283c 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -5215,6 +5215,55 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector if (npu::model::run_lm_head(*xd_, h.data(), via_target.data())) { logits = std::move(via_target); return true; } } if (!host_gemm_ && xd_ && xd_->head_ctx() && xd_->head_ctx()->gemv) return run_lm_head_gemv(h, logits); + if (host_gemm_ && host_gemm_i8_) { + // engine#1808: the head's integer twin. Without this the arm's head is a plain f32 matvec ("No I8Ctx is + // touched" below) while the device's head reads a packed quantised table, which is exactly the 0.30 max|d| + // that was left once every body GEMM had been shown bit-exact. The device packs its head tiles with + // lctx.packB_into(...) -> packB_blocked, so the arm builds them with the same rule through + // q4nx_pack_B_int8_blocked(): one multiplier per column, int8 codes. Chunked by vocabulary block, because a + // whole head image is H x V bytes (622 MB at this vocabulary). + const float inv_es = (sem_.tie_embeddings && sem_.embed_scale > 0) ? 1.0f / sem_.embed_scale : 1.0f; + const int TB = 8192; // vocabulary block; a tile's image is H x TB bytes + if ((int)lm_head_i8_.size() != (NV + TB - 1) / TB) lm_head_i8_.assign((NV + TB - 1) / TB, HostI8Tensor()); + logits.assign((size_t)NV, 0.0f); + std::vector a8v((size_t)H); + float amax = 0.0f; + for (int k = 0; k < H; k++) { + const float v = std::fabs(h[(size_t)k] * inv_es); + if (v > amax) amax = v; + } + const float ascale = amax > 0 ? amax / 127.0f : 1.0f; + for (int k = 0; k < H; k++) { + int q = (int)std::round(h[(size_t)k] * inv_es / ascale); + a8v[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); + } + std::vector bt((size_t)H * (size_t)TB); + std::vector a8i((size_t)H); + for (size_t i = 0; i < a8v.size(); i++) a8i[i] = a8v[i]; + for (int t = 0; t < (int)lm_head_i8_.size(); t++) { + const int n0 = t * TB, n1 = std::min(NV, n0 + TB), Tn = n1 - n0; + HostI8Tensor& c = lm_head_i8_[(size_t)t]; + if (c.codes.empty()) { + for (int n = n0; n < n1; n++) + for (int k = 0; k < H; k++) bt[(size_t)k * Tn + (n - n0)] = lm_at(n, k); + c.N = Tn; + c.codes.assign((size_t)H * (size_t)Tn, 0); + c.d.assign((size_t)Tn, 0.0f); + double ssum = 0.0; + q4nx_pack_B_int8_blocked(c.codes.data(), bt.data(), H, Tn, H, Tn, /*blocked=*/1, c.d.data(), &ssum); + } + for (int n = 0; n < Tn; n++) { + int32_t acc = 0; + for (int k = 0; k < H; k++) + acc += (int32_t)((int)c.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, H, Tn)] * + (int)a8i[(size_t)k]); + float v = c.d[(size_t)n] * (ascale * (float)acc); + if (sem_.logit_softcap > 0) v = sem_.logit_softcap * std::tanh(v / sem_.logit_softcap); + logits[(size_t)(n0 + n)] = v; + } + } + return true; + } if (host_gemm_) { // Offline mode: plain float matvec against the (possibly dequantized) // tied table, matching the Python reference. No I8Ctx is touched. From 7e6481e1cdc6e9930a3b9bd862382cd8cca7b5e5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 12:40:22 -0400 Subject: [PATCH 04/12] docs/npu: say what NPU_INFER_HOST_GEMM=i8 is for, and what it measured The integer arm was only discoverable from the code. It exists because an f32 host arm cannot gate a kernel family: the device's weights are int8 codes with one multiplier per column and int32 accumulators, so f32 differs by quantisation alone (cos 0.9960-0.9974, an argmax flip at step 3 on the h4096 family, neither of them a kernel fault). Documented: what it builds (the engine's own packer and blocked layout, so the reference shares the image rather than re-deriving one), that the LM head is included in vocabulary blocks, the measured BIT-EXACT result on kernels-h4096-b8 with its repeats and edge case, that it is a reference rather than a speed path, and what it does not claim. --- docs/npu.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index 98d287a95..99f7ca874 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1712,6 +1712,45 @@ usable beside another NPU engine, and what its limits are: not wait for other shared holders"; `LOCK_EX` = "I want the device to myself". A holder keeps its fd open for the life of the process; closing or exiting releases it. `LOCK_EX` and `LOCK_SH` never overlap, in either order, on one file. +### `NPU_INFER_HOST_GEMM=i8`: the integer arm, and why a float one cannot gate a kernel + +`NPU_INFER_HOST_GEMM=1` runs the forward on the host with **f32** GEMMs. That is enough to tell a structural +break (garbage output) from a working route, and it is *not* enough to check a kernel family: the device's +weights are int8 codes with one multiplier per column and its accumulators are int32, so an f32 arm differs from +it by quantisation alone -- measured on the h4096 family, cosine 0.9960-0.9974 and an argmax flip at step 3, +none of which is a kernel fault. A family gated against f32 can therefore never be *bit-exact*, which is the bar +`TISSUES-ALL` §4 asks for. + +`NPU_INFER_HOST_GEMM=i8` is the reference that can. It builds each body GEMM's weight image with the engine's +own packer (`q4nx_pack_B_int8_blocked()`: one multiplier per column, int8 codes, blocked microtile order), walks +it with `q4nx_layout::blocked_index()`, quantises the activation row to int8 with `amax/127`, accumulates in +int32 and applies `d[n] * (ascale * acc)`. **Same packer, same image, same order** -- that is the whole point: a +reference that re-quantises by a different rule computes a different model, and will disagree with a correct +device (measured: per-128-K int4 codes answer `ikt` to a question four other packs answer correctly on). + +The LM head is included, in vocabulary blocks (a whole head image is H x V bytes, 622 MB at 150k vocabulary). +Without it the body is exact and the head is not, and the residual shows up as `max|d| ~ 0.3` on logits that no +amount of body-side agreement can remove. + +**What it measures.** Run the device arm and the `=i8` arm on the same request with `--dump-logits`, then +`scripts/families/compare-logits.py` (npu-kernels). On `kernels-h4096-b8` with Mistral-7B-Instruct-v0.2 1.9: + +``` +step 0: argmax 3916 == 3916 cos 1.000000 max|d| 0 +step 1: argmax 278 == 278 cos 1.000000 max|d| 0 +step 2: argmax 28725 == 28725 cos 1.000000 max|d| 0 +step 3: argmax 22478 == 22478 cos 1.000000 max|d| 0 +BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 +``` +three repeats of the device arm, an edge case (`Hi`), and run-to-run determinism all `max|d| = 0`. + +**What it costs.** The arm warms up slowly (it builds one image per tensor, ~35 s on a 7B container) and is not +a speed path: it exists to be a reference. It is off by default and `=1` (f32) keeps its old meaning. + +**What it does not claim.** Nothing about throughput, and nothing about a family it has not been run on. The +same gate on the other four-h4096 sets needs a pack with *their* geometry, which is a download, not a code +change. + - **Who may take what.** Anything that honours the file interlocks with us: `forward-serve` (`--npu-streams` decides which mode), the in-process route (`LOCK_EX`, always -- `app/unified.cpp`), the ELF runner (`LOCK_EX`, always). **A run that never opens the device takes no lock**: with From c7c504309f3ba6910b05978a198122be0a1e4415 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 12:45:27 -0400 Subject: [PATCH 05/12] docs/npu: the two-arm command sequence for the bit-exact gate A documented reference that nobody can run is a claim, not an instrument: the section now carries the two forward-serve invocations, the dump naming scheme, and the three-step discipline (compare per position, repeat the device arm for determinism, add an edge case). --- docs/npu.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index 99f7ca874..0eaeb0620 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1744,6 +1744,18 @@ BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 ``` three repeats of the device arm, an edge case (`Hi`), and run-to-run determinism all `max|d| = 0`. +**How to run the two arms.** One ticket each, dumps compared per step: + +``` +1bit forward-serve -m /model.q4nx --kernels /npu -p 20081 --dump-logits /tmp/g/i8 \ + & # with NPU_INFER_HOST_GEMM=i8 in the environment +1bit forward-serve -m /model.q4nx --kernels /npu -p 20082 --dump-logits /tmp/g/dev +scripts/families/compare-logits.py /tmp/g/i8 /tmp/g/dev # BIT-EXACT at tolerance 0 +``` +`--dump-logits` writes `step__.bin` per generated step, so the comparison is per position rather than +per run. Repeat the device arm three times for determinism, and add an edge case (a two-token prompt), which is +what the table above did. + **What it costs.** The arm warms up slowly (it builds one image per tensor, ~35 s on a 7B container) and is not a speed path: it exists to be a reference. It is off by default and `=1` (f32) keeps its old meaning. From 043e58ba38990aebf4f83dfd87c81c99832eaa43 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:42:39 -0400 Subject: [PATCH 06/12] npu-pack: the h4096 36-layer fconst row, so the Qwen3-8B pack imports at all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `kFconst19Rules` held the 32-layer h4096 row and nothing for 36 layers, so `npu-pack info` refused `amd/Qwen3-8B_rai_1.9.0_npu` by design: "its block shape has not been measured for hidden 4096, 32 heads, 36 layers … the 1.8 pack of the same model is what establishes it". Measured now, with tools/fconst19_measure.py against amd/Qwen3-8B_rai_1.8.0_npu_16K: {4096, 32, 36, 12608, 5, 1, 2297856} 26,104 of 3,756,627,264 bytes differ (6.95e-06 of the file) -- content, not layout Every field but the layer count is identical to the 32-layer row (prefix 12,608, five tiles of 2,297,856, rotation 1), which is the first row here where the layer count moves nothing but the number of blocks: the h2048 pair moves both the prefix and the division with it (4,160/9 tiles at 36 layers against 8,512/3 at 24), so this row is measured rather than inferred from that one. The consequence beyond this pack: a kernel family and an fconst rule row are not the same partition. One family can span several rows, so each new pack of a family may cost a measurement before it costs a gate. Verified end to end: the import now completes -- 5,760/5,760 .fconst tiles survive decode->encode byte for byte, 144 attention projections plus 108 MLP projections with fp32 scale/zero exact for 100.00% of groups, 8 full_i8 ELFs linked from kernels-h4096-im12288-b8. --- npu/pack_import.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 248c2f6ce..0944afd02 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -449,6 +449,16 @@ constexpr Fconst19Rule kFconst19Rules[] = { // all 32 blocks. This is the largest family on #1503 (31 packs). Measured by another lane on main, whose // table is keyed on geometry; merged here in the same form. {4096, 32, 32, 12608, 5, 1, 2297856}, + // Qwen3-8B at hidden 4,096 / 32 heads / **36 layers**: measured with tools/fconst19_measure.py against + // amd/Qwen3-8B_rai_1.8.0_npu_16K, 26,104 of 3,756,627,264 bytes differ (6.95e-06 of the file, which the tool prints + // as 6.95e-04 %) -- content, not layout, and the largest residual in this table, so it is stated rather than + // rounded away. Every other field is IDENTICAL to the 32-layer row above (prefix 12,608, five tiles of 2,297,856, + // rotation 1): at this hidden size the layer count moves nothing but the number of blocks, which is the first row + // here where that is true -- the h2048 pair above moves both the prefix and the division with the layer count + // (4,160/9 tiles at 36 layers against 8,512/3 tiles at 24), so this row is measured, not inferred from that one. + // Without it the importer refuses the pack by design ("its block shape has not been measured for hidden 4096, 32 + // heads, 36 layers"), which is what it did until now: kernel family and fconst rule row are not the same partition. + {4096, 32, 36, 12608, 5, 1, 2297856}, // Llama-3.2-3B (and -Instruct, same geometry) at hidden 3,072 / 24 heads / 28 layers: measured with // tools/fconst19_measure.py against amd/Llama-3.2-3B_rai_1.8.0_npu_4K, 1,090 of 1,756,930,368 bytes differ // (6.20e-07 of the file, which the tool prints as 6.20e-05 %) -- content, not layout. The prefix is the Qwen3 line's 64 + 2*hidden + 4,352 = 10,560 at this hidden From 452975c68550f12ae8a18178243a36692ee87c17 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 14:06:36 -0400 Subject: [PATCH 07/12] npu-pack test: the new 36-layer row goes in BOTH places, as the guard's own comment requires CI caught the omission and said exactly why: "fconst19 rule: the table has exactly the 11 measured rows (12); a dropped row is a silent un-measure of a whole class, an added one must be added here too". The expected rows are literals copied from the measured table rather than read back from it, so that a wrong digit or a dropped row fails -- which is the arm working, not a nuisance. Added {4096, 32, 36, 12608, 5, 1, 2297856} to it, and the arm now passes. --- tests/npu_pack_import_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index c63b3fb93..fafc7f60a 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2791,6 +2791,7 @@ void test_fconst19_rule_table() { {3584, 28, 28, 7232, 8, 1, 1009664}, // the Qwen2.5-7B family, measured on two models {2048, 16, 36, 4160, 9, 1, 295936}, // Qwen2.5_3B_Instruct {4096, 32, 32, 12608, 5, 1, 2297856}, // Mistral-7B-Instruct-v0.2 + {4096, 32, 36, 12608, 5, 1, 2297856}, // Qwen3-8B (measured against ..._rai_1.8.0_npu_16K) {3072, 24, 28, 10560, 4, 1, 1740800}, // Llama-3.2-3B (and -Instruct) }; const int want_n = (int)(sizeof(want) / sizeof(want[0])); From 853021fcb57f42f7a01139f1e0a2dab8ce0413c0 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 14:22:49 -0400 Subject: [PATCH 08/12] npu-pack: pad the head's rows to the q4_1 tile quantum, so a 32,016-deep vocabulary imports CodeLlama-7b-Instruct's head is 32,016 rows and q4_1 tiles hold 32-row multiples, so the whole import refused with "q4_1 tiles need N % 32 == 0" -- a head the grid cannot hold, not a head that is wrong. The projections already had this rule (the pack's own padding stays, and the .fconst tail carries ceil(V/64)*64 rows: Hunyuan-0.5B has 120,832 for 120,818), but the graph-weight head path had no padding at all. pad_rows_to_tile_quantum() pads with rows that decode to zero (code 0, scale 0, zero 0 -- the quantised value is code*scale + zero, so a zeroed group is 0.0, not a weight) and LEAVES vocab_size at the vocabulary's own number, so the padding is never addressed. That invisibility is the part a test has to pin: widening N without it would put two plausible extra tokens at the end of every distribution. Measured on the pack that found it: the import now completes -- 387 tensors, 806,544 q4_1 tiles, container read back through q4nx::dequant_tile with rel_err 0.000e+00 and max|diff| 0.000e+00, 8 full_i8 ELFs linked from kernels-h4096-mha-b8 (the 4096x12288 family TNPU-ALL W3b asks to verify end to end). --- npu/pack_import.cpp | 24 ++++++++++++++++ npu/pack_import.h | 5 ++++ tests/npu_pack_import_test.cpp | 50 +++++++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 0944afd02..a4cd6bc61 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1774,6 +1774,29 @@ bool external_container_tensors(const std::string& role, int nh, int nkv, int hd return false; } +// engine#1990: q4_1 tiles hold 32-row multiples, and a pack's vocabulary need not be one. CodeLlama's is 32,016, which +// refused the whole import with "q4_1 tiles need N % 32 == 0" -- a head the grid cannot hold, not a head that is wrong. +// Where the pack pads its own head (the .fconst tail carries ceil(V/64)*64 rows: Hunyuan-0.5B has 120,832 for 120,818) +// nothing here fires; where it does not, pad with rows that decode to zero, and leave `vocab_size` at the vocabulary's +// own number so the padding is never addressed. codes 0 with scale 0 decode to 0, for the same reason a padded column +// does: the quantised value is code*scale + zero, and zero holds the group's `m`. +void pad_rows_to_tile_quantum(Q4& q, int vocab) { + if (q.N <= 0 || q.K <= 0 || q.block <= 0 || q.K % q.block) return; + if (q.N % q4nx::kRows == 0) return; + const int padded = (q.N + q4nx::kRows - 1) / q4nx::kRows * q4nx::kRows; + const size_t gpr = size_t(q.K / q.block); // groups per row + for (int r = q.N; r < padded; r++) { + q.codes.insert(q.codes.end(), size_t(q.K), 0); + q.scale.insert(q.scale.end(), gpr, 0.0f); + q.zero.insert(q.zero.end(), gpr, 0.0f); + } + std::fprintf(stderr, + "note: the head is %d rows for a vocabulary of %d (padded from %lld to the 32-row tile quantum); " + "vocab_size stays %d in the container\n", + padded, vocab, (long long)q.N, vocab); + q.N = padded; +} + Q4 container_from_external_q4(const Q4& pack, long long* m_inexact) { Q4 q = pack; // same codes, same geometry long long inexact = 0; @@ -3910,6 +3933,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons uint64_t qw_need = 0; // N * K/block * block/2 if (!mul_ok(groups_n, uint64_t(mm.block) / 2, qw_need)) throw std::runtime_error("lm_head: qweight length overflows"); Q4 q = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block); + pad_rows_to_tile_quantum(q, int(V)); // no qzeros tensor -> the implicit zero point (engine#1503: Qwen3-VL-4B's spelling, three-input nodes) if (!have_zp) implicit_zp_zero(q, mm.bits); diff --git a/npu/pack_import.h b/npu/pack_import.h index 01541c3d7..09a94f7e3 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -369,6 +369,11 @@ constexpr bool kExtQuantsRowMajor = false; // measurably NOT the same weights: on a real phi-4 tile it reproduced the pack's codes on only 2,826 of 8,192 // entries and moved weights by 2.5 % of the tensor's range (engine#1503). `m_inexact` reports how many group // offsets were not bf16-exact (86.3 % are, measured), because that rounding is the only loss this path has. +// engine#1990: a head whose row count is not a multiple of the 32-row tile quantum (CodeLlama: 32,016 rows) cannot be +// held by the q4_1 grid, so pad it with zero rows that decode to zero and leave the container's vocab_size alone. Exposed +// because the property -- padding is invisible to the vocabulary -- is what a test can pin without a pack. +void pad_rows_to_tile_quantum(Q4& q, int vocab); + Q4 container_from_external_q4(const Q4& pack, long long* m_inexact = nullptr); // The container role a matmul plays, read off the graph's own tensor name: `...qkv_proj...` -> "QKV", diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index fafc7f60a..ee583ac7c 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -918,6 +918,53 @@ void test_1671_plan_head_arm(const std::string& root) { } // ── 1. MatMulNBits: encode here, decode with the importer ──────────────────── +// engine#1990: a head whose row count is not a multiple of the 32-row tile quantum (CodeLlama's vocabulary is 32,016) is +// padded with rows that decode to zero, and the container's vocab_size is NOT changed -- the padding is invisible to the +// vocabulary. Both halves are pinned here because widening N without staying invisible is the failure that would show up +// as two plausible extra tokens at the end of every distribution. +void test_head_row_padding() { + const auto make = [](int N) { + pi::Q4 q; + q.N = N; + q.K = 128; // two 64-column tiles + q.block = 32; // four groups per row + q.codes.assign(size_t(N) * size_t(q.K), 1); // non-zero, so padding is distinguishable + q.scale.assign(size_t(N) * size_t(q.K / q.block), 0.5f); + q.zero.assign(size_t(N) * size_t(q.K / q.block), 0.25f); + return q; + }; + { + pi::Q4 q = make(32016); + pi::pad_rows_to_tile_quantum(q, 32016); + check(q.N == 32032 && q.N % 32 == 0, "head padding: 32,016 rows become 32,032 (the 32-row quantum)"); + check(q.codes.size() == size_t(32032) * 128 && q.scale.size() == size_t(32032) * 4, + "head padding: codes and scales grow with the rows, one group's worth each"); + bool zeros = true; + for (int r = 32016; r < q.N; r++) { + for (int k = 0; k < q.K; k++) zeros = zeros && q.codes[size_t(r) * 128 + size_t(k)] == 0; + for (int g = 0; g < 4; g++) + zeros = zeros && q.scale[size_t(r) * 4 + size_t(g)] == 0.0f && q.zero[size_t(r) * 4 + size_t(g)] == 0.0f; + } + check(zeros, "head padding: every padded row is code 0, scale 0, zero 0 -- which decodes to 0.0, not to a weight"); + bool kept = true; + for (int r = 0; r < 32016; r++) + for (int k = 0; k < 128; k++) kept = kept && q.codes[size_t(r) * 128 + size_t(k)] == 1; + check(kept, "head padding: the vocabulary's own rows are untouched"); + } + { // an aligned vocabulary is left exactly as it was: no note, no growth + pi::Q4 q = make(32000); + const size_t before = q.codes.size(); + pi::pad_rows_to_tile_quantum(q, 32000); + check(q.N == 32000 && q.codes.size() == before, "head padding: a 32-aligned vocabulary (32,000) is not touched"); + } + { // 16 rows of slack and 1 row of slack both round up to one tile boundary, not two + pi::Q4 a = make(32017), b = make(32031); + pi::pad_rows_to_tile_quantum(a, 32017); + pi::pad_rows_to_tile_quantum(b, 32031); + check(a.N == 32032 && b.N == 32032, "head padding: 32,017 and 32,031 both round to 32,032"); + } +} + void test_matmulnbits() { const int N = 64, K = 256, B = 128; std::vector qw(size_t(N) * (K / B) * (B / 2)); @@ -3687,7 +3734,8 @@ int main(int argc, char** argv) { test_crafted_refusals(golden_dir.empty() ? "." : golden_dir); test_modelbin_genre_refusal(golden_dir.empty() ? "." : golden_dir); test_1671_plan_head_arm(golden_dir.empty() ? "." : golden_dir); - test_matmulnbits(); + test_head_row_padding(); + test_matmulnbits(); test_tiles(golden_dir); test_q4_1f(); test_attn_tile(golden_dir); From 8932ac0089fc7caec1cdff5749c1213125e7d261 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 14:56:26 -0400 Subject: [PATCH 09/12] npu: a padded head is only invisible if the READER agrees -- scale 1.0, and dequant on the tensor's grid The first gate run on CodeLlama refused with "lm_head dequant failed", and it was right twice over. 1. The padding wrote SCALE 0 for its rows. Arithmetically code 0 * 0 + 0 = 0.0, but a zero scale is not a zero weight to a reader -- it is a malformed parameter block, and the dequant refuses it. The test asserted exactly the wrong property (scale == 0.0f), which is how a wrong assumption gets to look pinned. Both now say scale 1.0, with the reason written next to them. 2. The forward asked for exactly NV rows, and NV here is the VOCABULARY (32,016), which is not a multiple of the tile's 32 rows -- so dequant_any() refused a head that was perfectly well formed. The container's vocab_size is the vocabulary; the tensor carries the padding; the reader has to dequantise on the tensor's own grid and keep only the vocabulary's rows. That is the same convention the .fconst path already relies on (Hunyuan-0.5B: 120,832 rows for 120,818), and it was simply not implemented on the graph-weight path. The extra rows decode to zero and are never sampled: the logits vector stays NV long. So this commit is importer + reader, and the PR's claim is only true with both: "the padding is invisible to the vocabulary" is a property of the pair, not of the writer. --- npu/forward/src/q4nx_forward.cpp | 32 ++++++++++++++++++++++++++++++-- npu/pack_import.cpp | 7 ++++--- tests/npu_pack_import_test.cpp | 6 +++--- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index b587a1eb6..9dc0e139c 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1436,11 +1436,39 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, lm_head_f_.assign((size_t)NV * (size_t)kpad_, 0.0f); lm_w_ = kpad_; const uint8_t* lp = (const uint8_t*)model_tensor_data(mw_, ltd); - if (!lp || dequant_any(lp, (size_t)ltd->data_size, NV, kpad_, - lm_head_f_.data()) != 0) { + /* engine#1990: the container's head may carry MORE rows than the vocabulary, for the same reason a padded + * column exists -- q4_1 tiles hold 32-row multiples and a pack's vocabulary need not be one (CodeLlama's is + * 32,016). The importer pads those rows; the reader must therefore dequantise on the tensor's own grid and + * keep only the vocabulary's rows. Asking for exactly NV rows is what failed here: 32,016 is not a multiple + * of the tile's 32 rows, so dequant_any() refused a head that was perfectly well formed. + * + * The extra rows decode to zero and are never sampled: the logits vector stays NV long. */ + int head_rows = NV; + if (ltd->ndim >= 2 && kpad_ > 0 && kpad_ % Q4NX_TILE_COLS == 0 && ltd->shape[1] == kpad_) { + const int64_t per_block = kpad_ / Q4NX_TILE_COLS; // tiles per 32-row block + if (per_block > 0 && ltd->shape[0] % per_block == 0) { + const int64_t r = (ltd->shape[0] / per_block) * Q4NX_TILE_ROWS; + if (r >= NV && r % Q4NX_TILE_ROWS == 0) head_rows = int(r); + } + } + std::vector head_f32; + float* head_out = lm_head_f_.data(); + if (head_rows != NV) { + head_f32.assign((size_t)head_rows * (size_t)kpad_, 0.0f); + head_out = head_f32.data(); + } + if (!lp || dequant_any(lp, (size_t)ltd->data_size, head_rows, kpad_, head_out) != 0) { err_ = "lm_head dequant failed"; return false; } + if (head_rows != NV) { + for (int r = 0; r < NV; r++) + std::memcpy(&lm_head_f_[(size_t)r * (size_t)kpad_], &head_f32[(size_t)r * (size_t)kpad_], + sizeof(float) * (size_t)kpad_); + std::fprintf(stderr, + " [forward] lm head: %d row(s) stored for a vocabulary of %d; the %d padding row(s) are " + "not read (engine#1990)\n", head_rows, NV, head_rows - NV); + } fprintf(stderr, " [forward] separate lm_head.weight -> dequantized " "(%zu floats)\n", lm_head_f_.size()); } diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index a4cd6bc61..ec32a7e5b 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1778,8 +1778,9 @@ bool external_container_tensors(const std::string& role, int nh, int nkv, int hd // refused the whole import with "q4_1 tiles need N % 32 == 0" -- a head the grid cannot hold, not a head that is wrong. // Where the pack pads its own head (the .fconst tail carries ceil(V/64)*64 rows: Hunyuan-0.5B has 120,832 for 120,818) // nothing here fires; where it does not, pad with rows that decode to zero, and leave `vocab_size` at the vocabulary's -// own number so the padding is never addressed. codes 0 with scale 0 decode to 0, for the same reason a padded column -// does: the quantised value is code*scale + zero, and zero holds the group's `m`. +// own number so the padding is never addressed. codes 0 with SCALE 1 and zero 0 decode to exactly 0.0 -- and scale 1, +// not 0: a zero scale is not a zero weight to a reader, it is a malformed parameter block (the forward's dequant refuses +// it, which is how the first version of this padding was caught: "lm_head dequant failed"). void pad_rows_to_tile_quantum(Q4& q, int vocab) { if (q.N <= 0 || q.K <= 0 || q.block <= 0 || q.K % q.block) return; if (q.N % q4nx::kRows == 0) return; @@ -1787,7 +1788,7 @@ void pad_rows_to_tile_quantum(Q4& q, int vocab) { const size_t gpr = size_t(q.K / q.block); // groups per row for (int r = q.N; r < padded; r++) { q.codes.insert(q.codes.end(), size_t(q.K), 0); - q.scale.insert(q.scale.end(), gpr, 0.0f); + q.scale.insert(q.scale.end(), gpr, 1.0f); // 1.0, not 0: code 0 * 1 + 0 = 0.0, and a 0 scale is malformed q.zero.insert(q.zero.end(), gpr, 0.0f); } std::fprintf(stderr, diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index ee583ac7c..674960825 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -929,7 +929,7 @@ void test_head_row_padding() { q.K = 128; // two 64-column tiles q.block = 32; // four groups per row q.codes.assign(size_t(N) * size_t(q.K), 1); // non-zero, so padding is distinguishable - q.scale.assign(size_t(N) * size_t(q.K / q.block), 0.5f); + q.scale.assign(size_t(N) * size_t(q.K / q.block), 0.5f); // 1.0 in the padded rows; see below q.zero.assign(size_t(N) * size_t(q.K / q.block), 0.25f); return q; }; @@ -943,9 +943,9 @@ void test_head_row_padding() { for (int r = 32016; r < q.N; r++) { for (int k = 0; k < q.K; k++) zeros = zeros && q.codes[size_t(r) * 128 + size_t(k)] == 0; for (int g = 0; g < 4; g++) - zeros = zeros && q.scale[size_t(r) * 4 + size_t(g)] == 0.0f && q.zero[size_t(r) * 4 + size_t(g)] == 0.0f; + zeros = zeros && q.scale[size_t(r) * 4 + size_t(g)] == 1.0f && q.zero[size_t(r) * 4 + size_t(g)] == 0.0f; } - check(zeros, "head padding: every padded row is code 0, scale 0, zero 0 -- which decodes to 0.0, not to a weight"); + check(zeros, "head padding: every padded row is code 0, scale 1, zero 0 -- which decodes to 0.0, not to a weight"); bool kept = true; for (int r = 0; r < 32016; r++) for (int k = 0; k < 128; k++) kept = kept && q.codes[size_t(r) * 128 + size_t(k)] == 1; From 0f2d344110202473ceb74e1798783631d939a85d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 15:56:45 -0400 Subject: [PATCH 10/12] npu: the integer arm reads a padded activation row through the pad, so a non-256-aligned IM gates host_gemm_i8() read its activation row with a[k] for k < K, and K is the PADDED design width. chatglm3-6b's IM is 13,696 and its D design is K = 13,824, so the arm read 128 floats past the caller's row -- into the next row's data, or past the buffer on the last one. Measured before the fix on kernels-h4096-chatglm-b8: cos 0.40 -> -0.003 -> 0.06 -> 0.005 against the device, argmax unrelated, while the f32 arm tracked the device at 0.985-0.988. After: max|d| = 0 on all four steps, cos 1.000000. The f32 leg has always padded -- q4nx_host_gemm_f32_padded() "owns the pad-and-read pair so neither leg can be handed an unpadded A", a comment that records a review catching exactly this -- and the integer leg, written later, did not. One lambda now serves both loops (amax and quantise), so the two cannot drift again. Why this looks like three unrelated mysteries until it is one: the families whose shapes are all 256-aligned were bit-exact (Mistral, Qwen3-8B), the one with a padded K was wildly wrong (chatglm), and the one with no padding but a residual (CodeLlama) is a different, smaller defect. This is the first hybrid pack with a non-aligned IM. --- npu/forward/src/q4nx_forward.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 9dc0e139c..1c29eba6b 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1444,7 +1444,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, * * The extra rows decode to zero and are never sampled: the logits vector stays NV long. */ int head_rows = NV; - if (ltd->ndim >= 2 && kpad_ > 0 && kpad_ % Q4NX_TILE_COLS == 0 && ltd->shape[1] == kpad_) { + /* The head is stored as a TILE tensor (shape {tiles, bytes-per-tile}), not as {rows, cols}: the rows have to + * be recovered from the tile count and the grid, which is why comparing shape[1] against kpad_ was the wrong + * test and left this at NV. */ + if (ltd->ndim >= 2 && kpad_ > 0 && kpad_ % Q4NX_TILE_COLS == 0 && ltd->shape[0] > 0) { const int64_t per_block = kpad_ / Q4NX_TILE_COLS; // tiles per 32-row block if (per_block > 0 && ltd->shape[0] % per_block == 0) { const int64_t r = (ltd->shape[0] / per_block) * Q4NX_TILE_ROWS; @@ -3129,15 +3132,21 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, std::vector a8((size_t)K); for (int m = 0; m < M; m++) { const float* a = A + (size_t)m * (size_t)a_k; + // engine#845's pad, on THIS leg too: a padded design has K > a_k (chatglm3-6b's IM is 13,696 and its D design + // is K = 13,824), so reading A[k] for k < K runs 128 floats past the caller's row -- into the next row, or past + // the buffer on the last. The f32 leg has always padded (q4nx_host_gemm_f32_padded "owns the pad-and-read pair + // so neither leg can be handed an unpadded A"); this leg was written later and did not. Measured on chatglm + // before the fix: cos 0.40 -> -0.003 against the device, while the f32 arm tracked it at 0.985-0.988. + const auto act = [a, a_k](int k) { return k < a_k ? a[(size_t)k] : 0.0f; }; float amax = 0.0f; for (int k = 0; k < K; k++) { - const float v = std::fabs(a[k]); + const float v = std::fabs(act(k)); if (std::isfinite(v) && v > amax) amax = v; } const float ascale = (amax >= 1e-12f) ? amax / 127.0f : 1.0f; const float tis = 1.0f / ascale; for (int k = 0; k < K; k++) { - float v = a[k]; + float v = act(k); if (!std::isfinite(v)) v = 0.0f; int q = (int)std::round(v * tis); a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); From 7dcc361b7255811107a99dd23e4bf68b08ad5898 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 16:19:40 -0400 Subject: [PATCH 11/12] npu: restore the reader's padded-head block, which the merge dropped, and make the head block width a knob The merge that took main's unpadded-image work wholesale over q4nx_forward.cpp also took main's head dequant, which asks for exactly NV rows -- so the padded-head reader (10 references, reduced to 3) was silently lost and CodeLlama went back to "lm_head dequant failed" on the next run. Restored verbatim from 8932ac0. That is the second time a resolve-to-main has cost a fix on this file, which is why the block now carries its own measurement in the comment. Also: NPU_I8_HEAD_TB (default 8192, multiple of 128). CodeLlama's residual is confined to exactly sixteen logit positions starting at 16384, identical at every step -- a WEIGHT difference localised at a block edge, not accumulation drift -- and 16384 is where this arm's first head block ends. Moving the block width decides whether the chunking or the device's head read is responsible, in one ticket instead of an argument. --- npu/forward/src/q4nx_forward.cpp | 44 +++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 342c5ca07..a610222a8 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1439,11 +1439,41 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, lm_head_f_.assign((size_t)NV * (size_t)kpad_, 0.0f); lm_w_ = kpad_; const uint8_t* lp = (const uint8_t*)model_tensor_data(mw_, ltd); - if (!lp || dequant_any(lp, (size_t)ltd->data_size, NV, kpad_, - lm_head_f_.data()) != 0) { + /* engine#1990: the container's head may carry MORE rows than the vocabulary, for the same reason a padded + * column exists -- q4_1 tiles hold 32-row multiples and a pack's vocabulary need not be one (CodeLlama's is + * 32,016). The importer pads those rows; the reader must therefore dequantise on the tensor's own grid and + * keep only the vocabulary's rows. Asking for exactly NV rows is what failed here: 32,016 is not a multiple + * of the tile's 32 rows, so dequant_any() refused a head that was perfectly well formed. + * + * The extra rows decode to zero and are never sampled: the logits vector stays NV long. */ + int head_rows = NV; + if (ltd->ndim >= 2 && kpad_ > 0 && kpad_ % Q4NX_TILE_COLS == 0 && ltd->shape[1] == kpad_) { + const int64_t per_block = kpad_ / Q4NX_TILE_COLS; // tiles per 32-row block + if (per_block > 0 && ltd->shape[0] % per_block == 0) { + const int64_t r = (ltd->shape[0] / per_block) * Q4NX_TILE_ROWS; + if (r >= NV && r % Q4NX_TILE_ROWS == 0) head_rows = int(r); + } + } + std::vector head_f32; + float* head_out = lm_head_f_.data(); + if (head_rows != NV) { + head_f32.assign((size_t)head_rows * (size_t)kpad_, 0.0f); + head_out = head_f32.data(); + } + if (!lp || dequant_any(lp, (size_t)ltd->data_size, head_rows, kpad_, head_out) != 0) { err_ = "lm_head dequant failed"; return false; } + if (head_rows != NV) { + for (int r = 0; r < NV; r++) + std::memcpy(&lm_head_f_[(size_t)r * (size_t)kpad_], &head_f32[(size_t)r * (size_t)kpad_], + sizeof(float) * (size_t)kpad_); + std::fprintf(stderr, + " [forward] lm head: %d row(s) stored for a vocabulary of %d; the %d padding row(s) are " + "not read (engine#1990)\n", head_rows, NV, head_rows - NV); + } + fprintf(stderr, " [forward] separate lm_head.weight -> dequantized " + "(%zu floats)\n", lm_head_f_.size()); fprintf(stderr, " [forward] separate lm_head.weight -> dequantized " "(%zu floats)\n", lm_head_f_.size()); } @@ -5371,7 +5401,15 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector // q4nx_pack_B_int8_blocked(): one multiplier per column, int8 codes. Chunked by vocabulary block, because a // whole head image is H x V bytes (622 MB at this vocabulary). const float inv_es = (sem_.tie_embeddings && sem_.embed_scale > 0) ? 1.0f / sem_.embed_scale : 1.0f; - const int TB = 8192; // vocabulary block; a tile's image is H x TB bytes + // engine#1990: the block width is an experiment knob, because the residual on CodeLlama is confined to exactly + // 16 logit positions starting at 16384 -- the start of a block -- and is identical at every step, i.e. a WEIGHT + // difference localised at a block boundary rather than an accumulation difference. NPU_I8_HEAD_TB moves the + // boundaries: if the wrong positions move with it, the chunking is the cause; if they stay at 16384, it is not. + static const int TB = [] { + const char* e = std::getenv("NPU_I8_HEAD_TB"); + const int v = e ? std::atoi(e) : 8192; + return (v >= 1024 && v % 128 == 0) ? v : 8192; + }(); if ((int)lm_head_i8_.size() != (NV + TB - 1) / TB) lm_head_i8_.assign((NV + TB - 1) / TB, HostI8Tensor()); logits.assign((size_t)NV, 0.0f); std::vector a8v((size_t)H); From 0d629ec64aec7462631915593c7ca02c03b4b009 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 16:47:22 -0400 Subject: [PATCH 12/12] npu: the head's rows come from its TILE count, and that closes the fourth family The restored block was the v2 test (shape[1] == kpad_), which never holds for a tile tensor -- CodeLlama's head is {16016 tiles, 5120 B/tile}, so shape[1] is 5120 against kpad_ 4096 and head_rows stayed at NV, which is the exact refusal the block exists to prevent. Rows must come from shape[0] and the grid: (16016 / (4096/256)) * 32 = 32032. With that, kernels-h4096-mha-b8 is BIT-EXACT against the device: max|d| = 0 on all four steps, three repeats, run-to-run determinism, and the "Hi" edge case -- and it is BLOCK-WIDTH INVARIANT (NPU_I8_HEAD_TB 8192 and 16384 both exact), which also rules out the chunking as the source of the sixteen-position residual I was chasing. The previous reading of that residual -- "16 logit positions, a weight difference at a block edge, cause unknown" -- was this bug: the arm was reading the head's tile grid as if the vocabulary were the row count. Recorded because the wrong version of the story was in the issue thread for an hour. --- npu/forward/src/q4nx_forward.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index a610222a8..9a1ccd6d6 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1447,7 +1447,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, * * The extra rows decode to zero and are never sampled: the logits vector stays NV long. */ int head_rows = NV; - if (ltd->ndim >= 2 && kpad_ > 0 && kpad_ % Q4NX_TILE_COLS == 0 && ltd->shape[1] == kpad_) { + /* The head is a TILE tensor -- shape {tiles, bytes-per-tile} (CodeLlama: {16016, 5120}) -- not {rows, cols}, + * so the rows come from the tile count and the grid. Comparing shape[1] against kpad_ was the wrong test and + * left this at NV, which is exactly the refusal this block exists to prevent. */ + if (ltd->ndim >= 2 && kpad_ > 0 && kpad_ % Q4NX_TILE_COLS == 0 && ltd->shape[0] > 0) { const int64_t per_block = kpad_ / Q4NX_TILE_COLS; // tiles per 32-row block if (per_block > 0 && ltd->shape[0] % per_block == 0) { const int64_t r = (ltd->shape[0] / per_block) * Q4NX_TILE_ROWS;