From 1c0715d08e358142330911d824889ab85a341b4e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 04:14:21 -0400 Subject: [PATCH 1/3] npu: q/k norm is read from the container, so a qwen3_5 text model under another name is normalised (engine#1788) q4nx_semantics turned per-head q/k RMSNorm on by config.json's model_type (qwen3*, gemma*). MiniCPM-V 4.7 is 'minicpmv4_7' around a qwen3_5 text model, and its container ships self_attn.q_norm / k_norm in every full-attention layer; the forward ran those layers without the norm. Short prompts still answered; long ones drifted. A layer carrying both tensors now turns the norm on (never off), and the load prints which layer decided. Measured, 1B pack, host arm, against the original weights in transformers: a 149-token story question 'The question mentions that Maria buys' -> 'Pluto' (reference 'Pluto'); top-1 agreement over the prompt's rows 65.1 % -> 75.2 %; a sliced image described wrongly -> its three regions named correctly. Test: npu_rmsnorm_offset gains 9 arms on the rule (the published pack's model_type, the pair, q or k alone, range, already on, never off). Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 24 ++++++++++++++++ npu/forward/include/q4nx_semantics.h | 11 +++++++ npu/forward/src/q4nx_forward.cpp | 14 +++++++++ npu/forward/src/q4nx_semantics.c | 10 +++++++ tests/npu_rmsnorm_offset_test.c | 43 ++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index db140ca35..e220dea66 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1752,6 +1752,30 @@ the missing q/k norm above was found: - `--rows-argmax` (with `--none`) prints the model's own next-token choice at every row of a text prompt, from one teacher-forced pass. Compared with a reference's per-row choices, a low agreement that is flat across positions is an error in the layers; a step at some row is an error at a position or a batch edge. +**Per-head q/k normalisation is read from the container (engine#1788).** Whether the forward normalises q +and k per head used to be decided by `config.json`'s `model_type` alone: on for `qwen3*` and `gemma*`. +A pack of the same architecture under another name got it wrong in silence. FastFlowLM's MiniCPM-V 4.7 +is `minicpmv4_7` around a `qwen3_5` text model, and every full-attention layer of its container ships +`self_attn.q_norm` and `k_norm`; the forward ran those layers without the norm. The model still answered +short prompts (`Paris`, `red`) and drifted on longer ones. Now a layer that carries both tensors turns the +norm on, and the load says so: + +``` +[semantics] qk_norm=1 <- the container: layer 3 carries self_attn.q_norm and k_norm (config.json's model_type is not a family the table knows) +``` + +The rule never turns the norm off: a family the table knows keeps its answer. Measured on the 1B pack, +host arm, against the model's original weights in transformers (2026-11-11): + +| | before | after | reference | +|---|---|---|---| +| a 149-token story, "what is the name of Maria's cat?" | `The question mentions that Maria buys` | `Pluto` | `Pluto` | +| top-1 agreement with the reference over those 149 rows | 65.1 % | 75.2 % | | +| rows where the forward's choice is outside the reference's top 10 | 6 | 0 | | +| a sliced 448 × 896 image (red left, green top right, blue bottom right) | "three equal horizontal sections … white … red" | red left, green top right, blue bottom right | the same three regions | + +What is left of the disagreement (a quarter of the rows) is between a 4-bit container and the float +weights; it has not been separated into quantisation and anything else. ## Serving diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index 77393f1b1..5aebf8009 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -243,6 +243,17 @@ int q4nx_semantics_uses_ple(const Q4nxSemantics* s); * (1 + w) form, 0 for the plain w form. Exported so the rule is testable with no model present. */ int q4nx_rmsnorm_offset_from_norm(const float* w, int n); +/* Per-head q/k RMSNorm, decided by the container when config.json's name does not decide it (engine#1788). + * q4nx_semantics_from_config() turns qk_norm on for the families its table knows by model_type (qwen3*, gemma*). A + * pack of the same architecture under another name -- FastFlowLM's MiniCPM-V 4.7 is `minicpmv4_7` around a qwen3_5 + * text model -- came out as "no q/k norm" while shipping self_attn.q_norm and k_norm in every full-attention layer; + * those layers then ran without the norm, and the model answered short prompts and drifted on long ones. + * `has_q[l]` / `has_k[l]` say whether layer l carries a non-empty q_norm / k_norm tensor. A layer carrying BOTH is the + * container saying it has the norm: qk_norm becomes 1 and that layer's index is returned. Returns -1, changing + * nothing, when qk_norm is already on, when no layer carries both, or on null/empty input. It never turns the norm + * off: a family the table knows keeps its answer whatever the tensors are. Exported so the rule is testable. */ +int q4nx_semantics_qk_norm_from_tensors(Q4nxSemantics* s, const unsigned char* has_q, const unsigned char* has_k, int layers); + /* attention.mode for one container, from the two places that can state it (engine#796). * A d3 pack declares the mask it was trained with in its own `decision_config.json` diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index a5c56b8f1..389575a23 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1070,6 +1070,20 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, return false; } if (sem_.eps <= 0) sem_.eps = c_.eps; + // engine#1788: whether q and k are normalised per head is a fact about the container, not about its name + // (q4nx_semantics.h). A layer that carries both q_norm and k_norm turns it on for a model_type the table + // does not know; said out loud, because the dump below then prints 1 beside "<- model_type". + if (mw_ && mw_->config.num_layers > 0) { + std::vector has_q(size_t(mw_->config.num_layers), 0), has_k(has_q); + for (int l = 0; l < mw_->config.num_layers; l++) { + has_q[size_t(l)] = mw_->layers[l].q_norm_weight.name[0] && mw_->layers[l].q_norm_weight.data_size > 0; + has_k[size_t(l)] = mw_->layers[l].k_norm_weight.name[0] && mw_->layers[l].k_norm_weight.data_size > 0; + } + const int by = q4nx_semantics_qk_norm_from_tensors(&sem_, has_q.data(), has_k.data(), mw_->config.num_layers); + if (by >= 0) + fprintf(stderr, " [semantics] qk_norm=1 <- the container: layer %d carries self_attn.q_norm and k_norm " + "(config.json's model_type is not a family the table knows)\n", by); + } q4nx_semantics_dump(&sem_); // A layer kind nothing implements is refused BY NAME, here beside the other precise refusals (engine#1087's // shape): q4nx_semantics maps an unrecognised layer_types entry to Q4NX_LAYER_OTHER and the attention path diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index c623e660d..e5ee8f81d 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -535,6 +535,16 @@ void q4nx_semantics_layer_mix(const Q4nxSemantics* s, char* out, size_t out_sz) snprintf(out, out_sz, "full=%d sliding=%d linear=%d other=%d", full, slid, lin, oth); } +int q4nx_semantics_qk_norm_from_tensors(Q4nxSemantics* s, const unsigned char* has_q, const unsigned char* has_k, int layers) { + if (!s || !has_q || !has_k || layers <= 0 || s->qk_norm) return -1; + for (int l = 0; l < layers; l++) + if (has_q[l] && has_k[l]) { + s->qk_norm = 1; + return l; + } + return -1; +} + void q4nx_semantics_dump(const Q4nxSemantics* s) { if (!s) return; fprintf(stderr, " [semantics] rmsnorm_offset=%d <- %s\n", s->rmsnorm_offset, diff --git a/tests/npu_rmsnorm_offset_test.c b/tests/npu_rmsnorm_offset_test.c index b2735d2fe..d637d4015 100644 --- a/tests/npu_rmsnorm_offset_test.c +++ b/tests/npu_rmsnorm_offset_test.c @@ -112,6 +112,49 @@ int main(void) { check(q4nx_block_note(0, 16, 128)[0] == '\0' && q4nx_block_note(10, 16, 128)[0] == '\0', "a batch inside the block, or an empty one, prints nothing"); + /* engine#1788: per-head q/k norm, decided by the container when the name does not decide it. The config is the + * published MiniCPM-V 4.7 1B pack's: model_type minicpmv4_7 around a qwen3_5 text model whose full-attention + * layers (3, 7, 11, ...) ship q_norm and k_norm. */ + { + const char* path = "npu_rmsnorm_offset_test.config.json"; + const unsigned char full[8] = {0, 0, 0, 1, 0, 0, 0, 1}; /* layers 3 and 7 carry the tensor */ + const unsigned char none[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + Q4nxSemantics sem; + FILE* f = fopen(path, "w"); + if (f) { + fputs("{\"model_type\": \"minicpmv4_7\", \"architectures\": [\"MiniCPMV4_7ForConditionalGeneration\"], " + "\"hidden_size\": 1024, \"num_hidden_layers\": 8, \"hidden_act\": \"silu\"}", f); + fclose(f); + } + check(f && q4nx_semantics_from_config(path, ".", &sem) == 0 && sem.qk_norm == 0, + "minicpmv4_7: the name alone says no q/k norm (the gap)"); + check(q4nx_semantics_qk_norm_from_tensors(&sem, full, full, 8) == 3 && sem.qk_norm == 1, + "... and a layer carrying q_norm and k_norm turns it on, naming layer 3"); + check(q4nx_semantics_qk_norm_from_tensors(&sem, full, full, 8) == -1 && sem.qk_norm == 1, + "already on: nothing changes and no layer is named"); + sem.qk_norm = 0; + check(q4nx_semantics_qk_norm_from_tensors(&sem, none, none, 8) == -1 && sem.qk_norm == 0, + "no layer carries the tensors: it stays off (llama, phi)"); + check(q4nx_semantics_qk_norm_from_tensors(&sem, full, none, 8) == -1 && sem.qk_norm == 0, + "q_norm without k_norm is not the pair: it stays off"); + check(q4nx_semantics_qk_norm_from_tensors(&sem, none, full, 8) == -1 && sem.qk_norm == 0, + "k_norm without q_norm is not the pair either"); + check(q4nx_semantics_qk_norm_from_tensors(&sem, full, full, 3) == -1 && sem.qk_norm == 0, + "only the layers that exist are read (3 layers: the pair at layer 3 is out of range)"); + check(q4nx_semantics_qk_norm_from_tensors(&sem, NULL, full, 8) == -1 && q4nx_semantics_qk_norm_from_tensors(NULL, full, full, 8) == -1 && + q4nx_semantics_qk_norm_from_tensors(&sem, full, full, 0) == -1, + "null or empty input changes nothing"); + f = fopen(path, "w"); + if (f) { + fputs("{\"model_type\": \"qwen3_5\", \"hidden_size\": 1024, \"num_hidden_layers\": 8}", f); + fclose(f); + } + check(f && q4nx_semantics_from_config(path, ".", &sem) == 0 && sem.qk_norm == 1 && + q4nx_semantics_qk_norm_from_tensors(&sem, none, none, 8) == -1 && sem.qk_norm == 1, + "qwen3_5: on by name, and absent tensors never turn it off"); + remove(path); + } + printf(failures ? "npu_rmsnorm_offset: %d FAILED\n" : "npu_rmsnorm_offset: ok\n", failures); return failures ? 1 : 0; } From 8752fcc1f08c22f26be2bd92d239ce7e84c9f9c4 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 04:40:50 -0400 Subject: [PATCH 2/3] tests: npu_rmsnorm_offset creates its fixture through the private-file helper (CI at 926c730) CI's file_create_check and CodeQL both flagged the two fopen(path, "w") the new arms used for their config fixture. They go through onebit_fopen_private_wb now (owner-only, no symlink). Co-Authored-By: Claude Fable 5.2 --- tests/npu_rmsnorm_offset_test.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/npu_rmsnorm_offset_test.c b/tests/npu_rmsnorm_offset_test.c index d637d4015..6b0d17b0d 100644 --- a/tests/npu_rmsnorm_offset_test.c +++ b/tests/npu_rmsnorm_offset_test.c @@ -26,6 +26,7 @@ // own xn (cos 1.000000) and rmsnorm(embed, 1 + w) reproduces the reference's (cos 0.999999), so this // one bit is the whole 0.6367-token-probability gap in engine#796's causal arm. #include "q4nx_semantics.h" +#include "file_create.h" /* fixture files are created owner-only and never through a symlink */ #include #include @@ -120,7 +121,7 @@ int main(void) { const unsigned char full[8] = {0, 0, 0, 1, 0, 0, 0, 1}; /* layers 3 and 7 carry the tensor */ const unsigned char none[8] = {0, 0, 0, 0, 0, 0, 0, 0}; Q4nxSemantics sem; - FILE* f = fopen(path, "w"); + FILE* f = onebit_fopen_private_wb(path); if (f) { fputs("{\"model_type\": \"minicpmv4_7\", \"architectures\": [\"MiniCPMV4_7ForConditionalGeneration\"], " "\"hidden_size\": 1024, \"num_hidden_layers\": 8, \"hidden_act\": \"silu\"}", f); @@ -144,7 +145,7 @@ int main(void) { check(q4nx_semantics_qk_norm_from_tensors(&sem, NULL, full, 8) == -1 && q4nx_semantics_qk_norm_from_tensors(NULL, full, full, 8) == -1 && q4nx_semantics_qk_norm_from_tensors(&sem, full, full, 0) == -1, "null or empty input changes nothing"); - f = fopen(path, "w"); + f = onebit_fopen_private_wb(path); if (f) { fputs("{\"model_type\": \"qwen3_5\", \"hidden_size\": 1024, \"num_hidden_layers\": 8}", f); fclose(f); From b0a2a40dce9ed74ed9c0f104b6ba7da946ff0f59 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 05:51:46 -0400 Subject: [PATCH 3/3] docs(npu): a blank line between the probe's option list and the q/k-norm paragraph Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/npu.md b/docs/npu.md index e220dea66..3ef9bced6 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1752,6 +1752,7 @@ the missing q/k norm above was found: - `--rows-argmax` (with `--none`) prints the model's own next-token choice at every row of a text prompt, from one teacher-forced pass. Compared with a reference's per-row choices, a low agreement that is flat across positions is an error in the layers; a step at some row is an error at a position or a batch edge. + **Per-head q/k normalisation is read from the container (engine#1788).** Whether the forward normalises q and k per head used to be decided by `config.json`'s `model_type` alone: on for `qwen3*` and `gemma*`. A pack of the same architecture under another name got it wrong in silence. FastFlowLM's MiniCPM-V 4.7