npu: Hunyuan-7B answers on the forward route: use_qk_norm is read, tiles are q,k,v, RoPE base and norm order follow the model (Fixes #1889)

CMakeLists.txt

@@ -1655,6 +1655,14 @@ target_include_directories(q4nx_mrope_test PRIVATE npu/forward/include)
 target_link_libraries(q4nx_mrope_test PRIVATE m)
 add_test(NAME q4nx_mrope COMMAND q4nx_mrope_test)
 add_dependencies(npu_sampling_test q4nx_mrope_test)
+# Which models get the per-head q/k RMSNorm: config.json's use_qk_norm when the config states it, else the
+# model_type table (engine#1889: Hunyuan declares it under a model_type the table has no row for, so its gains were
+# loaded and never applied).  Plain C, links the semantics TU directly, no model and no device.
+add_executable(q4nx_qk_norm_test tests/q4nx_qk_norm_test.c npu/forward/src/q4nx_semantics.c)
+target_include_directories(q4nx_qk_norm_test PRIVATE npu/forward/include)
+target_link_libraries(q4nx_qk_norm_test PRIVATE m)
+add_test(NAME q4nx_qk_norm COMMAND q4nx_qk_norm_test)
+add_dependencies(npu_sampling_test q4nx_qk_norm_test)
 add_executable(q4nx_layer_kinds_test tests/q4nx_layer_kinds_test.c npu/forward/src/q4nx_semantics.c)
 target_include_directories(q4nx_layer_kinds_test PRIVATE npu/forward/include)
 target_link_libraries(q4nx_layer_kinds_test PRIVATE m)

docs/npu.md

@@ -915,6 +915,25 @@ forward route, which reads the block from `config.json`: `rope_type: "llama3"`
 declared-but-unusable `llama3` block, and every other declared type, at `1bit serve` /
 `forward-serve` load time rather than at request time.
 
+**Hunyuan (`hunyuan_v1_dense`) differs from the other q/k-norm families in four places: three in
+the forward, which reads them from the model rather than from a family list where it can, and one in
+the importer**
+([engine#1889](https://github.com/1bit-MONSTER/engine/issues/1889)). Each was measured against
+`Hunyuan-7B-Instruct`'s own pack:
+
+| what | rule | where it comes from |
+|---|---|---|
+| whether q and k are normalised | `use_qk_norm` in `config.json` decides when the config states it; the `qwen3*`/`gemma*` table answers only for configs that do not | the config key. The table has no Hunyuan row, so its gains were loaded and never applied |
+| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99. `1bit npu-pack import` tests this base against the pack's cache first and prints `note: RoPE base ... verified`; before, it warned that no plain theta explained the cache |
+| the order of norm and RoPE | RoPE first, then the per-head RMSNorm (Qwen3 and Gemma normalise first) | the graph: `q_proj -> RotaryEmbedding -> q_norm -> GroupQueryAttention` in every layer |
+| the order of the attention tiles in the pack (importer) | q tiles first, then k, then v. Every other family's block is k, q, v. `1bit npu-pack import` picks the order with the block shape that carries the packed q/k gains | measured: with k, q, v the imported model predicts 26 of 77 next tokens of a fixed text, with q, k, v 59 (the two projections were being read as each other) |
+
+A model whose config says q and k are normalised and whose container has no
+`self_attn.q_norm.weight` is refused at load: the gain would read as zeros and zero every head's q.
+That is the shape of a Hunyuan container imported before the importer carried the gains, so the
+answer is to re-import it. A Hunyuan container imported before the tile order was right has
+the gains and still answers filler words; re-import that one too.
+
 | GGUF type | To q4_1 | Error |
 |---|---|---|
 | Q4_0 | scale `d`, zero `-8d`, codes unchanged | bf16 rounding of scale and zero |

npu/forward/include/q4nx_rope.h

@@ -69,10 +69,26 @@ typedef struct {
     int on;                        /* config declared rope_type "dynamic" with usable numbers */
     double factor;                 /* 1.0 on Hunyuan-0.5B -- a factor of 1 is a no-op until pos > original */
     double original_max_position;  /* 262144 on Hunyuan-0.5B */
+    /* engine#1889: Hunyuan's own form of "dynamic". Its rope_scaling block also carries `alpha` (1000.0 on every
+     * Hunyuan/HY-MT 1.9 pack), and when alpha is set the model's rotary embedding does not move the base with the
+     * position at all: it uses the FIXED base
+     *
+     *     base = theta * alpha ^ (dim / (dim - 2))
+     *
+     * (HunYuanDenseV1RotaryEmbedding, "DynamicNTKAlphaRotary"). Measured against the pack rather than taken from the
+     * reference: Hunyuan-7B's model.onnx.data holds the graph's cos_cache/sin_cache (FP16 [32768, 64]), and they match
+     * base 10000 * 1000^(128/126) = 11,158,840 to 0.00024 at positions 1, 2, 7, 50 and 300, where plain theta 10000 is
+     * off by up to 1.99. Reading the block as factor-only made it a no-op, i.e. every position rotated by the wrong
+     * angle. 0 = the block has no alpha, and the factor rule above applies. */
+    double alpha;
 } Q4nxRopeDynamic;
 
 static inline double q4nx_rope_dynamic_base(double theta, const Q4nxRopeDynamic* d, double pos, int dim) {
     if (!d || !d->on) return theta;
+    if (d->alpha > 0.0) {                                       /* the alpha form: one fixed base, every position */
+        if (dim <= 2) return theta;
+        return theta * pow(d->alpha, (double)dim / (double)(dim - 2));
+    }
     if (!(d->factor > 0.0) || !(d->original_max_position > 0.0) || dim <= 2) return theta;
     if (!(pos >= d->original_max_position)) return theta;      /* inside the window: plain rope, bit for bit */
     const double scaled = d->factor * pos / d->original_max_position - (d->factor - 1.0);

npu/forward/include/q4nx_semantics.h

@@ -62,8 +62,16 @@ typedef struct {
      * Source: model_type/architectures == qwen3_5*, resolved against the weights. */
     int rmsnorm_offset_probe;
     /* Per-head RMSNorm applied to q and k before RoPE.
-     * Source: presence of self_attn.q_norm.weight / k_norm.weight. */
+     * Source: config.json's `use_qk_norm` when the config states it (engine#1889: Hunyuan), else the
+     * model_type/architectures table (qwen3*, gemma*). The forward refuses a model whose flag is set and whose
+     * layers carry no self_attn.q_norm.weight, since normalising with an absent gain zeroes q. */
     int qk_norm;
+    /* The q/k RMSNorm runs AFTER RoPE instead of before it (engine#1889). Qwen3 and Gemma normalise, then rotate;
+     * Hunyuan rotates, then normalises. With a per-dimension gain the two orders differ wherever a rotary pair's two
+     * gains differ, so the order is part of the model. Source: model_type/architectures (hunyuan*) -- no config key
+     * states it; the evidence is the pack's own graph, where every layer is
+     * q_proj -> RotaryEmbedding -> q_norm (SimplifiedLayerNormalization) -> GroupQueryAttention. */
+    int qk_norm_after_rope;
     /* RoPE base.  Source: rope_theta. */
     float rope_theta;
     /* Channels actually rotated; 0 means the full head_dim.  Partial rotary

npu/forward/src/q4nx_forward.cpp

@@ -204,9 +204,10 @@ static int dequant_any(const void* data, size_t nbytes, int rows, int cols, floa
 // transformers' _compute_llama3_parameters applies.  `on == 0` unless the config names the type, so
 // every other architecture builds exactly the table it built before.  Keys are searched with their
 // quotes, which is what keeps "factor" from matching inside "high_freq_factor".
-// Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`. Hunyuan's
-// 1.9 packs declare factor 1.0 with a 262,144 window, i.e. a no-op until a prompt passes that window -- which is why
-// accepting it is safe rather than a guess: below the window the base IS theta, so the served bytes are plain rope.
+// Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`: below the
+// window the base IS theta, so a block with factor 1.0 serves plain rope. Hunyuan's 1.9 packs declare that AND
+// `alpha: 1000.0`, which is a different rule (a fixed base theta * alpha^(dim/(dim-2)), engine#1889): reading only
+// the factor made their block a no-op and rotated every position by the wrong angle.
 Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) {
     Q4nxRopeDynamic d{};
     std::ifstream f(cfg_path);
@@ -223,7 +224,10 @@ Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) {
     };
     d.factor = number("factor");
     d.original_max_position = number("max_position_embeddings");
-    d.on = (d.factor > 0.0 && d.original_max_position > 0.0) ? 1 : 0;
+    // engine#1889: `alpha` selects Hunyuan's fixed-base form (q4nx_rope.h). The key is searched with its quotes and its
+    // colon, so "lora_alpha" or an "alpha" inside a string value cannot be taken for it.
+    d.alpha = number("alpha");
+    d.on = ((d.factor > 0.0 && d.original_max_position > 0.0) || d.alpha > 0.0) ? 1 : 0;
     return d;
 }
 
@@ -1784,6 +1788,17 @@ bool Q4nxNpuForward::load_norms() {
         layer_attn_dims(l, hd_l, nkv_l);
         if (!read_bf16(&mw_->layers[l].input_layernorm_weight, in_norm_[l].data(), H)) return false;
         if (!read_bf16(&mw_->layers[l].post_attention_layernorm_weight, post_norm_[l].data(), H)) return false;
+        // engine#1889: the config says q and k are normalised and this layer has no gain to normalise with. read_bf16
+        // would hand back zeros for the absent tensor and rmsnorm(q, 0) is 0 for every head, so the layer would attend
+        // uniformly and answer fluent nonsense. A Hunyuan container imported before the importer claimed the gains is
+        // exactly this shape, so it is refused by name here instead.
+        if (sem_.qk_norm && absent(&mw_->layers[l].q_norm_weight)) {
+            err_ = "config.json says q and k are normalised per head (" +
+                   std::string(q4nx_semantics_field(Q4NX_SEM_QK_NORM)) + ") and layer " + std::to_string(l) +
+                   " has no self_attn.q_norm.weight: normalising with an absent gain zeroes q, so this container "
+                   "would answer wrong text (engine#1889); re-import the pack with a build that carries the gains";
+            return false;
+        }
         if (!read_bf16(&mw_->layers[l].q_norm_weight, q_norm_[l].data(), hd_l)) return false;
         // Qwen2.5-style attention biases (q/k/v projection biases); absent on the
         // bias-free families, so a miss is not an error.  Loaded BEFORE the k_norm
@@ -4827,7 +4842,11 @@ bool Q4nxNpuForward::attention(const std::vector<float>& qkv, int pos, const int
 
     // per-head RMSNorm on q and k, only for architectures that have it.  A KV-shared
     // layer (Gemma4) computed no k at all, so its k is never normalized or used.
-    if (sem_.qk_norm) {
+    // engine#1889: WHERE it runs is per family. Qwen3 and Gemma normalise and then rotate; Hunyuan's graph rotates
+    // and then normalises (q_proj -> RotaryEmbedding -> q_norm -> attention), so for it the same code runs after the
+    // RoPE block below. The RMS itself is rotation-invariant; the per-dimension gain is not, so the order matters
+    // wherever a rotary pair's two gains differ.
+    auto qk_normalise = [&]() {
         std::vector<float> t(HD);
         for (int h = 0; h < NH; h++) {
             rmsnorm(&q[(size_t)h * HD], q_norm_[cur_layer_].data(), HD, t.data(), "q_norm");
@@ -4839,7 +4858,8 @@ bool Q4nxNpuForward::attention(const std::vector<float>& qkv, int pos, const int
                 memcpy(&k[(size_t)h * HD], t.data(), sizeof(float) * HD);
             }
         }
-    }
+    };
+    if (sem_.qk_norm && !sem_.qk_norm_after_rope) qk_normalise();
     // Gemma4 normalises v with a SCALE-FREE RMSNorm (its v_norm has no weight).
     if (sem_.v_norm && append_kv) {
         std::vector<float> tv(HD);
@@ -4859,7 +4879,8 @@ bool Q4nxNpuForward::attention(const std::vector<float>& qkv, int pos, const int
         std::vector<float> cos_(half), sin_(half);
         // Dynamic NTK: the base moves with the position, so it is one pow per position rather than one per dim pair.
         // Below the model's original window this returns theta unchanged, which is what makes a short prompt
-        // bit-identical to the same model served without the scaling.
+        // bit-identical to the same model served without the scaling. A block with `alpha` (Hunyuan) is the fixed
+        // base theta * alpha^(dim/(dim-2)) at every position instead (engine#1889).
         // engine#1792 review: the exponent is HF's `dim / (dim - 2)` over the ROTARY width (`dim = int(head_dim *
         // partial_rotary_factor)`), which is `rot` here -- passing `HD` was wrong for any model that declares dynamic
         // scaling together with partial_rotary_factor < 1. Unreachable in today's catalog, so this is a correctness
@@ -4906,6 +4927,7 @@ bool Q4nxNpuForward::attention(const std::vector<float>& qkv, int pos, const int
         for (int h = 0; h < NH; h++) rot_half(&q[(size_t)h * HD]);
         for (int h = 0; h < NKV; h++) rot_half(&k[(size_t)h * HD]);
     }
+    if (sem_.qk_norm && sem_.qk_norm_after_rope) qk_normalise();
 
     // append to cache (a KV-shared layer reuses its owner's, which the owner has
     // already appended earlier in this same step).

npu/forward/src/q4nx_semantics.c

@@ -26,7 +26,7 @@
 
 static const char* kFields[Q4NX_SEM_COUNT] = {
     /* RMSNORM_OFFSET  */ "model_type/architectures (gemma* uses the 1+w form)",
-    /* QK_NORM         */ "model_type/architectures (q_norm/k_norm tensors)",
+    /* QK_NORM         */ "use_qk_norm when stated, else model_type/architectures",
     /* ROPE_THETA      */ "rope_theta",
     /* ROTARY_DIM      */ "partial_rotary_factor * head_dim",
     /* MLP_ACT         */ "hidden_act (falls back to model_type)",
@@ -224,6 +224,18 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir,
 
     /* per-head q/k RMSNorm: qwen3*, gemma* and lfm2 have q_norm/k_norm; llama/phi do not */
     s->qk_norm = (is_qwen3 || is_gemma || is_lfm2) ? 1 : 0;
+    /* engine#1889: a config that STATES `use_qk_norm` decides it, whatever the name table says. The table is a list
+     * of the families measured so far, so a family it has never heard of got 0 silently: Hunyuan-7B is
+     * `hunyuan_v1_dense` / `HunYuanDenseV1ForCausalLM` with `use_qk_norm: true`, its container carries
+     * self_attn.{q,k}_norm.weight for all 32 layers, and the forward loaded them and never applied them -- the same
+     * text with the gains, without them and with the key set to false, because nothing read the key. Families that
+     * do not state the key (qwen3, gemma) keep the table's answer. */
+    {
+        double uq = 0;
+        if (get_num(json, "use_qk_norm", &uq)) s->qk_norm = uq != 0 ? 1 : 0;
+    }
+    /* Hunyuan rotates q and k first and normalises them after (see the header); everyone else normalises first. */
+    s->qk_norm_after_rope = (starts_with(model_type, "hunyuan") || starts_with(arch, "HunYuan")) ? 1 : 0;
 
     /* RoPE base.  Qwen3.5 nests it under rope_parameters (scalar); Gemma4 nests
      * it PER LAYER TYPE under rope_parameters.{full,sliding}_attention with
@@ -556,6 +568,9 @@ void q4nx_semantics_dump(const Q4nxSemantics* s) {
             q4nx_semantics_field(Q4NX_SEM_RMSNORM_OFFSET));
     fprintf(stderr, "  [semantics] qk_norm=%d  <- %s\n", s->qk_norm,
             q4nx_semantics_field(Q4NX_SEM_QK_NORM));
+    if (s->qk_norm)
+        fprintf(stderr, "  [semantics] qk_norm order: %s  <- model_type/architectures (hunyuan* normalises after RoPE)\n",
+                s->qk_norm_after_rope ? "RoPE, then norm" : "norm, then RoPE");
     fprintf(stderr, "  [semantics] rope_theta=%.6g  <- %s\n", s->rope_theta,
             q4nx_semantics_field(Q4NX_SEM_ROPE_THETA));
     fprintf(stderr, "  [semantics] rotary_dim=%d  <- %s\n", s->rotary_dim,