From e8842ab1ba8e4871bfb0a70d97db7cce7a4b143b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:17:48 -0400 Subject: [PATCH 1/2] npu: Hunyuan answers on the forward route: use_qk_norm is read, RoPE base and norm order follow the model, tiles are q,k,v (Fixes #1889) Four defects stood between a Hunyuan-7B pack and an answer. Each is measured on Hunyuan-7B-Instruct_rai_1.9.0_npu with the device-free forward. forward: - use_qk_norm in config.json decides whether q and k are normalised when the config states it. The model_type table (qwen3*, gemma*) has no Hunyuan row, so the imported gains were loaded and never applied. - rope_scaling {type: dynamic, alpha: A} is a fixed base theta * A^(dim/(dim-2)), as the pack's cos/sin cache and the reference implementation have it. It was read as plain theta. - Hunyuan applies RoPE first and the per-head RMSNorm second (the pack graph: q_proj -> RotaryEmbedding -> q_norm). Qwen3 and Gemma keep norm first. - a config that says q/k are normalised with no q_norm weight in the container is refused at load instead of zeroing every head's q. importer: - the Hunyuan attention block stores its tiles q, k, v. Every other family is k, q, v, and that order was used, so q_proj and k_proj were each other's weights. Top-1 next-token accuracy on a fixed 78-token text, teacher-forced: no q/k norm 33/77, q/k norm 62/77, all forward changes 59/77, all forward changes with the old k,q,v tiles 26/77. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 8 ++ docs/npu.md | 19 +++++ npu/forward/include/q4nx_rope.h | 16 ++++ npu/forward/include/q4nx_semantics.h | 10 ++- npu/forward/src/q4nx_forward.cpp | 36 ++++++-- npu/forward/src/q4nx_semantics.c | 17 +++- npu/pack_import.cpp | 24 +++++- npu/pack_import.h | 13 +++ tests/npu_pack_import_test.cpp | 16 ++++ tests/npu_rope_llama3_test.c | 29 +++++++ tests/q4nx_qk_norm_test.c | 119 +++++++++++++++++++++++++++ 11 files changed, 295 insertions(+), 12 deletions(-) create mode 100644 tests/q4nx_qk_norm_test.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 610142905..d2361d2a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1535,6 +1535,14 @@ target_include_directories(q4nx_mrope_test PRIVATE npu/forward/include) target_link_libraries(q4nx_mrope_test PRIVATE m) add_test(NAME q4nx_mrope COMMAND q4nx_mrope_test) add_dependencies(npu_sampling_test q4nx_mrope_test) +# Which models get the per-head q/k RMSNorm: config.json's use_qk_norm when the config states it, else the +# model_type table (engine#1889: Hunyuan declares it under a model_type the table has no row for, so its gains were +# loaded and never applied). Plain C, links the semantics TU directly, no model and no device. +add_executable(q4nx_qk_norm_test tests/q4nx_qk_norm_test.c npu/forward/src/q4nx_semantics.c) +target_include_directories(q4nx_qk_norm_test PRIVATE npu/forward/include) +target_link_libraries(q4nx_qk_norm_test PRIVATE m) +add_test(NAME q4nx_qk_norm COMMAND q4nx_qk_norm_test) +add_dependencies(npu_sampling_test q4nx_qk_norm_test) add_executable(q4nx_layer_kinds_test tests/q4nx_layer_kinds_test.c npu/forward/src/q4nx_semantics.c) target_include_directories(q4nx_layer_kinds_test PRIVATE npu/forward/include) target_link_libraries(q4nx_layer_kinds_test PRIVATE m) diff --git a/docs/npu.md b/docs/npu.md index 85857e81f..c112ae570 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -849,6 +849,25 @@ forward route, which reads the block from `config.json`: `rope_type: "llama3"` declared-but-unusable `llama3` block, and every other declared type, at `1bit serve` / `forward-serve` load time rather than at request time. +**Hunyuan (`hunyuan_v1_dense`) differs from the other q/k-norm families in four places: three in +the forward, which reads them from the model rather than from a family list where it can, and one in +the importer** +([engine#1889](https://github.com/1bit-MONSTER/engine/issues/1889)). Each was measured against +`Hunyuan-7B-Instruct`'s own pack: + +| what | rule | where it comes from | +|---|---|---| +| whether q and k are normalised | `use_qk_norm` in `config.json` decides when the config states it; the `qwen3*`/`gemma*` table answers only for configs that do not | the config key. The table has no Hunyuan row, so its gains were loaded and never applied | +| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99 | +| the order of norm and RoPE | RoPE first, then the per-head RMSNorm (Qwen3 and Gemma normalise first) | the graph: `q_proj -> RotaryEmbedding -> q_norm -> GroupQueryAttention` in every layer | +| the order of the attention tiles in the pack (importer) | q tiles first, then k, then v. Every other family's block is k, q, v. `1bit npu-pack import` picks the order with the block shape that carries the packed q/k gains | measured: with k, q, v the imported model predicts 26 of 77 next tokens of a fixed text, with q, k, v 59 (the two projections were being read as each other) | + +A model whose config says q and k are normalised and whose container has no +`self_attn.q_norm.weight` is refused at load: the gain would read as zeros and zero every head's q. +That is the shape of a Hunyuan container imported before the importer carried the gains, so the +answer is to re-import it. A Hunyuan container imported before the tile order was right has +the gains and still answers filler words; re-import that one too. + | GGUF type | To q4_1 | Error | |---|---|---| | Q4_0 | scale `d`, zero `-8d`, codes unchanged | bf16 rounding of scale and zero | diff --git a/npu/forward/include/q4nx_rope.h b/npu/forward/include/q4nx_rope.h index 946e41670..aee089a16 100644 --- a/npu/forward/include/q4nx_rope.h +++ b/npu/forward/include/q4nx_rope.h @@ -69,10 +69,26 @@ typedef struct { int on; /* config declared rope_type "dynamic" with usable numbers */ double factor; /* 1.0 on Hunyuan-0.5B -- a factor of 1 is a no-op until pos > original */ double original_max_position; /* 262144 on Hunyuan-0.5B */ + /* engine#1889: Hunyuan's own form of "dynamic". Its rope_scaling block also carries `alpha` (1000.0 on every + * Hunyuan/HY-MT 1.9 pack), and when alpha is set the model's rotary embedding does not move the base with the + * position at all: it uses the FIXED base + * + * base = theta * alpha ^ (dim / (dim - 2)) + * + * (HunYuanDenseV1RotaryEmbedding, "DynamicNTKAlphaRotary"). Measured against the pack rather than taken from the + * reference: Hunyuan-7B's model.onnx.data holds the graph's cos_cache/sin_cache (FP16 [32768, 64]), and they match + * base 10000 * 1000^(128/126) = 11,158,840 to 0.00024 at positions 1, 2, 7, 50 and 300, where plain theta 10000 is + * off by up to 1.99. Reading the block as factor-only made it a no-op, i.e. every position rotated by the wrong + * angle. 0 = the block has no alpha, and the factor rule above applies. */ + double alpha; } Q4nxRopeDynamic; static inline double q4nx_rope_dynamic_base(double theta, const Q4nxRopeDynamic* d, double pos, int dim) { if (!d || !d->on) return theta; + if (d->alpha > 0.0) { /* the alpha form: one fixed base, every position */ + if (dim <= 2) return theta; + return theta * pow(d->alpha, (double)dim / (double)(dim - 2)); + } if (!(d->factor > 0.0) || !(d->original_max_position > 0.0) || dim <= 2) return theta; if (!(pos >= d->original_max_position)) return theta; /* inside the window: plain rope, bit for bit */ const double scaled = d->factor * pos / d->original_max_position - (d->factor - 1.0); diff --git a/npu/forward/include/q4nx_semantics.h b/npu/forward/include/q4nx_semantics.h index 77393f1b1..abd6201fa 100644 --- a/npu/forward/include/q4nx_semantics.h +++ b/npu/forward/include/q4nx_semantics.h @@ -62,8 +62,16 @@ typedef struct { * Source: model_type/architectures == qwen3_5*, resolved against the weights. */ int rmsnorm_offset_probe; /* Per-head RMSNorm applied to q and k before RoPE. - * Source: presence of self_attn.q_norm.weight / k_norm.weight. */ + * Source: config.json's `use_qk_norm` when the config states it (engine#1889: Hunyuan), else the + * model_type/architectures table (qwen3*, gemma*). The forward refuses a model whose flag is set and whose + * layers carry no self_attn.q_norm.weight, since normalising with an absent gain zeroes q. */ int qk_norm; + /* The q/k RMSNorm runs AFTER RoPE instead of before it (engine#1889). Qwen3 and Gemma normalise, then rotate; + * Hunyuan rotates, then normalises. With a per-dimension gain the two orders differ wherever a rotary pair's two + * gains differ, so the order is part of the model. Source: model_type/architectures (hunyuan*) -- no config key + * states it; the evidence is the pack's own graph, where every layer is + * q_proj -> RotaryEmbedding -> q_norm (SimplifiedLayerNormalization) -> GroupQueryAttention. */ + int qk_norm_after_rope; /* RoPE base. Source: rope_theta. */ float rope_theta; /* Channels actually rotated; 0 means the full head_dim. Partial rotary diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index e3bebd1b2..b5c7639cd 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -198,9 +198,10 @@ static int dequant_any(const void* data, size_t nbytes, int rows, int cols, floa // transformers' _compute_llama3_parameters applies. `on == 0` unless the config names the type, so // every other architecture builds exactly the table it built before. Keys are searched with their // quotes, which is what keeps "factor" from matching inside "high_freq_factor". -// Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`. Hunyuan's -// 1.9 packs declare factor 1.0 with a 262,144 window, i.e. a no-op until a prompt passes that window -- which is why -// accepting it is safe rather than a guess: below the window the base IS theta, so the served bytes are plain rope. +// Dynamic NTK: `rope_scaling: {type: "dynamic"|"dynamic_ntk", factor: F}` and `max_position_embeddings`: below the +// window the base IS theta, so a block with factor 1.0 serves plain rope. Hunyuan's 1.9 packs declare that AND +// `alpha: 1000.0`, which is a different rule (a fixed base theta * alpha^(dim/(dim-2)), engine#1889): reading only +// the factor made their block a no-op and rotated every position by the wrong angle. Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) { Q4nxRopeDynamic d{}; std::ifstream f(cfg_path); @@ -217,7 +218,10 @@ Q4nxRopeDynamic read_rope_dynamic(const std::string& cfg_path) { }; d.factor = number("factor"); d.original_max_position = number("max_position_embeddings"); - d.on = (d.factor > 0.0 && d.original_max_position > 0.0) ? 1 : 0; + // engine#1889: `alpha` selects Hunyuan's fixed-base form (q4nx_rope.h). The key is searched with its quotes and its + // colon, so "lora_alpha" or an "alpha" inside a string value cannot be taken for it. + d.alpha = number("alpha"); + d.on = ((d.factor > 0.0 && d.original_max_position > 0.0) || d.alpha > 0.0) ? 1 : 0; return d; } @@ -1722,6 +1726,17 @@ bool Q4nxNpuForward::load_norms() { layer_attn_dims(l, hd_l, nkv_l); if (!read_bf16(&mw_->layers[l].input_layernorm_weight, in_norm_[l].data(), H)) return false; if (!read_bf16(&mw_->layers[l].post_attention_layernorm_weight, post_norm_[l].data(), H)) return false; + // engine#1889: the config says q and k are normalised and this layer has no gain to normalise with. read_bf16 + // would hand back zeros for the absent tensor and rmsnorm(q, 0) is 0 for every head, so the layer would attend + // uniformly and answer fluent nonsense. A Hunyuan container imported before the importer claimed the gains is + // exactly this shape, so it is refused by name here instead. + if (sem_.qk_norm && absent(&mw_->layers[l].q_norm_weight)) { + err_ = "config.json says q and k are normalised per head (" + + std::string(q4nx_semantics_field(Q4NX_SEM_QK_NORM)) + ") and layer " + std::to_string(l) + + " has no self_attn.q_norm.weight: normalising with an absent gain zeroes q, so this container " + "would answer wrong text (engine#1889); re-import the pack with a build that carries the gains"; + return false; + } if (!read_bf16(&mw_->layers[l].q_norm_weight, q_norm_[l].data(), hd_l)) return false; // Qwen2.5-style attention biases (q/k/v projection biases); absent on the // bias-free families, so a miss is not an error. Loaded BEFORE the k_norm @@ -4651,7 +4666,11 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int // per-head RMSNorm on q and k, only for architectures that have it. A KV-shared // layer (Gemma4) computed no k at all, so its k is never normalized or used. - if (sem_.qk_norm) { + // engine#1889: WHERE it runs is per family. Qwen3 and Gemma normalise and then rotate; Hunyuan's graph rotates + // and then normalises (q_proj -> RotaryEmbedding -> q_norm -> attention), so for it the same code runs after the + // RoPE block below. The RMS itself is rotation-invariant; the per-dimension gain is not, so the order matters + // wherever a rotary pair's two gains differ. + auto qk_normalise = [&]() { std::vector t(HD); for (int h = 0; h < NH; h++) { rmsnorm(&q[(size_t)h * HD], q_norm_[cur_layer_].data(), HD, t.data(), "q_norm"); @@ -4663,7 +4682,8 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int memcpy(&k[(size_t)h * HD], t.data(), sizeof(float) * HD); } } - } + }; + if (sem_.qk_norm && !sem_.qk_norm_after_rope) qk_normalise(); // Gemma4 normalises v with a SCALE-FREE RMSNorm (its v_norm has no weight). if (sem_.v_norm && append_kv) { std::vector tv(HD); @@ -4683,7 +4703,8 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int std::vector cos_(half), sin_(half); // Dynamic NTK: the base moves with the position, so it is one pow per position rather than one per dim pair. // Below the model's original window this returns theta unchanged, which is what makes a short prompt - // bit-identical to the same model served without the scaling. + // bit-identical to the same model served without the scaling. A block with `alpha` (Hunyuan) is the fixed + // base theta * alpha^(dim/(dim-2)) at every position instead (engine#1889). // engine#1792 review: the exponent is HF's `dim / (dim - 2)` over the ROTARY width (`dim = int(head_dim * // partial_rotary_factor)`), which is `rot` here -- passing `HD` was wrong for any model that declares dynamic // scaling together with partial_rotary_factor < 1. Unreachable in today's catalog, so this is a correctness @@ -4730,6 +4751,7 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int for (int h = 0; h < NH; h++) rot_half(&q[(size_t)h * HD]); for (int h = 0; h < NKV; h++) rot_half(&k[(size_t)h * HD]); } + if (sem_.qk_norm && sem_.qk_norm_after_rope) qk_normalise(); // append to cache (a KV-shared layer reuses its owner's, which the owner has // already appended earlier in this same step). diff --git a/npu/forward/src/q4nx_semantics.c b/npu/forward/src/q4nx_semantics.c index c623e660d..775994d7b 100644 --- a/npu/forward/src/q4nx_semantics.c +++ b/npu/forward/src/q4nx_semantics.c @@ -26,7 +26,7 @@ static const char* kFields[Q4NX_SEM_COUNT] = { /* RMSNORM_OFFSET */ "model_type/architectures (gemma* uses the 1+w form)", - /* QK_NORM */ "model_type/architectures (q_norm/k_norm tensors)", + /* QK_NORM */ "use_qk_norm when stated, else model_type/architectures", /* ROPE_THETA */ "rope_theta", /* ROTARY_DIM */ "partial_rotary_factor * head_dim", /* MLP_ACT */ "hidden_act (falls back to model_type)", @@ -221,6 +221,18 @@ int q4nx_semantics_from_config(const char* json_path, const char* model_dir, /* per-head q/k RMSNorm: qwen3* and gemma* have q_norm/k_norm; llama/phi do not */ s->qk_norm = (is_qwen3 || is_gemma) ? 1 : 0; + /* engine#1889: a config that STATES `use_qk_norm` decides it, whatever the name table says. The table is a list + * of the families measured so far, so a family it has never heard of got 0 silently: Hunyuan-7B is + * `hunyuan_v1_dense` / `HunYuanDenseV1ForCausalLM` with `use_qk_norm: true`, its container carries + * self_attn.{q,k}_norm.weight for all 32 layers, and the forward loaded them and never applied them -- the same + * text with the gains, without them and with the key set to false, because nothing read the key. Families that + * do not state the key (qwen3, gemma) keep the table's answer. */ + { + double uq = 0; + if (get_num(json, "use_qk_norm", &uq)) s->qk_norm = uq != 0 ? 1 : 0; + } + /* Hunyuan rotates q and k first and normalises them after (see the header); everyone else normalises first. */ + s->qk_norm_after_rope = (starts_with(model_type, "hunyuan") || starts_with(arch, "HunYuan")) ? 1 : 0; /* RoPE base. Qwen3.5 nests it under rope_parameters (scalar); Gemma4 nests * it PER LAYER TYPE under rope_parameters.{full,sliding}_attention with @@ -541,6 +553,9 @@ void q4nx_semantics_dump(const Q4nxSemantics* s) { q4nx_semantics_field(Q4NX_SEM_RMSNORM_OFFSET)); fprintf(stderr, " [semantics] qk_norm=%d <- %s\n", s->qk_norm, q4nx_semantics_field(Q4NX_SEM_QK_NORM)); + if (s->qk_norm) + fprintf(stderr, " [semantics] qk_norm order: %s <- model_type/architectures (hunyuan* normalises after RoPE)\n", + s->qk_norm_after_rope ? "RoPE, then norm" : "norm, then RoPE"); fprintf(stderr, " [semantics] rope_theta=%.6g <- %s\n", s->rope_theta, q4nx_semantics_field(Q4NX_SEM_ROPE_THETA)); fprintf(stderr, " [semantics] rotary_dim=%d <- %s\n", s->rotary_dim, diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index f2ad9c895..e6cdc9bef 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3305,6 +3305,14 @@ QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_b return o; } +AttnFirstTiles attn_first_tiles(int n_kv, int n_q, bool q_first) { + AttnFirstTiles f; + f.k = q_first ? n_q : 0; + f.q = q_first ? 0 : n_kv; + f.v = n_kv + n_q; + return f; +} + std::string header_witness_refusal(int vote_hdrs, bool declared_qk) { if (!declared_qk || vote_hdrs != 1) return std::string(); return "the epsilon-anchor vote recorded ONE in-block header, and this pack declares use_qk_norm, whose per-head q/k " @@ -3790,6 +3798,14 @@ Plan plan(const std::string& pack_dir) { (long long)rem); } handled = true; + // engine#1889: this block's projections are q, k, v -- not the k, q, v every other family + // writes. It cannot be read off the tiles (no biases here to identify the runs, and the + // round trip is byte-exact either way), so it was settled by the answer: Hunyuan-7B on the + // host forward, teacher-forced on "The capital of France is Paris. The capital of Germany is + // Berlin. The capital of Italy is", predicts ' Paris' / ' Berlin' / ' Rome' at the three + // "is" rows with q,k,v and ' the' / ' the' / ' the' with k,q,v. The same family puts q + // before k in its packed gains (engine#1934), which is the same writer's order. + p.attn_q_first = true; } } const int64_t o_tiles_h = H / kTileRows; @@ -5550,9 +5566,11 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // o_proj's K is q_proj's output width). struct Slot { const char* proj; int rows; int64_t base; int k; int first; }; const int64_t attn_off = attn_payload_off(H); - const Slot slots[] = {{"k_proj", kv_out, attn_off, int(H), 0}, - {"q_proj", q_out, attn_off, int(H), n_kv}, - {"v_proj", kv_out, attn_off, int(H), n_kv + n_q}, + // engine#1889: k, q, v on every family but the Hunyuan block, which is q, k, v (Plan::attn_q_first). + const AttnFirstTiles ft = attn_first_tiles(n_kv, n_q, p.attn_q_first); + const Slot slots[] = {{"k_proj", kv_out, attn_off, int(H), ft.k}, + {"q_proj", q_out, attn_off, int(H), ft.q}, + {"v_proj", kv_out, attn_off, int(H), ft.v}, {"o_proj", int(H), p.o_off, p.o_k, 0}}; for (int L = 0; L < p.dims.layers; ++L) { const int64_t block = int64_t(L) * g.stride; diff --git a/npu/pack_import.h b/npu/pack_import.h index 5fc37dc35..d8fa1a233 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -115,6 +115,14 @@ int64_t head_stored_rows(int64_t vocab, bool from_fconst_tail); // pinned without a 2.6 GB pack. std::string header_witness_refusal(int vote_hdrs, bool declared_qk); +// The first tile of k_proj, q_proj and v_proj inside the attention payload, for n_kv k/v tiles and n_q q tiles +// (64 rows each). `q_first` false is k, q, v; true is q, k, v (engine#1889, the Hunyuan block). v is last either way. +// Split out so the order is pinned by a test instead of living in an initializer list. +struct AttnFirstTiles { + int k = 0, q = 0, v = 0; +}; +AttnFirstTiles attn_first_tiles(int n_kv, int n_q, bool q_first); + // A hybrid pack's embedding table lives in its own graph (`embedding.onnx` + `.data`, gathered with // GatherBlockQuantized). Returns [V, H] f32 row-major, empty when the pack has no such component, with `why` naming the // reason. Exposed so the dequantisation can be tested against known codes and scales instead of only on a 3 GB pack. @@ -215,6 +223,11 @@ struct Plan { // none. Identified and verified byte-exact against the public Qwen3-1.7B/4B checkpoints (0 of 128 // values differ on either pack, and the swap differs on 127-128 of 128). int64_t knorm_off = 0, qnorm_off = 0; + // The order of the three projections in the block's first payload. Every family measured before Hunyuan writes + // k, q, v (identified on Qwen2.5-0.5B by the tiles' bias subvolumes). The Hunyuan block -- q/k/v at K=hidden, the + // packed per-head gains, o_proj -- writes q, k, v (engine#1889): with k,q,v the container's k_proj was q_proj's + // first rows, and the model answered filler words; with q,k,v it answers. See attn_first_tiles(). + bool attn_q_first = false; int64_t mlp_off = 0; // the MLP section starts just past the block's post-attention norm // What the pack's *graph* carries, reported so `npu-pack info` can answer a pack's genre without a // third-party parser: the rai_1.3/1.6/1.7.1 line ships its 4-bit weights as ordinary ONNX initializers, while diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 0dcad1ff0..a883fd303 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5683,6 +5683,21 @@ void test_header_witness_refusal() { std::printf(" header witness: 8 arms (one-header vote vs declaration refuses; 3/1-without-declaration/0 allowed)\n"); } +// engine#1889: which tile each projection starts at inside the attention payload. Every family before Hunyuan is +// k, q, v; the Hunyuan block is q, k, v, and reading it k, q, v handed the forward q_proj's first rows as k_proj +// (byte-exact round trip, filler-word answers). Hunyuan-7B's own counts: 16 k/v tiles, 64 q tiles. +void test_attn_first_tiles() { + const pi::AttnFirstTiles kqv = pi::attn_first_tiles(16, 64, false); + check(kqv.k == 0 && kqv.q == 16 && kqv.v == 80, "attn order: k,q,v puts k at tile 0, q at 16, v at 80"); + const pi::AttnFirstTiles qkv = pi::attn_first_tiles(16, 64, true); + check(qkv.q == 0 && qkv.k == 64 && qkv.v == 80, "attn order: q,k,v puts q at tile 0, k at 64, v at 80 (engine#1889)"); + check(qkv.k + 16 == qkv.v && kqv.q + 64 == kqv.v, "attn order: v follows the second run in both orders"); + const pi::AttnFirstTiles small = pi::attn_first_tiles(2, 14, false); // Qwen2.5-0.5B: k 2, q 14, v 2 + check(small.k == 0 && small.q == 2 && small.v == 16, "attn order: the Qwen2.5-0.5B run (2/14/2) is unchanged"); + pi::Plan p; + check(!p.attn_q_first, "attn order: a plan is k,q,v unless the block measured otherwise"); +} + int main(int argc, char** argv) { // engine#1907: every crafted 1.9 import writes a canonical copy into the scratch cache. Without this the suite left 0-byte // copies in the user's real ~/.cache/1bit/fconst19 and in /tmp on every run. Give the run its own directory, removed at exit. @@ -5695,6 +5710,7 @@ int main(int argc, char** argv) { test_header_witness_refusal(); test_encode_tile_code_range(); test_q4_writer_code_range(); + test_attn_first_tiles(); test_qk_norm_refusal(); test_head_stored_rows(); test_qk_gain_offsets(); diff --git a/tests/npu_rope_llama3_test.c b/tests/npu_rope_llama3_test.c index dd261d2df..63f310dcb 100644 --- a/tests/npu_rope_llama3_test.c +++ b/tests/npu_rope_llama3_test.c @@ -257,6 +257,35 @@ static void test_declared_type(void) { check(fabs(got64 - got128) > 1e3, "and the two widths differ by more than 1e3, so this arm cannot pass with the widths confused"); } + /* engine#1889: Hunyuan's block is `{"alpha": 1000.0, "factor": 1.0, "type": "dynamic"}`, and with alpha set its + * rotary embedding uses ONE fixed base, theta * alpha^(dim/(dim-2)), at every position. Read as factor-only the + * block was a no-op (base 10000), which rotated every position by the wrong angle. The expected numbers are the + * pack's, not the formula's: Hunyuan-7B's model.onnx.data carries the graph's cos_cache/sin_cache (FP16 + * [32768, 64]); at position 1 the angle of pair i is its inverse frequency, and the table reads + * pair 1: 0.77608 pair 2: 0.60209 pair 5: 0.28159 + * (FP16, so good to about 5e-4), against 0.86596 / 0.74989 / 0.48697 for plain theta 10000. */ + { + Q4nxRopeDynamic hy = {1, 1.0, 262144.0, 1000.0}; /* Hunyuan-7B's own numbers */ + const double base = q4nx_rope_dynamic_base(1e4, &hy, 1.0, 128); + check(fabs(base - 11158839.9) < 1.0, "alpha 1000 at theta 10000, dim 128: the base is 11,158,840"); + check(q4nx_rope_dynamic_base(1e4, &hy, 0.0, 128) == base && q4nx_rope_dynamic_base(1e4, &hy, 300000.0, 128) == base, + "... at every position, inside the window and past it (the alpha form does not move with pos)"); + check(fabs(pow(base, -1.0 / 64.0) - 0.77608) < 5e-4, "pair 1's inverse frequency is the pack table's 0.77608"); + check(fabs(pow(base, -2.0 / 64.0) - 0.60209) < 5e-4, "pair 2's is the pack table's 0.60209"); + check(fabs(pow(base, -5.0 / 64.0) - 0.28159) < 5e-4, "pair 5's is the pack table's 0.28159"); + check(fabs(pow(1e4, -1.0 / 64.0) - 0.77608) > 0.05, + "and plain theta 10000 misses pair 1 by more than 0.05, so these arms cannot pass on the no-op"); + Q4nxRopeDynamic hy64 = {1, 1.0, 262144.0, 1000.0}; + check(fabs(q4nx_rope_dynamic_base(1e4, &hy64, 1.0, 64) - 1e4 * pow(1000.0, 64.0 / 62.0)) < 1e-3, + "the exponent is dim/(dim-2) over the rotary width here too (dim 64 -> 64/62)"); + Q4nxRopeDynamic no_alpha = {1, 1.0, 262144.0, 0.0}; + check(q4nx_rope_dynamic_base(1e4, &no_alpha, 5.0, 128) == 1e4, + "alpha 0 is the factor rule: inside the window the base is theta, as before"); + Q4nxRopeDynamic off_alpha = {0, 1.0, 262144.0, 1000.0}; + check(q4nx_rope_dynamic_base(1e4, &off_alpha, 5.0, 128) == 1e4, "and an alpha on a block that is off does nothing"); + Q4nxRopeDynamic tiny = {1, 1.0, 262144.0, 1000.0}; + check(q4nx_rope_dynamic_base(1e4, &tiny, 5.0, 2) == 1e4, "a rotary width of 2 has no exponent, so the base stays theta"); + } } } diff --git a/tests/q4nx_qk_norm_test.c b/tests/q4nx_qk_norm_test.c new file mode 100644 index 000000000..1ea678b3e --- /dev/null +++ b/tests/q4nx_qk_norm_test.c @@ -0,0 +1,119 @@ +/* Copyright 2026 bong-water-water-bong + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* q4nx_qk_norm_test -- which models get the per-head q/k RMSNorm (engine#1889). + * + * The forward applies it only when the semantics flag is set, and the flag used to come from a model_type / + * architectures table alone (qwen3*, gemma*). Hunyuan-7B is `hunyuan_v1_dense` with `use_qk_norm: true`: the table had + * no row for it, so its container's self_attn.{q,k}_norm.weight were loaded and never applied. The config key now + * decides when the config states it; the table stays the answer for the families that do not state it. + * + * The strings below are the real keys of the configs on this box (Hunyuan-7B-Instruct's config.json, a Qwen3 and a + * Llama container's), cut down to the fields the rule reads. No model, no device. */ +#include "q4nx_semantics.h" + +#include +#include +#include +#include +#include + +static int failures = 0; + +static void check(int ok, const char* what) { + printf("%s %s\n", ok ? "ok " : "FAIL", what); + if (!ok) failures++; +} + +/* /config.json holding exactly `json`, 0600 (mkdtemp gives 0700; the file's own mode is what CodeQL reads). */ +static int after_rope = -1; /* the last config's qk_norm_after_rope */ + +static int qk_norm_of(const char* dir, const char* json) { + char path[512]; + snprintf(path, sizeof path, "%s/config.json", dir); + int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0600); + FILE* f = fd >= 0 ? fdopen(fd, "w") : NULL; + if (!f) { printf("FAIL cannot write %s\n", path); failures++; return -1; } + fputs(json, f); + fclose(f); + Q4nxSemantics s; + if (q4nx_semantics_from_config(path, dir, &s) != 0) { printf("FAIL cannot read %s\n", path); failures++; return -1; } + after_rope = s.qk_norm_after_rope; + return s.qk_norm; +} + +int main(void) { + char dir[] = "/tmp/q4nx-qk-norm-XXXXXX"; + if (!mkdtemp(dir)) { printf("FAIL mkdtemp\n"); return 1; } + + /* The measured case: a family the name table has never heard of, declaring the norm. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"num_attention_heads\": 32, \"num_key_value_heads\": 8, \"head_dim\": 128," + " \"use_qk_norm\": true}") == 1, + "hunyuan_v1_dense with use_qk_norm: true normalises q and k (engine#1889)"); + /* The same family saying it has none: the key is read, not assumed from the family. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"use_qk_norm\": false}") == 0, + "hunyuan_v1_dense with use_qk_norm: false does not"); + /* A family with no row and no key keeps the neutral convention. */ + check(qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"}") == 0, + "an unlisted family that does not state the key keeps the plain convention"); + + /* The table is unchanged for the families that never state the key. */ + check(qk_norm_of(dir, "{\"architectures\": [\"Qwen3ForCausalLM\"], \"model_type\": \"qwen3\"}") == 1, + "qwen3 without the key still normalises (the name table)"); + check(qk_norm_of(dir, "{\"architectures\": [\"Gemma3ForCausalLM\"], \"model_type\": \"gemma3_text\"}") == 1, + "gemma without the key still normalises (the name table)"); + check(qk_norm_of(dir, "{\"architectures\": [\"LlamaForCausalLM\"], \"model_type\": \"llama\"}") == 0, + "llama without the key still does not"); + check(qk_norm_of(dir, "{\"architectures\": [\"Phi3ForCausalLM\"], \"model_type\": \"phi3\"}") == 0, + "phi without the key still does not"); + + /* A stated key beats the table in both directions, and a null is not a statement. */ + check(qk_norm_of(dir, "{\"model_type\": \"llama\", \"use_qk_norm\": true}") == 1, + "a llama-named config that states use_qk_norm: true normalises"); + check(qk_norm_of(dir, "{\"model_type\": \"qwen3\", \"use_qk_norm\": false}") == 0, + "a qwen3-named config that states use_qk_norm: false does not"); + check(qk_norm_of(dir, "{\"model_type\": \"qwen3\", \"use_qk_norm\": null}") == 1, + "use_qk_norm: null states nothing, so the table answers"); + /* Only the top-level key counts: a nested one belongs to another component's config. */ + check(qk_norm_of(dir, "{\"model_type\": \"llama\", \"vision_config\": {\"use_qk_norm\": true}}") == 0, + "a use_qk_norm nested under another component's config is not this model's"); + + /* The ORDER (engine#1889). Hunyuan's graph is q_proj -> RotaryEmbedding -> q_norm -> attention in every layer, so + * its norm runs after RoPE; Qwen3 and Gemma keep norm-then-RoPE. */ + (void)qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"], \"model_type\": \"hunyuan_v1_dense\"," + " \"use_qk_norm\": true}"); + check(after_rope == 1, "hunyuan_v1_dense normalises q and k AFTER RoPE, as its graph does"); + (void)qk_norm_of(dir, "{\"architectures\": [\"HunYuanDenseV1ForCausalLM\"]}"); + check(after_rope == 1, "... and the architectures entry alone is enough to say so"); + (void)qk_norm_of(dir, "{\"architectures\": [\"Qwen3ForCausalLM\"], \"model_type\": \"qwen3\"}"); + check(after_rope == 0, "qwen3 keeps norm, then RoPE"); + (void)qk_norm_of(dir, "{\"architectures\": [\"Gemma3ForCausalLM\"], \"model_type\": \"gemma3_text\"}"); + check(after_rope == 0, "gemma keeps norm, then RoPE"); + (void)qk_norm_of(dir, "{\"model_type\": \"llama\", \"use_qk_norm\": true}"); + check(after_rope == 0, "a non-Hunyuan config that states use_qk_norm gets the default order"); + + check(strstr(q4nx_semantics_field(Q4NX_SEM_QK_NORM), "use_qk_norm") != NULL, + "the startup line names use_qk_norm as the flag's source"); + + char path[512]; + snprintf(path, sizeof path, "%s/config.json", dir); + unlink(path); + rmdir(dir); + printf(failures ? "q4nx_qk_norm: %d FAILED\n" : "q4nx_qk_norm: ok\n", failures); + return failures ? 1 : 0; +} From a43aea359ea7b284ef57aaf12ccb381ff6cb1d9c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:23:51 -0400 Subject: [PATCH 2/2] npu-pack: the RoPE cache check tries the rope_scaling alpha base first (engine#1889) Hunyuan-7B's import warned that no plain theta explains the pack's cos/sin cache (best 0.7222 at 1e7). The cache is theta * alpha^(dim/(dim-2)), which the forward now computes, so the check tries that base first and reports it verified. rope_theta stays as declared: the forward applies alpha itself. Co-Authored-By: Claude Fable 5.2 --- docs/npu.md | 2 +- npu/pack_import.cpp | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index c112ae570..d412a5fad 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -858,7 +858,7 @@ the importer** | what | rule | where it comes from | |---|---|---| | whether q and k are normalised | `use_qk_norm` in `config.json` decides when the config states it; the `qwen3*`/`gemma*` table answers only for configs that do not | the config key. The table has no Hunyuan row, so its gains were loaded and never applied | -| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99 | +| the RoPE base | `rope_scaling: {type: "dynamic", alpha: A}` is a **fixed** base `theta * A^(dim/(dim-2))` at every position (11,158,840 for theta 10000, alpha 1000, dim 128). Without `alpha` the block is the position-dependent NTK rule | the graph's `cos_cache`/`sin_cache` in `model.onnx.data` match that base to 0.00024; plain theta 10000 is off by up to 1.99. `1bit npu-pack import` tests this base against the pack's cache first and prints `note: RoPE base ... verified`; before, it warned that no plain theta explained the cache | | the order of norm and RoPE | RoPE first, then the per-head RMSNorm (Qwen3 and Gemma normalise first) | the graph: `q_proj -> RotaryEmbedding -> q_norm -> GroupQueryAttention` in every layer | | the order of the attention tiles in the pack (importer) | q tiles first, then k, then v. Every other family's block is k, q, v. `1bit npu-pack import` picks the order with the block shape that carries the packed q/k gains | measured: with k, q, v the imported model predicts 26 of 77 next tokens of a fixed text, with q, k, v 59 (the two projections were being read as each other) | diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index e6cdc9bef..b8e362361 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -5873,6 +5873,23 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons } // candidates: what the metadata says first, then the values these families actually use std::vector cands{p.dims.rope_theta, 1e4, 5e4, 1e5, 5e5, 1e6, 1e7, 1.6e6, 3.2e6}; + // engine#1889: a `rope_scaling: {type: "dynamic", alpha: A}` block (Hunyuan) is a FIXED base + // theta * A^(dim/(dim-2)), and the forward computes it from the block (q4nx_rope_dynamic_base). + // That base is tried first: on Hunyuan-7B it is 11,158,840 and no plain candidate comes near + // (best 0.7222 at 1e7), so the import warned "no plain theta explains" about a cache the + // model's own config explains. rope_theta is left as declared when this one wins -- writing the + // scaled base there would have the forward apply alpha twice. + double alpha_base = 0.0, alpha = 0.0; + if (c.contains("rope_scaling") && c["rope_scaling"].is_object() && + c["rope_scaling"].contains("type") && c["rope_scaling"]["type"].is_string() && + c["rope_scaling"]["type"].get() == "dynamic" && + c["rope_scaling"].contains("alpha") && c["rope_scaling"]["alpha"].is_number()) { + alpha = c["rope_scaling"]["alpha"].get(); + const double hd = double(half) * 2.0; + if (alpha > 0.0 && hd > 2.0 && p.dims.rope_theta > 0) + alpha_base = double(p.dims.rope_theta) * std::pow(alpha, hd / (hd - 2.0)); + } + if (alpha_base > 0.0) cands.insert(cands.begin(), alpha_base); double best_corr = -2.0, best_theta = p.dims.rope_theta; const int HD2 = int(half) * 2; // head_dim implied by the cache const int64_t pmax = std::min(rows, 4096); @@ -5903,7 +5920,21 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (corr == corr && corr > best_corr) { best_corr = corr; best_theta = th; } } } - if (best_corr > 0.99 && std::fabs(best_theta - p.dims.rope_theta) > 1e-9) { + if (best_corr > 0.99 && alpha_base > 0.0 && best_theta == alpha_base) { + std::fprintf(stderr, + "note: RoPE base %.8g (rope_theta %.6g, rope_scaling alpha %.6g) verified against " + "the pack's cos_cache (corr %.4f)\n", + alpha_base, (double)p.dims.rope_theta, alpha, best_corr); + c.erase("q4nx_rope_note"); + } else if (best_corr > 0.99 && alpha_base > 0.0) { + // A plain theta fits and the alpha base does not, while the config still carries alpha: the + // forward would scale whatever is written to rope_theta, so nothing is adopted. + std::fprintf(stderr, + "WARNING: this pack's RoPE caches fit plain theta %.6g (corr %.4f), not the base " + "%.8g its rope_scaling alpha %.6g implies; this forward applies alpha, so " + "attention may be wrong from the first token\n", + best_theta, best_corr, alpha_base, alpha); + } else if (best_corr > 0.99 && std::fabs(best_theta - p.dims.rope_theta) > 1e-9) { std::fprintf(stderr, "note: rope_theta %.6g declared, %.6g adopted (correlation %.4f against the " "pack's own cos_cache)\n", (double)p.dims.rope_theta, best_theta, best_corr);