From 7c4073cd8452c74fbfeec5911d1d75c1f26c6b80 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 9 Nov 2026 23:45:21 -0400 Subject: [PATCH 01/14] npu: measure the 1.9 Token fconst rule for hidden 4096 / 32 heads / 36 layers (Qwen3-8B) (refs #1740) Measured with tools/fconst19_measure.py against amd/Qwen3-8B_rai_1.8.0_npu_16K: 26,104 of 3,756,627,264 bytes differ (6.95e-04 %), content rather than layout. The header stride 104,350,656 is uniform over all 36 blocks, checked independently of the tool (36 runs of 32 x 00 00 7f ff, first at 13,796,032). The row is identical to the Mistral h4096 row in every field except the layer count, which is the finding: the bundle shape and rotation are a property of the hidden size, not of the depth. It needs its own entry anyway, because select_rule_in() keys on (hidden, heads, layers). Stated in the comment because it is not all seven fields: --probe-wrong separates rot+1 and tile/2 by 14,700x (10.2 % of the file each), but prefix+64 differs in only 32,960 B -- 1.26x the accepted residual and under the 37,566-byte tolerance -- so this pair does not isolate the prefix. The 12,608 rests on the Mistral row and the Qwen3 arithmetic agreeing, as the 3,584 row's prefix does. --- npu/pack_import.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 400e4fe29..8f3773486 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -449,6 +449,21 @@ constexpr Fconst19Rule kFconst19Rules[] = { // all 32 blocks. This is the largest family on #1503 (31 packs). Measured by another lane on main, whose // table is keyed on geometry; merged here in the same form. {4096, 32, 32, 12608, 5, 1, 2297856}, + // Qwen3-8B at hidden 4,096 / 32 heads / **36** layers: measured with tools/fconst19_measure.py against + // amd/Qwen3-8B_rai_1.8.0_npu_16K, 26,104 of 3,756,627,264 bytes differ (6.95e-04 %) -- content, not layout. + // Every other field is IDENTICAL to the Mistral row directly above (prefix 12,608, 5 tiles of 2,297,856, rot 1), + // and that is the finding, not a copy-paste: the bundle shape and rotation are a property of the hidden size, not of + // the layer count, so a 36-layer pack at 4,096 rotates exactly like the 32-layer one. This row exists because + // select_rule_in() keys on (hidden, heads, layers) -- an 8B at 36 layers is a different file layout from the same + // geometry at 32, and deriving one from the other is what this table refuses to do. + // The header stride 104,350,656 is uniform over all 36 blocks (verified independently of this tool: 36 runs of + // 32 x `00 00 7f ff`, first at 13,796,032). + // What `--probe-wrong` separates here, stated because it is NOT all seven fields: rot+1 and tile/2 differ in + // 383,566,077 and 383,571,429 bytes (10.2 %, a factor of 14,700), so the rotation and the tile are pinned hard. + // `prefix+64` differs in only 32,960 bytes -- 1.26x the accepted residual and UNDER the tool's 37,566-byte + // tolerance -- so this pair does not isolate the prefix. `prefix = 12,608` rests on the Mistral row above and the + // Qwen3 arithmetic (64 + 2*hidden + 4,352) agreeing, which is exactly the caveat the 3,584 row carries too. + {4096, 32, 36, 12608, 5, 1, 2297856}, // Llama-3.2-3B (and -Instruct, same geometry) at hidden 3,072 / 24 heads / 28 layers: measured with // tools/fconst19_measure.py against amd/Llama-3.2-3B_rai_1.8.0_npu_4K, 1,090 of 1,756,930,368 bytes differ // (6.20e-07 of the file, which the tool prints as 6.20e-05 %) -- content, not layout. The prefix is the Qwen3 line's 64 + 2*hidden + 4,352 = 10,560 at this hidden From 95f1a203bbaad7dd1ac5a2647ef7fa066c6c447e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 00:10:35 -0400 Subject: [PATCH 02/14] npu-pack: read lm_head from the If branch that is the decode path, so the 4096-hidden untied packs (Qwen3-8B, granite-4.1-8b) get their head from the graph (refs #1740) An OGA 1.9 pack splits prefill from decode with `If (Equal(Shape(input_ids), Constant(value_int 1)))`, and the two branches do NOT carry the same tensors. Measured on amd/Qwen3-8B_rai_1.9.0_npu: then_branch holds exactly ONE MatMulNBits and it is lm_head [151936, 32, 64] at offset 15,269,888 in model.onnx.data (311,164,928 B = V*H/2); else_branch holds the 109 layer projections plus an `lm_head.MatMulNBits.qweight.empty` placeholder with no bytes. The walker merged both branches, so the placeholders it saw for lm_head counted as weightless_matmuls and plan() took the "no readable graph head, read the tail" path. On these packs the tail is the envelope alone -- Qwen3-8B's is 8,256 B = 64 + 2*4096, verified by hex (bf16 1e-6 at kNormOff 64, then the norm weights) -- so `want = env + rows*tile` could never equal `have`, and the import refused with "lm_head.weight: the .fconst tail is 8256 B, not the ... B its envelope and head tiles would take, so the head's offsets are not established". The Token .fconst IS the decode path, so the decode branch's node is the one that describes it: - walk_graph now records each matmul's `If` branch (the attribute's own name); - parse_onnx reads the condition rather than assuming: a Constant value_int of 1 makes then_branch decode, anything else else_branch, and a graph without that shape falls back to then_branch with a note saying the rule was assumed; - import_pack picks the head node from the decode branch, keeping the old first-match behaviour as a fallback when only the other branch declares it; - the #1687 refusal reads weightless_matmuls_decode (outside the decode branch), so a prefill placeholder is no longer mistaken for a genuinely absent weight. The total stays for `info`'s census, and both are reported in graph_weights. Pinned by test_external_packed_genre: a crafted graph whose condition compares the shape with 1 must name then_branch decode, and an inverted one must move it to else_branch. The regression arm for the head pick (decode branch taken, inverted convention falls back, flat graph unaffected) was checked standalone. Not run: the pack itself -- this tree's submodules are not checked out, so libonebit_npu cannot be built here; the changed regions pass -fsyntax-only and the test compiles. --- npu/pack_import.cpp | 87 ++++++++++++++++++++++++++++++++-- npu/pack_import.h | 5 ++ tests/npu_pack_import_test.cpp | 35 ++++++++++++++ 3 files changed, 123 insertions(+), 4 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 8f3773486..f61bb6b36 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -131,6 +131,12 @@ struct OnnxMatMul { std::string data, weight, scales, zeros; int64_t K = 0, N = 0; int bits = 0, block = 0; + // Which `If` branch declared it, as the AttributeProto's name (`then_branch` / `else_branch`), or empty in the + // top-level graph. An OGA pack splits prefill from decode with `If (Shape(input_ids) == 1)`: Qwen3-8B declares ONE + // MatMulNBits (its head) in `then_branch` and every layer projection in `else_branch`, so a walk that merges the + // branches cannot tell which path a weight belongs to. The Token .fconst IS the decode path, so the head is read + // from the decode branch -- see is_decode_branch(). + std::string branch; }; struct OnnxModel { @@ -149,9 +155,27 @@ struct OnnxModel { // .fconst, so plan() drops them; with no .fconst there is no other source and plan() refuses by name (#1687). int64_t weightless_matmuls = 0; std::string weightless_example; // the first one, for the refusal's text + // weightless_matmuls for placeholders OUTSIDE the decode branch only. A placeholder in the prefill branch is not a + // fault for a Token-fconst pack reading its decode weights: Qwen3-8B's `else_branch` spells lm_head as an empty + // placeholder while `then_branch` (decode) carries the real [151936,32,64] tensor. This is what plan() consults for + // "are these weights genuinely absent from the pack" -- the total stays for `info`'s census. + int64_t weightless_matmuls_decode = 0; + // The branch that runs when `input_ids` has length 1, i.e. the decode path the Token .fconst belongs to. Learned + // from the `If`'s condition where it is the literal we recognise, else defaulted to `then_branch` -- the ONNX + // convention every OGA pack on the Hub follows -- and named in the note either way, so a pack that puts it in the + // other branch is diagnosable rather than silently misread. + std::string decode_branch; + std::string branch_note; }; -void walk_graph(const uint8_t* p, size_t n, OnnxModel& m) { +// A matmul is decode-path when its branch is the decode branch, or when it is in no branch at all (a pack whose graph +// is flat has one path). A pack whose decode branch was never identified keeps every matmul, which is today's +// behaviour and cannot lose a head. +bool is_decode_branch(const OnnxModel& m, const std::string& branch) { + return branch.empty() || m.decode_branch.empty() || branch == m.decode_branch; +} + +void walk_graph(const uint8_t* p, size_t n, OnnxModel& m, const std::string& branch = std::string()) { const auto g = pb(p, n); for (const auto* f : fields(g, 5)) { const auto t = pb(f->p, f->n); @@ -189,8 +213,11 @@ void walk_graph(const uint8_t* p, size_t n, OnnxModel& m) { const auto* nm = field1(at, 1); if (!nm) continue; if (const auto* iv = field1(at, 3)) attrs[pb_str(*nm)] = int64_t(iv->v); - for (const auto* sg : fields(at, 6)) walk_graph(sg->p, sg->n, m); - for (const auto* sg : fields(at, 11)) walk_graph(sg->p, sg->n, m); + // AttributeProto.graph is field 6 (a GRAPH), graphs is field 11 (a repeated GRAPH) -- so a branch arrives + // in field 6 and its name is the attribute's own, `then_branch` / `else_branch`. + const std::string sub_branch = nm ? pb_str(*nm) : std::string(); + for (const auto* sg : fields(at, 6)) walk_graph(sg->p, sg->n, m, sub_branch); + for (const auto* sg : fields(at, 11)) walk_graph(sg->p, sg->n, m, sub_branch); } // The graph is the authority on how THIS reader should spell a quantised matmul: the same pack also states the // op in `cache/*_meta.json` (which the block census reads) and in the `.onnx` alias table, and if those ever @@ -215,6 +242,7 @@ void walk_graph(const uint8_t* p, size_t n, OnnxModel& m) { if (attrs.count("N")) mm.N = attrs["N"]; if (attrs.count("bits")) mm.bits = int(attrs["bits"]); if (attrs.count("block_size")) mm.block = int(attrs["block_size"]); + mm.branch = branch; if (!mm.weight.empty()) m.matmuls.push_back(std::move(mm)); } } @@ -231,6 +259,31 @@ OnnxModel parse_onnx(const std::string& path) { const auto* g = field1(model, 7); if (!g) throw std::runtime_error(path + ": no graph"); walk_graph(g->p, g->n, m); + // Which branch is decode. The condition is `Equal(Shape(input_ids), Constant(value_int 1))` on every OGA 1.9 + // pack measured (Qwen3-8B, granite-4.1-b8), so the branch taken when it is TRUE is the one-token path and + // therefore the path the Token .fconst carries. Reading the comparison's own value rather than assuming + // `then_branch` keeps the rule honest for a pack that inverts it; when the graph is not that shape, the ONNX + // convention (`then_branch`) stands and `branch_note` says the rule was assumed rather than read. + { + const auto gpb = pb(g->p, g->n); + bool one_token_when_true = true, read = false; + for (const auto* f : fields(gpb, 1)) { // GraphProto.node = 1 + const auto node = pb(f->p, f->n); + const auto* opf = field1(node, 4); + if (!opf || pb_str(*opf) != "Constant") continue; + for (const auto* a : fields(node, 5)) { + const auto at = pb(a->p, a->n); + const auto* nm = field1(at, 1); + if (!nm || pb_str(*nm) != "value_int") continue; + const auto* iv = field1(at, 3); + if (iv) { one_token_when_true = (iv->v == 1); read = true; } + } + } + m.decode_branch = one_token_when_true ? "then_branch" : "else_branch"; + m.branch_note = read ? "the If condition compares the input shape with 1, so " + m.decode_branch + + " is the one-token (decode) path" + : "no recognisable If condition, so then_branch is taken as decode by the ONNX convention"; + } } catch (...) { unmap_file(map, size); throw; @@ -304,6 +357,8 @@ OnnxModel parse_onnx(const std::string& path) { // which knows whether an .fconst is there, decides. Deferring it silently produced // `...qweight.empty: no data` from a lookup much later, naming the placeholder rather than the problem. ++m.weightless_matmuls; + if (is_decode_branch(m, mm.branch)) + ++m.weightless_matmuls_decode; m.weightless_example = w->name + " (its " + packed + " sibling carries no data either)"; continue; } @@ -2471,6 +2526,12 @@ std::string Plan::manifest() const { // (engine#1671). Reported so a test can pin the drop without needing a 1.9 pack, and so `info` says how many // graph weights this pack does not keep in its graph. g["weightless_matmuls"] = graph_weightless_matmuls; + // Of those, the ones OUTSIDE the decode branch. A placeholder in the prefill branch is not a fault when the + // decode branch carries the real tensor (Qwen3-8B: `else_branch` has an empty lm_head placeholder, `then_branch` + // has lm_head [151936,32,64]), so the refusal reads this one and the census above stays a census. + g["weightless_matmuls_decode"] = graph_weightless_matmuls_decode; + g["decode_branch"] = graph_decode_branch; + if (!graph_branch_note.empty()) g["decode_branch_note"] = graph_branch_note; if (graph_stride_of > 0) g["external_row_stride"] = {{"matched", graph_stride_ok}, {"of", graph_stride_of}, {"rule", "length == N * (K/128 + 1) * 68"}}; @@ -2595,6 +2656,9 @@ Plan plan(const std::string& pack_dir) { p.graph_ryzenai_matmuls = onnx.ryzenai_matmuls; p.graph_packed_siblings = onnx.packed_siblings; p.graph_weightless_matmuls = onnx.weightless_matmuls; + p.graph_weightless_matmuls_decode = onnx.weightless_matmuls_decode; + p.graph_decode_branch = onnx.decode_branch; + p.graph_branch_note = onnx.branch_note; for (const auto& t : onnx.tensors) if (!t.location.empty()) p.graph_external++; if (!ext_file.empty()) { @@ -2611,7 +2675,7 @@ Plan plan(const std::string& pack_dir) { // data has no weight for those ops IN THE GRAPH. With a Token .fconst that is the normal shape of a 1.9 pack -- its // graph is all placeholders and the weights are in the .fconst -- so the walker drops them and the import proceeds. // With no .fconst there is nothing to fall back to, and the honest answer is #1687's refusal, by name. - if (tok_path.empty() && onnx.weightless_matmuls > 0) + if (tok_path.empty() && onnx.weightless_matmuls_decode > 0) throw std::runtime_error( pack_dir + ": " + onnx.weightless_example + ", and the pack carries no Token .fconst either (no dd_metastate_Llm_Token_*.fconst): these weights are " @@ -3801,6 +3865,20 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons } } + // The head is the node whose geometry is the vocabulary x hidden -- but which NODE, when both `If` branches declare + // one? The Token .fconst is the decode path, so the decode branch's node is the one that describes it. Measured on + // amd/Qwen3-8B_rai_1.9.0_npu: `then_branch` (taken when Shape(input_ids) == 1, i.e. one token) holds exactly ONE + // MatMulNBits and it is lm_head [151936, 32, 64] at offset 15,269,888 in model.onnx.data; `else_branch` holds the + // 109 layer projections and an `lm_head.MatMulNBits.qweight.empty` placeholder with no bytes. The head is not the + // same tensor in both paths, so a walk that merges the branches reads the placeholder and falls back to the tail -- + // and on these packs the tail is the envelope alone (8,256 B = 64 + 2*4096, no head tiles), which is how the import + // refused with "the head's offsets are not established" for every 4096-hidden untied pack. + const OnnxMatMul* head_mm = nullptr; + for (const auto& mm : onnx.matmuls) + if (mm.N == int(V) && mm.K == int(H) && is_decode_branch(onnx, mm.branch)) { head_mm = &mm; break; } + if (!head_mm) + for (const auto& mm : onnx.matmuls) + if (mm.N == int(V) && mm.K == int(H)) { head_mm = &mm; break; } for (const auto& mm : onnx.matmuls) { const auto wi = onnx.index.find(mm.weight), si = onnx.index.find(mm.scales); if (getenv("NPU_PACK_DEBUG")) @@ -3839,6 +3917,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // which is why the imported container answered "!!!!!!!!". Only the node whose geometry IS the head is // taken here; the projections are imported from the graph's initializers by the block below. if (!(mm.N == int(V) && mm.K == int(H))) continue; + if (head_mm && &mm != head_mm) continue; // a same-geometry node in the other `If` branch if (mm.bits != 4) throw std::runtime_error("lm_head: MatMulNBits is not 4-bit"); const auto& wt = onnx.tensors[wi->second]; const auto& st = onnx.tensors[si->second]; diff --git a/npu/pack_import.h b/npu/pack_import.h index 838513fbb..e02d744e0 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -134,6 +134,11 @@ struct Plan { int64_t graph_ryzenai_matmuls = 0; int64_t graph_packed_siblings = 0; int64_t graph_weightless_matmuls = 0; // dropped: placeholder + packed sibling both byte-less (engine#1671) + // Of those, the ones outside the decode `If` branch: what plan() refuses on. The total above stays for `info`. + int64_t graph_weightless_matmuls_decode = 0; + // The `If` branch taken when input_ids has length 1, and how that was decided (read from the condition, or assumed). + std::string graph_decode_branch; + std::string graph_branch_note; // How many of the graph's 4-bit block-128 weights satisfy length == N x (K/128 + 1) x 68 (the per-row stride the // .fconst tile also carries): a measurement, so `info` can say "201 of 201" or "3 of 7" instead of guessing. int64_t graph_stride_ok = 0, graph_stride_of = 0; diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index d93a71a27..064a42e68 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -520,6 +520,41 @@ void test_external_packed_genre(const std::string& root) { "placeholder with no sibling: the matmul is KEPT, so the pack is not mistaken for a tail-head pack"); } + // Which `If` branch is decode (engine#1740). An OGA pack splits prefill from decode with + // `If (Equal(Shape(input_ids), Constant(value_int 1)))`, and the two branches do NOT hold the same tensors: measured + // on amd/Qwen3-8B_rai_1.9.0_npu, `then_branch` holds exactly one MatMulNBits -- lm_head [151936,32,64] with real + // bytes -- while `else_branch` holds 109 layer projections and an `lm_head.MatMulNBits.qweight.empty` placeholder + // with none. The Token .fconst is the decode path, so a walk that merges the branches counts the prefill + // placeholder as "this pack has no head", falls back to the tail, and refuses (these packs' tails are the 8,256-byte + // envelope alone). The arm pins the reading of the condition, both ways round. + { + const auto graph_with_cond = [&](bool one_token_when_true, const std::string& head_branch) { + std::string named; + pb::str(named, 4, "Constant"); + std::string a; + pb::str(a, 1, "value_int"); + pb::ints(a, 3, one_token_when_true ? 1 : 2); + pb::tag(named, 5, 2); + pb::varint(named, a.size()); + named += a; + const std::string head = pb::matmulnbits("lm_head", {"x", "w", "w.scales", "w.zeros"}, K, N, 4, 128); + const std::string prefill = pb::graph({}, {head}); + const std::string decode = pb::graph({}, {head}); + return pb::model({}, {named, pb::if_node("c", "input_ids", "logits", head_branch, decode), + pb::if_node("c2", "input_ids", "logits", + head_branch == "then_branch" ? "else_branch" : "then_branch", prefill)}); + }; + put_file((dir / "model.onnx").string(), graph_with_cond(/*one_token_when_true=*/true, "then_branch")); + const json d1 = json::parse(pi::plan(dir.string()).manifest())["graph_weights"]; + check(d1.contains("decode_branch") && d1["decode_branch"].get() == "then_branch", + "decode branch: a condition comparing the shape with 1 makes then_branch the decode path"); + put_file((dir / "model.onnx").string(), graph_with_cond(/*one_token_when_true=*/false, "else_branch")); + const json d2 = json::parse(pi::plan(dir.string()).manifest())["graph_weights"]; + check(d2["decode_branch"].get() == "else_branch", + "decode branch: an inverted condition moves the decode path to else_branch"); + write_graph("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.packed", stride * N); + } + // A packed length that does not fit the stride must be reported as a mismatch, not crash and not claim a match. write_data(stride * N - 68); write_graph("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.packed", stride * N - 68); From e478d6c6596bca179396a144b314a451eef979d0 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 05:27:47 -0400 Subject: [PATCH 03/14] npu-pack: decide the decode If branch by which branch holds the weights, not only by the condition's constant The condition `Equal(Shape(input_ids), Constant(1))` is the pack's statement of which branch is one-token; the arithmetic is what must hold. A pack that inverts the condition without moving its weights answered the wrong branch, and the head vanished. Now the branch that actually declares MatMulNBits nodes wins when only one of them does, and the condition decides only when the graph is ambiguous (weights in both branches). branch_note names which rule was used and, when they disagree, how many matmuls each branch has. Also fixes the crafted test: it wrote model.onnx while this fixture's genai_config.json names eager-mm.onnx, so the arm was measuring the fixture's own graph rather than the crafted one -- and the graph needed its embedding tensor to be plannable at all. --- npu/pack_import.cpp | 44 ++++++++++++++++++++++++++++++---- tests/npu_pack_import_test.cpp | 14 +++++++---- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index f61bb6b36..78b135ca9 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -265,9 +265,10 @@ OnnxModel parse_onnx(const std::string& path) { // `then_branch` keeps the rule honest for a pack that inverts it; when the graph is not that shape, the ONNX // convention (`then_branch`) stands and `branch_note` says the rule was assumed rather than read. { - const auto gpb = pb(g->p, g->n); + // g IS the GraphProto (field 7 of ModelProto), so its nodes are at field 1 of g directly -- re-wrapping + // g's bytes in another pb() would read the graph as if it were a node list and find no Constant at all. bool one_token_when_true = true, read = false; - for (const auto* f : fields(gpb, 1)) { // GraphProto.node = 1 + for (const auto* f : fields(pb(g->p, g->n), 1)) { // GraphProto.node = 1 const auto node = pb(f->p, f->n); const auto* opf = field1(node, 4); if (!opf || pb_str(*opf) != "Constant") continue; @@ -279,9 +280,42 @@ OnnxModel parse_onnx(const std::string& path) { if (iv) { one_token_when_true = (iv->v == 1); read = true; } } } - m.decode_branch = one_token_when_true ? "then_branch" : "else_branch"; - m.branch_note = read ? "the If condition compares the input shape with 1, so " + m.decode_branch + - " is the one-token (decode) path" + // The branch whose graph actually declares a MatMulNBits. `value_int == 1` is the pack's convention, but the + // arithmetic is what must hold: the Token .fconst carries the graph's own quantised matmuls, so the branch + // that HAS them is the decode path whatever the constant says. A pack that inverts the condition without + // moving its weights stays readable; a pack whose weights sit in two branches is ambiguous, so the condition + // decides and the note says it did. + const auto branch_has_matmul = [&](const std::string& want) { + int n = 0; + for (const auto* f : fields(pb(g->p, g->n), 1)) { + const auto node = pb(f->p, f->n); + for (const auto* a : fields(node, 5)) { + const auto at = pb(a->p, a->n); + const auto* nm = field1(at, 1); + if (!nm || (pb_str(*nm) != "then_branch" && pb_str(*nm) != "else_branch")) continue; + if (pb_str(*nm) != want) continue; + for (const auto* sg : fields(at, 6)) { + for (const auto* nf : fields(pb(sg->p, sg->n), 1)) { + const auto sub = pb(nf->p, nf->n); + const auto* op = field1(sub, 4); + if (op && (pb_str(*op) == "MatMulNBits" || pb_str(*op) == "MatMulNBitsBf")) ++n; + } + } + } + } + return n; + }; + const std::string by_condition = one_token_when_true ? "then_branch" : "else_branch"; + const int then_mm = branch_has_matmul("then_branch"), else_mm = branch_has_matmul("else_branch"); + if (then_mm > 0 && else_mm == 0) m.decode_branch = "then_branch"; + else if (else_mm > 0 && then_mm == 0) m.decode_branch = "else_branch"; + else m.decode_branch = by_condition; + m.branch_note = read ? "the If condition compares the input shape with 1, so " + by_condition + + " is the one-token (decode) path" + + ((m.decode_branch == by_condition) ? "" : + "; its weights are in " + m.decode_branch + " (then " + + std::to_string(then_mm) + ", else " + std::to_string(else_mm) + + " quantised matmuls), which is where the Token .fconst's weights are read") : "no recognisable If condition, so then_branch is taken as decode by the ONNX convention"; } } catch (...) { diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 064a42e68..0c24fba03 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -537,18 +537,24 @@ void test_external_packed_genre(const std::string& root) { pb::tag(named, 5, 2); pb::varint(named, a.size()); named += a; + // A pack this fixture can actually plan: it needs its own embedding tensor in the graph, exactly as the + // write_graph lambda above gives it. + const std::string emb = pb::tensor("model.embed_tokens.weight", {64, 5120}, "eager-mm.onnx.data", 0, + uint64_t(embed_bytes)); const std::string head = pb::matmulnbits("lm_head", {"x", "w", "w.scales", "w.zeros"}, K, N, 4, 128); - const std::string prefill = pb::graph({}, {head}); - const std::string decode = pb::graph({}, {head}); + const std::string prefill = pb::graph({emb}, {head}); + const std::string decode = pb::graph({emb}, {head}); return pb::model({}, {named, pb::if_node("c", "input_ids", "logits", head_branch, decode), pb::if_node("c2", "input_ids", "logits", head_branch == "then_branch" ? "else_branch" : "then_branch", prefill)}); }; - put_file((dir / "model.onnx").string(), graph_with_cond(/*one_token_when_true=*/true, "then_branch")); + // The graph this pack imports is the one genai_config.json names (eager-mm.onnx); a crafted model.onnx beside + // it is not read at all, which is why the first version of this arm measured the fixture's own graph. + put_file((dir / "eager-mm.onnx").string(), graph_with_cond(/*one_token_when_true=*/true, "then_branch")); const json d1 = json::parse(pi::plan(dir.string()).manifest())["graph_weights"]; check(d1.contains("decode_branch") && d1["decode_branch"].get() == "then_branch", "decode branch: a condition comparing the shape with 1 makes then_branch the decode path"); - put_file((dir / "model.onnx").string(), graph_with_cond(/*one_token_when_true=*/false, "else_branch")); + put_file((dir / "eager-mm.onnx").string(), graph_with_cond(/*one_token_when_true=*/false, "else_branch")); const json d2 = json::parse(pi::plan(dir.string()).manifest())["graph_weights"]; check(d2["decode_branch"].get() == "else_branch", "decode branch: an inverted condition moves the decode path to else_branch"); From 1516de589d1914bb5a4bf6ae9e18710145113c7b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 06:04:21 -0400 Subject: [PATCH 04/14] npu: the integer arm's operands come from the engine's packer, not a private re-quantisation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The arm was merged in #1809 with a quantisation rule of its own: per 128-K group, int4 codes in [-8, 7]. The image the device reads is `q4nx_pack_B_int8_blocked()`'s: ONE multiplier per weight column, int8 codes in [-127, 127]. Both start from the same f32 matrix, so the arm computed a DIFFERENT MODEL -- measured on the Mistral-7B 1.9 container: it answered "ikt" to "The capital of France is" and then locked onto one token, while a per-GEMM A/B read cos 0.9999 against f32 on the same tensors. So the image is now built by that same packer (`blocked=1`, the layout the device reads and the fast path -- measured 174.5 s of warm-up blocked=0 versus 32-35 s blocked) and the MAC walks it with `q4nx_layout::blocked_index()`. Measured after, on current main: warm-up 35.2 s and the answer to "The capital of France is" is **'Paris,▁Question:'**, the same first token the device produces; before it was '%%%%...' / '▁/******/'. With the layer-aware cache key already in main (engine#1830 review), the arm now computes the same model as the device to int8 precision, which is what an integer reference has to do before any family can be gated against it. `npu_pack_test` (the ctest that pins the integer forms at K=130) builds and passes; copyright rc 0. --- npu/forward/include/q4nx_pack.h | 1 - npu/forward/src/q4nx_forward.cpp | 46 +++++++++++++++----------------- npu/forward/src/q4nx_pack.c | 1 + 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index 6f94b1f4c..1a36053b3 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -191,7 +191,6 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out); - /* Architecture dims a design set is derived from. */ typedef struct { int H; /* hidden_size */ diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 5a6b4b36a..c1d689e72 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3044,7 +3044,6 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector& h, std::vector amax) amax = v; - } - if (!(amax >= 1e-12f)) amax = 1.0f; - t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f; - const float tis = 127.0f / amax; - for (int k = k0; k < k1; k++) { - float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; - if (!std::isfinite(v)) v = 0.0f; - int q = (int)std::round(v * tis); - q = q > 7 ? 7 : (q < -8 ? -8 : q); - t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; - } - } - } + t.d.assign((size_t)N, 0.0f); + // THE OPERANDS MUST BE THE ENGINE'S, NOT A PRIVATE RE-QUANTISATION (engine#1808/#1830). This arm exists to be + // the integer reference a kernel family is gated against, and the device's weight image comes from + // q4nx_pack_B_int8_blocked(): ONE multiplier per column (amax over the whole K), int8 codes in [-127, 127]. + // Quantising here with a different rule -- per 128-K group, int4 [-8, 7] -- computes a different model: measured + // 2026-11-10, the arm answered "ikt" to "The capital of France is" and then locked onto one token, while the + // per-GEMM A/B read cos 0.9999 against f32 on the same tensors. Same packer, same bytes, same model. + double ssum = 0.0; + // blocked=1: the image is written in the blocked microtile order, which is the fast path AND the layout the + // device actually reads, so the MAC below walks it with q4nx_layout::blocked_index(). (blocked=0 was measured + // at 174.5 s of warm-up on this container versus 32.2 s blocked -- the row-major scatter is the slow one.) + q4nx_pack_B_int8_blocked(t.codes.data(), Bmat, K, N, K, N, /*blocked=*/1, t.d.data(), &ssum); if (!cacheable) { scratch = std::move(t); cached = &scratch; @@ -3109,8 +3099,14 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, int q = (int)std::round(v * tis); a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); } - q4nx_host_gemm_i8_grouped(a8.data(), K, t.codes.data(), t.d.data(), t.groups, N, ascale, - out + (size_t)m * (size_t)N); + float* dst = out + (size_t)m * (size_t)N; + for (int n = 0; n < N; n++) { + int32_t acc = 0; + for (int k = 0; k < K; k++) + acc += (int32_t)((int)t.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, K, N)] * + (int)a8[(size_t)k]); + dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc); + } } } diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 0914232e5..c021cca95 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -161,6 +161,7 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) { * * exactly the kernel's epilogue, with the inner sum in int32 and no float in it. `d` is the per-128-K-group * multiplier per column ([groups][N]), `ascale` the activation row's scale. */ + void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out) { const int G = 128; From 7904dfa2a07b05abcadd7566655e12041a25a013 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 06:47:15 -0400 Subject: [PATCH 05/14] npu: keyed dumps settle it -- the device's weight scales are BIT-IDENTICAL to the arm's (384/384) Dumps were paired by index first and that was meaningless: the two sides reach the GEMM a different number of times (1024 arm dumps against 384 device dumps on the same request). Keyed by (layer, N, K, ordinal) they pair exactly. Result, one token, Mistral-7B 1.9 container, one npu-hold ticket: arm dumps 1024, device dumps 384, paired 384 exactly equal weight-scale arrays: 384/384 worst relative difference: 0.000e+00 Together with the earlier measurements this closes the GEMM question: the activation scale is the same formula on both sides (`amax/127`, device at xdna_target.cpp:982), the epilogue's two multiplication orders were measured to differ by 0 (the probe returned 0 on every call), the int32 accumulators were identical for the first token's whole pass, and now the per-column weight scales are exactly equal. So the device's GEMMs are bit-exact against this arm. What that leaves is outside the GEMM: the arm and the device still differ by 0.30 max|d| in logits, and the non-GEMM work (RoPE, attention, norms) is shared host code, so the remaining candidates are the fused/selected kernel paths (fused_gu splits GU differently from the arm's one GEMM) and the head path. That is a different question from "are the kernels right", and the answer to that one is yes. Env-gated, off by default (`NPU_I8_GS_DUMP`); nothing in the shipped path changes. --- npu/forward/src/q4nx_forward.cpp | 9 +++++++++ npu/target/xdna/xdna_target.cpp | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index c1d689e72..4755c4d24 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3083,6 +3083,15 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, cached = &it->second; } const HostI8Tensor& t = *cached; + if (const char* gdir = getenv("NPU_I8_GS_DUMP")) { + static int n = 0; + char path[4096]; + std::snprintf(path, sizeof path, "%s/arm_L%d_N%d_K%d_%03d.bin", gdir, cur_layer_, N, K, n++); + if (std::FILE* f = std::fopen(path, "wb")) { + std::fwrite(t.d.data(), sizeof(float), std::min((size_t)N, t.d.size()), f); + std::fclose(f); + } + } std::vector a8((size_t)K); for (int m = 0; m < M; m++) { const float* a = A + (size_t)m * (size_t)a_k; diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index fe4c4ecae..4c45789d5 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -1149,6 +1149,18 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: } const auto tr = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; out.resize((size_t)M * N); + // engine#1808: the device's per-column weight scales, KEYED by (layer, N, K) so it can be paired with the host + // arm's dump exactly. Index pairing was tried first and is meaningless -- the two sides reach the GEMM a + // different number of times (384 device dumps against 1024 arm dumps on the same request). + if (const char* gdir = getenv("NPU_I8_GS_DUMP")) { + static int n = 0; + char path[4096]; + std::snprintf(path, sizeof path, "%s/dev_L%d_N%d_K%d_%03d.bin", gdir, ar.layer, N, K, n++); + if (std::FILE* f = std::fopen(path, "wb")) { + if (gs && !gs->empty()) std::fwrite(gs->data(), sizeof(float), (size_t)N, f); + std::fclose(f); + } + } for (int m = 0; m < M; m++) { const int32_t* cm = kctx.Cm + (size_t)m * kctx.ND; const float asc = ascales[(size_t)m]; From d5f7d48760eada1e9a422948934aa1d485a1e888 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 08:13:40 -0400 Subject: [PATCH 06/14] npu: the integer head, and the gate it unlocks -- device logits BIT-EXACT over four steps The head was the last place the host arm and the device computed the same tensor differently: in host mode `run_lm_head()` is a plain f32 matvec ("No I8Ctx is touched") while the device reads a packed quantised table (`lm_head_packed_`, `run_lm_head_gemv`). That was the whole of the remaining `max|d| = 0.30` once every body GEMM had been shown bit-exact (weight scales 384/384 identical, accumulators identical, epilogue order gap 0). The device packs its head tiles with `lctx.packB_into(...)` -> `packB_blocked`, so this builds them with the SAME rule through `q4nx_pack_B_int8_blocked()` -- one multiplier per column, int8 codes -- chunked by vocabulary block (8192 columns; a whole head image is 622 MB at this vocabulary) and cached per block. The activation row is quantised with `amax/127` after the same `1/embed_scale` factor the device applies, and the MAC is exact int32 walked with `q4nx_layout::blocked_index()`, then `d[n] * (ascale * acc)`. Measured, one npu-hold ticket, device vs this arm, logit dumps compared per step: step 0: argmax 3916 == 3916 cos 1.000000 max|d| 0 step 1: argmax 278 == 278 cos 1.000000 max|d| 0 step 2: argmax 28725 == 28725 cos 1.000000 max|d| 0 step 3: argmax 22478 == 22478 cos 1.000000 max|d| 0 BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 That is TNPU-ALL section 4's bar met literally for this family rather than read down to "at the accumulator": the device's arithmetic and the integer reference agree to the last bit, on every step, for a full request. --- npu/forward/include/q4nx_forward.h | 1 + npu/forward/src/q4nx_forward.cpp | 49 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 8586c0cea..c64be7992 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -248,6 +248,7 @@ class Q4nxNpuForward { // scratch buffer each layer is decoded into, so keying on it served layer 0's weights to every later layer // (engine#1830 review). Routed experts pass wkey < 0 and are never cached: their weights change per token. std::map, HostI8Tensor> host_i8_cache_; + std::vector lm_head_i8_; // integer arm's head image, one per vocabulary block void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 4755c4d24..c43ea283c 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -5215,6 +5215,55 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector if (npu::model::run_lm_head(*xd_, h.data(), via_target.data())) { logits = std::move(via_target); return true; } } if (!host_gemm_ && xd_ && xd_->head_ctx() && xd_->head_ctx()->gemv) return run_lm_head_gemv(h, logits); + if (host_gemm_ && host_gemm_i8_) { + // engine#1808: the head's integer twin. Without this the arm's head is a plain f32 matvec ("No I8Ctx is + // touched" below) while the device's head reads a packed quantised table, which is exactly the 0.30 max|d| + // that was left once every body GEMM had been shown bit-exact. The device packs its head tiles with + // lctx.packB_into(...) -> packB_blocked, so the arm builds them with the same rule through + // q4nx_pack_B_int8_blocked(): one multiplier per column, int8 codes. Chunked by vocabulary block, because a + // whole head image is H x V bytes (622 MB at this vocabulary). + const float inv_es = (sem_.tie_embeddings && sem_.embed_scale > 0) ? 1.0f / sem_.embed_scale : 1.0f; + const int TB = 8192; // vocabulary block; a tile's image is H x TB bytes + if ((int)lm_head_i8_.size() != (NV + TB - 1) / TB) lm_head_i8_.assign((NV + TB - 1) / TB, HostI8Tensor()); + logits.assign((size_t)NV, 0.0f); + std::vector a8v((size_t)H); + float amax = 0.0f; + for (int k = 0; k < H; k++) { + const float v = std::fabs(h[(size_t)k] * inv_es); + if (v > amax) amax = v; + } + const float ascale = amax > 0 ? amax / 127.0f : 1.0f; + for (int k = 0; k < H; k++) { + int q = (int)std::round(h[(size_t)k] * inv_es / ascale); + a8v[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); + } + std::vector bt((size_t)H * (size_t)TB); + std::vector a8i((size_t)H); + for (size_t i = 0; i < a8v.size(); i++) a8i[i] = a8v[i]; + for (int t = 0; t < (int)lm_head_i8_.size(); t++) { + const int n0 = t * TB, n1 = std::min(NV, n0 + TB), Tn = n1 - n0; + HostI8Tensor& c = lm_head_i8_[(size_t)t]; + if (c.codes.empty()) { + for (int n = n0; n < n1; n++) + for (int k = 0; k < H; k++) bt[(size_t)k * Tn + (n - n0)] = lm_at(n, k); + c.N = Tn; + c.codes.assign((size_t)H * (size_t)Tn, 0); + c.d.assign((size_t)Tn, 0.0f); + double ssum = 0.0; + q4nx_pack_B_int8_blocked(c.codes.data(), bt.data(), H, Tn, H, Tn, /*blocked=*/1, c.d.data(), &ssum); + } + for (int n = 0; n < Tn; n++) { + int32_t acc = 0; + for (int k = 0; k < H; k++) + acc += (int32_t)((int)c.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, H, Tn)] * + (int)a8i[(size_t)k]); + float v = c.d[(size_t)n] * (ascale * (float)acc); + if (sem_.logit_softcap > 0) v = sem_.logit_softcap * std::tanh(v / sem_.logit_softcap); + logits[(size_t)(n0 + n)] = v; + } + } + return true; + } if (host_gemm_) { // Offline mode: plain float matvec against the (possibly dequantized) // tied table, matching the Python reference. No I8Ctx is touched. From 297d9ba950c317e252889333d7bbf319b91c3855 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 11:32:32 -0400 Subject: [PATCH 07/14] npu-pack: read the fused SSMLP/GQO FFN attributes, and the 4096/32/40 shape rule (refs #1740) granite-4.1-8b states its FFN nowhere the reader looked: config.json is literally {}, genai_config.json's decoder block has only hidden_size / heads / layers, and its FFN is not MatMulNBits nodes at all -- it is one `SSMLP` op with gate_K 4096, gate_N 12800, up_K 4096, up_N 12800, down_K 12800, down_N 4096. So the import refused with "the pack states no intermediate_size", which is correct for what it could see and wrong about the pack. Recording those attributes as matmuls lets intermediate_size_from_graph find w by the same witness it already uses for the separate-node spelling (gate and up reading one activation) instead of adding a second rule that could disagree with the first. With the width resolved the pack reaches the shape gate, which had no row for hidden 4,096 / 32 heads / 40 layers. Its 1.9 header is clean -- 40 runs of 32 x `00 00 7f ff`, first at 13,795,392, every gap exactly 107,728,192 -- and its tail is 225,198,144 B, which is exactly 8,256 + 1,568 x 143,616: the h4096 head grid (rows = ceil(100,352 / 64)). So unlike Qwen3-8B the head IS in the tail here, and the bundle/rotation agree with the Qwen3-8B row at the same hidden size and head count. amd publishes no 1.8 twin for this model, so that agreement plus the head-grid arithmetic is the whole evidence for the rotation -- the row says so, and says the prefix inherits the Qwen3 row's own caveat. --- npu/pack_import.cpp | 35 ++++++++++++++++++++++++++++++++++ tests/npu_pack_import_test.cpp | 12 ++++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 78b135ca9..865ac2f8f 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -245,6 +245,30 @@ void walk_graph(const uint8_t* p, size_t n, OnnxModel& m, const std::string& bra mm.branch = branch; if (!mm.weight.empty()) m.matmuls.push_back(std::move(mm)); } + // The Ryzen AI fused MLPs: `SSMLP` is gate+up+down as ONE node and `GQO` is grouped-query attention, so neither + // has a MatMulNBits node for the FFN to pair up -- which is why amd/granite-4.1-8b_rai_1.9.0_npu states no + // intermediate_size anywhere (`config.json` is literally {}, and genai_config.json's decoder block has only + // hidden_size / heads / layers) and was refused. The node carries the same numbers as attributes, measured on + // that pack: gate_K 4096, gate_N 12800, up_K 4096, up_N 12800, down_K 12800, down_N 4096 -- the plain + // (K == hidden, N == w) gate/up and (K == w, N == hidden) down triple, w = 12800. Recording them as matmuls lets + // intermediate_size_from_graph find them by the SAME witness it uses for the separate-node spelling (gate and up + // reading one activation) rather than adding a second rule that could disagree with the first. + if (op == "SSMLP" || op == "GQO") { + for (const char* sub : {"gate", "up", "down"}) { + const auto k = attrs.find(std::string(sub) + "_K"), n = attrs.find(std::string(sub) + "_N"); + if (k == attrs.end() || n == attrs.end() || k->second <= 0 || n->second <= 0) continue; + OnnxMatMul mm; + // gate and up share the node's activation input; down reads the gated product, so it gets its own name. + // That shared activation is the very witness separate gate/up nodes provide. + mm.data = ins.empty() ? std::string() : ins[0]; + if (std::string(sub) == "down") mm.data += ".down"; + mm.weight = sub; // the role name intermediate_size_from_graph's excluded() filter reads + mm.K = k->second; + mm.N = n->second; + mm.branch = branch; + m.matmuls.push_back(std::move(mm)); + } + } } } @@ -553,6 +577,17 @@ constexpr Fconst19Rule kFconst19Rules[] = { // tolerance -- so this pair does not isolate the prefix. `prefix = 12,608` rests on the Mistral row above and the // Qwen3 arithmetic (64 + 2*hidden + 4,352) agreeing, which is exactly the caveat the 3,584 row carries too. {4096, 32, 36, 12608, 5, 1, 2297856}, + // granite-4.1-8b at hidden 4,096 / 32 heads / 40 layers: the same bundle shape and rotation as the Qwen3-8B row + // above -- every field identical but the layer count -- which again says the bundle is a property of the hidden size + // and its head count, not the depth. It must still be its own row because select_rule_in() keys on all three. + // Unlike Qwen3-8B this pack's head is NOT in the tail's envelope: its tail is 225,198,144 B after 40 x 107,728,192, + // and that is exactly 8,256 + 1,568 x 143,616 (the h4096 tile, rows = ceil(100,352 / 64)), i.e. a full head grid, + // so the tail carries the head itself. The header stride is uniform over all 40 blocks (40 runs of 32 x + // `00 00 7f ff`, first at 13,795,392, every gap 107,728,192). + // Honest limit: amd publishes no 1.8 twin for this model, so the rotation and the tile are corroborated by the head + // grid arithmetic and by agreement with the Qwen3-8B row at the same geometry rather than by a direct diff. The + // prefix is the same 64 + 2*hidden + 4,352 = 12,608 the Qwen3 row uses, with the same caveat that row states. + {4096, 32, 40, 12608, 5, 1, 2297856}, // Llama-3.2-3B (and -Instruct, same geometry) at hidden 3,072 / 24 heads / 28 layers: measured with // tools/fconst19_measure.py against amd/Llama-3.2-3B_rai_1.8.0_npu_4K, 1,090 of 1,756,930,368 bytes differ // (6.20e-07 of the file, which the tool prints as 6.20e-05 %) -- content, not layout. The prefix is the Qwen3 line's 64 + 2*hidden + 4,352 = 10,560 at this hidden diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 0c24fba03..face7a604 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -1648,8 +1648,16 @@ void test_pack(const std::string& pack, const std::string& golden_dir) { } check(attributed, "pack: each runtime-refused tensor is marked imported=false in the manifest"); } - check(r.lmhead_corr > 0.999, "pack: the decode reproduces the fp16 embedding (corr " + - std::to_string(r.lmhead_corr) + ")"); + // Only a TIED pack can be checked this way: there lm_head IS the embedding, so correlating them measures the decoder. + // An untied pack's lm_head is a separate matrix and the number is meaningless by construction (Qwen3-8B: 0.0043 on a + // head that decodes correctly -- std 0.026, plausible per-row scale, weakly correlated rows). Asserting 0.999 + // unconditionally demanded every pack be tied, which no untied pack can satisfy. + if (r.selfcheck_tied) + check(r.lmhead_corr > 0.999, "pack: the decode reproduces the fp16 embedding, tied (corr " + + std::to_string(r.lmhead_corr) + ")"); + else + check(true, "pack: untied, so the embedding correlation is not a decoder check (corr " + + std::to_string(r.lmhead_corr) + ")"); check(r.ctrl_corr_nibble < 0.5 && r.ctrl_corr_nozp < 0.95, "pack: both wrong decoders are rejected as controls"); // engine#1185: an imported container carries fp32 scale/zero (layout q4nx-2, npu/q4nx.h From 7e6481e1cdc6e9930a3b9bd862382cd8cca7b5e5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 12:40:22 -0400 Subject: [PATCH 08/14] docs/npu: say what NPU_INFER_HOST_GEMM=i8 is for, and what it measured The integer arm was only discoverable from the code. It exists because an f32 host arm cannot gate a kernel family: the device's weights are int8 codes with one multiplier per column and int32 accumulators, so f32 differs by quantisation alone (cos 0.9960-0.9974, an argmax flip at step 3 on the h4096 family, neither of them a kernel fault). Documented: what it builds (the engine's own packer and blocked layout, so the reference shares the image rather than re-deriving one), that the LM head is included in vocabulary blocks, the measured BIT-EXACT result on kernels-h4096-b8 with its repeats and edge case, that it is a reference rather than a speed path, and what it does not claim. --- docs/npu.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index 98d287a95..99f7ca874 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1712,6 +1712,45 @@ usable beside another NPU engine, and what its limits are: not wait for other shared holders"; `LOCK_EX` = "I want the device to myself". A holder keeps its fd open for the life of the process; closing or exiting releases it. `LOCK_EX` and `LOCK_SH` never overlap, in either order, on one file. +### `NPU_INFER_HOST_GEMM=i8`: the integer arm, and why a float one cannot gate a kernel + +`NPU_INFER_HOST_GEMM=1` runs the forward on the host with **f32** GEMMs. That is enough to tell a structural +break (garbage output) from a working route, and it is *not* enough to check a kernel family: the device's +weights are int8 codes with one multiplier per column and its accumulators are int32, so an f32 arm differs from +it by quantisation alone -- measured on the h4096 family, cosine 0.9960-0.9974 and an argmax flip at step 3, +none of which is a kernel fault. A family gated against f32 can therefore never be *bit-exact*, which is the bar +`TISSUES-ALL` §4 asks for. + +`NPU_INFER_HOST_GEMM=i8` is the reference that can. It builds each body GEMM's weight image with the engine's +own packer (`q4nx_pack_B_int8_blocked()`: one multiplier per column, int8 codes, blocked microtile order), walks +it with `q4nx_layout::blocked_index()`, quantises the activation row to int8 with `amax/127`, accumulates in +int32 and applies `d[n] * (ascale * acc)`. **Same packer, same image, same order** -- that is the whole point: a +reference that re-quantises by a different rule computes a different model, and will disagree with a correct +device (measured: per-128-K int4 codes answer `ikt` to a question four other packs answer correctly on). + +The LM head is included, in vocabulary blocks (a whole head image is H x V bytes, 622 MB at 150k vocabulary). +Without it the body is exact and the head is not, and the residual shows up as `max|d| ~ 0.3` on logits that no +amount of body-side agreement can remove. + +**What it measures.** Run the device arm and the `=i8` arm on the same request with `--dump-logits`, then +`scripts/families/compare-logits.py` (npu-kernels). On `kernels-h4096-b8` with Mistral-7B-Instruct-v0.2 1.9: + +``` +step 0: argmax 3916 == 3916 cos 1.000000 max|d| 0 +step 1: argmax 278 == 278 cos 1.000000 max|d| 0 +step 2: argmax 28725 == 28725 cos 1.000000 max|d| 0 +step 3: argmax 22478 == 22478 cos 1.000000 max|d| 0 +BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 +``` +three repeats of the device arm, an edge case (`Hi`), and run-to-run determinism all `max|d| = 0`. + +**What it costs.** The arm warms up slowly (it builds one image per tensor, ~35 s on a 7B container) and is not +a speed path: it exists to be a reference. It is off by default and `=1` (f32) keeps its old meaning. + +**What it does not claim.** Nothing about throughput, and nothing about a family it has not been run on. The +same gate on the other four-h4096 sets needs a pack with *their* geometry, which is a download, not a code +change. + - **Who may take what.** Anything that honours the file interlocks with us: `forward-serve` (`--npu-streams` decides which mode), the in-process route (`LOCK_EX`, always -- `app/unified.cpp`), the ELF runner (`LOCK_EX`, always). **A run that never opens the device takes no lock**: with From c7c504309f3ba6910b05978a198122be0a1e4415 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 12:45:27 -0400 Subject: [PATCH 09/14] docs/npu: the two-arm command sequence for the bit-exact gate A documented reference that nobody can run is a claim, not an instrument: the section now carries the two forward-serve invocations, the dump naming scheme, and the three-step discipline (compare per position, repeat the device arm for determinism, add an edge case). --- docs/npu.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index 99f7ca874..0eaeb0620 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1744,6 +1744,18 @@ BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 ``` three repeats of the device arm, an edge case (`Hi`), and run-to-run determinism all `max|d| = 0`. +**How to run the two arms.** One ticket each, dumps compared per step: + +``` +1bit forward-serve -m /model.q4nx --kernels /npu -p 20081 --dump-logits /tmp/g/i8 \ + & # with NPU_INFER_HOST_GEMM=i8 in the environment +1bit forward-serve -m /model.q4nx --kernels /npu -p 20082 --dump-logits /tmp/g/dev +scripts/families/compare-logits.py /tmp/g/i8 /tmp/g/dev # BIT-EXACT at tolerance 0 +``` +`--dump-logits` writes `step__.bin` per generated step, so the comparison is per position rather than +per run. Repeat the device arm three times for determinism, and add an edge case (a two-token prompt), which is +what the table above did. + **What it costs.** The arm warms up slowly (it builds one image per tensor, ~35 s on a 7B container) and is not a speed path: it exists to be a reference. It is off by default and `=1` (f32) keeps its old meaning. From 172178de84da1e02d8d9e61a1fde2e05b35a85c7 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:29:25 -0400 Subject: [PATCH 10/14] npu-pack: withdraw the 4096/32/40 row -- a prefix cannot be inferred, only measured against a twin I added this row on granite-4.1-8b because its 1.9 header is clean (40 uniform runs at stride 107,728,192) and its tail is exactly 8,256 + 1,568 x 143,616, which pins the TILE by head-grid arithmetic. That was the mistake: the tile is not the whole rule. `prefix` is defined by the twin diff -- tools/fconst19_measure.py:104 walks `prefix += 1` while the 1.9 block still equals the 1.8 one, so prefix is the first byte where they diverge, and the tile is found by where that first-differing run reappears. Neither number is named anywhere in the 1.9 pack alone. The row's own in-block checks rejected it, which is the argument for removing it rather than for keeping it with a caveat: scale/zero exactness 0 %, second header 0, and 0 of 80 RMSNorm anchors found. A row that fails the pack it was written for is not a measured row. Confirmed against the twin I already had: Qwen3-8B's prefix 12,608 is exactly the first byte where its 1.9 block diverges from its 1.8 block (1.9 `8782 0777...` vs 1.8 `6756 ea78...`, 0 differing bytes in [0, 12608)). amd publishes no 1.8 granite, so granite stays refused by the shape gate -- the honest answer for a layout nothing can establish. The table now says that, so the next person does not re-derive the row from the tail arithmetic the way I did. Qwen3-8B is unaffected: still "all npu-pack checks passed (including the pack leg)". --- npu/pack_import.cpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 865ac2f8f..4aba5973e 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -491,6 +491,14 @@ constexpr double kTieCorr = 0.99; // Qwen3's per-head QK norm gains, but Llama-3.2-1B -- which has no QK norms at all -- carries the same +4,352 at hidden // 2,048, so the extra is a property of the geometry, not of the architecture. Measured anyway, because a plausible // explanation that survives one model is how this table would get a wrong row. +// A geometry with no row is refused, and for a pack with no 1.8 twin that is the right answer rather than a gap to +// fill by inference. `prefix` and the tile are defined by a diff against the twin (`tools/fconst19_measure.py`: prefix +// is the first byte where the 1.9 block diverges from the 1.8 one, and `probe-what-follows` locates the tile by finding +// where that first-differing run reappears at a displacement). Nothing in the 1.9 pack alone names either number: +// amd/granite-4.1-8b_rai_1.9.0_npu has 40 clean uniform headers and a tail of exactly 8,256 + 1,568 x 143,616, which +// pins its TILE by head-grid arithmetic and says nothing about its prefix -- and adding a row from that alone was +// refused by the pack's own in-block checks (scale/zero exactness 0 %, no second header, 0/80 norm anchors). No 1.8 +// granite exists on the Hub, so the row cannot be measured until one does. constexpr Fconst19Rule kFconst19Rules[] = { // Qwen2.5-0.5B-Instruct: normalises to its 1.8 twin byte for byte, whole file, all 24 blocks. {896, 14, 24, 1856, 16, 2, 34816}, @@ -577,17 +585,6 @@ constexpr Fconst19Rule kFconst19Rules[] = { // tolerance -- so this pair does not isolate the prefix. `prefix = 12,608` rests on the Mistral row above and the // Qwen3 arithmetic (64 + 2*hidden + 4,352) agreeing, which is exactly the caveat the 3,584 row carries too. {4096, 32, 36, 12608, 5, 1, 2297856}, - // granite-4.1-8b at hidden 4,096 / 32 heads / 40 layers: the same bundle shape and rotation as the Qwen3-8B row - // above -- every field identical but the layer count -- which again says the bundle is a property of the hidden size - // and its head count, not the depth. It must still be its own row because select_rule_in() keys on all three. - // Unlike Qwen3-8B this pack's head is NOT in the tail's envelope: its tail is 225,198,144 B after 40 x 107,728,192, - // and that is exactly 8,256 + 1,568 x 143,616 (the h4096 tile, rows = ceil(100,352 / 64)), i.e. a full head grid, - // so the tail carries the head itself. The header stride is uniform over all 40 blocks (40 runs of 32 x - // `00 00 7f ff`, first at 13,795,392, every gap 107,728,192). - // Honest limit: amd publishes no 1.8 twin for this model, so the rotation and the tile are corroborated by the head - // grid arithmetic and by agreement with the Qwen3-8B row at the same geometry rather than by a direct diff. The - // prefix is the same 64 + 2*hidden + 4,352 = 12,608 the Qwen3 row uses, with the same caveat that row states. - {4096, 32, 40, 12608, 5, 1, 2297856}, // Llama-3.2-3B (and -Instruct, same geometry) at hidden 3,072 / 24 heads / 28 layers: measured with // tools/fconst19_measure.py against amd/Llama-3.2-3B_rai_1.8.0_npu_4K, 1,090 of 1,756,930,368 bytes differ // (6.20e-07 of the file, which the tool prints as 6.20e-05 %) -- content, not layout. The prefix is the Qwen3 line's 64 + 2*hidden + 4,352 = 10,560 at this hidden From 043e58ba38990aebf4f83dfd87c81c99832eaa43 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:42:39 -0400 Subject: [PATCH 11/14] npu-pack: the h4096 36-layer fconst row, so the Qwen3-8B pack imports at all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `kFconst19Rules` held the 32-layer h4096 row and nothing for 36 layers, so `npu-pack info` refused `amd/Qwen3-8B_rai_1.9.0_npu` by design: "its block shape has not been measured for hidden 4096, 32 heads, 36 layers … the 1.8 pack of the same model is what establishes it". Measured now, with tools/fconst19_measure.py against amd/Qwen3-8B_rai_1.8.0_npu_16K: {4096, 32, 36, 12608, 5, 1, 2297856} 26,104 of 3,756,627,264 bytes differ (6.95e-06 of the file) -- content, not layout Every field but the layer count is identical to the 32-layer row (prefix 12,608, five tiles of 2,297,856, rotation 1), which is the first row here where the layer count moves nothing but the number of blocks: the h2048 pair moves both the prefix and the division with it (4,160/9 tiles at 36 layers against 8,512/3 at 24), so this row is measured rather than inferred from that one. The consequence beyond this pack: a kernel family and an fconst rule row are not the same partition. One family can span several rows, so each new pack of a family may cost a measurement before it costs a gate. Verified end to end: the import now completes -- 5,760/5,760 .fconst tiles survive decode->encode byte for byte, 144 attention projections plus 108 MLP projections with fp32 scale/zero exact for 100.00% of groups, 8 full_i8 ELFs linked from kernels-h4096-im12288-b8. --- npu/pack_import.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 248c2f6ce..0944afd02 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -449,6 +449,16 @@ constexpr Fconst19Rule kFconst19Rules[] = { // all 32 blocks. This is the largest family on #1503 (31 packs). Measured by another lane on main, whose // table is keyed on geometry; merged here in the same form. {4096, 32, 32, 12608, 5, 1, 2297856}, + // Qwen3-8B at hidden 4,096 / 32 heads / **36 layers**: measured with tools/fconst19_measure.py against + // amd/Qwen3-8B_rai_1.8.0_npu_16K, 26,104 of 3,756,627,264 bytes differ (6.95e-06 of the file, which the tool prints + // as 6.95e-04 %) -- content, not layout, and the largest residual in this table, so it is stated rather than + // rounded away. Every other field is IDENTICAL to the 32-layer row above (prefix 12,608, five tiles of 2,297,856, + // rotation 1): at this hidden size the layer count moves nothing but the number of blocks, which is the first row + // here where that is true -- the h2048 pair above moves both the prefix and the division with the layer count + // (4,160/9 tiles at 36 layers against 8,512/3 tiles at 24), so this row is measured, not inferred from that one. + // Without it the importer refuses the pack by design ("its block shape has not been measured for hidden 4096, 32 + // heads, 36 layers"), which is what it did until now: kernel family and fconst rule row are not the same partition. + {4096, 32, 36, 12608, 5, 1, 2297856}, // Llama-3.2-3B (and -Instruct, same geometry) at hidden 3,072 / 24 heads / 28 layers: measured with // tools/fconst19_measure.py against amd/Llama-3.2-3B_rai_1.8.0_npu_4K, 1,090 of 1,756,930,368 bytes differ // (6.20e-07 of the file, which the tool prints as 6.20e-05 %) -- content, not layout. The prefix is the Qwen3 line's 64 + 2*hidden + 4,352 = 10,560 at this hidden From 452975c68550f12ae8a18178243a36692ee87c17 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 14:06:36 -0400 Subject: [PATCH 12/14] npu-pack test: the new 36-layer row goes in BOTH places, as the guard's own comment requires CI caught the omission and said exactly why: "fconst19 rule: the table has exactly the 11 measured rows (12); a dropped row is a silent un-measure of a whole class, an added one must be added here too". The expected rows are literals copied from the measured table rather than read back from it, so that a wrong digit or a dropped row fails -- which is the arm working, not a nuisance. Added {4096, 32, 36, 12608, 5, 1, 2297856} to it, and the arm now passes. --- tests/npu_pack_import_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index c63b3fb93..fafc7f60a 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2791,6 +2791,7 @@ void test_fconst19_rule_table() { {3584, 28, 28, 7232, 8, 1, 1009664}, // the Qwen2.5-7B family, measured on two models {2048, 16, 36, 4160, 9, 1, 295936}, // Qwen2.5_3B_Instruct {4096, 32, 32, 12608, 5, 1, 2297856}, // Mistral-7B-Instruct-v0.2 + {4096, 32, 36, 12608, 5, 1, 2297856}, // Qwen3-8B (measured against ..._rai_1.8.0_npu_16K) {3072, 24, 28, 10560, 4, 1, 1740800}, // Llama-3.2-3B (and -Instruct) }; const int want_n = (int)(sizeof(want) / sizeof(want[0])); From 62cfd5ed3f2f0e9a379d75c568d434f00bae6e97 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 15:51:08 -0400 Subject: [PATCH 13/14] npu-pack: fused-MLP attributes are not graph matmuls, and the 4096/32/36 row is pinned in the test Two things found by running the tests and real packs on main + this branch. 1. main's row-count arm (#1861) pins kFconst19Rules as literals; the Qwen3-8B row was not in it, so the merged tree failed 11 != 12. Added. 2. The SSMLP/GQO gate/up/down attributes were pushed into OnnxModel::matmuls. matmuls.empty() is how the importer knows a 1.9 pack keeps its head in the .fconst tail, and every 1.9 pack has a fused MLP, so the block-32 tail head (#1948) was no longer read: Qwen-2.5 1.5B 1.9 imported 338 tensors instead of 339, with no lm_head, exit 0 and no refusal. The attributes now go to OnnxModel::fused_mlp, which only intermediate_size_from_graph() reads. New arm 2b builds the tied block-32 fixture with an SSMLP node; with the old push it fails three assertions. Co-Authored-By: Claude Fable 5.2 --- npu/pack_import.cpp | 18 ++++++++++++---- tests/npu_pack_import_test.cpp | 38 ++++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 1da78c128..022278eb7 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -144,6 +144,9 @@ struct OnnxMatMul { struct OnnxModel { std::vector tensors; std::vector matmuls; + // The fused-MLP nodes' (SSMLP / GQO) gate/up/down shapes, from their attributes. Kept apart from `matmuls` because they + // name no weight tensor: `matmuls` answers "does the graph carry quantised weights", and these must not. + std::vector fused_mlp; std::map index; // `com.ryzenai:MatMulNBitsBf` is the same quantised matmul as `com.microsoft:MatMulNBits`, and on the 1.9 // blob/external-data packs it is the ONLY one: phi-4's 201 weights are all `MatMulNBitsBf`, so a walker that @@ -268,7 +271,12 @@ void walk_graph(const uint8_t* p, size_t n, OnnxModel& m, const std::string& bra mm.K = k->second; mm.N = n->second; mm.branch = branch; - m.matmuls.push_back(std::move(mm)); + // NOT into `matmuls`: these carry no weight tensor, and `matmuls.empty()` is how the importer knows a + // 1.9 pack keeps its head in the .fconst tail. Counted as graph matmuls they turned that off for every + // pack with a fused MLP -- measured on Qwen-2.5 1.5B 1.9, which then imported 338 tensors with no + // lm_head, exit 0 and no refusal. They are geometry witnesses only, read by + // intermediate_size_from_graph(). + m.fused_mlp.push_back(std::move(mm)); } } } @@ -1717,14 +1725,16 @@ int intermediate_size_from_graph(const std::string& model_path, int64_t hidden) }; std::map> shared; // activation tensor -> N -> how many nodes read it std::set down_k, gate_n; // down: N == hidden, K > hidden; gate/up: K == hidden, N > hidden - for (const auto& mm : m.matmuls) { - if (mm.K <= 0 || mm.N <= 0 || excluded(mm.weight)) continue; + const auto witness = [&](const OnnxMatMul& mm) { + if (mm.K <= 0 || mm.N <= 0 || excluded(mm.weight)) return; if (mm.K == hidden && mm.N > hidden) { shared[mm.data][mm.N]++; gate_n.insert(mm.N); } if (mm.N == hidden && mm.K > hidden) down_k.insert(mm.K); - } + }; + for (const auto& mm : m.matmuls) witness(mm); + for (const auto& mm : m.fused_mlp) witness(mm); // SSMLP / GQO attributes: shapes without a weight tensor // separate gate/up first: it is the witness attention cannot produce std::set separate; for (const int64_t dk : down_k) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index d7af19124..42c0f60e5 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -1850,7 +1850,25 @@ void test_block32_tail_head(const std::string& root) { check(std::fabs(ncorr(emb, other)) < 0.3, "block-32 tail: the fixture's two heads are unrelated (corr " + std::to_string(ncorr(emb, other)) + ")"); - const auto build = [&](bool tied, const char* tag, const std::string& tail_after_env) { + // A Ryzen AI fused MLP node as the 1.9 graphs spell it: op SSMLP, the gate/up/down shapes as int attributes, and + // NO weight tensor (the weights are in the .fconst). Every 1.9 pack measured carries one per layer. + const auto ssmlp_node = [&]() { + std::string n; + pb::str(n, 1, "/model/layers.0/x"); + pb::str(n, 4, "SSMLP"); + const std::pair attrs[] = {{"gate_K", 128}, {"gate_N", 256}, {"up_K", 128}, + {"up_N", 256}, {"down_K", 256}, {"down_N", 128}}; + for (const auto& [name, value] : attrs) { + std::string a; + pb::str(a, 1, name); + pb::ints(a, 3, value); + pb::tag(n, 5, 2); + pb::varint(n, a.size()); + n += a; + } + return n; + }; + const auto build = [&](bool tied, const char* tag, const std::string& tail_after_env, bool fused_mlp = false) { const fs::path dir = fs::path(root) / tag; fs::remove_all(dir); fs::create_directories(dir / "cache"); @@ -1895,7 +1913,7 @@ void test_block32_tail_head(const std::string& root) { pb::placeholder("lm_head.weight.MatMulNBits.qweight"), pb::placeholder("lm_head.weight.MatMulNBits.scales"), pb::placeholder("lm_head.weight.MatMulNBits.zeros")}, - {})); + fused_mlp ? std::vector{ssmlp_node()} : std::vector{})); return dir; }; struct Run { @@ -1973,6 +1991,21 @@ void test_block32_tail_head(const std::string& root) { "block-32 tail, tied: the container's head is the embedding (corr " + std::to_string(x.head.empty() ? 0.0 : ncorr(x.head, emb)) + ")"); } + { // arm 2b: the same pack with a fused-MLP node in its graph, as every real 1.9 pack has. The node names no weight, + // so it must not make the graph look like it carries quantised matmuls: that is the switch for "the head is in + // the .fconst tail", and with it off the import wrote a container with no lm_head at all and exit 0 + // (measured on Qwen-2.5 1.5B 1.9 with the SSMLP attributes counted as graph matmuls: 338 tensors, not 339). + const Run x = run(build(true, "b32-tied-ssmlp", encode_block32_units(emb, V, H), true), "ssmlp"); + check(x.said.empty(), "block-32 tail, fused MLP in the graph: imports (" + x.said.substr(0, 160) + ")"); + check(x.has_head && !x.head.empty() && ncorr(x.head, emb) > 0.99, + "block-32 tail, fused MLP in the graph: the container still carries the head read from the tail"); + check(x.r.selfcheck_tied && x.r.head_read_note.find("reproduces") != std::string::npos, + "block-32 tail, fused MLP in the graph: the tied self-check still runs (" + x.r.head_read_note + ")"); + const json g = json::parse(pi::plan(build(true, "b32-tied-ssmlp-plan", encode_block32_units(emb, V, H), true) + .string()).manifest()); + check(g.value("graph_weights", json::object()).value("matmulnbits_ops", int64_t(-1)) == 0, + "block-32 tail, fused MLP in the graph: SSMLP attributes are not counted as graph matmuls"); + } { // arm 3: TIED, but the tail's head is some other matrix -> the read is NOT taken, the synthesis stands, says why const Run x = run(build(true, "b32-tied-mismatch", encode_block32_units(other, V, H)), "mismatch"); check(x.r.head_read_note.empty() && x.r.head_synthesized_note.find("correlates") != std::string::npos, @@ -3291,6 +3324,7 @@ void test_fconst19_rule_table() { {3584, 28, 28, 7232, 8, 1, 1009664}, // the Qwen2.5-7B family, measured on two models {2048, 16, 36, 4160, 9, 1, 295936}, // Qwen2.5_3B_Instruct {4096, 32, 32, 12608, 5, 1, 2297856}, // Mistral-7B-Instruct-v0.2 + {4096, 32, 36, 12608, 5, 1, 2297856}, // Qwen3-8B (the Mistral row at 36 layers; the key includes layers) {3072, 24, 28, 10560, 4, 1, 1740800}, // Llama-3.2-3B (and -Instruct) }; const int want_n = (int)(sizeof(want) / sizeof(want[0])); From f03b84a87fe736df2301e0c4fefb4b2bfca957df Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 15:51:43 -0400 Subject: [PATCH 14/14] npu-pack test: arm 2b removes its second fixture directory It was left under the golden directory after a run. Co-Authored-By: Claude Fable 5.2 --- tests/npu_pack_import_test.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 42c0f60e5..0ed1f8809 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2001,8 +2001,9 @@ void test_block32_tail_head(const std::string& root) { "block-32 tail, fused MLP in the graph: the container still carries the head read from the tail"); check(x.r.selfcheck_tied && x.r.head_read_note.find("reproduces") != std::string::npos, "block-32 tail, fused MLP in the graph: the tied self-check still runs (" + x.r.head_read_note + ")"); - const json g = json::parse(pi::plan(build(true, "b32-tied-ssmlp-plan", encode_block32_units(emb, V, H), true) - .string()).manifest()); + const fs::path plan_dir = build(true, "b32-tied-ssmlp-plan", encode_block32_units(emb, V, H), true); + const json g = json::parse(pi::plan(plan_dir.string()).manifest()); + fs::remove_all(plan_dir); check(g.value("graph_weights", json::object()).value("matmulnbits_ops", int64_t(-1)) == 0, "block-32 tail, fused MLP in the graph: SSMLP attributes are not counted as graph matmuls"); }