diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index b5334363a..86d89a06b 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -1439,11 +1439,50 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, lm_head_f_.assign((size_t)NV * (size_t)kpad_, 0.0f); lm_w_ = kpad_; const uint8_t* lp = (const uint8_t*)model_tensor_data(mw_, ltd); - if (!lp || dequant_any(lp, (size_t)ltd->data_size, NV, kpad_, - lm_head_f_.data()) != 0) { + /* engine#1990: the container's head may carry MORE rows than the vocabulary, for the same reason a padded + * column exists -- q4_1 tiles hold 32-row multiples and a pack's vocabulary need not be one (CodeLlama's is + * 32,016). The importer pads those rows; the reader must therefore dequantise on the tensor's own grid and + * keep only the vocabulary's rows. Asking for exactly NV rows is what failed here: 32,016 is not a multiple + * of the tile's 32 rows, so dequant_any() refused a head that was perfectly well formed. + * + * The extra rows decode to zero and are never sampled: the logits vector stays NV long. */ + int head_rows = NV; + if (ltd->ndim >= 2 && kpad_ > 0 && kpad_ % Q4NX_TILE_COLS == 0) { + const int64_t per_block = kpad_ / Q4NX_TILE_COLS; // tiles per 32-row block + // engine#1990: the tile COUNT is shape[0] when the tensor is stored as [tiles, tile bytes] -- which is + // what this container does (CodeLlama-7b: I8 [16016, 6144] for a 32,032-row head at kpad_ 4,096) -- and + // it is shape[0] / per_block when it is stored as a row grid. Gating on `shape[1] == kpad_` saw only the + // second form, so a padded head in the first form kept head_rows = NV and dequant_any() refused it: + // "lm_head dequant failed" on a head that is exactly what the importer was asked to write. + // Both spellings count TILE-ROWS in shape[0] -- [rows, kpad_] and [tiles, tile bytes] -- so the number of + // 32-row blocks is shape[0] / per_block either way; only the row WIDTH in shape[1] differs. + int64_t blocks = 0; + if (per_block > 0 && ltd->shape[0] % per_block == 0) blocks = ltd->shape[0] / per_block; + if (blocks > 0) { + const int64_t r = blocks * Q4NX_TILE_ROWS; + if (r >= NV && r % Q4NX_TILE_ROWS == 0) head_rows = int(r); + } + } + std::vector head_f32; + float* head_out = lm_head_f_.data(); + if (head_rows != NV) { + head_f32.assign((size_t)head_rows * (size_t)kpad_, 0.0f); + head_out = head_f32.data(); + } + if (!lp || dequant_any(lp, (size_t)ltd->data_size, head_rows, kpad_, head_out) != 0) { err_ = "lm_head dequant failed"; return false; } + if (head_rows != NV) { + for (int r = 0; r < NV; r++) + std::memcpy(&lm_head_f_[(size_t)r * (size_t)kpad_], &head_f32[(size_t)r * (size_t)kpad_], + sizeof(float) * (size_t)kpad_); + std::fprintf(stderr, + " [forward] lm head: %d row(s) stored for a vocabulary of %d; the %d padding row(s) are " + "not read (engine#1990)\n", head_rows, NV, head_rows - NV); + } + fprintf(stderr, " [forward] separate lm_head.weight -> dequantized " + "(%zu floats)\n", lm_head_f_.size()); fprintf(stderr, " [forward] separate lm_head.weight -> dequantized " "(%zu floats)\n", lm_head_f_.size()); } @@ -3188,15 +3227,21 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, std::vector a8((size_t)K); for (int m = 0; m < M; m++) { const float* a = A + (size_t)m * (size_t)a_k; + // engine#845's pad on THIS leg too (kept across the merge: main fixed the IMAGE side of the padded-geometry + // problem -- unpadded images, row-major when blocked does not fit -- but the activation read still ran past the + // row). A padded design has K > a_k: chatglm3-6b's IM is 13,696 and its D design is K = 13,824, so reading A[k] + // for k < K runs 128 floats into the next row, or past the buffer. Measured before this: cos 0.40 -> -0.003 + // against the device while the f32 arm tracked it at 0.985-0.988; after, max|d| = 0 on all four steps. + const auto act = [a, a_k](int k) { return k < a_k ? a[(size_t)k] : 0.0f; }; float amax = 0.0f; for (int k = 0; k < K; k++) { - const float v = std::fabs(a[k]); + const float v = std::fabs(act(k)); if (std::isfinite(v) && v > amax) amax = v; } const float ascale = (amax >= 1e-12f) ? amax / 127.0f : 1.0f; const float tis = 1.0f / ascale; for (int k = 0; k < K; k++) { - float v = a[k]; + float v = act(k); if (!std::isfinite(v)) v = 0.0f; int q = (int)std::round(v * tis); a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); @@ -5365,7 +5410,15 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector // q4nx_pack_B_int8_blocked(): one multiplier per column, int8 codes. Chunked by vocabulary block, because a // whole head image is H x V bytes (622 MB at this vocabulary). const float inv_es = (sem_.tie_embeddings && sem_.embed_scale > 0) ? 1.0f / sem_.embed_scale : 1.0f; - const int TB = 8192; // vocabulary block; a tile's image is H x TB bytes + // engine#1990: the block width is an experiment knob, because the residual on CodeLlama is confined to exactly + // 16 logit positions starting at 16384 -- the start of a block -- and is identical at every step, i.e. a WEIGHT + // difference localised at a block boundary rather than an accumulation difference. NPU_I8_HEAD_TB moves the + // boundaries: if the wrong positions move with it, the chunking is the cause; if they stay at 16384, it is not. + static const int TB = [] { + const char* e = std::getenv("NPU_I8_HEAD_TB"); + const int v = e ? std::atoi(e) : 8192; + return (v >= 1024 && v % 128 == 0) ? v : 8192; + }(); if ((int)lm_head_i8_.size() != (NV + TB - 1) / TB) lm_head_i8_.assign((NV + TB - 1) / TB, HostI8Tensor()); logits.assign((size_t)NV, 0.0f); std::vector a8v((size_t)H); diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index df92b468e..dd8a10786 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -451,6 +451,16 @@ constexpr Fconst19Rule kFconst19Rules[] = { // all 32 blocks. This is the largest family on #1503 (31 packs). Measured by another lane on main, whose // table is keyed on geometry; merged here in the same form. {4096, 32, 32, 12608, 5, 1, 2297856}, + // Qwen3-8B at hidden 4,096 / 32 heads / **36 layers**: measured with tools/fconst19_measure.py against + // amd/Qwen3-8B_rai_1.8.0_npu_16K, 26,104 of 3,756,627,264 bytes differ (6.95e-06 of the file, which the tool prints + // as 6.95e-04 %) -- content, not layout, and the largest residual in this table, so it is stated rather than + // rounded away. Every other field is IDENTICAL to the 32-layer row above (prefix 12,608, five tiles of 2,297,856, + // rotation 1): at this hidden size the layer count moves nothing but the number of blocks, which is the first row + // here where that is true -- the h2048 pair above moves both the prefix and the division with the layer count + // (4,160/9 tiles at 36 layers against 8,512/3 tiles at 24), so this row is measured, not inferred from that one. + // Without it the importer refuses the pack by design ("its block shape has not been measured for hidden 4096, 32 + // heads, 36 layers"), which is what it did until now: kernel family and fconst rule row are not the same partition. + {4096, 32, 36, 12608, 5, 1, 2297856}, // Llama-3.2-3B (and -Instruct, same geometry) at hidden 3,072 / 24 heads / 28 layers: measured with // tools/fconst19_measure.py against amd/Llama-3.2-3B_rai_1.8.0_npu_4K, 1,090 of 1,756,930,368 bytes differ // (6.20e-07 of the file, which the tool prints as 6.20e-05 %) -- content, not layout. The prefix is the Qwen3 line's 64 + 2*hidden + 4,352 = 10,560 at this hidden @@ -2103,6 +2113,30 @@ bool external_container_tensors(const std::string& role, int nh, int nkv, int hd return false; } +// engine#1990: q4_1 tiles hold 32-row multiples, and a pack's vocabulary need not be one. CodeLlama's is 32,016, which +// refused the whole import with "q4_1 tiles need N % 32 == 0" -- a head the grid cannot hold, not a head that is wrong. +// Where the pack pads its own head (the .fconst tail carries ceil(V/64)*64 rows: Hunyuan-0.5B has 120,832 for 120,818) +// nothing here fires; where it does not, pad with rows that decode to zero, and leave `vocab_size` at the vocabulary's +// own number so the padding is never addressed. codes 0 with SCALE 1 and zero 0 decode to exactly 0.0 -- and scale 1, +// not 0: a zero scale is not a zero weight to a reader, it is a malformed parameter block (the forward's dequant refuses +// it, which is how the first version of this padding was caught: "lm_head dequant failed"). +void pad_rows_to_tile_quantum(Q4& q, int vocab) { + if (q.N <= 0 || q.K <= 0 || q.block <= 0 || q.K % q.block) return; + if (q.N % q4nx::kRows == 0) return; + const int padded = (q.N + q4nx::kRows - 1) / q4nx::kRows * q4nx::kRows; + const size_t gpr = size_t(q.K / q.block); // groups per row + for (int r = q.N; r < padded; r++) { + q.codes.insert(q.codes.end(), size_t(q.K), 0); + q.scale.insert(q.scale.end(), gpr, 1.0f); // 1.0, not 0: code 0 * 1 + 0 = 0.0, and a 0 scale is malformed + q.zero.insert(q.zero.end(), gpr, 0.0f); + } + std::fprintf(stderr, + "note: the head is %d rows for a vocabulary of %d (padded from %lld to the 32-row tile quantum); " + "vocab_size stays %d in the container\n", + padded, vocab, (long long)q.N, vocab); + q.N = padded; +} + Q4 container_from_external_q4(const Q4& pack, long long* m_inexact) { // A PASS-THROUGH since the signed-int4 fix: `decode_external_tile` produces the container's own convention directly // (code = c_signed + 8, zero = (-8 - z_signed) * scale), so converting `zero` again applies the mapping TWICE -- @@ -4511,6 +4545,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons uint64_t qw_need = 0; // N * K/block * block/2 if (!mul_ok(groups_n, uint64_t(mm.block) / 2, qw_need)) throw std::runtime_error("lm_head: qweight length overflows"); Q4 q = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block); + pad_rows_to_tile_quantum(q, int(V)); // no qzeros tensor -> the implicit zero point (engine#1503: Qwen3-VL-4B's spelling, three-input nodes) if (!have_zp) implicit_zp_zero(q, mm.bits); diff --git a/npu/pack_import.h b/npu/pack_import.h index 775eed018..43a90a425 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -517,6 +517,11 @@ constexpr bool kExtQuantsRowMajor = false; // measurably NOT the same weights: on a real phi-4 tile it reproduced the pack's codes on only 2,826 of 8,192 // entries and moved weights by 2.5 % of the tensor's range (engine#1503). `m_inexact` reports how many group // offsets were not bf16-exact (86.3 % are, measured), because that rounding is the only loss this path has. +// engine#1990: a head whose row count is not a multiple of the 32-row tile quantum (CodeLlama: 32,016 rows) cannot be +// held by the q4_1 grid, so pad it with zero rows that decode to zero and leave the container's vocab_size alone. Exposed +// because the property -- padding is invisible to the vocabulary -- is what a test can pin without a pack. +void pad_rows_to_tile_quantum(Q4& q, int vocab); + Q4 container_from_external_q4(const Q4& pack, long long* m_inexact = nullptr); // The container role a matmul plays, read off the graph's own tensor name: `...qkv_proj...` -> "QKV", diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index a2656f574..8fb944a7f 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -1264,6 +1264,53 @@ void test_1671_plan_head_arm(const std::string& root) { } // ── 1. MatMulNBits: encode here, decode with the importer ──────────────────── +// engine#1990: a head whose row count is not a multiple of the 32-row tile quantum (CodeLlama's vocabulary is 32,016) is +// padded with rows that decode to zero, and the container's vocab_size is NOT changed -- the padding is invisible to the +// vocabulary. Both halves are pinned here because widening N without staying invisible is the failure that would show up +// as two plausible extra tokens at the end of every distribution. +void test_head_row_padding() { + const auto make = [](int N) { + pi::Q4 q; + q.N = N; + q.K = 128; // two 64-column tiles + q.block = 32; // four groups per row + q.codes.assign(size_t(N) * size_t(q.K), 1); // non-zero, so padding is distinguishable + q.scale.assign(size_t(N) * size_t(q.K / q.block), 0.5f); // 1.0 in the padded rows; see below + q.zero.assign(size_t(N) * size_t(q.K / q.block), 0.25f); + return q; + }; + { + pi::Q4 q = make(32016); + pi::pad_rows_to_tile_quantum(q, 32016); + check(q.N == 32032 && q.N % 32 == 0, "head padding: 32,016 rows become 32,032 (the 32-row quantum)"); + check(q.codes.size() == size_t(32032) * 128 && q.scale.size() == size_t(32032) * 4, + "head padding: codes and scales grow with the rows, one group's worth each"); + bool zeros = true; + for (int r = 32016; r < q.N; r++) { + for (int k = 0; k < q.K; k++) zeros = zeros && q.codes[size_t(r) * 128 + size_t(k)] == 0; + for (int g = 0; g < 4; g++) + zeros = zeros && q.scale[size_t(r) * 4 + size_t(g)] == 1.0f && q.zero[size_t(r) * 4 + size_t(g)] == 0.0f; + } + check(zeros, "head padding: every padded row is code 0, scale 1, zero 0 -- which decodes to 0.0, not to a weight"); + bool kept = true; + for (int r = 0; r < 32016; r++) + for (int k = 0; k < 128; k++) kept = kept && q.codes[size_t(r) * 128 + size_t(k)] == 1; + check(kept, "head padding: the vocabulary's own rows are untouched"); + } + { // an aligned vocabulary is left exactly as it was: no note, no growth + pi::Q4 q = make(32000); + const size_t before = q.codes.size(); + pi::pad_rows_to_tile_quantum(q, 32000); + check(q.N == 32000 && q.codes.size() == before, "head padding: a 32-aligned vocabulary (32,000) is not touched"); + } + { // 16 rows of slack and 1 row of slack both round up to one tile boundary, not two + pi::Q4 a = make(32017), b = make(32031); + pi::pad_rows_to_tile_quantum(a, 32017); + pi::pad_rows_to_tile_quantum(b, 32031); + check(a.N == 32032 && b.N == 32032, "head padding: 32,017 and 32,031 both round to 32,032"); + } +} + void test_matmulnbits() { const int N = 64, K = 256, B = 128; std::vector qw(size_t(N) * (K / B) * (B / 2)); @@ -3393,6 +3440,7 @@ void test_fconst19_rule_table() { {3584, 28, 28, 7232, 8, 1, 1009664}, // the Qwen2.5-7B family, measured on two models {2048, 16, 36, 4160, 9, 1, 295936}, // Qwen2.5_3B_Instruct {4096, 32, 32, 12608, 5, 1, 2297856}, // Mistral-7B-Instruct-v0.2 + {4096, 32, 36, 12608, 5, 1, 2297856}, // Qwen3-8B (measured against ..._rai_1.8.0_npu_16K) {3072, 24, 28, 10560, 4, 1, 1740800}, // Llama-3.2-3B (and -Instruct) }; const int want_n = (int)(sizeof(want) / sizeof(want[0])); @@ -4849,7 +4897,8 @@ int main(int argc, char** argv) { test_crafted_refusals(golden_dir.empty() ? "." : golden_dir); test_modelbin_genre_refusal(golden_dir.empty() ? "." : golden_dir); test_1671_plan_head_arm(golden_dir.empty() ? "." : golden_dir); - test_matmulnbits(); + test_head_row_padding(); + test_matmulnbits(); test_tiles(golden_dir); test_q4_1f(); test_attn_tile(golden_dir);