diff --git a/docs/npu.md b/docs/npu.md index 98d287a95..0eaeb0620 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1712,6 +1712,57 @@ usable beside another NPU engine, and what its limits are: not wait for other shared holders"; `LOCK_EX` = "I want the device to myself". A holder keeps its fd open for the life of the process; closing or exiting releases it. `LOCK_EX` and `LOCK_SH` never overlap, in either order, on one file. +### `NPU_INFER_HOST_GEMM=i8`: the integer arm, and why a float one cannot gate a kernel + +`NPU_INFER_HOST_GEMM=1` runs the forward on the host with **f32** GEMMs. That is enough to tell a structural +break (garbage output) from a working route, and it is *not* enough to check a kernel family: the device's +weights are int8 codes with one multiplier per column and its accumulators are int32, so an f32 arm differs from +it by quantisation alone -- measured on the h4096 family, cosine 0.9960-0.9974 and an argmax flip at step 3, +none of which is a kernel fault. A family gated against f32 can therefore never be *bit-exact*, which is the bar +`TISSUES-ALL` ยง4 asks for. + +`NPU_INFER_HOST_GEMM=i8` is the reference that can. It builds each body GEMM's weight image with the engine's +own packer (`q4nx_pack_B_int8_blocked()`: one multiplier per column, int8 codes, blocked microtile order), walks +it with `q4nx_layout::blocked_index()`, quantises the activation row to int8 with `amax/127`, accumulates in +int32 and applies `d[n] * (ascale * acc)`. **Same packer, same image, same order** -- that is the whole point: a +reference that re-quantises by a different rule computes a different model, and will disagree with a correct +device (measured: per-128-K int4 codes answer `ikt` to a question four other packs answer correctly on). + +The LM head is included, in vocabulary blocks (a whole head image is H x V bytes, 622 MB at 150k vocabulary). +Without it the body is exact and the head is not, and the residual shows up as `max|d| ~ 0.3` on logits that no +amount of body-side agreement can remove. + +**What it measures.** Run the device arm and the `=i8` arm on the same request with `--dump-logits`, then +`scripts/families/compare-logits.py` (npu-kernels). On `kernels-h4096-b8` with Mistral-7B-Instruct-v0.2 1.9: + +``` +step 0: argmax 3916 == 3916 cos 1.000000 max|d| 0 +step 1: argmax 278 == 278 cos 1.000000 max|d| 0 +step 2: argmax 28725 == 28725 cos 1.000000 max|d| 0 +step 3: argmax 22478 == 22478 cos 1.000000 max|d| 0 +BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 +``` +three repeats of the device arm, an edge case (`Hi`), and run-to-run determinism all `max|d| = 0`. + +**How to run the two arms.** One ticket each, dumps compared per step: + +``` +1bit forward-serve -m /model.q4nx --kernels /npu -p 20081 --dump-logits /tmp/g/i8 \ + & # with NPU_INFER_HOST_GEMM=i8 in the environment +1bit forward-serve -m /model.q4nx --kernels /npu -p 20082 --dump-logits /tmp/g/dev +scripts/families/compare-logits.py /tmp/g/i8 /tmp/g/dev # BIT-EXACT at tolerance 0 +``` +`--dump-logits` writes `step__.bin` per generated step, so the comparison is per position rather than +per run. Repeat the device arm three times for determinism, and add an edge case (a two-token prompt), which is +what the table above did. + +**What it costs.** The arm warms up slowly (it builds one image per tensor, ~35 s on a 7B container) and is not +a speed path: it exists to be a reference. It is off by default and `=1` (f32) keeps its old meaning. + +**What it does not claim.** Nothing about throughput, and nothing about a family it has not been run on. The +same gate on the other four-h4096 sets needs a pack with *their* geometry, which is a download, not a code +change. + - **Who may take what.** Anything that honours the file interlocks with us: `forward-serve` (`--npu-streams` decides which mode), the in-process route (`LOCK_EX`, always -- `app/unified.cpp`), the ELF runner (`LOCK_EX`, always). **A run that never opens the device takes no lock**: with diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 8586c0cea..c64be7992 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -248,6 +248,7 @@ class Q4nxNpuForward { // scratch buffer each layer is decoded into, so keying on it served layer 0's weights to every later layer // (engine#1830 review). Routed experts pass wkey < 0 and are never cached: their weights change per token. std::map, HostI8Tensor> host_i8_cache_; + std::vector lm_head_i8_; // integer arm's head image, one per vocabulary block void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index 70dd13248..a634a64c7 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -202,7 +202,6 @@ void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, co void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out); - /* Architecture dims a design set is derived from. */ typedef struct { int H; /* hidden_size */ diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index f487b66cb..b587a1eb6 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3044,7 +3044,6 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector& h, std::vector amax) amax = v; - } - /* engine#1839: the weights here are the container's INT4 codes, so the scale that maps amax onto - * that range is amax/7 -- pairing amax/127 (an int8 scale) with the [-8, 7] clamp saturated every code - * but the largest and made the host reference's weights ~18x too small. q4nx_quant_int4_*() own the - * pair so this site and q4nx_pack.c's cannot drift apart again. */ - const float amax_eff = (amax >= 1e-12f) ? amax : 1.0f; - t.d[(size_t)g * (size_t)N + (size_t)n] = q4nx_quant_int4_scale(amax_eff); - for (int k = k0; k < k1; k++) - t.codes[(size_t)k * (size_t)N + (size_t)n] = - q4nx_quant_int4_code(Bmat[(size_t)k * (size_t)N + (size_t)n], amax_eff); - } - } + // RESOLVED toward the int8 packer (this branch) rather than main's corrected int4 rule (engine#1839). + // Both are self-consistent; the question is what the DEVICE consumes. The device's body image comes from + // packB_blocked() -- one multiplier per column, int8 codes -- and this branch measured the consequence: + // device logits BIT-EXACT (max|d| = 0) over four steps, three repeats, plus an edge case. So the int8 packer + // is the rule the reference has to share with the device, and q4nx_quant_int4_*() stays for the paths that + // genuinely hold int4 codes. + t.d.assign((size_t)N, 0.0f); + // THE OPERANDS MUST BE THE ENGINE'S, NOT A PRIVATE RE-QUANTISATION (engine#1808/#1830). This arm exists to be + // the integer reference a kernel family is gated against, and the device's weight image comes from + // q4nx_pack_B_int8_blocked(): ONE multiplier per column (amax over the whole K), int8 codes in [-127, 127]. + // Quantising here with a different rule -- per 128-K group, int4 [-8, 7] -- computes a different model: measured + // 2026-11-10, the arm answered "ikt" to "The capital of France is" and then locked onto one token, while the + // per-GEMM A/B read cos 0.9999 against f32 on the same tensors. Same packer, same bytes, same model. + double ssum = 0.0; + // blocked=1: the image is written in the blocked microtile order, which is the fast path AND the layout the + // device actually reads, so the MAC below walks it with q4nx_layout::blocked_index(). (blocked=0 was measured + // at 174.5 s of warm-up on this container versus 32.2 s blocked -- the row-major scatter is the slow one.) + q4nx_pack_B_int8_blocked(t.codes.data(), Bmat, K, N, K, N, /*blocked=*/1, t.d.data(), &ssum); if (!cacheable) { scratch = std::move(t); cached = &scratch; @@ -3092,6 +3089,15 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, cached = &it->second; } const HostI8Tensor& t = *cached; + if (const char* gdir = getenv("NPU_I8_GS_DUMP")) { + static int n = 0; + char path[4096]; + std::snprintf(path, sizeof path, "%s/arm_L%d_N%d_K%d_%03d.bin", gdir, cur_layer_, N, K, n++); + if (std::FILE* f = std::fopen(path, "wb")) { + std::fwrite(t.d.data(), sizeof(float), std::min((size_t)N, t.d.size()), f); + std::fclose(f); + } + } std::vector a8((size_t)K); for (int m = 0; m < M; m++) { const float* a = A + (size_t)m * (size_t)a_k; @@ -3108,8 +3114,14 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, int q = (int)std::round(v * tis); a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); } - q4nx_host_gemm_i8_grouped(a8.data(), K, t.codes.data(), t.d.data(), t.groups, N, ascale, - out + (size_t)m * (size_t)N); + float* dst = out + (size_t)m * (size_t)N; + for (int n = 0; n < N; n++) { + int32_t acc = 0; + for (int k = 0; k < K; k++) + acc += (int32_t)((int)t.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, K, N)] * + (int)a8[(size_t)k]); + dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc); + } } } @@ -5233,6 +5245,55 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector if (npu::model::run_lm_head(*xd_, h.data(), via_target.data())) { logits = std::move(via_target); return true; } } if (!host_gemm_ && xd_ && xd_->head_ctx() && xd_->head_ctx()->gemv) return run_lm_head_gemv(h, logits); + if (host_gemm_ && host_gemm_i8_) { + // engine#1808: the head's integer twin. Without this the arm's head is a plain f32 matvec ("No I8Ctx is + // touched" below) while the device's head reads a packed quantised table, which is exactly the 0.30 max|d| + // that was left once every body GEMM had been shown bit-exact. The device packs its head tiles with + // lctx.packB_into(...) -> packB_blocked, so the arm builds them with the same rule through + // q4nx_pack_B_int8_blocked(): one multiplier per column, int8 codes. Chunked by vocabulary block, because a + // whole head image is H x V bytes (622 MB at this vocabulary). + const float inv_es = (sem_.tie_embeddings && sem_.embed_scale > 0) ? 1.0f / sem_.embed_scale : 1.0f; + const int TB = 8192; // vocabulary block; a tile's image is H x TB bytes + if ((int)lm_head_i8_.size() != (NV + TB - 1) / TB) lm_head_i8_.assign((NV + TB - 1) / TB, HostI8Tensor()); + logits.assign((size_t)NV, 0.0f); + std::vector a8v((size_t)H); + float amax = 0.0f; + for (int k = 0; k < H; k++) { + const float v = std::fabs(h[(size_t)k] * inv_es); + if (v > amax) amax = v; + } + const float ascale = amax > 0 ? amax / 127.0f : 1.0f; + for (int k = 0; k < H; k++) { + int q = (int)std::round(h[(size_t)k] * inv_es / ascale); + a8v[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); + } + std::vector bt((size_t)H * (size_t)TB); + std::vector a8i((size_t)H); + for (size_t i = 0; i < a8v.size(); i++) a8i[i] = a8v[i]; + for (int t = 0; t < (int)lm_head_i8_.size(); t++) { + const int n0 = t * TB, n1 = std::min(NV, n0 + TB), Tn = n1 - n0; + HostI8Tensor& c = lm_head_i8_[(size_t)t]; + if (c.codes.empty()) { + for (int n = n0; n < n1; n++) + for (int k = 0; k < H; k++) bt[(size_t)k * Tn + (n - n0)] = lm_at(n, k); + c.N = Tn; + c.codes.assign((size_t)H * (size_t)Tn, 0); + c.d.assign((size_t)Tn, 0.0f); + double ssum = 0.0; + q4nx_pack_B_int8_blocked(c.codes.data(), bt.data(), H, Tn, H, Tn, /*blocked=*/1, c.d.data(), &ssum); + } + for (int n = 0; n < Tn; n++) { + int32_t acc = 0; + for (int k = 0; k < H; k++) + acc += (int32_t)((int)c.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, H, Tn)] * + (int)a8i[(size_t)k]); + float v = c.d[(size_t)n] * (ascale * (float)acc); + if (sem_.logit_softcap > 0) v = sem_.logit_softcap * std::tanh(v / sem_.logit_softcap); + logits[(size_t)(n0 + n)] = v; + } + } + return true; + } if (host_gemm_) { // Offline mode: plain float matvec against the (possibly dequantized) // tied table, matching the Python reference. No I8Ctx is touched. diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 533996f52..1c6006948 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -161,6 +161,7 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) { * * exactly the kernel's epilogue, with the inner sum in int32 and no float in it. `d` is the per-128-K-group * multiplier per column ([groups][N]), `ascale` the activation row's scale. */ + void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, int groups, int N, float ascale, float* out) { const int G = 128; diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 8cbf8f51b..5a3621a93 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -1157,6 +1157,18 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: } const auto tr = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; out.resize((size_t)M * N); + // engine#1808: the device's per-column weight scales, KEYED by (layer, N, K) so it can be paired with the host + // arm's dump exactly. Index pairing was tried first and is meaningless -- the two sides reach the GEMM a + // different number of times (384 device dumps against 1024 arm dumps on the same request). + if (const char* gdir = getenv("NPU_I8_GS_DUMP")) { + static int n = 0; + char path[4096]; + std::snprintf(path, sizeof path, "%s/dev_L%d_N%d_K%d_%03d.bin", gdir, ar.layer, N, K, n++); + if (std::FILE* f = std::fopen(path, "wb")) { + if (gs && !gs->empty()) std::fwrite(gs->data(), sizeof(float), (size_t)N, f); + std::fclose(f); + } + } for (int m = 0; m < M; m++) { const int32_t* cm = kctx.Cm + (size_t)m * kctx.ND; const float asc = ascales[(size_t)m];