npu: the integer host arm a kernel gate needs, because f32 cannot be bit-exact with int8 (Refs #1808)

CMakeLists.txt

@@ -847,6 +847,10 @@ add_test(NAME route_port_lock COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/
     # the whole run loop against canned completions, and re-derives the summary from the written transcript.  Stdlib
     # only, with no route, no socket, no dataset fetch and no device.
     add_test(NAME gsm8k_eval COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/gsm8k_eval_test.py)
+    # engine#1808's integer host arm is pinned inside tests/npu_pack_test.cpp (ctest npu_pack): that binary is
+    # already built by ci.yml, and a new ctest target would need a `.github/workflows/ci.yml` edit, which this
+    # lane's token cannot push (the OAuth app has no `workflow` scope) -- the ci_targets check caught exactly
+    # that, with both failures ("ci_targets" and "npu_host_gemm_i8 Not Run").
     # The repack's tile-order claim (engine#1110): `npu_plan.py` writes container.tile_order=engine-1 for every
     # plan, so the load-time guard (#1130) is green on a container whose TILES are another order.  This pins the
     # mapping from q4nx_diff's verdict to (stamp value, fatal), stdlib only: no model, no numpy, no device.
@@ -1095,13 +1099,18 @@ if(ONEBIT_OGA_PACK)
     add_test(NAME npu_pack_import_oga
              COMMAND npu_pack_import_test ${CMAKE_SOURCE_DIR}/tests/golden ${ONEBIT_OGA_PACK})
 endif()
-add_executable(npu_pack_test tests/npu_pack_test.cpp)
+# q4nx_pack.c is listed here rather than reached through onebit_npu: the integer host arm's checks
+# (engine#1808) call q4nx_host_gemm_i8*() directly, and those live in that file.
+add_executable(npu_pack_test tests/npu_pack_test.cpp npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c)
 add_executable(flm_head_import tools/flm_head_import.cpp)
 target_link_libraries(flm_head_import PRIVATE onebit_npu)
 add_executable(npu_flm_head_import_test tests/npu_flm_head_import_test.cpp)
 target_link_libraries(npu_flm_head_import_test PRIVATE onebit_npu)
 add_test(NAME npu_flm_head_import COMMAND npu_flm_head_import_test ${CMAKE_SOURCE_DIR}/tests/golden/flm-i8)
 target_link_libraries(npu_pack_test PRIVATE onebit_npu)
+# engine#1808: the integer host arm's checks live in this binary (see above), so it needs the header and the
+# golden fixture that pins them.
+target_include_directories(npu_pack_test PRIVATE npu/forward/include ${CMAKE_SOURCE_DIR}/tests)
 
 # The device weight layout contract (q4nx_weight_layout): pure index math, no device
 # and no runtime.  CI builds an explicit target list, so the check compiles itself with

npu/forward/include/q4nx_forward.h

@@ -233,6 +233,15 @@ class Q4nxNpuForward {
     bool init_impl(xrt::device* dev, const char* model_path,
                    const std::string& artifact_dir);
     bool host_gemm_ = false;
+    // NPU_INFER_HOST_GEMM=i8 (engine#1808): the host arm accumulates int8 x int4 in EXACT int32, which is
+    // the only arm a kernel family can be bit-exact against -- f32 cannot match an int8 device by
+    // construction. It cannot see a wrong weight layout (the f32 arm covers that); it can see a wrong
+    // quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator.
+    bool host_gemm_i8_ = false;
+    // Weight codes + per-128-K multipliers for the integer host arm, quantised once per tensor (engine#1809).
+    struct HostI8Tensor { int groups = 0; int N = 0; std::vector<int8_t> codes; std::vector<float> d; };
+    std::map<int, HostI8Tensor> host_i8_cache_;
+    void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out);
     bool load_layer(int l);
     // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or
     // intermediate size).  It is K whenever the model is already on the container's 256-column K

npu/forward/include/q4nx_pack.h

@@ -170,6 +170,27 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out);
 void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, const float* Bmat,
                               float* out);
 
+/* The integer arm of the same host path (engine#1808): int8 activation x int4 weight codes -> exact int32,
+ * which is what the kernel computes. Written for the bit-exactness gate, because the f32 host path cannot
+ * be bit-exact with an int8 device arm by construction; scales are applied by the caller on both sides so
+ * a scale mistake cannot hide inside the integer comparison. `codes` is [K][N], one int8 per element
+ * holding an int4 value in [-8, 7], exactly as the importer's packB lays it down. */
+void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32);
+
+/* The whole GEMM in int8 x int4 for the gate's `i8` host arm, quantising both sides here with the engine's
+ * own rule. It cannot see a wrong weight LAYOUT (the f32 arm covers that); what it can see is a wrong int4
+ * quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator -- i.e. the
+ * kernel's arithmetic, exactly. int32 out, M*N, no scaling applied. */
+void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, const float* Bmat,
+                                int32_t* out32);
+
+/* The reusable form: `codes` and `d` are quantised once per tensor by the caller (a per-call quantisation of
+ * the weights costs ~7.5 G scalar operations per 7B token, measured — it never finished a token in a
+ * 9-minute ticket), and this applies the kernel's epilogue: ascale * sum_g d[g][n] * sum_{k in g} code*a8,
+ * with the inner sum exact int32 and no float inside it. */
+void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d,
+                               int groups, int N, float ascale, float* out);
+
 
 /* Architecture dims a design set is derived from. */
 typedef struct {

npu/forward/src/q4nx_forward.cpp

@@ -424,7 +424,14 @@ bool Q4nxNpuForward::init_host(const char* model_path, const std::string& dir) {
 
 bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
                                const std::string& dir) {
-    if (getenv("NPU_INFER_HOST_GEMM")) host_gemm_ = true;
+    if (const char* hg = getenv("NPU_INFER_HOST_GEMM")) {
+        host_gemm_ = true;
+        // engine#1808: "i8" selects the integer arm, whose int32 accumulation is the only thing a
+        // kernel family can be bit-exact against (an int8 device cannot match an f32 host by
+        // construction). Any other value keeps the f32 path, which is what every existing caller and
+        // every row measured so far used.
+        host_gemm_i8_ = (hg[0] == 'i' && hg[1] == '8');
+    }
     // The launch layer.  A1: everything below that touches a device talks to this, and the settings the
     // container declares (weight layout, pb16i image dir, Hadamard fold) are pushed into it as they are
     // parsed.  A2 turns this into NpuTarget::open(plan, kernels_dir).
@@ -3025,6 +3032,65 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector<float>& h, std::vector<f
 // packed-weight BO cache, pb16i images, Hadamard fold.  What stays here is the model's own fallback, the
 // plain float GEMM of init_host() -- that is a property of the model, not of any NPU.
 // --------------------------------------------------------------------------
+// The integer host arm (engine#1808/#1809). Two decisions here, both measured:
+//  * the weight codes and their per-128-K multipliers are quantised ONCE per tensor (`wkey`) and cached --
+//    doing it per call made a 7B token ~7.5 G scalar operations and the arm never finished a token inside a
+//    9-minute ticket;
+//  * the inner sum is exact int32 with no float in it (q4nx_host_gemm_i8_grouped), so the comparison against
+//    the device is about accumulation and epilogue, not about a float both sides happen to share.
+void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat,
+                                  int wkey, float* out) {
+    const int G = 128, groups = (K + G - 1) / G;
+    auto it = host_i8_cache_.find(wkey);
+    if (it == host_i8_cache_.end()) {
+        HostI8Tensor t;
+        t.groups = groups; t.N = N;
+        t.codes.assign((size_t)K * (size_t)N, 0);
+        t.d.assign((size_t)groups * (size_t)N, 0.0f);
+        for (int g = 0; g < groups; g++) {
+            const int k0 = g * G, k1 = (k0 + G < K) ? k0 + G : K;
+            for (int n = 0; n < N; n++) {
+                float amax = 0.0f;
+                for (int k = k0; k < k1; k++) {
+                    const float v = std::fabs(Bmat[(size_t)k * (size_t)N + (size_t)n]);
+                    if (std::isfinite(v) && v > amax) amax = v;
+                }
+                if (!(amax >= 1e-12f)) amax = 1.0f;
+                t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f;
+                const float tis = 127.0f / amax;
+                for (int k = k0; k < k1; k++) {
+                    float v = Bmat[(size_t)k * (size_t)N + (size_t)n];
+                    if (!std::isfinite(v)) v = 0.0f;
+                    int q = (int)std::round(v * tis);
+                    q = q > 7 ? 7 : (q < -8 ? -8 : q);
+                    t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q;
+                }
+            }
+        }
+        it = host_i8_cache_.emplace(wkey, std::move(t)).first;
+    }
+    const HostI8Tensor& t = it->second;
+    std::vector<int8_t> a8((size_t)K);
+    for (int m = 0; m < M; m++) {
+        const float* a = A + (size_t)m * (size_t)a_k;
+        float amax = 0.0f;
+        for (int k = 0; k < K; k++) {
+            const float v = std::fabs(a[k]);
+            if (std::isfinite(v) && v > amax) amax = v;
+        }
+        const float ascale = (amax >= 1e-12f) ? amax / 127.0f : 1.0f;
+        const float tis = 1.0f / ascale;
+        for (int k = 0; k < K; k++) {
+            float v = a[k];
+            if (!std::isfinite(v)) v = 0.0f;
+            int q = (int)std::round(v * tis);
+            a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q));
+        }
+        q4nx_host_gemm_i8_grouped(a8.data(), K, t.codes.data(), t.d.data(), t.groups, N, ascale,
+                                  out + (size_t)m * (size_t)N);
+    }
+}
+
 void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const float* Bmat,
                               std::vector<float>& out, int wkey, const char* sc, int a_k) {
     if (host_gemm_) {
@@ -3033,6 +3099,10 @@ void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const floa
         // unpadded A. (In review of this PR the split moved the pad into the target and left this
         // branch reading past the caller's buffer at a padded design.)
         out.assign((size_t)N, 0.0f);
+        if (host_gemm_i8_) {
+            host_gemm_i8(A, a_k, 1, K, N, Bmat, wkey, out.data());
+            return;
+        }
         q4nx_host_gemm_f32_padded(A, a_k, 1, K, N, Bmat, out.data());
         return;
     }
@@ -3052,6 +3122,10 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N
         // Same pad, and here it is also a stride: the rows form used to index A at `m * K` while the
         // caller's buffer is laid out at `m * a_k`, so every row past the first read the wrong row.
         out.assign((size_t)M * N, 0.0f);
+        if (host_gemm_i8_) {
+            host_gemm_i8(A, a_k, M, K, N, Bmat, wkey, out.data());
+            return;
+        }
         q4nx_host_gemm_f32_padded(A, a_k, M, K, N, Bmat, out.data());
         return;
     }

npu/forward/src/q4nx_pack.c

@@ -132,6 +132,95 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) {
     }
 }
 
+/* The integer arm of the same contract (engine#1808): the activation is already quantised to int8 by the
+ * engine's own rule (q4nx_quant_i8::row, which is byte-identical to its scalar reference), the weights are
+ * the container's int4 codes as the importer packed them, and the accumulation is EXACT int32 -- no float
+ * anywhere. That is what the kernel computes, so this is the only host reference a bit-exactness gate can
+ * be written against: the f32 path (below) cannot be bit-exact with int8 by construction.
+ *
+ * out32[n] = sum_k codes[k][n] * a8[k],  codes in [-8, 7], a8 in [-127, 127].
+ * Scaling (per-128-K group multiplier and the per-row activation scale) is deliberately NOT applied here:
+ * the gate compares the int32 exactly, and applies scales on both sides from the same inputs, so a scale
+ * mistake cannot hide inside the comparison. */
+/* The whole GEMM in int8 x int4, for the gate's `NPU_INFER_HOST_GEMM=i8` arm: quantise the activation row
+ * and every weight column with the engine's own rule, accumulate exactly, and hand back the int32 -- the
+ * caller applies the multiplier and the activation scale, the same two factors the kernel's epilogue uses.
+ *
+ * LIMIT, stated because it decides what this arm can and cannot prove: the weights are quantised HERE from
+ * the dequantised f32 matrix, not read from the container's packed b8 image. So this arm isolates the
+ * kernel's ACCUMULATION and epilogue from its weight-layout handling: a wrong tile order will not show up
+ * here (the f32 arm covers that by construction), but a wrong int4 quantisation, a wrong per-128-K group
+ * boundary, an int8 intermediate or a saturated accumulator will. That is the split the MoE oracle already
+ * uses (`expected_int32` with the scales factored out), and it is what makes an int32 bit-exact comparison
+ * meaningful at all. */
+/* The per-call cost that matters (engine#1809, measured): quantising the WEIGHTS inside the GEMM made one 7B
+ * token ~7.5 G scalar operations, so the arm never finished a token in a 9-minute ticket. The weights are the
+ * same matrix on every call, so they are quantised once per tensor by the caller and reuse this form:
+ *
+ *   out[n] = ascale * sum_g d[g][n] * ( sum_{k in group g} code[k][n] * a8[k] )
+ *
+ * exactly the kernel's epilogue, with the inner sum in int32 and no float in it. `d` is the per-128-K-group
+ * multiplier per column ([groups][N]), `ascale` the activation row's scale. */
+void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d,
+                               int groups, int N, float ascale, float* out) {
+    const int G = 128;
+    for (int n = 0; n < N; n++) out[n] = 0.0f;
+    for (int g = 0; g < groups; g++) {
+        const int k0 = g * G, k1 = (k0 + G < K) ? k0 + G : K;
+        for (int n = 0; n < N; n++) {
+            int32_t acc = 0;
+            for (int k = k0; k < k1; k++) acc += (int32_t)((int)codes[(size_t)k * (size_t)N + (size_t)n] * (int)a8[k]);
+            out[n] += d[(size_t)g * (size_t)N + (size_t)n] * (ascale * (float)acc);
+        }
+    }
+}
+
+void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, const float* Bmat,
+                                int32_t* out32) {
+    const int G = 128;                       /* the arena's multiplier group: one d per 128 K-rows per column */
+    const int kg = (K + G - 1) / G;
+    for (int m = 0; m < M; m++) {
+        const float* a = A + (size_t)m * (size_t)a_k;
+        int32_t* out = out32 + (size_t)m * (size_t)N;
+        for (int n = 0; n < N; n++) out[n] = 0;
+        for (int g = 0; g < kg; g++) {
+            const int k0 = g * G, k1 = (k0 + G < K) ? k0 + G : K;
+            /* the activation's own per-row scale is the caller's; here only the integer codes matter, and the
+             * kernel receives them already scaled, so quantise with a per-group unit scale to keep the
+             * comparison about the accumulation rather than about a float that both sides share */
+            for (int n = 0; n < N; n++) {
+                float amax = 0.0f;
+                for (int k = k0; k < k1; k++) {
+                    const float v = fabsf(Bmat[(size_t)k * (size_t)N + (size_t)n]);
+                    if (v > amax && isfinite(v)) amax = v;
+                }
+                if (!(amax >= 1e-12f)) amax = 1.0f;
+                const float tis = 127.0f / amax;
+                for (int k = k0; k < k1; k++) {
+                    float v = Bmat[(size_t)k * (size_t)N + (size_t)n];
+                    if (!isfinite(v)) v = 0.0f;
+                    int q = (int)roundf(v * tis);
+                    if (q > 7) q = 7;
+                    else if (q < -8) q = -8;    /* int4 codes are [-8, 7] in this format */
+                    int aq = (int)roundf(a[k] * 127.0f);
+                    if (aq > 127) aq = 127;
+                    else if (aq < -127) aq = -127;
+                    out[n] += (int32_t)(q * aq);
+                }
+            }
+        }
+    }
+}
+
+void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32) {
+    for (int n = 0; n < N; n++) out32[n] = 0;
+    for (int k = 0; k < K; k++) {
+        const int a = (int)a8[k];
+        const int8_t* row = codes + (size_t)k * (size_t)N;
+        for (int n = 0; n < N; n++) out32[n] += (int32_t)((int)row[n] * a);
+    }
+}
+
 void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, const float* Bmat,
                               float* out) {
     if (!A || !Bmat || !out || M <= 0 || K <= 0 || N <= 0) return;