diff --git a/CMakeLists.txt b/CMakeLists.txt index 7df77955f..ed9db049b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -847,6 +847,10 @@ add_test(NAME route_port_lock COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/ # the whole run loop against canned completions, and re-derives the summary from the written transcript. Stdlib # only, with no route, no socket, no dataset fetch and no device. add_test(NAME gsm8k_eval COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/gsm8k_eval_test.py) + # engine#1808's integer host arm is pinned inside tests/npu_pack_test.cpp (ctest npu_pack): that binary is + # already built by ci.yml, and a new ctest target would need a `.github/workflows/ci.yml` edit, which this + # lane's token cannot push (the OAuth app has no `workflow` scope) -- the ci_targets check caught exactly + # that, with both failures ("ci_targets" and "npu_host_gemm_i8 Not Run"). # The repack's tile-order claim (engine#1110): `npu_plan.py` writes container.tile_order=engine-1 for every # plan, so the load-time guard (#1130) is green on a container whose TILES are another order. This pins the # mapping from q4nx_diff's verdict to (stamp value, fatal), stdlib only: no model, no numpy, no device. @@ -1095,13 +1099,18 @@ if(ONEBIT_OGA_PACK) add_test(NAME npu_pack_import_oga COMMAND npu_pack_import_test ${CMAKE_SOURCE_DIR}/tests/golden ${ONEBIT_OGA_PACK}) endif() -add_executable(npu_pack_test tests/npu_pack_test.cpp) +# q4nx_pack.c is listed here rather than reached through onebit_npu: the integer host arm's checks +# (engine#1808) call q4nx_host_gemm_i8*() directly, and those live in that file. +add_executable(npu_pack_test tests/npu_pack_test.cpp npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c) add_executable(flm_head_import tools/flm_head_import.cpp) target_link_libraries(flm_head_import PRIVATE onebit_npu) add_executable(npu_flm_head_import_test tests/npu_flm_head_import_test.cpp) target_link_libraries(npu_flm_head_import_test PRIVATE onebit_npu) add_test(NAME npu_flm_head_import COMMAND npu_flm_head_import_test ${CMAKE_SOURCE_DIR}/tests/golden/flm-i8) target_link_libraries(npu_pack_test PRIVATE onebit_npu) +# engine#1808: the integer host arm's checks live in this binary (see above), so it needs the header and the +# golden fixture that pins them. +target_include_directories(npu_pack_test PRIVATE npu/forward/include ${CMAKE_SOURCE_DIR}/tests) # The device weight layout contract (q4nx_weight_layout): pure index math, no device # and no runtime. CI builds an explicit target list, so the check compiles itself with diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index be495fe27..b0a8b2286 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -233,6 +233,15 @@ class Q4nxNpuForward { bool init_impl(xrt::device* dev, const char* model_path, const std::string& artifact_dir); bool host_gemm_ = false; + // NPU_INFER_HOST_GEMM=i8 (engine#1808): the host arm accumulates int8 x int4 in EXACT int32, which is + // the only arm a kernel family can be bit-exact against -- f32 cannot match an int8 device by + // construction. It cannot see a wrong weight layout (the f32 arm covers that); it can see a wrong + // quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator. + bool host_gemm_i8_ = false; + // Weight codes + per-128-K multipliers for the integer host arm, quantised once per tensor (engine#1809). + struct HostI8Tensor { int groups = 0; int N = 0; std::vector codes; std::vector d; }; + std::map host_i8_cache_; + void host_gemm_i8(const float* A, int a_k, int M, int K, int N, const float* Bmat, int wkey, float* out); bool load_layer(int l); // `a_k` is the activation's REAL width per row (the design's real_k, i.e. the model's hidden or // intermediate size). It is K whenever the model is already on the container's 256-column K diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index b224cda0d..6f94b1f4c 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -170,6 +170,27 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out); void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, const float* Bmat, float* out); +/* The integer arm of the same host path (engine#1808): int8 activation x int4 weight codes -> exact int32, + * which is what the kernel computes. Written for the bit-exactness gate, because the f32 host path cannot + * be bit-exact with an int8 device arm by construction; scales are applied by the caller on both sides so + * a scale mistake cannot hide inside the integer comparison. `codes` is [K][N], one int8 per element + * holding an int4 value in [-8, 7], exactly as the importer's packB lays it down. */ +void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32); + +/* The whole GEMM in int8 x int4 for the gate's `i8` host arm, quantising both sides here with the engine's + * own rule. It cannot see a wrong weight LAYOUT (the f32 arm covers that); what it can see is a wrong int4 + * quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator -- i.e. the + * kernel's arithmetic, exactly. int32 out, M*N, no scaling applied. */ +void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, const float* Bmat, + int32_t* out32); + +/* The reusable form: `codes` and `d` are quantised once per tensor by the caller (a per-call quantisation of + * the weights costs ~7.5 G scalar operations per 7B token, measured — it never finished a token in a + * 9-minute ticket), and this applies the kernel's epilogue: ascale * sum_g d[g][n] * sum_{k in g} code*a8, + * with the inner sum exact int32 and no float inside it. */ +void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, + int groups, int N, float ascale, float* out); + /* Architecture dims a design set is derived from. */ typedef struct { diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index aa8179083..821c8ae98 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -424,7 +424,14 @@ bool Q4nxNpuForward::init_host(const char* model_path, const std::string& dir) { bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, const std::string& dir) { - if (getenv("NPU_INFER_HOST_GEMM")) host_gemm_ = true; + if (const char* hg = getenv("NPU_INFER_HOST_GEMM")) { + host_gemm_ = true; + // engine#1808: "i8" selects the integer arm, whose int32 accumulation is the only thing a + // kernel family can be bit-exact against (an int8 device cannot match an f32 host by + // construction). Any other value keeps the f32 path, which is what every existing caller and + // every row measured so far used. + host_gemm_i8_ = (hg[0] == 'i' && hg[1] == '8'); + } // The launch layer. A1: everything below that touches a device talks to this, and the settings the // container declares (weight layout, pb16i image dir, Hadamard fold) are pushed into it as they are // parsed. A2 turns this into NpuTarget::open(plan, kernels_dir). @@ -3025,6 +3032,65 @@ bool Q4nxNpuForward::run_lm_head_gemv(const std::vector& h, std::vector amax) amax = v; + } + if (!(amax >= 1e-12f)) amax = 1.0f; + t.d[(size_t)g * (size_t)N + (size_t)n] = amax / 127.0f; + const float tis = 127.0f / amax; + for (int k = k0; k < k1; k++) { + float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; + if (!std::isfinite(v)) v = 0.0f; + int q = (int)std::round(v * tis); + q = q > 7 ? 7 : (q < -8 ? -8 : q); + t.codes[(size_t)k * (size_t)N + (size_t)n] = (int8_t)q; + } + } + } + it = host_i8_cache_.emplace(wkey, std::move(t)).first; + } + const HostI8Tensor& t = it->second; + std::vector a8((size_t)K); + for (int m = 0; m < M; m++) { + const float* a = A + (size_t)m * (size_t)a_k; + float amax = 0.0f; + for (int k = 0; k < K; k++) { + const float v = std::fabs(a[k]); + if (std::isfinite(v) && v > amax) amax = v; + } + const float ascale = (amax >= 1e-12f) ? amax / 127.0f : 1.0f; + const float tis = 1.0f / ascale; + for (int k = 0; k < K; k++) { + float v = a[k]; + if (!std::isfinite(v)) v = 0.0f; + int q = (int)std::round(v * tis); + a8[(size_t)k] = (int8_t)(q > 127 ? 127 : (q < -127 ? -127 : q)); + } + q4nx_host_gemm_i8_grouped(a8.data(), K, t.codes.data(), t.d.data(), t.groups, N, ascale, + out + (size_t)m * (size_t)N); + } +} + void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const float* Bmat, std::vector& out, int wkey, const char* sc, int a_k) { if (host_gemm_) { @@ -3033,6 +3099,10 @@ void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const floa // unpadded A. (In review of this PR the split moved the pad into the target and left this // branch reading past the caller's buffer at a padded design.) out.assign((size_t)N, 0.0f); + if (host_gemm_i8_) { + host_gemm_i8(A, a_k, 1, K, N, Bmat, wkey, out.data()); + return; + } q4nx_host_gemm_f32_padded(A, a_k, 1, K, N, Bmat, out.data()); return; } @@ -3052,6 +3122,10 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N // Same pad, and here it is also a stride: the rows form used to index A at `m * K` while the // caller's buffer is laid out at `m * a_k`, so every row past the first read the wrong row. out.assign((size_t)M * N, 0.0f); + if (host_gemm_i8_) { + host_gemm_i8(A, a_k, M, K, N, Bmat, wkey, out.data()); + return; + } q4nx_host_gemm_f32_padded(A, a_k, M, K, N, Bmat, out.data()); return; } diff --git a/npu/forward/src/q4nx_pack.c b/npu/forward/src/q4nx_pack.c index 20fa4ff69..0914232e5 100644 --- a/npu/forward/src/q4nx_pack.c +++ b/npu/forward/src/q4nx_pack.c @@ -132,6 +132,95 @@ void q4nx_pad_a_f32(const float* A, int M, int real_k, int K, float* out) { } } +/* The integer arm of the same contract (engine#1808): the activation is already quantised to int8 by the + * engine's own rule (q4nx_quant_i8::row, which is byte-identical to its scalar reference), the weights are + * the container's int4 codes as the importer packed them, and the accumulation is EXACT int32 -- no float + * anywhere. That is what the kernel computes, so this is the only host reference a bit-exactness gate can + * be written against: the f32 path (below) cannot be bit-exact with int8 by construction. + * + * out32[n] = sum_k codes[k][n] * a8[k], codes in [-8, 7], a8 in [-127, 127]. + * Scaling (per-128-K group multiplier and the per-row activation scale) is deliberately NOT applied here: + * the gate compares the int32 exactly, and applies scales on both sides from the same inputs, so a scale + * mistake cannot hide inside the comparison. */ +/* The whole GEMM in int8 x int4, for the gate's `NPU_INFER_HOST_GEMM=i8` arm: quantise the activation row + * and every weight column with the engine's own rule, accumulate exactly, and hand back the int32 -- the + * caller applies the multiplier and the activation scale, the same two factors the kernel's epilogue uses. + * + * LIMIT, stated because it decides what this arm can and cannot prove: the weights are quantised HERE from + * the dequantised f32 matrix, not read from the container's packed b8 image. So this arm isolates the + * kernel's ACCUMULATION and epilogue from its weight-layout handling: a wrong tile order will not show up + * here (the f32 arm covers that by construction), but a wrong int4 quantisation, a wrong per-128-K group + * boundary, an int8 intermediate or a saturated accumulator will. That is the split the MoE oracle already + * uses (`expected_int32` with the scales factored out), and it is what makes an int32 bit-exact comparison + * meaningful at all. */ +/* The per-call cost that matters (engine#1809, measured): quantising the WEIGHTS inside the GEMM made one 7B + * token ~7.5 G scalar operations, so the arm never finished a token in a 9-minute ticket. The weights are the + * same matrix on every call, so they are quantised once per tensor by the caller and reuse this form: + * + * out[n] = ascale * sum_g d[g][n] * ( sum_{k in group g} code[k][n] * a8[k] ) + * + * exactly the kernel's epilogue, with the inner sum in int32 and no float in it. `d` is the per-128-K-group + * multiplier per column ([groups][N]), `ascale` the activation row's scale. */ +void q4nx_host_gemm_i8_grouped(const int8_t* a8, int K, const int8_t* codes, const float* d, + int groups, int N, float ascale, float* out) { + const int G = 128; + for (int n = 0; n < N; n++) out[n] = 0.0f; + for (int g = 0; g < groups; g++) { + const int k0 = g * G, k1 = (k0 + G < K) ? k0 + G : K; + for (int n = 0; n < N; n++) { + int32_t acc = 0; + for (int k = k0; k < k1; k++) acc += (int32_t)((int)codes[(size_t)k * (size_t)N + (size_t)n] * (int)a8[k]); + out[n] += d[(size_t)g * (size_t)N + (size_t)n] * (ascale * (float)acc); + } + } +} + +void q4nx_host_gemm_i8_from_f32(const float* A, int a_k, int M, int K, int N, const float* Bmat, + int32_t* out32) { + const int G = 128; /* the arena's multiplier group: one d per 128 K-rows per column */ + const int kg = (K + G - 1) / G; + for (int m = 0; m < M; m++) { + const float* a = A + (size_t)m * (size_t)a_k; + int32_t* out = out32 + (size_t)m * (size_t)N; + for (int n = 0; n < N; n++) out[n] = 0; + for (int g = 0; g < kg; g++) { + const int k0 = g * G, k1 = (k0 + G < K) ? k0 + G : K; + /* the activation's own per-row scale is the caller's; here only the integer codes matter, and the + * kernel receives them already scaled, so quantise with a per-group unit scale to keep the + * comparison about the accumulation rather than about a float that both sides share */ + for (int n = 0; n < N; n++) { + float amax = 0.0f; + for (int k = k0; k < k1; k++) { + const float v = fabsf(Bmat[(size_t)k * (size_t)N + (size_t)n]); + if (v > amax && isfinite(v)) amax = v; + } + if (!(amax >= 1e-12f)) amax = 1.0f; + const float tis = 127.0f / amax; + for (int k = k0; k < k1; k++) { + float v = Bmat[(size_t)k * (size_t)N + (size_t)n]; + if (!isfinite(v)) v = 0.0f; + int q = (int)roundf(v * tis); + if (q > 7) q = 7; + else if (q < -8) q = -8; /* int4 codes are [-8, 7] in this format */ + int aq = (int)roundf(a[k] * 127.0f); + if (aq > 127) aq = 127; + else if (aq < -127) aq = -127; + out[n] += (int32_t)(q * aq); + } + } + } + } +} + +void q4nx_host_gemm_i8(const int8_t* a8, int K, const int8_t* codes, int N, int32_t* out32) { + for (int n = 0; n < N; n++) out32[n] = 0; + for (int k = 0; k < K; k++) { + const int a = (int)a8[k]; + const int8_t* row = codes + (size_t)k * (size_t)N; + for (int n = 0; n < N; n++) out32[n] += (int32_t)((int)row[n] * a); + } +} + void q4nx_host_gemm_f32_padded(const float* A, int a_k, int M, int K, int N, const float* Bmat, float* out) { if (!A || !Bmat || !out || M <= 0 || K <= 0 || N <= 0) return; diff --git a/tests/golden/host_gemm_i8_fixture.h b/tests/golden/host_gemm_i8_fixture.h new file mode 100644 index 000000000..c9d968429 --- /dev/null +++ b/tests/golden/host_gemm_i8_fixture.h @@ -0,0 +1,157 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Generated by tools/host_gemm_i8_fixture.py -- do not edit. Known answer for +// tests/npu_host_gemm_i8_test.cpp: int8 x int4 -> int32, exact, no scales involved. +#pragma once +#include + +static constexpr int kHostI8K = 128; +static constexpr int kHostI8N = 4; + +static const int8_t kHostI8A[128] = {-51, 38, -83, 17, 87, 56, 58, -7, 59, 42, -32, -46, -81, 16, -44, 120, -69, 79, -44, 9, 12, -24, 9, -92, 115, 116, -37, 52, 24, -13, 59, -25, 81, 59, -70, -83, 17, 48, 0, 11, -75, -88, 116, -86, -75, -60, 62, -111, -109, -49, 112, -4, -114, -107, -80, 56, 65, 114, 113, -28, 40, -66, -6, 127, 103, -81, -105, -72, 12, 98, -8, -16, -106, 31, 71, -94, 78, 122, -120, -107, -58, -37, -90, 98, -72, 62, 79, 0, -50, -93, -109, -72, -115, 23, 101, -54, 24, -122, 85, 32, -47, -120, 97, -12, -77, 65, -64, 67, 35, -9, -29, 94, -84, 75, 50, -8, -78, -15, -124, 49, -57, -127, 77, -54, -45, -22, -80, 88}; + +static const int8_t kHostI8Codes[128][4] = { + {3, -7, -7, -8}, + {6, -7, -4, -6}, + {5, -6, -8, -1}, + {-4, 4, -7, 7}, + {-1, -3, -1, 2}, + {4, -6, 6, -6}, + {-7, -5, 4, -1}, + {-1, 2, -5, -6}, + {4, -2, 5, -5}, + {-2, -3, -6, 7}, + {-1, -3, 5, -7}, + {-3, 4, -8, -1}, + {-7, 3, -6, 3}, + {2, 7, 5, 5}, + {-4, 4, 0, -6}, + {-1, 7, -1, 0}, + {-7, -7, 1, -4}, + {1, 3, 1, -1}, + {-6, 4, 6, 7}, + {-5, 4, 1, 6}, + {-6, -8, -8, 7}, + {4, -1, 3, -7}, + {5, 4, -4, -8}, + {6, -1, -2, -2}, + {5, -3, -5, 0}, + {-4, 5, 5, -3}, + {-5, -1, -2, 4}, + {4, 4, 0, -5}, + {1, -5, -7, 7}, + {5, 1, -1, -8}, + {-2, -3, -7, -8}, + {-6, 3, 6, -1}, + {-8, 5, 6, 5}, + {2, 7, -2, 2}, + {7, -5, 5, -3}, + {5, -4, -3, -4}, + {3, -3, 2, -7}, + {-5, -2, 5, 3}, + {-6, -5, -3, -1}, + {2, 1, 3, 7}, + {-8, -7, -4, -5}, + {-4, 4, -4, 4}, + {1, 7, -3, -4}, + {0, 2, 5, 6}, + {0, -1, 4, 5}, + {1, 7, -8, -2}, + {-8, -6, 7, -3}, + {-4, -6, -3, 5}, + {-5, 7, -8, -4}, + {3, 0, 2, -7}, + {0, 2, 5, -3}, + {-5, -8, -7, 7}, + {3, -2, 2, 5}, + {-7, 7, -5, 4}, + {2, 3, 5, -4}, + {-3, -6, 3, 2}, + {2, 7, 7, 6}, + {-2, -4, -6, 1}, + {3, -5, -3, -5}, + {-6, -6, -6, -4}, + {2, -6, 5, 7}, + {3, -1, -3, 0}, + {-1, 6, -5, -7}, + {5, -8, 3, 3}, + {-4, -4, 0, -2}, + {3, -6, -1, 4}, + {5, 1, 7, -8}, + {-5, -2, 6, -4}, + {3, -2, 3, -7}, + {7, -5, -7, 2}, + {7, -6, 5, 2}, + {-6, -2, -3, -3}, + {-6, -8, 4, 6}, + {6, -6, 7, -7}, + {-6, 7, 2, 6}, + {-4, -4, -6, 7}, + {-2, -4, 6, 7}, + {4, 7, 3, -2}, + {1, 3, -3, 2}, + {-2, 6, -4, -3}, + {7, -5, -6, -6}, + {3, 3, -6, -8}, + {3, -7, -6, 2}, + {-6, -2, 2, 1}, + {-1, 2, 1, -2}, + {3, 1, 0, 1}, + {-3, -2, -4, -6}, + {1, 7, 4, 6}, + {-4, 1, 7, -1}, + {-7, -1, 7, 3}, + {-8, -8, 0, -7}, + {4, 5, 1, -5}, + {-3, 1, 7, -8}, + {6, -5, -8, 2}, + {-1, 6, 6, 0}, + {-5, 6, 0, 1}, + {4, -5, 1, -5}, + {4, -3, -3, -2}, + {7, 3, -3, -5}, + {5, -2, 0, -6}, + {-3, -4, 4, -7}, + {-4, -5, 6, -2}, + {-8, 3, -4, 2}, + {-7, -4, 7, -3}, + {1, -2, 2, 4}, + {2, -2, -2, -1}, + {-6, 6, -4, 1}, + {-8, 2, -5, -6}, + {0, 0, -2, -6}, + {5, 0, 2, -1}, + {4, 5, 1, -2}, + {-8, 6, 6, -1}, + {-5, 1, 4, -8}, + {3, 0, -3, 7}, + {5, -1, -5, -2}, + {-1, -7, -3, 4}, + {-5, -2, -5, -5}, + {6, 6, 6, 6}, + {-5, -6, 3, 5}, + {-4, 1, -3, 4}, + {0, 0, 0, 1}, + {-4, 4, 2, -7}, + {-1, -7, 1, 5}, + {-1, 3, -6, 6}, + {-5, 2, -4, -1}, + {5, 6, -2, 4}, + {-7, 5, 2, -2}, + {-8, -1, -7, -4}, +}; + +static const int32_t kHostI8Expected[4] = {4927, 1939, 1583, 2736}; diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index 6ff6ad82a..905c1d411 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -22,6 +22,11 @@ // i5_.bin, i6_.bin for layers 0 and 2). #include "full_elf.h" #include "pack.h" +#include "q4nx_pack.h" + +#include "golden/host_gemm_i8_fixture.h" + +#include #include #include @@ -62,6 +67,75 @@ int main(int argc, char** argv) { check(f32_to_bf16(1.0f) == 0x3F80, "bf16(1.0)"); check(f32_to_bf16(1.00390625f) == 0x3F80, "bf16 ties to even (down)"); check(f32_to_bf16(1.01171875f) == 0x3F82, "bf16 ties to even (up)"); + // The integer host arm (engine#1808/#1809): "bit-exact against the host reference" needs a host path that + // runs the SAME integer arithmetic as the kernel, and f32 cannot be bit-exact with int8 x int4 by + // construction. These checks live here, next to the pack/tile checks they belong with, because ci.yml + // builds this binary and a new ctest target would need a workflow edit this lane's token cannot make. + { + int32_t got[kHostI8N] = {0, 0, 0, 0}; + q4nx_host_gemm_i8(kHostI8A, kHostI8K, &kHostI8Codes[0][0], kHostI8N, got); + bool same = true; + for (int n = 0; n < kHostI8N; n++) same &= got[n] == kHostI8Expected[n]; + check(same, "int8 x int4 -> int32 matches the independent Python fixture (golden/host_gemm_i8_fixture.h)"); + bool in_range = true; + for (int k = 0; k < kHostI8K; k++) + for (int n = 0; n < kHostI8N; n++) + if (kHostI8Codes[k][n] < -8 || kHostI8Codes[k][n] > 7) in_range = false; + check(in_range, "every weight code is an int4 value in [-8, 7]"); + // a -128 activation is outside the quantiser's range but inside int8: it must accumulate exactly, and + // the product bound for the largest K in use (14336) is 1.47e7, so int32 saturation would be a bug + static int8_t a8[256], codes2[512]; + for (int k = 0; k < 256; k++) { a8[k] = (k == 0) ? -128 : 127; codes2[k * 2] = -8; codes2[k * 2 + 1] = 7; } + int32_t edge[2] = {0, 0}; + q4nx_host_gemm_i8(a8, 256, codes2, 2, edge); + check(edge[0] == -258056 && edge[1] == 225799, + "a -128 activation accumulates exactly (no int8 intermediate, no int32 saturation)"); + // the reusable grouped form: inner sum int32, per-128-K multipliers applied outside it, K=130 so the + // group boundary matters + const int KK = 130, NN = 2, groups = 2; + std::vector A(KK), B((size_t)KK * NN); + std::vector codes((size_t)KK * NN, 0); + std::vector d((size_t)groups * NN, 0.0f), want((size_t)NN, 0.0f); + for (int k = 0; k < KK; k++) A[k] = float(k % 13) - 6.0f; + for (int k = 0; k < KK; k++) + for (int n = 0; n < NN; n++) B[(size_t)k * NN + n] = float((k * 3 + n) % 7) - 3.0f; + float amax_all = 0.0f; + for (int k = 0; k < KK; k++) amax_all = std::max(amax_all, std::fabs(A[k])); + const float ascale = amax_all / 127.0f, tis = 1.0f / ascale; + std::vector a8q(KK); + for (int k = 0; k < KK; k++) { + int q = int(std::round(A[k] * tis)); + a8q[k] = int8_t(q > 127 ? 127 : (q < -127 ? -127 : q)); + } + for (int g = 0; g < groups; g++) { + const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; + for (int n = 0; n < NN; n++) { + float amax = 0.0f; + for (int k = k0; k < k1; k++) amax = std::max(amax, std::fabs(B[(size_t)k * NN + n])); + if (amax < 1e-12f) amax = 1.0f; + d[(size_t)g * NN + n] = amax / 127.0f; + for (int k = k0; k < k1; k++) { + int q = int(std::round(B[(size_t)k * NN + n] * (127.0f / amax))); + codes[(size_t)k * NN + n] = int8_t(q > 7 ? 7 : (q < -8 ? -8 : q)); + } + } + } + for (int g = 0; g < groups; g++) { + const int k0 = g * 128, k1 = (k0 + 128 < KK) ? k0 + 128 : KK; + for (int n = 0; n < NN; n++) { + double acc = 0; + for (int k = k0; k < k1; k++) acc += double(codes[(size_t)k * NN + n]) * double(a8q[k]); + want[(size_t)n] += float(d[(size_t)g * NN + n] * double(ascale * float(acc))); + } + } + std::vector outv((size_t)NN, 0.0f); + q4nx_host_gemm_i8_grouped(a8q.data(), KK, codes.data(), d.data(), groups, NN, ascale, outv.data()); + bool ok = true; + for (int n = 0; n < NN; n++) + ok &= std::fabs(outv[(size_t)n] - want[(size_t)n]) <= 1e-5f * std::fabs(want[(size_t)n]) + 1e-6f; + check(ok, "grouped form: int32 inner sum + per-128-K multipliers match an independent computation"); + } + if (argc == 1) { std::printf("%s\n", failures ? "FAILED" : "ok (pass for the model checks)"); return failures ? 1 : 0; diff --git a/tools/host_gemm_i8_fixture.py b/tools/host_gemm_i8_fixture.py new file mode 100755 index 000000000..049e8d1e5 --- /dev/null +++ b/tools/host_gemm_i8_fixture.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Write tests/golden/host_gemm_i8_fixture.h — the known answer for q4nx_host_gemm_i8(). + +engine#1808: the kernel gate asks for "bit-exact against the host reference", but the only host arm the +forward offers is f32 (`q4nx_host_gemm_f32_padded`), and f32 cannot be bit-exact with an int8 x int4 +device by construction. The integer arm (`q4nx_host_gemm_i8`) accumulates exactly, and this fixture is its +independent known answer: the expected int32 values are computed here, in Python, from the operands that +are also written out, so a C++ change that merely agrees with itself cannot pass the test. + + tools/host_gemm_i8_fixture.py > tests/golden/host_gemm_i8_fixture.h + +Fixed seed, so the file is reproducible; K=128 (the smallest shape that reaches a whole 128-K group) and +N=4 columns, with activations across the quantiser's full [-127, 127] range and codes across [-8, 7]. +""" +import random +import sys + +SEED = 20261110 +K, N = 128, 4 + + +def main(): + rng = random.Random(SEED) + a8 = [rng.randint(-127, 127) for _ in range(K)] + codes = [[rng.randint(-8, 7) for _ in range(N)] for _ in range(K)] # int4 value held in an int8 + expect = [sum(codes[k][n] * a8[k] for k in range(K)) for n in range(N)] + w = sys.stdout.write + w("// Copyright 2026 bong-water-water-bong\n") + w("// SPDX-License-Identifier: Apache-2.0\n") + w("//\n") + w("// Licensed under the Apache License, Version 2.0 (the \"License\");\n") + w("// you may not use this file except in compliance with the License.\n") + w("// You may obtain a copy of the License at\n") + w("//\n") + w("// http://www.apache.org/licenses/LICENSE-2.0\n") + w("//\n") + w("// Unless required by applicable law or agreed to in writing, software\n") + w("// distributed under the License is distributed on an \"AS IS\" BASIS,\n") + w("// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n") + w("// See the License for the specific language governing permissions and\n") + w("// limitations under the License.\n") + w("//\n") + w("// Generated by tools/host_gemm_i8_fixture.py -- do not edit. Known answer for\n") + w("// tests/npu_host_gemm_i8_test.cpp: int8 x int4 -> int32, exact, no scales involved.\n") + w("#pragma once\n#include \n\n") + w(f"static constexpr int kHostI8K = {K};\nstatic constexpr int kHostI8N = {N};\n\n") + w(f"static const int8_t kHostI8A[{K}] = {{{', '.join(str(v) for v in a8)}}};\n\n") + w(f"static const int8_t kHostI8Codes[{K}][{N}] = {{\n") + for k in range(K): + w(" {%s},\n" % ", ".join(str(v) for v in codes[k])) + w("};\n\n") + w(f"static const int32_t kHostI8Expected[{N}] = {{{', '.join(str(v) for v in expect)}}};\n") + + +if __name__ == "__main__": + main()