diff --git a/CMakeLists.txt b/CMakeLists.txt index 1985ce08f..05924ea8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1106,7 +1106,10 @@ add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp) target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) add_test(NAME npu_q4nx COMMAND npu_q4nx_test ${CMAKE_SOURCE_DIR}/tests/golden/q4nx) -add_executable(npu_pack_import_test tests/npu_pack_import_test.cpp) +# npu/forward/src/q4nx_dequant.c carries q4nx_dequant_tile_row, which the external-data writer's success-path arm +# reads a written tile back with (engine#1849). Sources added to an existing target, so ci_targets sees no new test. +add_executable(npu_pack_import_test tests/npu_pack_import_test.cpp npu/forward/src/q4nx_dequant.c) +target_include_directories(npu_pack_import_test PRIVATE npu/forward/include) target_link_libraries(npu_pack_import_test PRIVATE onebit_npu) add_test(NAME npu_pack_import COMMAND npu_pack_import_test ${CMAKE_SOURCE_DIR}/tests/golden) # engine#1726: NPU_INFER_DUMP_DIR's residual write must SAY when it cannot write -- a missing or unwritable directory, or a diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 019347612..e7f7a78ad 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -37,6 +37,7 @@ #include "pack.h" #include "pack_import.h" #include "q4nx.h" +#include "forward/include/q4nx_dequant.h" #include "q4nx_device.h" #include @@ -453,6 +454,123 @@ void test_external_packed_genre(const std::string& root) { check(g["external_row_stride"]["matched"] == 1 && g["external_row_stride"]["of"] == 1, "external-packed: the bytes fit length == N x (K/128 + 1) x 68, reported as a count"); + // engine#1849: the writer's SUCCESS path. Every other arm on this genre expects a refusal, so three merged changes + // to import_external (#1712, #1751, #1793) have only ever run against real packs on the box. + // + // Its own directory and its own geometry, because the container split has to FIT: the earlier fixture here is + // 768 wide, and a qkv split of 40 heads x 128 needs N 5120 for q alone ("container range [0, 5120) does not fit + // N 768"). With 40 heads and 40 kv heads the fused qkv is 3 x 5120 = 15360 wide, so every role's slice is 5120. + // + // The blob cannot be the all-0x22 the other fixtures write: a 68-byte stride is 64 B of codes then 4 B of zero + // point and bf16 scale, so all-0x22 makes every weight (2 - 2) x s = 0 -- and an all-zero decode is exactly what a + // broken writer produces too (engine#1871, engine#1875). These bytes decode to a value that pins all three parts: + // codes 0xCC -> nibble 12 = two's-complement int4 -4; zero point 0x11 -> four nibbles of 1; scale bf16 + // 0x4000 = 2.0; the pack's fields are SIGNED, so w = (sx(c) - sx(zp)) * s = (-4 - 1) * 2 = -10.0. + // + // This arm reads 22.0 today, and that is engine#1932, not an arithmetic choice: decode_external_tile() keeps raw + // nibbles 0..15 and container_from_external_q4() pairs them with m = -zp*s, so every code with the high bit set + // comes back +16 codes high (+16*s = +4.0 at s=0.25; +32.0 at s=2.0 here: -10 -> 22). The expectation below is the + // CORRECT value, so this arm is RED until #1932's fix (c_u = sx(c) + 8, z_c = (-8 - sx(zp)) * s) lands. Do not + // "fix" the expectation to 22.0: that turns the bug into the spec. + { + const fs::path sdir = fs::path(root) / "external-success"; + fs::remove_all(sdir); + fs::create_directories(sdir / "cache"); + const int64_t K = 5120, N = 15360, heads = 40, hd = 128; + put_file((sdir / "config.json").string(), + R"({"model_type":"phi3","hidden_size":5120,"intermediate_size":17920,"num_hidden_layers":1,)" + R"("num_attention_heads":40,"num_key_value_heads":40,"head_dim":128,"vocab_size":64,)" + R"("rope_theta":10000.0,"tie_word_embeddings":false})"); + put_file((sdir / "genai_config.json").string(), + R"({"model":{"decoder":{"filename":"eager-mm.onnx","hidden_size":5120,"num_hidden_layers":1,)" + R"("num_attention_heads":40,"num_key_value_heads":40,"head_size":128,"intermediate_size":17920}}})"); + const int64_t embed_bytes = 64 * 5120 * 4; + const std::string qc(2048, '\xCC'); + const std::string zs(64, '\x11'); + std::string sc(64, '\0'); + for (size_t i = 0; i + 1 < sc.size(); i += 2) sc[i + 1] = '\x40'; // bf16 2.0 + std::string blob; + const int64_t blocks = K / 128, per_tile = 2 * (blocks + 1); + for (int64_t tile = 0; tile < N / 64; ++tile) + for (int64_t s2 = 0; s2 < per_tile; ++s2) + blob += (s2 / 2 == 0) ? std::string(2048 + 128, '\0') : (qc + zs + sc); // kk 0 is the bias pair + put_file((sdir / "eager-mm.onnx.data").string(), std::string(size_t(embed_bytes), '\x22') + blob); + put_file((sdir / "eager-mm.onnx").string(), + pb::model({pb::tensor("model.embed_tokens.weight", {64, 5120}, "eager-mm.onnx.data", 0, + uint64_t(embed_bytes)), + pb::placeholder("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.empty"), + pb::placeholder("model.layers.0.attn.qkv_proj.MatMulNBits.scales.empty"), + pb::tensor("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.packed", {int64_t(blob.size())}, + "eager-mm.onnx.data", uint64_t(embed_bytes), uint64_t(blob.size()))}, + {pb::matmulnbits("qkv", {"x", "/model/layers.0/attn/qkv_proj/MatMulNBits.qweight.empty", + "/model/layers.0/attn/qkv_proj/MatMulNBits.scales.empty", "y"}, + K, N, 4, 128, "MatMulNBitsBf")})); + const fs::path sout = sdir / "out"; + pi::Options ok_opt; + bool imported = false; + std::string why_imp; + try { + (void)pi::import_pack(sdir.string(), sout.string(), ok_opt); + imported = true; + } catch (const std::exception& e) { why_imp = e.what(); } + check(imported, "external writer: a crafted pack with a satisfiable split imports" + (imported ? "" : " (" + why_imp + ")")); + if (imported) { + check(fs::exists(sout / "model.q4nx"), "external writer: and writes model.q4nx"); + // ALL THREE projections, not just q: container_from_external_q4() runs on the FUSED qkv (15360 rows), so a + // slot it gets wrong can land in k or v while q stays clean -- measured, doubling scale[size/2] of the + // fused pack leaves q_proj at 22.0 and lands in v_proj, which a q-only arm never sees. + for (const char* role : {"q_proj", "k_proj", "v_proj"}) { + std::ifstream f(sout / "model.q4nx", std::ios::binary); + uint64_t hlen = 0; + f.read(reinterpret_cast(&hlen), 8); + std::string hdr(static_cast(hlen), '\0'); + f.read(&hdr[0], std::streamsize(hlen)); + const json h = json::parse(hdr); + std::string name; + uint64_t off = 0, nbytes = 0, chunk = 0; + for (auto it = h.begin(); it != h.end(); ++it) { + if (!it.value().is_object() || !it.value().contains("data_offsets")) continue; + if (it.key().find(role) == std::string::npos) continue; + name = it.key(); + off = it.value()["data_offsets"][0].get(); + nbytes = it.value()["data_offsets"][1].get() - off; + if (it.value().contains("shape") && it.value()["shape"].size() >= 2) + chunk = it.value()["shape"][1].get(); + } + if (name.empty()) { check(false, std::string("external writer: the container carries ") + role); continue; } + // chunk_of() reads shape.back(), so the Tensor is shaped {ntiles, chunk_bytes} -- what the writer stores. + onebit::npu::Tensor q4f; + q4f.dtype = "I8"; + q4f.shape = {int64_t(nbytes / chunk), int64_t(chunk)}; + std::vector raw(static_cast(nbytes)); + f.seekg(std::streamoff(8 + hlen + off)); + f.read(reinterpret_cast(raw.data()), std::streamsize(raw.size())); + q4f.data = raw.data(); + q4f.bytes = q4f.avail = nbytes; + const std::vector conv = onebit::npu::q4nx::as_q4_1(q4f); + const int64_t rows = heads * hd, cols = K; + std::vector row(static_cast(cols)); + size_t bad = 0; + int64_t bad_row = -1; + for (int64_t r = 0; r < rows; ++r) { + if (q4nx_dequant_tile_row(conv.data(), conv.size(), int(rows), int(cols), Q4NX_TILE_BYTES, + int(r), row.data()) != 0) { ++bad; continue; } + for (int64_t c = 0; c < cols; ++c) + if (std::fabs(row[size_t(c)] - (-10.0f)) > 1e-6f) { ++bad; if (bad_row < 0) bad_row = r; } + } + std::printf(" writer success: %s %llu B in %llu B chunks -> %zu tiles, rows %lld\n", name.c_str(), + (unsigned long long)nbytes, (unsigned long long)chunk, + conv.size() / size_t(Q4NX_TILE_BYTES), (long long)rows); + char msg[240]; + std::snprintf(msg, sizeof msg, + "external writer: every one of %s %lld rows decodes to -10.0 ((sx(12) - sx(1)) * 2.0 = -10); %zu differ, first bad row %lld", + role, (long long)rows, bad, (long long)bad_row); + check(bad == 0, msg); + } + } + } + + // engine#1671: when the `*.packed` sibling is ALSO a placeholder (dims [0], no data), this graph carries no // weight for that matmul at all -- and the node must be DROPPED rather than left naming the `.empty` // placeholder. Left naming it, the reader calls bytes_of() on a byte-less tensor and throws