diff --git a/CMakeLists.txt b/CMakeLists.txt index 7df77955f..c2ec55326 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1250,8 +1250,10 @@ target_link_libraries(npu_derive_designs_dump PRIVATE m) # the test compares it against the two-step dequant+pack, so both live here. # q4nx_pack_blocked.cpp is here because the head-tile identity arm (engine#1716) compares the blocked packer the forward # actually calls against the reference; the test links both. It is C++, so CMake switches this target's linker to CXX. +# q4nx_head_image.c is the image cache's file format (engine#1716's cache slice, this branch): the identity arm above pins +# what the image must contain, this one pins that what is written and read back is byte-identical to it. add_executable(npu_design_grid_test tests/npu_design_grid_test.c npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c - npu/forward/src/q4nx_pack_blocked.cpp) + npu/forward/src/q4nx_pack_blocked.cpp npu/forward/src/q4nx_head_image.c) target_include_directories(npu_design_grid_test PRIVATE npu/forward/include) target_link_libraries(npu_design_grid_test PRIVATE m) add_test(NAME npu_design_grid COMMAND npu_design_grid_test) diff --git a/npu/forward/include/q4nx_head_image.h b/npu/forward/include/q4nx_head_image.h new file mode 100644 index 000000000..56d292261 --- /dev/null +++ b/npu/forward/include/q4nx_head_image.h @@ -0,0 +1,63 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The packed lm-head image (engine#1716): the bytes `I8Ctx::packB_into` writes into a weight BO, on disk. +// +// The forward packs the head per tile on EVERY serve -- `memset`, `q4nx_pack_B_int8_blocked`, a BO sync -- which the issue +// measured at 10-17 s quiet and minutes contended for the h4096 shapes. Nothing device-side enters that image, so it can be +// produced on the host, written once at import and copied on later serves. +// +// What makes that safe is the identity, not the speed. The image is written with a format version, its geometry, and a hash +// over the payload; `read` recomputes the hash and refuses rather than returning bytes it cannot vouch for. A packer change, +// a geometry change or a truncated write are therefore a loud refusal and never a plausible-looking image -- the same rule the +// .fconst canonical copy follows after engine#1662/#1669. +// +// Absence is not an error: a container without an image simply packs as it did before. +#ifndef Q4NX_HEAD_IMAGE_H +#define Q4NX_HEAD_IMAGE_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Bumped whenever the packer's output changes in any way. A reader that does not recognise the version refuses. +#define Q4NX_HEAD_IMAGE_VERSION 1u + +typedef struct { + uint32_t kd; // rows of the image (K padded, as the BO holds it) + uint32_t nd; // columns of the image + uint64_t hash; // FNV-1a over the codes and then the scales, in that order +} Q4nxHeadImageInfo; + +// Writes header + codes + scales. Returns 1 on success, 0 on failure (no partial file is left: the write goes to a +// temporary name beside `path` and is renamed into place only after it is fully written). +int q4nx_head_image_write(const char *path, const int8_t *codes, const float *scales, int kd, int nd); + +// Reads into caller-provided buffers: `codes` must hold kd*nd int8 and `scales` nd floats (the packer's contract is one +// scale per COLUMN). Returns 1 on success, 0 on refusal with `why` set -- wrong version, wrong geometry, short payload, +// trailing bytes, or a hash that does not reproduce. Never a silent success with partial data. +int q4nx_head_image_read(const char *path, int8_t *codes, int codes_n, float *scales, int scales_n, Q4nxHeadImageInfo *info, + char *why, int why_n); + +// The hash the writer would compute, so a caller that built an image itself can compare against a file it read. +uint64_t q4nx_head_image_hash(const int8_t *codes, const float *scales, int kd, int nd); + +#ifdef __cplusplus +} +#endif + +#endif // Q4NX_HEAD_IMAGE_H diff --git a/npu/forward/src/q4nx_head_image.c b/npu/forward/src/q4nx_head_image.c new file mode 100644 index 000000000..6dce8c0b9 --- /dev/null +++ b/npu/forward/src/q4nx_head_image.c @@ -0,0 +1,148 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The packed lm-head image, on disk (engine#1716). See the header for why the identity matters more than the speed: a +// head image is only ever a copy of what the packer would produce for the same tile, so the file carries a version, its +// geometry and a hash, and a reader that cannot reproduce the hash refuses instead of returning bytes it cannot vouch for. + +#include "q4nx_head_image.h" + +#include +#include + +// "Q4HI" then the version as four ASCII digits, so a hexdump of the first eight bytes says what the file is. +#define Q4NX_HEAD_IMAGE_MAGIC "Q4HI" + +typedef struct { + char magic[4]; + char version[4]; + uint32_t kd; + uint32_t nd; + uint64_t hash; +} HeadImageHeader; + +static uint64_t fnv1a(uint64_t h, const void *p, size_t n) { + const unsigned char *b = (const unsigned char *)p; + for (size_t i = 0; i < n; ++i) { + h ^= (uint64_t)b[i]; + h *= 1099511628211ull; + } + return h; +} + +uint64_t q4nx_head_image_hash(const int8_t *codes, const float *scales, int kd, int nd) { + if (kd <= 0 || nd <= 0) return 0; + uint64_t h = 1469598103934665603ull; + h = fnv1a(h, codes, (size_t)kd * (size_t)nd); + h = fnv1a(h, scales, (size_t)nd * sizeof(float)); + return h; +} + +static void version_digits(char out[4]) { + const unsigned v = Q4NX_HEAD_IMAGE_VERSION; + out[0] = (char)('0' + (v / 100) % 10); + out[1] = (char)('0' + (v / 10) % 10); + out[2] = (char)('0' + v % 10); + out[3] = 0; +} + +int q4nx_head_image_write(const char *path, const int8_t *codes, const float *scales, int kd, int nd) { + if (!path || !codes || !scales || kd <= 0 || nd <= 0) return 0; + HeadImageHeader h; + memcpy(h.magic, Q4NX_HEAD_IMAGE_MAGIC, 4); + version_digits(h.version); + h.kd = (uint32_t)kd; + h.nd = (uint32_t)nd; + h.hash = q4nx_head_image_hash(codes, scales, kd, nd); + // Written beside the destination and renamed into place: a truncated image must never be readable as a valid one, + // because the reader would then copy it into a weight BO and the model would answer wrongly with no message. + char tmp[4096]; + if (snprintf(tmp, sizeof tmp, "%s.tmp", path) <= 0) return 0; + FILE *f = fopen(tmp, "wb"); + if (!f) return 0; + int ok = fwrite(&h, sizeof h, 1, f) == 1 && fwrite(codes, (size_t)kd * (size_t)nd, 1, f) == 1 && + fwrite(scales, (size_t)nd * sizeof(float), 1, f) == 1; + ok = ok && fflush(f) == 0; + if (fclose(f) != 0) ok = 0; + if (!ok || rename(tmp, path) != 0) { + remove(tmp); + return 0; + } + return 1; +} + +int q4nx_head_image_read(const char *path, int8_t *codes, int codes_n, float *scales, int scales_n, Q4nxHeadImageInfo *info, + char *why, int why_n) { + if (why && why_n > 0) why[0] = 0; + char dv[4]; + version_digits(dv); + FILE *f = path ? fopen(path, "rb") : NULL; + if (!f) { + if (why) snprintf(why, (size_t)why_n, "the head image is not there (%s)", path ? path : "(no path)"); + return 0; + } + HeadImageHeader h; + if (fread(&h, sizeof h, 1, f) != 1) { + fclose(f); + if (why) snprintf(why, (size_t)why_n, "the head image is shorter than its header"); + return 0; + } + if (memcmp(h.magic, Q4NX_HEAD_IMAGE_MAGIC, 4) != 0) { + fclose(f); + if (why) snprintf(why, (size_t)why_n, "the head image does not start with %s", Q4NX_HEAD_IMAGE_MAGIC); + return 0; + } + if (memcmp(h.version, dv, 4) != 0) { + fclose(f); + if (why) + snprintf(why, (size_t)why_n, "the head image is format version %.4s, this build writes %s", h.version, dv); + return 0; + } + if (h.kd == 0 || h.nd == 0 || (int64_t)h.kd * (int64_t)h.nd != (int64_t)codes_n || (int)h.nd != scales_n) { + fclose(f); + if (why) + snprintf(why, (size_t)why_n, + "the head image is %ux%u (so %d codes and %u scales), not the %d x %d this container needs", + h.kd, h.nd, (int)(h.kd * h.nd), h.nd, codes_n > 0 && scales_n > 0 ? codes_n / scales_n : 0, scales_n); + return 0; + } + const size_t n_codes = (size_t)h.kd * (size_t)h.nd, n_scales = (size_t)h.nd * sizeof(float); + int ok = fread(codes, n_codes, 1, f) == 1 && fread(scales, n_scales, 1, f) == 1; + if (ok) { + // anything after the payload means the file is not the one the header describes + char extra; + ok = fread(&extra, 1, 1, f) == 0; + } + fclose(f); + if (!ok) { + if (why) snprintf(why, (size_t)why_n, "the head image payload is short or has trailing bytes"); + return 0; + } + const uint64_t got = q4nx_head_image_hash(codes, scales, (int)h.kd, (int)h.nd); + if (got != h.hash) { + if (why) + snprintf(why, (size_t)why_n, + "the head image does not match its own hash (%016llx in the file, %016llx over the bytes): it was " + "written by a different packer or damaged", + (unsigned long long)h.hash, (unsigned long long)got); + return 0; + } + if (info) { + info->kd = h.kd; + info->nd = h.nd; + info->hash = h.hash; + } + return 1; +} diff --git a/tests/npu_design_grid_test.c b/tests/npu_design_grid_test.c index 2df24e848..b92314fcd 100644 --- a/tests/npu_design_grid_test.c +++ b/tests/npu_design_grid_test.c @@ -27,14 +27,24 @@ // 1024-column tensor at 896 columns (which the chunk decoder refuses outright). #include "q4nx_dequant.h" #include "q4nx_pack.h" +#include "q4nx_head_image.h" #include #include +#include #include #include static int failures = 0; + +static void truncate_file(const char* path, long bytes) { + FILE* f = fopen(path, "r+b"); + if (!f) return; + if (ftruncate(fileno(f), bytes) != 0) { /* best effort: the arm asserts the refusal either way */ } + fclose(f); +} + static void check(int ok, const char* what) { printf("%s %s\n", ok ? "ok " : "FAIL", what); if (!ok) failures++; @@ -490,7 +500,91 @@ static void activation_pad(void) { free(src); free(dst); } + +static void test_head_image(void) { + /* THE HEAD IMAGE ROUND TRIP (engine#1716, the cache's correctness bar). The image a serve copies into the weight BO + * must be byte-identical to what the packer produces for the same tile, and a file that cannot prove it is the same + * image must be REFUSED rather than copied -- a stale or damaged image copied into a BO answers wrongly with no + * message at all, which is the failure this whole contract exists to make impossible. */ + { + const int kd = 96, nd = 64; /* the same head-tile shape the layout contract above uses */ + float* bt = malloc((size_t)kd * nd * sizeof(float)); + int8_t* codes = malloc((size_t)kd * nd); + int8_t* back = malloc((size_t)kd * nd); + float* scales = malloc((size_t)nd * sizeof(float)); + float* sc_back = malloc((size_t)nd * sizeof(float)); + check(bt && codes && back && scales && sc_back, "head image: the fixture allocates"); + if (bt && codes && back && scales && sc_back) { + for (int k = 0; k < kd; k++) + for (int n = 0; n < nd; n++) bt[(size_t)k * nd + n] = (float)((k * 29 + n * 11) % 101) * 0.013f - 0.6f; + q4nx_pack_B_int8(bt, kd, nd, codes, nd, scales); + + const char* path = "head_image_roundtrip.bin"; + remove(path); + check(q4nx_head_image_write(path, codes, scales, kd, nd) == 1, "head image: the image is written"); + Q4nxHeadImageInfo info; + char why[256]; + check(q4nx_head_image_read(path, back, kd * nd, sc_back, nd, &info, why, (int)sizeof why) == 1, + "head image: it reads back"); + check_int((int)info.kd, kd, "head image: the geometry in the header is the geometry written"); + check_int((int)info.nd, nd, "head image: and so is the column count"); + /* THE BAR: byte-identical to a fresh pack, not merely close */ + check(memcmp(codes, back, (size_t)kd * nd) == 0, "head image: the codes are byte-identical to a fresh pack"); + check(memcmp(scales, sc_back, (size_t)nd * sizeof(float)) == 0, + "head image: and so are the scales"); + + /* a caller that rebuilt the image itself can compare hashes rather than bytes */ + check(q4nx_head_image_hash(codes, scales, kd, nd) == info.hash, + "head image: the file's hash is the hash of the image the caller would build"); + + /* (a) a corrupted byte must refuse */ + { + FILE* f = fopen(path, "r+b"); + if (f) { + /* the payload starts after the 24-byte header this writer uses */ + fseek(f, 24 + 5, SEEK_SET); + int c = fgetc(f); + fseek(f, 24 + 5, SEEK_SET); + fputc((c ^ 0x5a) & 0xff, f); + fclose(f); + } + why[0] = 0; + check(q4nx_head_image_read(path, back, kd * nd, sc_back, nd, &info, why, (int)sizeof why) == 0, + "head image: a single corrupted payload byte is REFUSED, not copied"); + check(why[0] != 0 && strstr(why, "hash") != NULL, "... and the refusal says the hash did not reproduce"); + } + /* (b) the wrong geometry must refuse even when the file is intact */ + { + check(q4nx_head_image_write(path, codes, scales, kd, nd) == 1, "head image: rewritten intact"); + why[0] = 0; + check(q4nx_head_image_read(path, back, kd * nd, sc_back, nd - 1, &info, why, (int)sizeof why) == 0, + "head image: a container with a different column count refuses it"); + } + /* (c) absence is not an error at this layer: it is a refusal with a reason the caller can treat as "pack" */ + { + remove(path); + why[0] = 0; + check(q4nx_head_image_read(path, back, kd * nd, sc_back, nd, &info, why, (int)sizeof why) == 0, + "head image: no image reads as a refusal"); + check(why[0] != 0 && strstr(why, "not there") != NULL, "... whose reason is that it is not there"); + } + /* (d) a truncated file refuses rather than returning a partial image */ + { + check(q4nx_head_image_write(path, codes, scales, kd, nd) == 1, "head image: written for the truncation arm"); + FILE* f = fopen(path, "r+b"); + if (f) { fseek(f, 0, SEEK_END); const long sz = ftell(f); fclose(f); if (sz > 40) truncate_file(path, sz - 40); } + why[0] = 0; + check(q4nx_head_image_read(path, back, kd * nd, sc_back, nd, &info, why, (int)sizeof why) == 0, + "head image: a truncated image is refused"); + remove(path); + } + } + free(bt); free(codes); free(back); free(scales); free(sc_back); + } +} + int main(void) { + test_head_image(); oga_geometry(); on_grid_geometry(); activation_pad();