diff --git a/docs/npu.md b/docs/npu.md
index efe1e3d93..db140ca35 100644
--- a/docs/npu.md
+++ b/docs/npu.md
@@ -1741,6 +1741,18 @@ engine#1788. On the published packs the text model only loads once the padded-he
engine#2026 is in: their head is stored 2 and 16 rows past the vocabulary, and without that rule the
probe stops at `text model: lm_head dequant failed`.
+Three options exist to bisect a wrong answer against a reference run of the same prompt, and are how
+the missing q/k norm above was found:
+
+- `--dump
` writes what the text model is about to be given, in `hybrid_prompt_probe`'s file forms:
+ `ids.i32`, `positions.i32` (the t, h and w planes one after another) and `pooled.f32` (the tower's rows).
+ If these match a reference pipeline's, the fault is past the inputs.
+- `--chunk N` sends the prompt in calls of at most N rows, as the served route sends a prefill; `0` (the
+ default) sends it in one call. Different answers at different N point at the batched body.
+- `--rows-argmax` (with `--none`) prints the model's own next-token choice at every row of a text prompt,
+ from one teacher-forced pass. Compared with a reference's per-row choices, a low agreement that is flat
+ across positions is an error in the layers; a step at some row is an error at a position or a batch edge.
+
## Serving
The 1bit NPU engine runs the model on the XDNA 2 NPU. With a release package that carries the NPU
diff --git a/npu/forward/minicpm_image_probe.cpp b/npu/forward/minicpm_image_probe.cpp
index cc161bdcb..dfb8e5c45 100644
--- a/npu/forward/minicpm_image_probe.cpp
+++ b/npu/forward/minicpm_image_probe.cpp
@@ -20,13 +20,16 @@
// -> the text model: step_batch_embeds() with the tower's rows where the placeholders are, then greedy decode.
//
// minicpm_image_probe (--image file.ppm | --solid R,G,B [--size HxW] | --halves R,G,B:R,G,B | --none)
-// --prompt "question" [--decode N] [--expect-ids ids.i32]
+// --prompt "question" [--decode N] [--expect-ids ids.i32] [--dump dir] [--chunk N] [--rows-argmax]
//
// A probe, not a route: the tower runs on the CPU (about half a minute for one 448 x 448 view on one thread), the image is
// a binary PPM or a generated colour so that no image decoder is involved, and the prompt is the one the pack's own chat
// template renders for a single user turn with one image and thinking off, written out here rather than rendered:
// <|im_start|>user\n0[image]\n{question}<|im_end|>\n<|im_start|>assistant\n\n\n\n\n
// --expect-ids compares the token ids built here with a file of int32 ids (the reference processor's), and says so.
+// --dump writes what the text model is about to be given (ids.i32, positions.i32, pooled.f32), --chunk N sends the
+// prompt in calls of at most N rows as the served route does, and --rows-argmax prints the model's next-token choice at
+// every row of a text-only prompt. The three exist to bisect a wrong answer against a reference run.
// The text model opens the device unless NPU_INFER_HOST_GEMM is set, so on the device this runs inside npu-hold.
#include "q4nx_forward.h"
@@ -90,14 +93,14 @@ static bool read_ppm(const std::string& path, std::vector& rgb, int& h,
int main(int argc, char** argv) {
if (argc < 3) {
std::fprintf(stderr, "usage: %s (--image f.ppm | --solid R,G,B [--size HxW] | --halves R,G,B:R,G,B | "
- "--none) --prompt \"question\" [--decode N] [--expect-ids ids.i32]\n", argv[0]);
+ "--none) --prompt \"question\" [--decode N] [--expect-ids ids.i32] [--dump dir] [--chunk N] [--rows-argmax]\n", argv[0]);
return 2;
}
const std::string pack_dir = argv[1], kernels = argv[2];
- std::string image_path, prompt, expect_ids;
- bool none = false, have_solid = false, have_halves = false;
+ std::string image_path, prompt, expect_ids, dump_dir;
+ bool none = false, have_solid = false, have_halves = false, rows_argmax = false;
uint8_t solid[3] = {0, 0, 0}, left[3] = {0, 0, 0}, right[3] = {0, 0, 0};
- int height = 448, width = 448, decode = 8;
+ int height = 448, width = 448, decode = 8, chunk = 0;
for (int i = 3; i < argc; ++i) {
const std::string a = argv[i];
const auto value = [&]() -> std::string {
@@ -116,8 +119,12 @@ int main(int argc, char** argv) {
else if (a == "--prompt") prompt = value();
else if (a == "--decode") decode = std::atoi(value().c_str());
else if (a == "--expect-ids") expect_ids = value();
+ else if (a == "--dump") dump_dir = value();
+ else if (a == "--chunk") chunk = std::atoi(value().c_str());
+ else if (a == "--rows-argmax") rows_argmax = true;
else { std::fprintf(stderr, "unknown option %s\n", a.c_str()); return 2; }
}
+ if (chunk < 0) { std::fprintf(stderr, "--chunk wants a row count (0: one call)\n"); return 2; }
if (prompt.empty() || decode < 1 || decode > 512 || int(none) + int(have_solid) + int(have_halves) + int(!image_path.empty()) != 1) {
std::fprintf(stderr, "give --prompt and exactly one of --image, --solid, --halves, --none\n");
return 2;
@@ -201,6 +208,25 @@ int main(int argc, char** argv) {
std::printf("prompt ids vs %s: %s (%zu here, %zu there)\n", expect_ids.c_str(), same ? "IDENTICAL" : "DIFFERENT", ids.size(), want.size());
}
+ // --dump : what the text model is about to be given, in hybrid_prompt_probe's file forms, so a wrong answer
+ // can be bisected against a reference that writes the same files (ids.i32, positions.i32 as t|h|w, pooled.f32).
+ if (!dump_dir.empty()) {
+ const auto put = [&](const char* name, const void* data, size_t bytes) {
+ std::ofstream f(dump_dir + "/" + name, std::ios::binary);
+ f.write(static_cast(data), std::streamsize(bytes));
+ if (!f) throw std::runtime_error("cannot write " + dump_dir + "/" + name);
+ };
+ std::vector i32(ids.begin(), ids.end());
+ put("ids.i32", i32.data(), i32.size() * 4);
+ std::vector pl;
+ pl.insert(pl.end(), pos.t.begin(), pos.t.end());
+ pl.insert(pl.end(), pos.h.begin(), pos.h.end());
+ pl.insert(pl.end(), pos.w.begin(), pos.w.end());
+ put("positions.i32", pl.data(), pl.size() * 4);
+ put("pooled.f32", rows.data(), rows.size() * sizeof(float));
+ std::printf("dumped %zu ids, 3 x %zu positions and %zu row values to %s\n", ids.size(), ids.size(), rows.size(), dump_dir.c_str());
+ }
+
// ── the text model ──
Q4nxNpuForward f;
std::unique_ptr dev;
@@ -240,9 +266,44 @@ int main(int argc, char** argv) {
Q4nxNpuForward::HybridInputs in;
in.position_ids = &planes;
std::vector logits;
+ // --rows-argmax (text-only prompts): the model's own next-token choice at every prompt row, from one teacher-forced
+ // pass over the prompt's ids. Compared row by row with a reference run of the same ids, it says whether a wrong
+ // answer is one bad position or a uniform error in the layers; that is how engine#2149 was found (65 % agreement,
+ // flat across positions). Images are refused: step_batch_all() takes token ids, not rows.
+ if (rows_argmax && !none) throw std::runtime_error("--rows-argmax needs --none: the per-row pass takes token ids");
+ if (rows_argmax) {
+ std::vector all;
+ f.reset();
+ if (!f.step_batch_all(ids, 0, all)) throw std::runtime_error(std::string("step_batch_all: ") + f.last_error());
+ const size_t V = all.size() / size_t(n);
+ std::printf("rows-argmax:");
+ for (int r = 0; r < n; ++r) {
+ const float* row = all.data() + size_t(r) * V;
+ size_t b = 0;
+ for (size_t i = 1; i < V; ++i) if (row[i] > row[b]) b = i;
+ std::printf(" %zu", b);
+ }
+ std::printf("\n");
+ }
f.reset();
const auto t4 = now();
- if (!f.step_batch_embeds(embeds, 0, logits, in)) throw std::runtime_error(std::string("prefill: ") + f.last_error());
+ // The prompt goes in as chunks of at most `chunk` rows, as the served route sends a prefill (--chunk N; 0 sends
+ // it as one call). Each call gets its own rows' embeddings and planes and starts at its cache slot.
+ if (chunk <= 0) {
+ if (!f.step_batch_embeds(embeds, 0, logits, in)) throw std::runtime_error(std::string("prefill: ") + f.last_error());
+ } else {
+ for (int off = 0; off < n; off += chunk) {
+ const int take = off + chunk < n ? chunk : n - off;
+ const std::vector part(embeds.begin() + long(size_t(off) * H), embeds.begin() + long(size_t(off + take) * H));
+ std::vector pl;
+ pl.insert(pl.end(), pos.t.begin() + off, pos.t.begin() + off + take);
+ pl.insert(pl.end(), pos.h.begin() + off, pos.h.begin() + off + take);
+ pl.insert(pl.end(), pos.w.begin() + off, pos.w.begin() + off + take);
+ Q4nxNpuForward::HybridInputs ci;
+ ci.position_ids = &pl;
+ if (!f.step_batch_embeds(part, off, logits, ci)) throw std::runtime_error(std::string("prefill: ") + f.last_error());
+ }
+ }
const auto t5 = now();
const auto argmax = [](const std::vector& v) {
int b = 0;