From 407c0c5fa47bdffa5460eb4b16b55c21c7467b79 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 12 Nov 2026 23:53:07 -0400 Subject: [PATCH] npu: audio row on current main, with the split line formatted from the run (engine#1783) --- CMakeLists.txt | 39 +- docs/npu.md | 738 +++-------------------- npu/audio_exec.cpp | 24 +- npu/audio_pack.cpp | 178 ++++++ npu/audio_pack.h | 53 ++ npu/audio_parallel.h | 25 +- npu/forward/include/audio_device_gemm.h | 112 ++++ npu/forward/include/audio_kernel_roles.h | 97 +++ npu/forward/include/audio_launch_plan.h | 117 ++++ npu/forward/include/q4nx_pack.h | 18 +- npu/forward/include/whisper_ids.h | 46 ++ npu/onnx_graph.h | 4 + npu/pack_import.cpp | 575 +----------------- npu/pack_import.h | 103 ---- scripts/npu-kernel-bitx.sh | 133 ++++ tests/audio_device_gemm_test.cpp | 97 +++ tests/audio_exec_test.cpp | 34 +- tests/audio_kernel_plan.sh | 53 ++ tests/audio_kernel_roles_test.cpp | 102 ++++ tests/audio_launch_plan_test.cpp | 83 +++ tests/audio_pack_test.cpp | 143 +++++ tests/data/whisper-jfk-ids.json | 31 + tests/onnx_graph_test.cpp | 26 +- tests/whisper_ids_test.cpp | 62 ++ tests/whisper_lv3_row.sh | 26 + tools/audio_kernel_shapes.cpp | 249 ++++++++ tools/whisper_transcribe.cpp | 231 +++++-- 27 files changed, 2028 insertions(+), 1371 deletions(-) create mode 100644 npu/audio_pack.cpp create mode 100644 npu/audio_pack.h create mode 100644 npu/forward/include/audio_device_gemm.h create mode 100644 npu/forward/include/audio_kernel_roles.h create mode 100644 npu/forward/include/audio_launch_plan.h create mode 100644 npu/forward/include/whisper_ids.h create mode 100755 scripts/npu-kernel-bitx.sh create mode 100644 tests/audio_device_gemm_test.cpp create mode 100755 tests/audio_kernel_plan.sh create mode 100644 tests/audio_kernel_roles_test.cpp create mode 100644 tests/audio_launch_plan_test.cpp create mode 100644 tests/audio_pack_test.cpp create mode 100644 tests/data/whisper-jfk-ids.json create mode 100644 tests/whisper_ids_test.cpp create mode 100755 tests/whisper_lv3_row.sh create mode 100644 tools/audio_kernel_shapes.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b81dca211..946d1ef57 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -904,7 +904,7 @@ if(ONEBIT_NPU_HOST) find_package(PkgConfig REQUIRED) pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp - npu/audio_exec.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp + npu/audio_exec.cpp npu/audio_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) target_include_directories(onebit_npu PUBLIC npu) target_link_libraries(onebit_npu PUBLIC nlohmann_json::nlohmann_json onebit_q4nx_cache PRIVATE PkgConfig::PCRE2) @@ -1904,6 +1904,43 @@ add_test(NAME audio_mel COMMAND audio_mel_test) set_tests_properties(audio_ops PROPERTIES SKIP_RETURN_CODE 77) add_test(NAME onnx_graph COMMAND onnx_graph_test ${ONEBIT_AUDIO_ONNX}) add_test(NAME audio_exec COMMAND audio_exec_test ${ONEBIT_AUDIO_ONNX}) +# engine#1783: the audio pack -- two graphs by their exported names (never `decoder_with_past`), mel bins from the encoder's +# own first Conv, config.json as a cross-check. +add_executable(audio_pack_test tests/audio_pack_test.cpp) +target_include_directories(audio_pack_test PRIVATE npu/forward/include) +target_link_libraries(audio_pack_test PRIVATE onebit_npu) +# The shape-to-role rule AND the engine's staging grid the ELF names are built on (K % 256, N % 512). +add_executable(audio_kernel_roles_test tests/audio_kernel_roles_test.cpp) +target_include_directories(audio_kernel_roles_test PRIVATE npu/forward/include) +add_executable(whisper_ids_test tests/whisper_ids_test.cpp) +target_include_directories(whisper_ids_test PRIVATE npu/forward/include) +# Which shapes a pack asks for, whether a kernel dir covers them, and the launch plan a route would read. +add_executable(audio_kernel_shapes tools/audio_kernel_shapes.cpp) +target_include_directories(audio_kernel_shapes PRIVATE npu/forward/include) +target_link_libraries(audio_kernel_shapes PRIVATE onebit_npu) +# One projection on the device, and the launch loop that routes device-first and COUNTS what fell back. +add_executable(audio_device_gemm_test tests/audio_device_gemm_test.cpp) +target_include_directories(audio_device_gemm_test PRIVATE npu/forward/include npu/target npu/plan) +target_link_libraries(audio_device_gemm_test PRIVATE nlohmann_json::nlohmann_json) +add_executable(audio_launch_plan_test tests/audio_launch_plan_test.cpp) +target_include_directories(audio_launch_plan_test PRIVATE npu/forward/include npu/target npu/plan) +target_link_libraries(audio_launch_plan_test PRIVATE nlohmann_json::nlohmann_json) +add_test(NAME whisper_ids COMMAND whisper_ids_test) +add_test(NAME audio_kernel_roles COMMAND audio_kernel_roles_test) +add_test(NAME audio_pack COMMAND audio_pack_test) +add_test(NAME audio_device_gemm COMMAND audio_device_gemm_test) +add_test(NAME audio_launch_plan COMMAND audio_launch_plan_test) +add_test(NAME whisper_transcribe_jfk COMMAND whisper_transcribe) +set_tests_properties(whisper_transcribe_jfk PROPERTIES + ENVIRONMENT "ONEBIT_TEST_WHISPER_IDS_FILE=${CMAKE_SOURCE_DIR}/tests/data/whisper-jfk-ids.json" + SKIP_RETURN_CODE 77 TIMEOUT 1800) +add_test(NAME whisper_transcribe_jfk_lv3 + COMMAND ${CMAKE_SOURCE_DIR}/tests/whisper_lv3_row.sh $ + ${CMAKE_SOURCE_DIR}/tests/data/whisper-jfk-ids.json) +set_tests_properties(whisper_transcribe_jfk_lv3 PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 3600) +add_test(NAME audio_kernel_plan + COMMAND bash ${CMAKE_SOURCE_DIR}/tests/audio_kernel_plan.sh $) +set_tests_properties(audio_kernel_plan PROPERTIES SKIP_RETURN_CODE 77) add_test(NAME audio_parallel COMMAND audio_parallel_test) set_tests_properties(audio_exec PROPERTIES SKIP_RETURN_CODE 77) set_tests_properties(onnx_graph PROPERTIES SKIP_RETURN_CODE 77) diff --git a/docs/npu.md b/docs/npu.md index 9eedb6c1d..59a107d5a 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -68,14 +68,6 @@ curl -s http://127.0.0.1:8000/v1/chat/completions -H 'Content-Type: application/ npu/ the model's NPU kernels ``` -The NPU route reads `tokenizer.json` with its own reader (`npu/tokenizer.cpp`). Four vocabulary shapes are handled: -byte-level BPE; SentencePiece pieces with no merges; SentencePiece pieces with merges and a normalizer that prepends -the word-boundary symbol (Phi-3.5); and BPE with merges whose normalizer only rewrites spaces to that symbol and -prepends none (the Gemma family, engine#2215). The last one is encoded by rank over the whole run with no prefix, and -the symbol decodes back to a space. Before that shape was recognised a Gemma4 prompt encoded to 31 ids where the -reference library gives 21, and replies kept the raw symbols. With `-DONEBIT_GEMMA_TOKENIZER=` ctest -`tokenizer_gemma4` holds the reader to the 17 reference cases kept for that file (2 of 17 before, 17 of 17 now). - **What ends an answer.** The serve stops on every id in `config.json`'s `eos_token_id` (one integer or a list), on `<|endoftext|>` when the tokenizer has it, on `tokenizer_config.json`'s `eos_token` when that file is beside the tokenizer, and on the marker the chat route's prompts close an assistant turn with: `<|im_end|>` on the ChatML @@ -273,46 +265,6 @@ Measured on this box through that route, 1,000 fixed sentences (the gate's own s and STS-B through the same route: **Spearman ×100 82.9412** (the model card's 82.87, its own runtime 82.75 on the same evaluator). -### Sentence embeddings from a q4nx encoder (Embedding-Gemma-300M-NPU2, engine#1786) - -FastFlowLM's `Embedding-Gemma-300M-NPU2` is a `Gemma3TextModel` with `use_bidirectional_attention: true` plus the -sentence-transformers head (`dense_2` 768 -> 3072, `dense_3` 3072 -> 768). The same flags serve it: - -``` -1bit serve --embedding -m --device npu|cpu # ONEBIT_NPU_KERNELS, /kernels or the build's set -``` - -`/v1/embeddings` has the shape described above. The per-op forward encodes the input in one batched pass at -position 0 (`Q4nxNpuForward::embed_pooled`). Every layer reads the whole input in both directions, sliding layers -included. That equals the band a sliding layer was trained with only while the input fits half the window, so an -input longer than 256 tokens (the window is 512) is refused with a 400 naming the cap. The rows are mean-pooled -after the final norm, then pass through `dense_2` and `dense_3` (linear, no bias) and are L2-normalised. Ids are -framed as the pack's `tokenizer_config.json` says (`add_bos_token`, `add_eos_token`: ` text `), which gives -the reference tokenizer's ids exactly. - -Four things about this pack differ from the other gemma3 packs, and each is read from the pack: -- the tensors carry no `model.` prefix (`layers.N.*`, `embed_tokens`, `norm`), and the loader prefixes them; -- the projections are unquantised BF16, read as they are; -- the embedding is stored pre-scaled by sqrt(H), so the forward applies no input scale; -- the norms are HF's raw `w`. FastFlowLM's quantised gemma3 packs store `1 + w`; this one is a copy of the HF - checkpoint, so a gemma3 pack with BF16 projections is run as `1 + w` (the load prints `rmsnorm_offset=1 <- gemma3 - with BF16 projections`). - -Measured against transformers' own `Gemma3TextModel` on `unsloth/embeddinggemma-300m` (the ungated copy of google's -checkpoint, mean pooling and the two dense modules as sentence-transformers applies them), five sentences, the same -ids: - -| route | cos to the reference (min of 5) | 5 inputs | -|---|---|---| -| `--device cpu` (host GEMM) | 0.999990 | | -| `--device npu` | 0.999093 | 0.88 s | - -The similarity matrix is the reference's to three decimals on the host (the two cat sentences 0.885, the unrelated -pairs 0.09 to 0.28). Each rule matters: with the norm rule off the host reads cos 0.09, and with the sliding layers -left causal 0.70 to 0.90. `tests/embed_forward_ci.py` (`embed_forward`) pins the encoder on a fabricated two-layer -container in this layout against a numpy Gemma3 encoder (cos 1.0000000). The same two mutants fail it (0.99 and -0.24). - ### Q4NX chunks Q4NX is [FastFlowLM](https://github.com/ROCm/FastFlowLM)'s NPU model format (MIT), and its models on @@ -343,15 +295,7 @@ the bf16 rounding reappears for the device, never in the container. **The cache is bounded, and the bound is least-recently-used (engine#1956).** After writing an entry, `1bit serve` removes the least-recently-used entries until the cache is within -`ONEBIT_Q4NX_CACHE_MAX_GIB` (default 60; `0` or a negative number turns the sweep off). The -forward's device-form conversion (`npu/q4nx_device.cpp`) sweeps the same way after it writes a new -entry (engine#2263), so a box that converts through the forward and never starts a serve stays -within the bound too. A forward load (`1bit forward-serve`, `1bit npu-run` on a sealed set, the -host-only route too) also sweeps once it has loaded, because a pack that needs no conversion still -writes its head images (below) into the cache. The sweep itself is `npu/q4nx_cache_sweep.h`, -shared by every writer and by `1bit q4nx-cache evict`, which without `--root` sweeps the directory -every writer uses (`$ONEBIT_Q4NX_CACHE`, else `~/.cache/1bit/q4nx`). A box that writes nothing never -sweeps; `1bit q4nx-cache evict` is the manual trigger for that case. The entry +`ONEBIT_Q4NX_CACHE_MAX_GIB` (default 60; `0` or a negative number turns the sweep off). The entry just written and any entry used in the last ten minutes are never removed. "Used" means reused, not only written: a cache hit — a repacked GGUF found in the cache, or a device-form conversion reused at load — marks the entry's directory as used now, so a container imported once and served @@ -359,16 +303,6 @@ every day outlives a one-off imported yesterday. The value is read as a whole nu number too large to be a byte count is the largest bound that is one (it does not wrap to a small one), and a value that is not a number gets the default rather than an unbounded cache. -**A box that never serves never sweeps (engine#2263).** Both triggers — the write-time sweep and the manual -`1bit q4nx-cache evict` — need the binary to run, so a review, CI or build box that holds the cache and never starts a -serve keeps every entry it ever wrote. Measured on this box: **59 GiB in 12 h** with the command merged and nothing -invoking it, which put free space 3 GiB under the 175 GiB build gate; one manual 24 h age cut took it to 24 GiB and it -regrew. `scripts/install-q4nx-cache-timer.sh` installs the invoker as a `systemd --user` timer — hourly, -`--max-gib 40 --min-age 86400`, opt-in, nothing outside `~/.config/systemd/user` — and refuses to install against a -binary that predates the command (`1bit q4nx-cache` printing *unknown command*), because a timer naming a command the -binary lacks fails hourly and silently. `--dry-run` prints the units and verifies them without changing anything; -`--uninstall` removes them and leaves the cache alone. - **The packed lm-head is cached beside it (engine#1716).** A model whose head runs as int8 tiles packs those tiles on the host — gather `kpad × T` weights per tile, quantise each column to int8 with its own scale — and until this cache that happened on every serve. The image is a function @@ -393,11 +327,10 @@ The serve log line says what happened: `lm head: N packed weight tile(s) residen image cache, P packed (W written for the next serve) in S s`. `ONEBIT_HEAD_IMAGE_CACHE=0` turns the cache off (nothing is read or written). The files are `kd × nd` bytes plus `nd` scales per tile, so a head costs about `kpad × vocabulary` bytes on disk per kernel set. The Q4NX cache sweep -(`ONEBIT_Q4NX_CACHE_MAX_GIB`, above) counts each key in `head-images/` (`headimg-`, all its -tiles) as one entry of the cache (engine#2263; it used to count the directory as one entry, so the -sweep could only keep every pack's head or remove all of them). A key's age is its newest tile, and -a cache hit marks the tile it read as used, so a head served every day is not aged from the day it -was packed. Losing a key costs the next serve of that pack one pack of its head, nothing else. Ternary (`pb16i`) heads and the gemv head do not +(`ONEBIT_Q4NX_CACHE_MAX_GIB`, above) counts `head-images/` as one entry of the cache and removes it +whole when the cache is over its bound and that directory is the least recently written one. Its +age is the last time a tile was written into it: a serve that only reads its tiles does not refresh +it. Losing the directory costs the next serve one pack of its head, nothing else. Ternary (`pb16i`) heads and the gemv head do not use it: their image already is a file. Each writer builds the file under its own name (`.tmp.`) and renames it into place, so two serves that start cold on the same pack both finish their write and a reader only ever sees a whole image. The behaviour is pinned host-side in @@ -411,20 +344,6 @@ apart. Decoding tile (0, 0) of each lm_head and comparing with a GGUF of the sam gives correlation 0.9998 with the right layout and about 0.01 with the other. `npu/q4nx` decodes the first layout. -The forward's decoder (`npu/forward/src/q4nx_dequant.c`) reads the layout from the codes -themselves (`q4nx_q8_split16`, engine#1786). Each (row, 32-column group) of a symmetric Q8 tensor is -scaled by its own absmax, so in the right layout every group that holds a nonzero code holds one -of magnitude 127; read in the other layout the groups mix rows, and most miss it. Measured on 64 -chunks: 1.000 against 0.50-0.77 on Qwen3.5-0.8B-NPU2's `lm_head`, `ssm_alpha_proj` and -`ssm_out_proj`, and the reverse on the local Qwen3.5-2B-NPU2. The second layout is chosen only -when it alone holds every group at 127; otherwise the decoder keeps the first, as before. The -hub's Qwen3.5 packs (`flm_version` 0.9.45: 0.8B, 2B, 4B, 9B) are in the second layout and also -store `ssm_out_proj` as Q8. A FastFlowLM install rewrites a pack it fetches (the local 2B copy -reports 1.0.3, stores `ssm_out_proj` as Q4_K and uses the first layout), so a row measured on -`~/.config/flm/models` is not a row on the hub's files. Before this, Qwen3.5-0.8B-NPU2 from the -hub answered "What is the capital of France?" with noise on the host and on the NPU. Its -`blk.0.ssm_out` against the BF16 GGUF of the same checkpoint: corr 0.0054 before, 0.9993 after. - `npu/q4nx.h` decodes all five (the two q4_1 parameter forms included), and repacks Q4_K into q4_1 (exact apart from rounding `S * scale` and `M * min` to bf16), so code that reads q4_1 runs Q4_K weights unchanged. Q8 does @@ -551,16 +470,15 @@ paragraphs up — which is the reason it exists: the target was self-consistent, own scalar reference, and decoding a different matrix than the container meant. -**The decode head runs on the host (engine#1785); `NPU_HEAD_HOST=0` puts it back on the device.** A decode step sends one row +**The decode head can run on the host (engine#1785), `NPU_HEAD_HOST=1`.** A decode step sends one row through every tile of the int8 head, and a launch costs the same weight pass for one row as for a full -batch, so those launches are the most expensive use of the device in a token. The -single-row head is therefore computed on the host, from the same image the device reads (the resident weight +batch, so those launches are the most expensive use of the device in a token. With `NPU_HEAD_HOST=1` the +single-row head is computed on the host instead, from the same image the device reads (the resident weight buffer's host mapping, no second copy), with the device path's own activation quantiser and the same epilogue. The sums are int32, so the logits are the device head's bit for bit, not an approximation of them. `NPU_HEAD_HOST_THREADS` sets the threads the tiles are split over (default 4, at most 16). It applies to the int8 tiled head only: the ternary (`pb16i`) head, the gemv head and batched head rows (prefill, -verify) stay on the device. The serve log says which it is: `lm head: on the host (default; NPU_HEAD_HOST=0 for the -device), N tile(s), T thread(s)`. If the image cannot be read on the host the head goes to the device, with one line. +verify) stay on the device. Off by default. Measured on the NPU with FastFlowLM's `Qwen3.5-2B-NPU2` container (vocabulary 248,320, width 2,048, 21 head tiles), same binary, device head and host head interleaved, 24 greedy tokens per request, on a busy @@ -585,15 +503,9 @@ serve (96 of 96): | head plus following layers | 386 ms (2.6 tok/s) | 334 ms (3.0 tok/s) | So the host head takes 68 ms off the head and the next token's layers give 15 ms of it back, a net 52 ms -(13 %) per token on this model. On `MiniCPM-V-4.7-1B-NPU2` (width 1,024, 35 head tiles) the same alternation, 100 -tokens at load 23, read 89.2 ms against 5.3 ms for the head and 320.4 against 321.2 ms for the following layers, so -nothing is given back there: about 84 ms of a 410 ms token. Its 96 logits dumps are byte-identical to a device-only -serve's. - -It is the default because the logits are the device's on both models and the gain is measured inside one serve on -both. The profile table then has one `HEAD` row at M=1 in place of the M=16 launches; -`scripts/npu-window11.sh --compare` accepts exactly that difference against a device-head run -([npu-plan.md](npu-plan.md)). The layers are the other four fifths of a token and are untouched. +(13 %) per token on this model. It stays off by default because it changes the profile table's row set (one +`HEAD` row at M=1 in place of the M=16 launches), and `scripts/npu-window11.sh --compare` requires the two +binaries it compares to print the same rows. The layers are the other four fifths of a token and are untouched. ### The t2s stream (what the XDNA decode kernel reads) @@ -764,17 +676,6 @@ carrying every instance (`main:gu|d|qkv|o|z|qh|kv`), named by the container's ow a pb16i ELF would read 4,608-byte tiles out of 2,176-byte images). `tools/npu_plan.py` writes the key into `plan.json`'s `decode.sealed_elf`, and the engine runs every decode role inside that one context. -**A b8 (int8) container can use a sealed set too (#2208).** The 24-layer Qwen3.5 packs spend most of a -decode token's device wait switching between per-role hw_contexts, which is the same tax. A b8 container -names one sealed int8 ELF with its own key, `q4nx_b8_sealed_elf` (or `NPU_I8_SEALED_ELF=` for a -run), in the kernels directory. Its instances are `main:qkv` (linear-attention QKV), `main:qkvfa` (the -fused full-attention QKV with its output gate), `main:z`, `main:o`, `main:gu` (fused gate/up) and `main:d` -(`q4nx_i8_sealed_instance`). A role the ELF has no instance for, such as split `G`/`U`, keeps its per-role -`_b8_m16.elf` and the load logs `b8: the sealed ELF has no instance for role ...`. `tools/npu_plan.py` -copies the key into `decode.sealed_elf`, and the plan accepts a sealed ELF for `container.layout` b8 as -well as pb16i. The ELF is built for one model's shapes; naming it for a pack with other shapes is a -container error, as with pb16i. - #### The t2s family is M=16 today, and that is what caps prefill (#1416) Measured, not inferred: a t2s container's kernel set contains **only `*_t2s2_m16.elf`** for 8 of the nine decode roles, and the engine reports the @@ -798,70 +699,6 @@ An **M≥256 t2s family** is what removes the 8x launch overhead; it is private- family passes, and the acceptance row is: both Llama3-8B prefill shapes (`K=4096 N=14336`, `K=14336 N=4096`) at M = 256 / 1024 / 16384, ≥ 4 TFLOP/s, byte-exact logits against the current path on a fixed prompt set, `amdxdna` counter 0. -#### A pack with both families: a 17-row chunk no longer pays for 128 rows (#1959) - -An imported int8 pack (the `b8` layout) carries a 16-row twin **and** a 128-row prefill design for each projection, and -`max_batch_rows()` is 128 there. A chunk of up to 16 rows runs the twin; anything wider ran the wide design, which walks -all 128 of its baked rows whatever the chunk holds. So one more prompt token cost 5.9 s: on -`Qwen2.5-7B-Instruct_rai_1.9.0_npu`, a 16-row pass is 1073 ms and a 17-row pass was 6992 ms. - -`npu_gemm_rows()` now decides per projection, from the launch costs this process has measured, whether a chunk of -17..128 rows runs as one wide launch or is walked at the twin's width (`npu/forward/include/batch_split.h`): it splits -when `ceil(M/16) x narrow cost` is at least a tenth below the wide cost, and keeps the wide launch otherwise or while -either side is unmeasured (the first wide launch of a projection is what measures it). The rule is measured, not a -constant, because the ratio belongs to the kernel family: here a wide launch costs about six narrow ones, and on a -family where it costs two the wide launch keeps every chunk above 32 rows. - -Same binary, `NPU_PREFILL_SPLIT=0` (the wide launch kept) against the default, fresh serve per arm, one ticket, -two prompts per length; the pass wall is the serve's own `[bs] M= start=0` line: - -| batched rows | wide launch kept | default (measured split) | launches per pass | -|---:|---:|---:|---| -| 16 | 1077 / 1075 ms | 1137 / 1060 ms | 112 / 112 | -| 17 | 6948 / 6956 ms | **2226 / 1938 ms** | 112 / 220 | -| 24 | 6973 / 8124 ms | **1968 / 1955 ms** | 112 / 224 | -| 48 | 7041 / 7084 ms | **3032 / 2892 ms** | 112 / 336 | -| 96 | 7295 / 7309 ms | **5909 / 5874 ms** | 112 / 392 | -| 112 | 7323 / 7437 ms | 6885 / 6872 ms | 112 / 328 | -| 128 | 7421 / 7492 ms | 7430 / 7343 ms | 112 / 308 | - -A request of 18 prompt ids and 3 output tokens went from 10.0 s to 5.0-5.3 s. The first 17-row pass of a serve is the -slower of its pair (2226 ms) because it contains the one wide launch per projection that takes the measurement. The -launch counts are not multiples of one factor because the decision is per projection: at 112 and 128 rows only the -projections whose wide launch costs more than eight or nine narrow ones are split. - -Nothing in the output moves. Every row of a batched GEMM is computed independently of the others, which is the -property the block walk for chunks wider than the design already relies on: the greedy text of all 14 requests is -identical between the two arms, and the per-row bitwise oracle (`NPU_BATCH_CHECK=1` on a 17-id `npu-run`) reports -`rows M=17 identical=17 differing=0` with the split forced (`NPU_PREFILL_SPLIT=1`) and with it off. A pack with one -family only (every t2s/pb16i container: `max_batch_rows()` is 16 there) never reaches the rule, and weights that are -re-packed per launch (routed MoE experts) are never split. The serve says once what it did: -`[batch] a 17-row chunk runs as 2 launch(es) of 16 rows instead of one 128-row launch: measured ...`. - -Two things this rule had wrong until it met a pack with a wide sealed family (#1990), both found on -`bonsai2-27b-t2s2` with a 32-row set and both fixed here: - -- **A walked chunk stepped through its rows by the wrong distance.** The walk advanced the activation pointer by - `a_k`, the design's real width. A design that does not pad never sets it, so on such a design `a_k` is 0 and every - block after the first read the first block's rows. `NPU_BATCH_CHECK=1 npu-run` on 32 ids with the split forced - reported `rows M=32 identical=16 differing=16 first row=16`, twice, and `identical=32` with the split off; the serve - generated different text. The int8 pack above pads (`real_k` 3584 under K 4096), which is why its 17-row check - passed. The stride is now `row_stride(a_k, K)` (`batch_split.h`): `a_k` only when it is a real, narrower width, the - rule the target itself applies. The older block walk for chunks wider than the design had the same line and is - fixed with it. After the fix the same run reports `identical=32 differing=0` forced, default and off. -- **One start-up launch was taken for the wide launch's cost.** The first 32-row launch of a projection measured - 4.17 ms against 0.51 ms for a 16-row one, so the rule split every 32-row chunk of that projection, on a family whose - wide launch is the cheaper one. The wide cost now counts only after three launches were timed - (`kSplitWideSamples`, `split_wide_cost()`); until then that side is "not measured" and the wide launch is kept, - which is also what collects the samples. -- **With a wide sealed family the rule stands down.** Even with three samples it split a few 32-row launches: - `measured 0.50 ms per 16-row launch against 1.23 ms per 32-row launch (attn_output)`. A narrow launch is mostly - timed carrying one row (a decode step), and its host share grows with the rows it carries, so that 0.50 ms is not - what a full 16-row launch costs. A pack that names a wide family has already stated which side is cheaper, so the - measured rule does not apply to it; `NPU_PREFILL_SPLIT=1` still forces the walk, which is how it is checked. The - rule keeps its job on the packs it was written for, where the wide design is the 128-row prefill kernel and the - gap is a factor of six. - #### Where a prefill block's time goes now (measured), and why it is no longer the pack (#1416) #1416 was filed on a `[prof] pre` table whose `pack` column read 24 ms per launch against 1.3 ms of device wait. That @@ -885,31 +722,7 @@ blocks per arm, two arms per binary run A B B A in one ticket): | of which GDN recurrence | 30.5-33.4 ms | 27.0-30.0 ms | | So prefill on this route is bound by host work outside the launches and by the M=16 launch geometry above, in that -order, and not by packing. - -**The `pack` column no longer carries the weight-cache fill (#1783).** The fill moved before the ready line, but the -warm-up pass is itself a profiled prefill, so a short request still printed a `[prof] pre` table where half the -launches were fills: `pack` read 2.7-3.5 ms per launch on the 768 family and 2.9-7.2 ms on the 1280 family, against -0.00 in the decode rows of the same process. An encoder window at those numbers was extrapolated as twelve launches per -projection per layer, each paying the one-time fill again. `NPU_PROFILE=1` now times each fill on its own and leaves -it out of `pack`. `pack` is what every launch pays, and a separate line gives the fills. The same request on the same -768 container and kernel set (2026-11-13, one `npu-hold` ticket, `amdxdna` error count 632 before and after, same -greedy text): - -| `[prof] pre`, M=128, 24 launches per role | QKV | O | GU | D | -|---|---:|---:|---:|---:| -| `pack` before (ms/launch) | 3.10 | 3.09 | 3.49 | 2.73 | -| `pack` after (ms/launch) | 0.00 | 0.00 | 0.00 | 0.00 | -| `ms/launch` after | 1.269 | 1.359 | 1.935 | 1.595 | - -``` -[prof] pre weight-cache fills (one-time, not in pack): 48, 310.4 ms, 6.466 ms/fill -``` - -The 48 fills are 12 layers x 4 roles, all in the warm-up pass, and they account for the old `pack` totals (24 x 12.41 -= 298 ms). - -An uncached launch (`NPU_NO_WCACHE=1`) packs its weight on every launch, and that stays in `pack`. "After" is one change: the per-(row, head) alpha/beta dot products of a GDN layer ran on one +order, and not by packing. "After" is one change: the per-(row, head) alpha/beta dot products of a GDN layer ran on one thread inside the row loop (0.099 ms per row and layer, 48 GDN layers), and they now run as a second worksharing loop inside the parallel region that layer already opens for the recurrence. No region is added, because the region count is what a loaded box pays for. Each dot product is the same sequential loop as before and the build has no `-ffast-math`, @@ -1375,24 +1188,6 @@ stride the `.fconst` attention tile already carries (2,176 B subvolume = 32 rows from the node's own `N`/`K` (`gate_proj`/`up_proj` → N, `down_proj` → K), because the weight is a 1-D byte blob with no shape to read it from (engine#1503). -**The 1.8 hybrid line (`*_rai_1.8.0_hybrid`) is not importable, and that is a decision, not a gap still to be filled (engine#1503).** These packs pair a `model.onnx` with a `model.bin`: 1.3 GB on `Llama-3.2-1B-Instruct_rai_1.8.0_hybrid`, alongside a 542 MB `model.onnx.data` that holds only the embedding and the norms. In the graph, every `.MatMulNBits.*` initializer is a `[0]` placeholder with no bytes, and so is its `.packed` sibling (162 of 162 on that pack). The weights exist only in `model.bin`, a compiled container that the vendor's runtime reads. Writing a reader for it means decoding that container, which this engine does not do, so `npu-pack info` and `npu-pack import` refuse these packs by name before reading anything: - -``` -lm_head.MatMulNBits.qweight (its lm_head.MatMulNBits.qweight.packed sibling carries no data either), and the pack carries -no Token .fconst either (...): these weights are genuinely absent from the pack's external data, so this pack cannot be -imported -- not a tile-layout or block-size problem. This pack's weights are in its model.bin, which this importer does -not read (engine#1503) -``` - -The same models ship in an earlier generation that this engine reads. Use that pack instead. The last column says what was measured on it (11-11): - -| 1.8 hybrid pack | instead | measured | -|---|---|---| -| `Llama-3.2-1B-Instruct_rai_1.8.0_hybrid` | the `rai_1.7.1` hybrid | imported and served on the NPU on 11-10, before the tied-head gate existed; not re-run since | -| `SmolLM2-135M-Instruct_rai_1.8.0_hybrid` | the `rai_1.7.1` hybrid | `npu-pack import` rc 0 with the head-fit check (head 1.07× its 4-bit floor); the earlier rel ≤ 0.02 rule refused it at 0.104 | -| `Qwen2.5-0.5B-Instruct_rai_1.8.0_hybrid` | the `rai_1.7.1` hybrid, or `rai_1.9.0_npu` | both `npu-pack import` rc 0 | -| `Phi-3.5-mini-instruct_rai_1.8.0_hybrid` | the `rai_1.7.1` hybrid | `npu-pack info` rc 0 (plan clean); not imported here | - **A pack can be the weights without the text.** AMD's `_rai_1.8.0_npu_4K` repos do publish the model's own `tokenizer.json`, `tokenizer_config.json`, `vocab.json`, `merges.txt`, `chat_template.jinja`, `special_tokens_map.json`, `added_tokens.json` and `generation_config.json`, @@ -1624,26 +1419,6 @@ graph: `model.onnx` carries `attn.q_norm.layernorm.weight` and `attn.k_norm.laye FP16 `[128]` initializers, and on Hunyuan-7B-Instruct their bf16 rounding is the first and the second block bit for bit in 32 of 32 layers. `qk_gain_offsets()` holds that arithmetic. -The same gap exists when q_proj is wider than hidden. Hunyuan-0.5B-Instruct (hidden 1024, 16 heads -of 128, so q_proj has 2,048 rows) has `[64 q/k/v tiles at K=1024][640 B][16 o_proj tiles at -K=2048]`, and its graph's gains are in that gap in the same order in 24 of 24 layers (q_norm at -block offset 2,508,928, k_norm 320 B later). `wide_q_attn_section()` holds that division and names -the gains; before it did, this pack was refused as declaring gains the layout did not hold. - -Where the gains are, by layout, in one place: a 4-header block keeps them in their own two sections, -k first (Qwen3, above); a 2-header block keeps them in the 640 B gap between q/k/v and o_proj, q -first, whether o_proj's K is hidden (Hunyuan-7B, Hy-MT2-1.8B) or q_proj's width (Hunyuan-0.5B). A -pack that declares `use_qk_norm` and has them in none of these is refused (`qk_norm_refusal()`). -The gap's offsets are block arithmetic, one number for every layer, so the import checks each -layer's bytes to be `[64 B zeros][finite gain][64 B zeros][finite gain]` (`qk_gap_block_why()`) -and refuses a layer that is not, by name, instead of writing whatever is there. - -Hunyuan-0.5B's first payload is in the order q, k, v, like Hunyuan-7B's (`Plan::attn_q_first`). -Checked against `tencent/Hunyuan-0.5B-Instruct` on the host: read as k, q, v, q_proj and k_proj -correlate 0.01 with the checkpoint in layers 0, 11 and 23; read as q, k, v they correlate 0.96 to -0.99, and in chat format the container picks the checkpoint's next token at 39 of 42 positions -(mean KL 0.030 nats). - **Every tile read is its own check.** Before anything is written, each tile is decoded and re-encoded; the encoder has to reproduce the 34,816 bytes it came from, bias slots included. That one comparison holds every field, index and nibble order at once, and it is why a tile whose @@ -2057,231 +1832,6 @@ the missing q/k norm above was found: from one teacher-forced pass. Compared with a reference's per-row choices, a low agreement that is flat across positions is an error in the layers; a step at some row is an error at a position or a batch edge. -**Gemma3 text decoders: the forward reads them (engine#1786).** FastFlowLM's `Medgemma-4B-NPU2` (`gemma3_text`, 34 -layers) now answers correctly through the per-op forward on the host (`NPU_INFER_HOST_GEMM=1`, no device). Before this -change it was refused at load: `config.json declares rope scaling "linear", which this forward does not implement`. -Five things were missing. All are read from the config or decided by the family: - -- `sliding_window_pattern`: the layer mix, when there is no `layer_types` list. Every 6th layer is full attention - (HF's `(i + 1) % 6`), so Medgemma has 5 full and 29 sliding layers. Before, all 34 ran as full attention. -- `rope_local_base_freq`: the sliding layers' own RoPE base (10000, against 1e6 for the full layers). A sliding layer - with no base of its own now takes the model's base. Before, it took 0, which was the case for Gemma3-270M's - stated `layer_types`. -- `rope_scaling` `{"rope_type": "linear", "factor": 8}`: the frequency is divided by the factor, on the full layers - only, because Gemma3's sliding layers rotate unscaled. A model without a local base scales every layer, as HF does. - A `linear` block with no usable factor is still refused. -- `query_pre_attn_scalar`: the attention scale is `1/sqrt(query_pre_attn_scalar)`. It equals `head_dim` on Medgemma - but not on every Gemma3. -- The sandwich norms: `post_attention_layernorm` on the attention output, plus `pre_feedforward_layernorm` and - `post_feedforward_layernorm` (`extra_layer_norms = 2`: Gemma4's set without its post-PLE norm and layer scalar). - -The norm form is the one non-obvious rule. HF's Gemma3 RMSNorm is `x/rms * (1 + w)`, but these containers store -`w` with the 1 already added. Every norm kind of the Medgemma pack (input, post-attention, pre- and post-feedforward, -q, final) equals llama.cpp's GGUF of the same model, whose converter adds the 1, to bf16 rounding. With the 1 added -again, the known-answer prompt answered `*,,,,,,,`; with the plain form it answers `Paris`. So gemma3 takes the plain -form, and gemma2 keeps `(1 + w)` (not measured here). `tests/q4nx_gemma3_semantics_test.c` pins each rule with -Medgemma's own keys. - -A prefill runs batched (`step_batch`) with the same three extra norms. `batch_check_state_test` on the Medgemma -pack: batched equals sequential, rel-L1 0. - -**Gemma4 runs batched too (engine#1780).** It used to fall back to one `step()` per row, so verifying k drafted -tokens cost k weight streams, and a DFlash pair on a Gemma4 target could not beat plain decode. The batched body -now does what `step()` does for each row: -- it builds the row's per-layer inputs; -- it runs the PLE gate and projection as M-row GEMMs, then the layer scalar; -- on a KV-shared layer it reads the owner layer's cache and appends nothing (the owner already appended every row - of the batch earlier in the same pass); -- a shared layer's double-wide MLP uses `GU2`/`D2`. - -A batch with injected embeddings is still refused on a PLE model, because the per-layer inputs need token ids. - -The KV cache's row width is now per layer, from `kv_widths()`. A model with two attention geometries holds more -floats per position on its full layers: on Gemma4-E4B, 2 × 512 against 2 × 256 on the sliding layers. Snapshot, -restore, a verify's rollback and `extend_context` used to count every layer at `NKV * HD`. That reported twice the -positions and cut the full layers at half on a snapshot or a partial accept, which corrupts the prefix cache and -any verify on such a model. - -`batch_check_state_test` on `Nemotron-3.5-Bolt-E4B` (host GEMM) checks all of this: -- chunks [0,3), [3,5), [5,6): every chunk's argmax matches; the last chunk's logits equal the sequential ones - (rel-L1 0); the positions held are exact after each chunk; -- a verify arm: a 4-row `step_batch_all` is bit-identical to 4 sequential steps (the oracle reports - `identical=4`); after `commit_verify_rows(2)` exactly 5 positions are held, and the next step equals a forward - that saw only those rows. - -Before the width fix, the first arm failed 4 of 4. With the old single width for the rollback, the verify arm fails. -`q4nx_state_planes_test` holds the per-layer cut to Gemma4-E4B's 42-layer geometry. - -The verify arm also runs on the CI's fabricated containers. There it found that `step_batch_all()` on a model that -still falls back to one `step()` per row (gpt-oss, a MoE with a leading dense layer) returned no per-row logits: the -fallback filled only the last row and left `logits_all` as it found it. The fallback now appends every step's row. - -The serve also needs the BOS. Medgemma's `config.json` states neither `bos_token_id` nor `add_bos_token`, and its -`tokenizer_config.json` states both (``, true), as HF reads them. forward-serve now takes what `config.json` -does not state from `tokenizer_config.json`. Served without it, the reply to "Explain in three sentences why the sky -is blue." was "The sky is blue because of the way the light is." With it, the reply is "The sky appears blue due to a -phenomenon called Rayleigh scattering", the same as the host forward with the BOS. - -On the device, with a locally built kernel set of the four shapes (`QKV K2560 N4096`, `O K2048 N2560`, -`GU K2560 N20480`, `D K10240 N2560`) and their 16-row twins, in one `npu-hold` ticket, the known answer came back -`Paris` three times, identical. That is not yet a leaderboard row: decode is 0.68 tok/s, and no kernel set for these -shapes ships. - -**Gemma3-270M-NPU2 and the 1,280-byte quarter tile.** This pack's hidden size, 640, is not a multiple of the -5,120-byte tile's 256 columns. FastFlowLM stores every I8 tensor of it in a quarter tile instead: 16 rows x 128 -columns, with scales and zero-points at `g*16 + r` and codes at `c*8 + r/2` (`Q4NX_CHUNK_Q4H`, `q4nx_dequant.h`). -Both kinds cost 0.625 bytes a weight, so the byte count cannot tell them apart. The tensor's declared shape does: -`[chunks, 1280]` against `[tiles, 5120]`, and the reader goes by it. The reading was settled against an independent -GGUF of the same model (`v_proj` against its Q5_0 `attn_v`: correlation 0.9962; the 5,120-tile reading gives 0.09) -and against the pack's own embedding (`lm_head`: 0.9963, at most 0.57 of a step off). On the host the pack answers -the known-answer prompt `Paris` and explains Rayleigh scattering in coherent sentences. Before, it failed at load -with `chunk 4266 B is not a kind`. No kernel set for its shapes was built or run (the NPU is off). - -**LFM3-1.7B-NPU2: the forward reads LFM3 (engine#1786).** LiquidAI's LFM3 is LFM2's hybrid of gated short -convolutions and attention, plus four rules from its `modeling_lfm3.py`. The forward now runs all four: - -- `layer_types` names the conv layers `"conv"`, beside `full_attention` and `sliding_attention`. Each `"conv"` entry - has to match a layer whose conv tensors are in the container, and a mismatch either way is refused by name. - `"conv"` is an exact match: `conv_attention` is still an unknown kind. -- `use_sandwich_norm`: a norm on the conv or attention output before the residual add, and one on the MLP output. - It is the same pair the Gemma3 path runs (`extra_layer_norms` 2), and here the conv layers take it too. -- A per-head output gate. Every attention layer has `self_attn.gate_proj` `[heads, hidden]` (bf16 in the pack), and - each head's context is multiplied by `sigmoid(gate_proj . x)` before `o_proj`, `x` being the block's normed input. -- Zero-tail partial RoPE on the full layers. With `partial_rotary_factor` 0.5 the first 32 frequencies follow the - progression of the whole 128-channel head and the other 32 are zero, with `rotate_half` pairing across the whole - head. That is the rule Gemma4's "proportional" RoPE already had. The vendor GGUF writes the same rule as - `rope_freqs` (1 for 32 factors, 1e30 for the rest). - -The pack keeps the upstream tensor names (`conv.*`, `operator_norm`, `ffn_norm`, `feed_forward.w1/w2/w3`, -`post_operator_norm`, `post_ffn_norm`, `embedding_norm`). `model_load` renames them to the canonical ones, and only -where the canonical name is absent (`model_alias_lfm_names`, `model.h`). `ffn_norm` is the MLP input's norm, so it -becomes `pre_feedforward_layernorm` in a sandwich layer and `post_attention_layernorm` otherwise. - -Checked on the host, 2026-11-12, against a numpy transcription of `modeling_lfm3.py`. Run on the bf16 checkpoint, -the reference's last-position logits correlate 0.939 with ours. Run on the dequantized weights of the vendor's -`LFM3-1.7B-QAD-Q4_0` GGUF, which this pack's I8 tensors come from, they correlate 0.9995, with the same top 10. -The two references differ from each other by the same 0.94, so that gap is the quantization-aware weights, not the -arithmetic. Each rule matters on this prompt: with the gate off the bf16 reference falls to 0.907 against ours, -with the usual partial RoPE to 0.822 (and its top token changes), and without the sandwich to 0.134. -`batch_check_state_test` on the pack: batched equals sequential, rel-L1 0. The known-answer prompt, with thinking -off (the template's ``), gives `ParisParis<|im_end|>`. The reference makes the same three -choices: `` after the first `Paris`, then `Paris` again. Asked why the sky is blue, it explains Rayleigh -scattering in correct sentences (80 tokens, greedy). No kernel set for its shapes was built or run (the NPU is off). - -**Qwen2's attention biases are read from the container (engine#1786).** The forward adds the q, k and v projection -biases only when `attention_bias` is on. A config that states the key decides it. Qwen2's `config.json` never states -it, though every Qwen2 layer has the three biases. With the key absent the forward loaded the biases and never added -them, so FastFlowLM's `Qwen2.5-3B-Instruct-NPU2` answered the known-answer prompt with noise -(`UNS �leon�leon�ucz …`). The rule is now the one q/k normalisation follows: with the key absent, a layer that carries -a q, k or v bias turns the biases on, and the startup line says so (`attention_bias=1 <- the container`). A config -that says `attention_bias: false` while the container carries biases is refused by name rather than guessed. -After: `Paris<|im_end|>` on the host. `batch_check_state_qwen2` fabricates a Qwen2 with biases and no key and -compares the forward's logits with a numpy Qwen2 forward on the container's own weights: corr 1.0000000, max |d| -6.6e-06 of 9.6. The binary without the fix fails it (corr 0.444, another argmax), and batched == sequential passed -on that binary all the same. - -Qwen2.5-VL's text tower is the same Qwen2, and it was refused one step earlier. Its config declares -`rope_scaling: {"type": "mrope", "mrope_section": [16, 24, 24]}`, and the forward read `"mrope"` as a frequency scaling -it does not implement. It is a position layout instead: plain RoPE frequencies, each taking its position from the t, -h or w plane. Qwen2-VL and Qwen2.5-VL lay the sections out CONTIGUOUSLY (transformers' `apply_multimodal_rotary_pos_emb` -splits cos by `mrope_section * 2`): frequencies 0-15 are t, 16-39 h and 40-63 w. Qwen3-VL interleaves them -instead. The forward now reads that form (`mrope_sectioned`, `q4nx_mrope_axis_sectioned`), and refuses a block whose -three sections do not sum to the rotary half-width. On a text-only prompt the three planes are equal, so it is plain -RoPE. `Qwen2.5-VL-3B-Instruct-NPU2`'s text tower now answers `Paris<|im_end|>` on the host; its vision tower is not -loaded here. `batch_check_state_qwen2vl` is the Qwen2 above declaring that block: the same logits as the plain-RoPE -reference (corr 1.0000000). With `"mrope"` refused again (main's behaviour) the arm fails. - -**Gemma4-12B-IT-NPU2: a dense Gemma4 and two new chunk kinds (engine#1786).** The text tower loads and answers on the -host after three changes: - -- **The wide chunks.** Every I8 tensor is declared `[rows/16, K/256, bytes]`, with `bytes` either 2304 (`Q4NX_CHUNK_Q4W`) - or 4352 (`Q4NX_CHUNK_Q8W`, the tied head). A chunk is 16 rows x 256 columns. It starts with 128 bf16 scales at - `g*16 + r` (one per row and 32-column group, symmetric, no zero-point). Then come either 4-bit two's-complement - codes at `c*8 + r/2` (even row in the low nibble) or int8 codes at `c*16 + r`. The reading was settled against - independent weights: Google's own QAT Q4_0 GGUF of the model agrees at corr 0.9986 on seven tensors over layers 0, - 5 and 47 (the 5% rel-rms is FastFlowLM's re-quantization, the same in each), and the Q8W head reproduces the - pack's bf16 embedding at 1.00000. A Q4W tensor has exactly the byte count of a 4608-byte Q4S grid, which is how it - was read before. Layer 0 then went non-finite at its first attention. The declared shape now picks the kind. -- **`attention_k_eq_v`.** The global layers have no `v_proj`: their values are the k projection, taken before - `k_norm` and then given Gemma4's scale-free v norm. This is our llama.cpp fork's rule (`gemma4.cpp`: - `Vcur = wv ? wv x : Kcur`). The forward fills those v rows from k. A layer with no `v_proj` and no such key is - refused by name. -- **No per-layer embedding.** `post_layernorm` belongs to the PLE block, so it is read only when the model has one. - Before, init failed on its absence with an empty error. Those lookups now name the missing tensor. - -On the host the known-answer prompt gives `<|channel>thought\nParis`: the model writes the -template's own empty-thought prefix, then answers. Gemma4-E2B, the PLE model, still answers `Paris`. No kernel set -for the 12B's shapes was built or run (the NPU is off). - -**GPT-OSS-20B-NPU2: the forward reads gpt_oss, on the host (engine#1786).** FastFlowLM's pack stores the model in -two chunk kinds this forward did not read, and the model adds five rules. All of them are now in the per-op forward, -with the MoE run on the host: - -- **D2560** (I8 `[RB, CB, S, 2560]`, attention q/k/v/o and the head) and **MX2560** (U8 `[E, 69, CB, S, 2560]`, the - experts as MXFP4). One chunk is 32 rows x 128 columns: parameters at `g*32 + r`, then 4-bit codes in two 16-row - halves at `h*1024 + c*8 + (r%16)/2` (`q4nx_dequant.h`). D2560 is asymmetric (bf16 scale and zero-point). MX2560 is an - E8M0 exponent and an E2M1 code, with a weight of `E2M1 * 2^(e-127)`. Per expert, the 69 row-blocks are gate and up - alternating by 128 rows, then 23 of down. Against `ggml-org/gpt-oss-20b-GGUF`: D2560 corr 0.995-0.997 against its - Q8_0 (4-bit against 8-bit), MX2560 exact against its MXFP4 read through ggml's own rule (`kvalues_fp4`, the E2M1 - table doubled, times `2^(e-128)`). -- **Attention sinks**: one logit per head that joins the softmax and carries no value - (`q4nx_attn::head_context`'s `sink`; a null sink leaves every other model bit-identical). **o_proj bias.** -- **YaRN** (`rope_type: "yarn"`, `Q4nxRopeYarn` in `q4nx_rope.h`): the frequency ramp of transformers' - `_compute_yarn_parameters` (`truncate: false` honoured) and the attention factor on cos and sin. -- **The MoE** (`gptoss_moe`): top-k of the biased router logits, a softmax over those k (llama.cpp's - `SOFTMAX_WEIGHT`), and per expert ggml's `swiglu_oai` (`min(gate, 7) * sigmoid(1.702 * gate) * (clamp(up, -7, 7) + 1)`) - with the gate, up and down biases. The experts are read from the MXFP4 chunks in place, never expanded. -- The sliding window on alternate layers was already read from `layer_types`. - -Checked on the host, 2026-11-12. A numpy GPT-OSS forward of the same weights (transcribed from our llama.cpp fork's -`openai-moe.cpp`) puts `<|channel|>` first after `<|start|>assistant` (38.3 against 18.7 for the next token). The -forward's logits match it at corr 1.000000, max |d| 6e-5 of 38.3. `gptoss_forward` (CI) writes a two-layer container -in this format and holds the forward to the same reference on its bytes: corr 1.0000000. Removing any one rule fails -it, while batched equals sequential passes every time: the sinks (0.61), the o bias (0.29), the MXFP4 bias (0.84), -the SwiGLU clamp (0.92), the YaRN attention factor (0.994), the YaRN frequency rule (0.99989), and the top-k softmax -(0.99967). A batched prefill of gpt_oss takes the per-token path. No kernel set for its shapes was built or run (the -NPU is off). - -**Hy-Memory-30B-A3B-NPU2: hy_v3 runs on the existing MoE path (engine#1786).** Tencent's hy_v3 routes as DeepSeek-V3 -does (our llama.cpp fork's `src/models/hy-v3.cpp`): sigmoid router probabilities, the top 8 of 128 chosen on probability -plus an expert bias, weighted by the unbiased probabilities normalised to sum 1 and scaled, one ungated shared expert, -and a dense first layer. The forward already ran that routing. What it did not do was read the pack's way of saying -it. Each of these, alone, made the reply ` of of of`: - -- **The config keys.** `moe_router_use_sigmoid` is the sigmoid (it was read as softmax), `router_scaling_factor` is the - weight scale, `first_k_dense_replace` is the leading dense count with the dense width from `intermediate_size`, and - `expert_hidden_dim` (768) is the expert width. `moe_intermediate_size` says 1536, but the experts and the shared - expert are `[768][2048]` in `tencent/Hy-Memory-30B-A3B` itself. -- **The router** is stored `[H][E]`, the transpose of upstream's `[E][H]`, checked by a ranged read of the checkpoint. - A router of that shape is transposed at load. **The expert bias** is the F32 `moe_expert_bias`, equal to upstream's - `mlp.expert_bias`. -- **The expert slices are `[N][K]`.** FastFlowLM's Qwen MoE packs transpose each expert, and the forward read every - `*_exps_proj` stack that way. This pack does not transpose: its gate stack is `[3072][8][4736]`, 24 x 8 tiles per - expert for 768 x 2048. The grid is now decided from the first MoE layer's own gate stack. Against upstream expert - 0 of layer 1, `[N][K]` gives gate and down corr 0.9974, and the old reading gives 0.008. The same rule also fires - on `MiniCPM-V-4.7-35B-A3B-NPU2` (the only other container on the box it fires on). That reading is right there too: - its layer-0 expert 0 against `openbmb/MiniCPM-V-4.7-35B-A3B`'s own `gate_up_proj` gives corr 0.997 as `[N][K]` and - 0.024 as `[K][N]`. That pack does not run on this forward either way ("MoE designs missing", engine#1788), so no - answer changes. -- **A leading dense layer without MLA** gets the DG/DU/DD designs and `dense_mlp` the MLA path already had. A batched - prefill of such a model takes the per-token path. - -On the host the known-answer prompt (no-think template) answers `Paris` and ``. Against a numpy hy_v3 forward -of the same weights, decoded through `q4nx_dequant.c`, the last position's logits match at corr 1.000000, max |d| -3e-5 of 26.6, with the same top 5. `batch_check_state_hy_v3` fabricates a two-layer pack written this way and holds -the forward to that reference: corr 1.0000000. Each of these mutants fails it: - -| mutant | corr | -|---|---| -| `[K][N]` slices | 0.59 | -| no router transpose | 0.78 | -| softmax gating | 0.9965 | -| no scale | 0.94 | -| bias ignored | 0.87 | -| expert width from `moe_intermediate_size` | refused at the expert gate | - -No kernel set for its shapes was built or run (the NPU is off). - **Per-head q/k normalisation is read from the container (engine#1788).** Whether the forward normalises q and k per head used to be decided by `config.json`'s `model_type` alone: on for `qwen3*` and `gemma*`. A pack of the same architecture under another name got it wrong in silence. FastFlowLM's MiniCPM-V 4.7 @@ -2373,7 +1923,7 @@ engine#587 measured the route on it): | `dflash` | `FastFlowLM/Nemotron-3.5-Bolt-E4B-DFlash-NPU2` | hrx | not served: a draft head, not a decoder; "The draft loop, and whether to serve it" below covers what the HRX route does with it | | `gemma4` | `FastFlowLM/Nemotron-3.5-Bolt-E4B-NPU2`, `FastFlowLM/Gemma4-12B-IT-NPU2` | hrx | not served: no gemma4 model type or kernels in the route | | `hunyuan-dense` | `FastFlowLM/Hy-MT2-1.8B-NPU2` | hrx, npu | **served and measured** on the per-op route ("FastFlowLM's own containers" below) | -| `gemma3` | `FastFlowLM/medgemma-1.5-4b-it-NPU2` | hrx | not served on the device by a stock install: no kernel set for a gemma3 geometry ships. The forward reads the family and answers the known-answer prompt on the host ("Gemma3 text decoders" above) for four of its packs, checked 2026-11-12: `Gemma3-270M-NPU2`, `Gemma3-1B-NPU2`, `Gemma3-4B-Text-NPU2` and `Medgemma-4B-NPU2` each answer `Paris` and then stop. `medgemma-1.5-4b-it`, `Gemma3-4B` and `Translategemma-4B-Instruct` have the same `config.json` text keys and `model.q4nx` size as `Medgemma-4B`. The shapes each geometry needs: 4B `QKV K2560 N4096`, `O K2048 N2560`, `GU K2560 N20480`, `D K10240 N2560`; 1B `QKV K1280 N1536`, `O K1024 N1536`, `GU K1280 N13824`, `D K6912 N1536` | +| `gemma3` | `FastFlowLM/medgemma-1.5-4b-it-NPU2` | hrx | not served | | `lfm2` | `FastFlowLM/LFM2-2.6B-NPU2`, `FastFlowLM/LFM2-1.2B-NPU2`, `FastFlowLM/LFM2.5-1.2B-NPU2` | hrx, npu | `LFM2.5-1.2B-NPU2` **served and measured** on the per-op route ("FastFlowLM's own containers" below); `LFM2-1.2B-NPU2` serves too (one paired ticket, same section); `LFM2-2.6B-NPU2` was not run | | `gpt-oss` | `FastFlowLM/GPT-OSS-20B-NPU2`, `FastFlowLM/GPT-OSS-Safeguard-20b-NPU2` | hrx | not served | | `hy_v3` | `FastFlowLM/Hy-Memory-30B-A3B-NPU2` | hrx | not served; the first table above records this one | @@ -2401,7 +1951,7 @@ pack declares (a config with no `architectures` declares its `model_type`): | `DFlashDraftModel` | Nemotron-3.5-Bolt-E4B-DFlash | not served: a draft head, not a decoder | engine#1780 | | `Gemma4ForConditionalGeneration`, `Gemma4UnifiedForConditionalGeneration` | Nemotron-3.5-Bolt-E4B, Gemma4-E4B-IT, Gemma4-E2B-IT, Gemma4-12B-IT | not served: `gemma4`, hrx only | engine#1786 | | `HunYuanDenseV1ForCausalLM` | Hy-MT2-1.8B | not served: `hunyuan-dense`, hrx only | engine#1786 | -| `Gemma3ForCausalLM`, `gemma3_text` | Gemma3-270M, Gemma3-1B, Gemma3-4B, Gemma3-4B-Text, medgemma-4b-it, medgemma-1.5-4b-it, Translategemma-4B-Instruct | not served: `gemma3`, hrx only (the per-op forward reads the family on the host; the device needs a kernel set per geometry, see the `gemma3` row above) | engine#1786 | +| `Gemma3ForCausalLM`, `gemma3_text` | Gemma3-270M, Gemma3-1B, Gemma3-4B, Gemma3-4B-Text, medgemma-4b-it, medgemma-1.5-4b-it, Translategemma-4B-Instruct | not served: `gemma3`, hrx only | engine#1786 | | `Lfm2ForCausalLM` | LFM2-1.2B, LFM2-2.6B, LFM2-1.2B-Tool, LFM2-2.6B-Transcript, LFM2.5-1.2B, LFM2.5-1.2B-Thinking | not in the registry's NPU set: `lfm2`, hrx only (LFM2.5-1.2B on the per-op route is engine#2170, which adds no `NPU_ARCHS` entry) | engine#1786 | | `GptOssForCausalLM` | GPT-OSS-20B, GPT-OSS-Safeguard-20b | not served: `gpt-oss`, hrx only | engine#1786 | | `Qwen3_5MoeForConditionalGeneration` | Qwen3.6-35B-A3B | not served for this class: the route serves `qwen3_5_moe` through `Qwen3_5MoeForCausalLM` (GGUF on the NPU); the VLM wrapper class is hrx only, and FastFlowLM's own pack is not measured on the route | engine#1786 | @@ -2459,8 +2009,7 @@ does on a geometry that has a sealed set: `Qwen3-1.7B-NPU2` is 3.0 tok/s here an | `Qwen3-VL-4B-Instruct-NPU2`, text tower only | `qwen3` | identical, `Paris` | 1.4 to 1.5 tok/s | | `Hy-MT2-1.8B-NPU2` | `hunyuan_v1_dense` | identical, `Paris`; 4 of 5 translations byte-identical to FastFlowLM's runtime (below) | 2.3 to 2.8 tok/s | | `LFM2.5-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | identical, `Paris`; against FastFlowLM's runtime below | 3.5 to 4.6 tok/s | -| `LFM2-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | identical, `Paris`; against FastFlowLM's runtime below | 3.7 to 4.3 tok/s | -| `LFM2.5-1.2B-Thinking-NPU2` | `lfm2`, hybrid, thinking | identical; `Paris` after 287 reasoning tokens, no clean stop (below) | 4.3 tok/s | +| `LFM2-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | `Paris`; one request per prompt, against FastFlowLM's runtime below | 3.7 tok/s | `D3-Nano-NPU2` is a decision pack: it loads on the `Qwen3.5-2B` kernel set, its readout converts with `tools/safetensors_to_q4nx.py decision`, and `/v1/systemone` returned three identical answers to the documented @@ -2515,19 +2064,13 @@ the gap "Per-op route" above describes. at the merged head of that change: two of five answers byte-identical (`Paris`; the count from one to ten), three that share 21, 58 and 53 characters with FastFlowLM's and then continue differently, each a fluent answer to the question (a Fibonacci function that returns `0` for `n <= 0` where theirs returns a message). Decode 3.7 tok/s -against 42 to 47. Its logprobs at the three points were not taken. A later ticket at the merged code (main `1131acf0`) asked both -packs the known-answer prompt three times each: `Paris`, identical, with decode at 4.2 to 4.3 tok/s; those four -figures are records in `bench/npu-leaderboard/`. - -`LFM2.5-1.2B-Thinking-NPU2` serves on the same directory too: its template is read from the pack's -`chat_template.jinja` and the reasoning is split out. Asked the known-answer prompt three times with 320 tokens -of room, it reasoned for 287 tokens and then answered, the three replies identical in reasoning and content, at -4.3 tok/s. The content begins `Paris` and does not end there: it goes on (`\boxed{Paris}`, then a restatement) -until the token limit, so the row has the answer and not a clean stop. FastFlowLM's own runtime, given the same -file, prompt and budget, returned no content at all: 320 tokens of reasoning that begin `Okay derivative of the -function f(x) = x^2 + 3x - 4. Find its vertex.`, a problem nobody asked, the same on both of two requests. So -on this pack the two runtimes' texts are not comparable and the row is not a parity row; it records that this -route reasons about the question that was asked. +against 42 to 47. Its logprobs at the three points were not taken, and it was asked each prompt once, so this row +does not carry the three-identical-answers condition. + +`LFM2.5-1.2B-Thinking-NPU2` loads on the same directory too: its template is read from the pack's +`chat_template.jinja`, the reasoning is split out, and at 96 tokens every one of the five replies was still +reasoning, so there is no answer text to compare and no row; the reasoning differs from FastFlowLM's from its +first words. **`Hy-MT2-1.8B-NPU2`: a third tile.** This pack's projections are stored 4,608 bytes to a tile where every other pack on the box uses 5,120, and no reader knew the size, so it was refused at the first dequant. The tile is @@ -2557,33 +2100,6 @@ one request still different is `Me gustaría una taza de café…` here against the same code three requests of the known-answer prompt returned `Paris` three times, and decode was 2.3 to 2.8 tok/s (32 layers on the per-op route) against FastFlowLM's 50 to 53. -**`Qwen3.5-9B-NPU2`: a head the config does not declare.** This pack's `config.json` has no -`tie_word_embeddings` key, and the forward read an absent key as "tied", the usual default. So it took the logits -off the embedding, although the pack ships a separate `lm_head.weight` and the model is untied (upstream states -`false`). The answer was noise on the host and on the NPU, from weights that all decode correctly. When the key is -absent and the container holds both tables, the forward now samples 16 rows of each and lets the weights decide. -A tied head is the embedding's quantised copy (correlation near 1), and an untied one is unrelated. The load says -which, for example `tie_word_embeddings not stated: lm_head.weight vs the embedding on 16 rows, corr 0.0142 -> -untied (the separate head is used)`. A stated key is still obeyed. `batch_check_state_qwen2_untied` pins this: the -Qwen2 container with the key removed must match the numpy reference that uses its `lm_head` (corr 1.0000000). With -the check disabled it reads corr 0.03 and argmax 6 against 7. - -**`Qwen3.6-35B-A3B-NPU2`: two spellings of the same model.** This pack differs from the hub's Qwen3.5 dense packs -in two places, and each made the reply noise on the host: -- **`ssm_alpha_proj` / `ssm_beta_proj` are BF16 `[2048, 32]`**, i.e. `[in, out]`, like the pack's `[H][E]` router. - Against the GGUF of the same checkpoint (unsloth's Q8_0), read transposed they match at corr 0.99999; the GGUF - orders the 32 value heads interleaved. `npu/forward/include/bf16_matrix.h` reads a BF16 matrix whose declared shape - is exactly the transpose of the grid asked for that way, and a square one as `[out, in]`. -- **The shared expert's sigmoid gate is `shared_expert_gate.weight`**, beside the router, where the FastFlowLM name - table had `mlp.share_router.weight`. The gate was never read, so the shared expert was added ungated in every layer. - The forward reads either name. - -With both fixes the host forward reasons ("Thinking Process: 1. **…"). The other tensor kinds decode against the GGUF: -routed expert 0's gate 0.9958 and the shared gate 1.0000; `ssm_out` and the GDN `qkv` match per value head. Pinned by -`bf16_matrix_test` (both orientations, the padded grid, the refusals) and `batch_check_state_hy_v3_sgate`: the hy_v3 -fixture with the gate tensor, against a reference that applies it. With the gate name missed, that arm reads corr -0.989. - Not established by any row here: that a pack goes through the importer (`npu-pack` still reports these containers as recognised, not importable), anything about a vision or audio tower, and any pack whose geometry has no kernel set on the box. Of the 57 `*-NPU2` repos with a readable geometry on 2026-11-11, 19 had all four @@ -2825,7 +2341,7 @@ got before. | `top_k` | keep the `k` highest logits; `0` or absent keeps all of them | | `top_p` | nucleus sampling, `1` or absent keeps all of them (`min_keep` candidates always stay) | | `seed` | the same seed and the same request give the same text; absent or `-1` seeds from the clock | -| `logprobs` | `true` reports each generated token's log-probability on `/v1/chat/completions` (streamed or not); the values are the raw distribution, before `top_k`/`top_p`/`temperature` and before any grammar mask, as OpenAI and vLLM report them; normalised over the tokenizer's ids only, so a head padded past the vocabulary (Qwen3: 151,936 rows against 151,669 ids) neither shifts them nor puts a padding id in `top_logprobs` (engine#2272) | +| `logprobs` | `true` reports each generated token's log-probability on `/v1/chat/completions` (streamed or not); the values are the raw distribution, before `top_k`/`top_p`/`temperature` and before any grammar mask, as OpenAI and vLLM report them | | `top_logprobs` | `0`-`100` alternatives per token, highest raw logit first; needs `logprobs: true` | | `stop` | up to 4 strings: generation stops before the first match, and the tail is held back so a stop split across two tokens still matches | | `max_tokens` / `max_completion_tokens` | the reply's cap (`max_completion_tokens` wins); a value that is not a number is a 400 naming the field, and a fractional one is truncated | @@ -3345,28 +2861,6 @@ can fail a run. variable and never sees that line is running a binary built before the dump existed (#1977): rebuild it, rather than looking for a code path that skips the dump. -**A projection that is right on the host and wrong on the device: check its weight-cache key.** The device target -keeps one packed weight image per (layer, key), and the host arms do not use that cache. Two projections of one layer -under one key therefore launch with the same weights on the NPU only: a Gemma4 pack answered correctly in host-GEMM -mode and returned noise on the device, with every kernel bit-exact, because its per-layer input gate and projection -used the attention projections' keys (engine#2213). `NPU_INFER_DUMP_OPS=1` on both sides showed it in one token: the -op lines agree up to the first projection that shares a key. A layer's keys are named in -`npu/forward/include/weight_keys.h` and held distinct at compile time; a new projection takes a new name there. - -A caller inside the process that needs the same stream as data, not files, uses -`Q4nxNpuForward::set_layer_taps(layers, &sink)` (engine#1780): every pass then appends one row per position to the -sink, `taps x H` wide, the listed layers side by side. It works for `step()` and for the batched passes, and it is what -a DFlash draft's `target_layer_ids` are read with (docs/amd-drafters.md). Layer `l`'s block is the stream after layer -`l`, the same point `layer_.f32` records; `layer_taps_model_test` holds the two to each other bit for bit. - -**A Gemma4 (per-layer-embedding) pack no longer re-reads its layer weights every token.** `step()` skips the -per-token Q4NX dequant once a dense layer's projections are packed on the device; a per-layer-embedding model was -excluded from that, although its gate and projection are read once at init and are packed like the rest. Measured on -`Nemotron-3.5-Bolt-E4B` (42 layers), one ticket, one binary, arms old / new / old / new (`NPU_PLE_RELOAD=1` is the old -behaviour): 3377 and 3410 ms per step with the reload, 2486 and 1914 ms without, same reply in all four. What is left -is the device: on this per-op int8 family a 16-row launch streams the whole weight image (`GU` 21.5 ms, `D` 13.6 ms, -`QKV` 4.0 ms per launch), which is about 1.9 s per token over 42 layers. - The files these dumps write (`NPU_INFER_DUMP_DIR`'s `layer_.f32`, `NPU_I8_GS_DUMP`'s weight-scale pairs) are created owner-only (0600) and never through a symlink at the final path component, whatever the umask, and a dump file an older build left at 0666 is narrowed to 0600 when it is rewritten; the head-image cache's files are at most 0644. Both go through `npu/forward/include/file_create.h`, so no write site uses `fopen(…, "w")`, @@ -3673,123 +3167,6 @@ to 6.9 %** — the head needs the hidden of the token it drafts past, which a on the hidden without lagging the cursor is worse: it accepts **0 of 92** while the mismatch counter stays at 0, i.e. a silently broken loop rather than a slow one. -**The one-pass round, for a head that does not need that hidden** (engine#1781). The refutation above is about the -MTP head. A DFlash draft drafts from the base's *committed* rows plus the last token's embedding, so it never needs the -hidden after the token it drafts past. Such a head says so (`DraftHead::needs_last_hidden()` returns false), and -`mtp::run()` then verifies `[t1, d1..dk]` as one batch of `k + 1` rows: row `i` checks draft `i + 1`, the round commits -`t1` plus the accepted drafts, the drafter's `commit_rows()` gets those rows (`t1`'s first), and the next `t1` is the -base's choice in the first row past them. `k` is held to `max_batch_rows() - 1`, because `t1` takes a row. The MTP head -keeps the two-pass round (the default). `tests/mtp_draft_test.cpp` [17] pins it on the toy base: the text and the -recurrent state are plain decode's for perfect, garbage and half-right heads at `k` = 1 to 12; a fully accepted chain -takes one pass per `k + 1` tokens and no step (k = 7: 0.125 base passes per token against the two-pass round's 0.250); -and every committed row reaches the drafter once, in order. This is what the "one pass (verify only)" rows of -[docs/oga-drafter-path.md](oga-drafter-path.md) price. - -The DFlash draft is that head: `npu/dflash_head.h` (`DFlashHead`) drafts from the forward's tap sink -(`set_layer_taps`), fused into a `DraftKvCache` as rows are committed, and trims the sink after each verify batch to the -rows the round committed, because the forward writes a tap row for every position a pass runs, rejected drafts -included. The target's embedding and the rule that turns a drafted row into a token (the head's argmax or a DFlash 2 -selector) are hooks. `tests/dflash_head_test.cpp` runs it through `mtp::run()` on a toy base that writes taps the same -way: the text equals plain decode's, the sink ends holding exactly the committed rows, and every draft equals -`draft_forward()` over those rows, bit for bit. - -Two pieces for a caller that is not a test. `mtp::run_from()` starts the loop from a base that already ran its prompt -(a batched prefill: the positions it holds and the logits after the last), where `run()` prefills one token at a time. -And `Options::fallback_below` / `fallback_window` apply the speculation fallback (`SpecFallback`, -`npu/forward/include/spec_greedy.h`) inside the loop, on either round: once the last `fallback_window` rounds emitted -fewer than `fallback_below` tokens on average, the run finishes with plain steps (`Stats::fell_back_at` says after which -round). `tools/dflash2_probe loop` with `DFLASH2_MTP_RUN=1` drives the served 27B through exactly these pieces, with -`DFlashHead` and the DFlash 2 selector as its pick hook, after the same batched prefill as its hand-written loop. - -**Serving it: `forward-serve --spec-lookup K`** (off by default; `--spec-fallback X` sets the fallback, which defaults to 2 tokens per round with this drafter and 3 with `--spec-draft`, and `--spec-fallback 0` turns it off: on the served 27B a prose reply that keeps 2.24 tokens per DFlash round ran at 2.51 tok/s without one and 3.88 with the default, A B A B). A request that -is greedy (on a server without `--dump-logits`) decodes through -`mtp::run_from()` with the prompt-lookup drafter (`npu/mtp/prompt_lookup.h`). That drafter needs no model: it proposes -the up to `K` tokens that followed the latest earlier occurrence of the stream's last one to three tokens. Every -other request takes the plain loop. The rule is that speculation is used only where it cannot change a token: each -token is still the tokenizer-bounded argmax, and both loops hand every token to the same code for end tokens, stop -sequences and the stop hold-back. The reply's last token is not run through the model (`Options::consume_last = -false`), as in the plain loop. The server logs one `[spec]` line per request (rounds, drafts accepted, plain steps). -`tests/serve_spec_lookup_test.py` (ctest `serve_spec_lookup`, on a build with `ONEBIT_NPU`) holds it to that end to -end. It fabricates a container, serves it host-GEMM twice (plain, then with `--spec-lookup 4`), and requires the same -text, `finish_reason` and `completion_tokens` for: -- every budget; -- stop sequences cut from the plain replies, so they do end them; -- a seeded sampled request, which must stay on the plain loop (without `--spec-sample`, below). - -In host-GEMM mode the verify batch is 16 rows, the device's width. Host GEMMs have no width of their own, and the 1 -the forward reports there would leave no row for a draft. The device route has not been measured with it yet. - -The verify pass returns its rows at the head's width, which is wider than the tokenizer whenever the head carries -padding ids (Qwen3: 151,936 against 151,669). The loop chooses among the tokenizer's ids but steps through the batch -by `BaseModel::row_stride()`, the head's width; stepping by the tokenizer's bound reads every row after the first -misaligned. The test's container has a 512-row head against its 256-id tokenizer for this reason. With the wrong -stride its speculative replies leave plain greedy within 24 tokens. - -A request with logprobs takes the speculative path as well. `Options::on_token_row` hands every emitted token the -row it was chosen from or checked against: the base's own logits at that position, at the head's full width. The -serve reports the token's log-probability and its `top_logprobs` from that row, as the plain loop does from its step's -row. The test sends chat requests with `top_logprobs` 3 to every server, one of them a prompt whose drafts the lookup -keeps. Each speculative server reports the plain server's tokens, log-probabilities and alternatives exactly -(difference 0 on the host forward). - -A request with a grammar (`grammar`, or `response_format` json) takes the path too, its choices made through the -sampler (`npu::SpecSampler` with a `TokenConstraint`): -- the round's own token is chosen inside the mask of the grammar state the emitted tokens left; -- draft `i` is checked inside the mask of that state advanced past drafts `0..i-1`. `RowSampler::begin_round()` - starts each round from the committed state, and a kept draft advances the round's copy; -- greedy, a draft is kept iff it is the masked argmax, so a constrained greedy reply is the plain loop's token for - token; -- sampled (with `--spec-sample`), the rule above runs on the masked distribution; -- a state that allows no token ends the reply (`Result::sampler_ended`), as it does in the plain loop. - -`mtp_sampled [10]` holds this on a toy grammar: -- greedy replies equal the constrained plain loop; -- 200 sampled replies per round shape obey the rule; -- sampled replies sit within TV 0.005-0.006 of the exact masked product, against 0.056 for a sampler that checks - every draft in the round's first state; -- a constraint with nothing left ends the run cleanly. - -The serve test adds two greedy constrained requests to every server, a forced-alternation GBNF and a json_object. -Both equal plain's, and the lookup keeps 11 of their drafts. A sampled constrained reply on the `--spec-sample` server -stays inside its grammar. - -**`--spec-sample`** (with either drafter) takes sampled requests (temperature above 0, with or without a grammar) onto -the same path. `Options::sampler` (`mtp::RowSampler`, `npu/mtp/draft.h`) turns the loop's choices into draws. -`npu::SpecSampler` (`app/npu_spec_sampler.h`) is forward-serve's sampler: -- `pick()` is the plain loop's `sample_token()`; -- `check()` keeps a draft `x` with probability `p(x)`, where `p` is what `sample_token()` draws from under the same - temperature, top_k and top_p (`sample_probs()`, `app/npu_sampling.h`). Otherwise it emits a draw from `p` with `x` - removed, and that draw ends the round. - -This is `npu/mtp/spec_accept.h`'s rule for a drafter whose own distribution is one-hot at its proposal, which both -drafters are. Each emitted token then follows `p` at its position, as in the plain loop. It does not follow the plain -loop's random stream: the same seed gives the same reply again, but a different one than without the flag. That is why -the flag is off by default. - -`tests/mtp_sampled_test.cpp` (ctest `mtp_sampled`) holds the distribution on a toy Markov chain. It runs 200,000 -seeded 3-token replies per arm and compares them with the exact product of the chain's conditionals; plain sampling is -the control: -- both round shapes land at TV 0.008-0.009 against the control's 0.009, and forward-serve's own sampler (temperature - 0.8, top_k 4, top_p 0.9) at 0.006; -- a sampler that keeps every draft lands at 0.92; -- every transition of 24-token replies matches its row at k = 1 and 4, which catches a refusal carried into a later - round. - -`npu_sampling_test` holds `sample_probs()` to `sample_token()`'s own draws. The serve test adds a `--spec-sample` -server: its greedy requests equal plain's; a seeded sampled request repeats itself; at temperature 0.001 a sampled -reply is plain greedy's. - -**`forward-serve --spec-draft `** (`--spec-draft-k K`, default the package's block minus one) serves the same path -with a DFlash draft package as the drafter, in place of the prompt lookup. At load the package is decoded once and -checked against the target, before `ready`: the widths must match, and its target layers must exist. The forward then -writes those layers' hidden states into a tap sink on every pass. The sink is emptied when a request takes the engine, -so its rows start at the request's first position. `DFlashHead` drafts from it, picking with the target's own head: -DFlash 2's candidate selector when the package has one (`dflash2_select`, `npu/dflash_head.h`), else the bounded argmax. -A request that resumed from a saved prefix has no rows for the positions it skipped, so its head refuses to draft and -it decodes with plain steps: the same text, just no speedup. The same test serves a generation-1 package it writes -(`tests/dflash_draft_test.cpp`'s synthetic layout, sized to the fabricated target). On it every greedy request of two -or more tokens drafts, and every reply equals the plain server's. - **What would change the answer**: a cheaper verify batch (the array wait is ~60 % of a batched pass — a kernel lever) or a head that reliably produces a chain above 3.9 (widening `k` is not it: `k = 6` bought +0.05, and depth 6 measured 0 %). @@ -3908,6 +3285,73 @@ prints both, so "95 % with 2 of 40 cut off" cannot be read as "95 %". The rows i Yes. The two were measured running together for forty repetitions with no hang and no corruption, and `--device hybrid` splits a single request across them ([hybrid.md](hybrid.md)). +## The audio chain on the host (Whisper), and what it measures + +The engine can read a Whisper pack's ONNX graphs and run the whole chain on the host — log-mel front end, the encoder, and a +greedy decode to token ids — with nothing from outside the tree. It is a **reference**, not a serving route: it exists so the +numbers a device route will be judged against come from the same code that ships (engine#1783). + +``` +tools/whisper_transcribe [expected ids ...] +tools/whisper_transcribe [expected ids ...] +``` + +The pack form reads both graphs, **and the mel-bin count, from the pack**: `80` for whisper-small and `128` for +whisper-large-v3, taken from the encoder graph's own first `Conv`. Passing the wrong count produces a graph that runs and +answers wrongly, so `npu/audio_pack.h` derives it rather than trusting a config field, and reports a disagreement instead of +resolving it silently. + +Measured on this box, 2026-11-11, load 0.69 (a quiet box; the same binary read 41 s for the encoder at load 40, so **quote the +load with any rate**): + +| | encoder / 30 s window | whole chain, 25-token answer | +|---|---|---| +| serial (`ONEBIT_AUDIO_THREADS=1`) | 13.6 s | 123.4 s (4.30 s/token) | +| default (8 threads) | **7.4 s** | **68.3 s** (2.35 s/token) | + +Two things follow, and both matter more than the speed itself: + +- **Decode is the wall, not the encoder.** The executor re-runs all 786 decoder nodes over all **128 positions** per token + (there is no KV cache, and the pack ships no `decoder_with_past` graph), so the cost is O(positions) per token and no amount + of threading removes it. +- **0.6 s per 30 s window** (FastFlowLM 1.0.12's published figure, their device, their kernels) is a *device* number: the host + reference above is 7.4 s, so the gap this row is about is closed by putting the encoder on the NPU, not by tuning the host. + +### Checking a pack is servable before a device row + +``` +tools/audio_kernel_shapes # the shapes the encoder's projections ask for +tools/audio_kernel_shapes --kernels # ... and whether both M tiers of each are present +``` + +Roles are decided by **shape against the pack's hidden width** (h×h is q/k/v/o — one ELF serves all four — 3h is fused QKV, +4h is GU/D), because a packed export names its weights `val_N` and the human name sits on a tensor the MatMul never touches. + +**Whisper's encoders ask for three of those four, not four.** Their attention is three separate `[h, h]` MatMuls, so q/k/v/o +all land on the one `h×h` kernel and no `3h` shape exists in the graph; what the encoder needs is `O (h×h)`, `GU (h→4h)` and +`D (4h→h)`. A fused `QKV` kernel belongs to a decoder-style export, and emitting one for an encoder is harmless but misleading — +`audio_kernel_shapes` lists what the graph actually contains, which for whisper-small is `O K768 N768`, `GU K768 N3072`, +`D K3072 N768` and for large-v3 `O K1280 N1280`, `GU K1280 N5120`, `D K5120 N1280`. + +**The widths in those names are the engine's STAGED ones, not the model's**: a container's K is a multiple of 256 and its N a +multiple of 512 (`npu/forward/src/q4nx_pack.c`, `q4nx_pad_k` / `q4nx_pad_n`), and the forward asks for the ELF at that width, so +hidden 768 is `N1024` and a `3h` of 2304 is `N2560`. A set emitted at the model's own widths is a set the forward cannot find — +measured on the first device run of a Whisper family (engine#1783), after `audio_kernel_shapes` had called it covered. +A +shape that is none of the four is reported as `unknown`: asking for the wrong ELF wastes a device ticket. + +### Running the chain as a test + +`ctest -R whisper` covers the pieces; the end-to-end row needs a pack and a wav: + +``` +cmake -S . -B build -DONEBIT_TEST_WHISPER_PACK=$HOME/models/whisper-small +ONEBIT_TEST_WHISPER_WAV=$HOME/jfk.wav ctest -R whisper +``` + +The expected ids live in `tests/data/whisper-jfk-ids.json`; without a pack or a wav the rows **skip (77)** rather than pass. +`ctest -R "audio|onnx_graph"` reports `0 skipped` only when a pack is configured — a run with skips has not checked the chain. + ### Do I need a special kernel or driver? The NPU needs the `amdxdna` kernel driver. The [1bit OS](os-v3.md) appliance ships a kernel built diff --git a/npu/audio_exec.cpp b/npu/audio_exec.cpp index edb394ad7..86da3dd89 100644 --- a/npu/audio_exec.cpp +++ b/npu/audio_exec.cpp @@ -18,6 +18,7 @@ #include "audio_exec.h" #include "audio_parallel.h" +#include #include #include #include @@ -221,9 +222,14 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con return t.count(name) != 0 ? need(name) : init_f32(name); }; + // ONEBIT_AUDIO_TIMING=1 prints one line per node: index, op, output shape, milliseconds. Off by default and off in every + // test. It exists because "the encoder takes N seconds" is not actionable for a device route: the route needs to know WHICH + // launches, and `audio_kernel_shapes --write-plan` names them by order, so the two lists line up. + const bool timing = std::getenv("ONEBIT_AUDIO_TIMING") != nullptr; int node_index = 0; for (const auto& n : g.nodes) { ++node_index; + const auto t_node = std::chrono::steady_clock::now(); const auto where = [&]() { return "node " + std::to_string(node_index) + " (" + n.op + ")"; }; if (n.outs.empty()) fail(where() + " has no output"); const std::string& oname = n.outs[0]; @@ -263,11 +269,16 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con } const float* ap = a.data.data() + oa * size_t(M * K); const float* bpp = b.data.data() + ob * size_t(K * N); - float* rr = r.data.data() + bi * size_t(M * N) + size_t(i) * size_t(N); + float* __restrict rr = r.data.data() + bi * size_t(M * N) + size_t(i) * size_t(N); + // The row pointer is hoisted out of k (it does not depend on it) and `restrict` tells the compiler rr and + // br cannot overlap, so the inner loop vectorises. The accumulation ORDER is untouched -- still k + // ascending into the same rr[j] -- so the result is bit-identical: measured on a 128x768x768 projection, + // 22.57 -> 33.84 GFLOP/s (1.50x) with max |difference| exactly 0 against the previous loop. + const float* __restrict arow = ap + size_t(i) * size_t(K); for (int64_t k = 0; k < K; ++k) { - const float av = ap[size_t(i) * size_t(K) + size_t(k)]; + const float av = arow[k]; if (av == 0.0f) continue; - const float* br = bpp + size_t(k) * size_t(N); + const float* __restrict br = bpp + size_t(k) * size_t(N); for (int64_t j = 0; j < N; ++j) rr[j] += av * br[j]; } }); @@ -526,6 +537,13 @@ bool run_graph_impl(const OnnxGraphInfo& g, std::map t, con } else { fail(where() + " is not implemented; the pack needs an op this executor does not have"); } + if (timing) { + const double ms = + std::chrono::duration(std::chrono::steady_clock::now() - t_node).count(); + std::string shape; + for (int64_t d : r.dims) shape += (shape.empty() ? "" : "x") + std::to_string(d); + std::fprintf(stderr, "audio-node %d %-22s %-18s %.3f ms\n", node_index, n.op.c_str(), shape.c_str(), ms); + } t[oname] = std::move(r); } diff --git a/npu/audio_pack.cpp b/npu/audio_pack.cpp new file mode 100644 index 000000000..c21e3549c --- /dev/null +++ b/npu/audio_pack.cpp @@ -0,0 +1,178 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// audio_pack.cpp -- see audio_pack.h. The discovery is deliberately boring: known names, in a fixed order, and a refusal that +// names what is missing. The one interesting step is the mel-bin count, which comes from the encoder graph rather than a config. + +#include "audio_pack.h" + +#include +#include +#include +#include + +#include "onnx_graph.h" + +namespace fs = std::filesystem; + +namespace onebit::audio { +namespace { + +bool is_file(const std::string& p) { + std::error_code ec; + return fs::is_regular_file(fs::path(p), ec); +} + +// The names exporters actually use, most specific first. A pack that carries both spellings is a pack someone assembled by +// hand; the first match wins and the other is ignored, which is why `dir` is recorded. +const char* kEncoderNames[] = {"encoder_model.onnx", "encoder.onnx"}; +const char* kDecoderNames[] = {"decoder_model.onnx", "decoder.onnx"}; +const char* kTokenizerNames[] = {"tokenizer.json", "tokenizer_config.json"}; + +int mel_bins_from_encoder(const std::string& path, std::string* why) { + using namespace onebit::npu::pack_import; + OnnxGraphInfo g; + try { + g = parse_onnx_graph(path); + } catch (const std::exception& e) { + if (why != nullptr) *why = e.what(); + return 0; + } + // The mel front-end is the graph's first Conv; its input-channel count is the bin count the encoder was exported for. + for (const auto& n : g.nodes) { + if (n.op != "Conv" || n.ins.size() < 2) continue; + for (const auto& ti : g.init_tensors) { + if (ti.name != n.ins[1] || ti.dims.size() != 3) continue; + return int(ti.dims[1]); + } + } + if (why != nullptr) *why = "no Conv with a weight tensor found, so the mel-bin count cannot be derived"; + return 0; +} + +} // namespace + +bool discover_audio_pack(const std::string& dir, AudioPack* out, std::string* why) { + if (out == nullptr) return false; + *out = AudioPack{}; + out->dir = dir; + std::error_code ec; + if (!fs::is_directory(fs::path(dir), ec)) { + if (why != nullptr) *why = dir + ": not a directory"; + return false; + } + for (const char* name : kEncoderNames) { + const std::string p = dir + "/" + name; + if (is_file(p)) { out->encoder = p; break; } + } + for (const char* name : kDecoderNames) { + const std::string p = dir + "/" + name; + if (is_file(p)) { out->decoder = p; break; } + } + for (const char* name : kTokenizerNames) { + const std::string p = dir + "/" + name; + if (is_file(p)) { out->tokenizer_json = p; break; } + } + // A pack an operator actually has may name its graphs after the model (large_v3_encoder.onnx), so after the exact names a + // scan is allowed -- but it never picks a `with_past` decoder, which is a different graph with a different input list, and + // whatever it picks is reported as a concern so the choice is visible rather than silent. + if (out->encoder.empty() || out->decoder.empty()) { + std::vector candidates; + for (const auto& entry : fs::directory_iterator(fs::path(dir), ec)) { + if (!entry.is_regular_file(ec)) continue; + const std::string name = entry.path().filename().string(); + if (name.size() < 6 || name.compare(name.size() - 5, 5, ".onnx") != 0) continue; + candidates.push_back(name); + } + std::sort(candidates.begin(), candidates.end()); + const auto pick = [&](const char* needle, bool exclude_with_past) -> std::string { + for (const auto& name : candidates) { + if (name.find(needle) == std::string::npos) continue; + if (exclude_with_past && name.find("with_past") != std::string::npos) continue; + return dir + "/" + name; + } + return std::string(); + }; + if (out->encoder.empty()) { + out->encoder = pick("encoder", false); + if (!out->encoder.empty()) { + out->concern += "encoder graph taken from its name: " + out->encoder; + } + } + if (out->decoder.empty()) { + out->decoder = pick("decoder", true); + if (!out->decoder.empty()) { + out->concern += (out->concern.empty() ? "" : "; ") + std::string("decoder graph taken from its name: ") + out->decoder; + } + } + } + if (out->encoder.empty()) out->missing.push_back("an encoder graph (encoder_model.onnx, encoder.onnx, or a *encoder*.onnx)"); + if (out->decoder.empty()) out->missing.push_back("a decoder graph (decoder_model.onnx, decoder.onnx, or a *decoder*.onnx without with_past)"); + if (!out->missing.empty()) { + if (why != nullptr) *why = dir + ": missing " + out->missing.front(); + return false; + } + + std::string graph_why; + const int from_graph = mel_bins_from_encoder(out->encoder, &graph_why); + if (from_graph > 0) { + out->mel_bins = from_graph; + out->mel_bins_source = "encoder graph"; + } else { + out->concern += (out->concern.empty() ? std::string() : std::string("; ")) + + std::string("the encoder graph could not be read for its mel-bin count (") + graph_why + ")"; + } + + // config.json is a cross-check, never the authority: a disagreement is reported, and the graph wins because it is what the + // weights were exported with. + const std::string cfg_path = dir + "/config.json"; + if (is_file(cfg_path)) { + const std::string text = [&] { + std::string body; + FILE* f = std::fopen(cfg_path.c_str(), "rb"); + if (f == nullptr) return body; + char buf[4096]; + size_t n; + while ((n = std::fread(buf, 1, sizeof buf, f)) > 0) body.append(buf, n); + std::fclose(f); + return body; + }(); + const auto number_after = [&](const char* key) -> int { + const std::string k = std::string("\"") + key + "\""; + const size_t at = text.find(k); + if (at == std::string::npos) return 0; + const size_t colon = text.find(':', at); + if (colon == std::string::npos) return 0; + return std::atoi(text.c_str() + colon + 1); + }; + const int cfg_bins = number_after("num_mel_bins"); + if (cfg_bins > 0) { + if (out->mel_bins == 0) { + out->mel_bins = cfg_bins; + out->mel_bins_source = "config.json"; + } else if (cfg_bins != out->mel_bins) { + out->concern += (out->concern.empty() ? "" : "; ") + std::string("config.json says ") + + std::to_string(cfg_bins) + " mel bins but the encoder graph's Conv takes " + + std::to_string(out->mel_bins) + " (the graph is what the weights were exported with)"; + } + } + const int sr = number_after("sampling_rate"); + if (sr > 0) out->sample_rate = sr; + const int fft = number_after("n_fft"); + if (fft > 0) out->n_fft = fft; + const int hop = number_after("hop_length"); + if (hop > 0) out->hop = hop; + const int frames = number_after("nb_max_frames"); + if (frames > 0) out->frames = frames; + } + if (out->mel_bins == 0) { + if (why != nullptr) { + *why = dir + ": the mel-bin count is in neither the encoder graph nor config.json, and guessing it is how a 128-bin " + "window reaches an 80-bin encoder"; + } + return false; + } + return true; +} + +} // namespace onebit::audio diff --git a/npu/audio_pack.h b/npu/audio_pack.h new file mode 100644 index 000000000..74d997d4f --- /dev/null +++ b/npu/audio_pack.h @@ -0,0 +1,53 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// audio_pack.h -- what an audio pack is, discovered rather than configured (engine#1783). +// +// Until now the host chain was driven by two file paths on a command line, which is enough for a bench and wrong for a route: +// a pack directory is what an operator has, and the two graphs inside it are named by whoever exported it. This finds them, +// derives the front-end's mel-bin count from the encoder graph's own first Conv instead of trusting a config field, and +// refuses by name when a pack is not one. +// +// The mel-bin derivation matters: whisper-small exports 80 bins and large-v3 exports 128, and feeding a 128-bin window to an +// 80-bin encoder produces a graph that runs and answers wrongly -- the failure mode this whole tree has spent the night +// removing. The Conv's input-channel count is the pack's own statement; a `config.json` is only consulted when the graph +// cannot be read, and a disagreement between the two is reported rather than resolved silently. + +#pragma once + +#include +#include + +namespace onebit::audio { + +struct AudioPack { + std::string dir; + std::string encoder; // absolute paths, empty when absent + std::string decoder; + std::string tokenizer_json; + int mel_bins = 0; // from the encoder's first Conv; 0 when it could not be derived + std::string mel_bins_source; // "encoder graph" | "config.json" | "" -- so a caller can say where the number came from + int sample_rate = 16000; // the front-end's defaults, overridable by config.json + int n_fft = 400; + int hop = 160; + int frames = 3000; // the fixed 30 s window the encoder was exported for + std::vector missing; // what a complete pack needs and this one lacks, named + std::string concern; // a disagreement or an oddity worth printing, never a refusal on its own +}; + +// True when the directory yields both graphs. `why` carries the first concrete reason when it does not. +bool discover_audio_pack(const std::string& dir, AudioPack* out, std::string* why); + +} // namespace onebit::audio diff --git a/npu/audio_parallel.h b/npu/audio_parallel.h index 30ebfc05c..e019b6d7b 100644 --- a/npu/audio_parallel.h +++ b/npu/audio_parallel.h @@ -35,6 +35,10 @@ #include #include +#ifdef __linux__ +#include +#endif + namespace onebit::npu::pack_import { // How many threads the audio executor may use. ONEBIT_AUDIO_THREADS=N sets it; anything that is not a positive number @@ -45,7 +49,26 @@ inline size_t audio_thread_count() { const long n = std::strtol(env, nullptr, 10); return n > 0 ? size_t(n) : 1; } - const size_t hw = std::thread::hardware_concurrency(); + // hardware_concurrency() answers "how many CPUs does this machine have", not "how many may this process use". Under a + // cpuset -- a thermal-run scope, a container, a systemd AllowedCPUs= -- it over-reports: measured on this box it says 32 + // while the scope allows 4, so the capped-at-8 default spawned 8 threads for 4 CPUs. The affinity mask is what the + // scheduler enforces, so that is what this follows. + // What this is NOT: a measured speedup. Three interleaved repetitions at 4 CPUs (8 threads / 4 threads) gave encoder + // 10.38/10.40 s and decode 2.58/2.69 s per token -- within the box's own noise, and my first, non-interleaved comparison + // of 8 threads against serial (5.21 against 4.24 s/token) was load, not threading. The reason to prefer the mask is that + // oversubscribing a cpuset cannot help and can only queue, not that it measured faster. + size_t hw = 0; +#ifdef __linux__ + { + cpu_set_t set; + CPU_ZERO(&set); + if (sched_getaffinity(0, sizeof set, &set) == 0) { + const int allowed = CPU_COUNT(&set); + if (allowed > 0) hw = size_t(allowed); + } + } +#endif + if (hw == 0) hw = std::thread::hardware_concurrency(); return hw == 0 ? 1 : (hw > 8 ? 8 : hw); } diff --git a/npu/forward/include/audio_device_gemm.h b/npu/forward/include/audio_device_gemm.h new file mode 100644 index 000000000..e237ff857 --- /dev/null +++ b/npu/forward/include/audio_device_gemm.h @@ -0,0 +1,112 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// audio_device_gemm.h -- one staged projection of an audio encoder, on the NPU (engine#1783). +// +// Why this exists as a header and not as code inside the audio executor: the device side of a projection is the TARGET's job +// (`NpuTarget::gemv` takes f32 activations and f32 back, and does the per-row int8 quantisation, the b8 pack and the +// dequantisation itself), so what the executor needs is the small, testable decision layer around it -- which shape may go to +// the device, what the caller's tensor is versus what the kernel writes, and what to say when it cannot. That is a pure +// function over a `NpuTarget&`, so it is driven against a fake target with no device, the same shape as every other rule here. +// +// Two facts it encodes, both learned from measurements on this box (engine#1783): +// * the kernel is named at the ENGINE'S STAGED width -- K a multiple of 256, N of 512 -- so the kernel writes N padded +// (whisper-small's O is N768 named N1024). The caller's tensor is the model's width, so the result is de-padded here rather +// than by every caller; forgetting that is how a padded column becomes a plausible wrong activation. +// * a `Plan` that loads has told you NOTHING about its designs: `Plan::designs_agree`/`check_artifacts` both return true when +// the plan names no designs. So the shape this header trusts is the one the CALLER states (role/K/N taken from the pack), and +// it refuses a shape the engine's grid cannot name instead of deriving one. + +#pragma once + +#include +#include + +#include "audio_kernel_roles.h" +#include "target.h" // ::npu::target::{NpuTarget, GemvArgs, Caps, Op} (target.h opens npu::target at global scope) + +namespace onebit::audio { + +namespace nput = ::npu::target; + +// One projection as the ENCODER states it (the model's own K and N, not the staged widths). +struct DeviceProjection { + int role = 0; // index into the target's design table + int layer = -1; // layer in flight, for residency and profiling + int k = 0; // the model's contracted width + int n = 0; // the model's output width (the CALLER's tensor width) + int rows = 0; // M: rows to run in one launch + const float* x = nullptr; // M x k row-major f32 + int a_k = 0; // the activation's real width per row (0 = k) + const char* tag = nullptr; // container tag, for the target's log +}; + +// One projection on the device: `out` receives rows x n (the CALLER's width), de-padded from the staged write. +// Returns false and fills `why` for: a shape the engine's grid cannot name, a row count the target cannot launch, +// a target without gemv, a target failure, or an output whose size is not rows x staged_n (which is the target +// disagreeing with the plan, and is never repaired here). +inline bool device_projection(nput::NpuTarget& target, const DeviceProjection& p, std::vector* out, + std::string* why) { + if (out == nullptr) { + if (why != nullptr) *why = "no output vector"; + return false; + } + if (p.x == nullptr || p.k <= 0 || p.n <= 0 || p.rows <= 0) { + if (why != nullptr) *why = "a projection needs x, rows > 0, k > 0 and n > 0"; + return false; + } + if (!target.supports(nput::Op::gemv)) { + if (why != nullptr) *why = std::string("the target \"") + target.name() + "\" does not implement gemv"; + return false; + } + const ProjectionShape shape{p.k, p.n}; + if (role_of_shape(shape, 0) == std::string("unknown") && p.tag == nullptr) { + // Not a refusal: the caller states the role index directly. But an unnamed shape with no tag is undebuggable, and + // this header's job is to keep the device log answerable. + if (why != nullptr) *why = "an unnamed projection needs a tag so the target's log can name it"; + return false; + } + const ProjectionShape staged_shape = staged(shape); + const nput::Caps caps = target.caps(); + const bool wide = p.rows > caps.max_rows; + if (wide && p.rows > caps.prefill_rows) { + if (why != nullptr) + *why = "rows=" + std::to_string(p.rows) + " is more than the target launches in one call (prefill_rows=" + + std::to_string(caps.prefill_rows) + ", max_rows=" + std::to_string(caps.max_rows) + ")"; + return false; + } + + nput::GemvArgs a; + a.role = p.role; + a.layer = p.layer; + a.M = p.rows; + a.K = staged_shape.k; + a.N = staged_shape.n; + a.x = p.x; + a.a_k = p.a_k > 0 ? p.a_k : p.k; + a.rows = wide; + a.tag = p.tag; + + std::vector got; + std::string err; + if (!target.gemv(a, got, err)) { + if (why != nullptr) *why = err.empty() ? "the target refused the projection" : err; + return false; + } + const size_t want = size_t(p.rows) * size_t(staged_shape.n); + if (got.size() != want) { + if (why != nullptr) + *why = "the target wrote " + std::to_string(got.size()) + " floats, the plan's staged shape is " + + std::to_string(p.rows) + " x " + std::to_string(staged_shape.n) + " = " + std::to_string(want) + + " (the target and the plan disagree; this is not repaired here)"; + return false; + } + // De-pad: the kernel's rows are staged_n wide, the caller's tensor is n wide. Drop the padding columns per row rather + // than returning the staged buffer, which would hand the caller a row whose stride is not its own width. + out->assign(size_t(p.rows) * size_t(p.n), 0.0f); + for (int r = 0; r < p.rows; ++r) + for (int c = 0; c < p.n; ++c) (*out)[size_t(r) * size_t(p.n) + size_t(c)] = got[size_t(r) * size_t(staged_shape.n) + size_t(c)]; + return true; +} + +} // namespace onebit::audio diff --git a/npu/forward/include/audio_kernel_roles.h b/npu/forward/include/audio_kernel_roles.h new file mode 100644 index 000000000..bca9977b7 --- /dev/null +++ b/npu/forward/include/audio_kernel_roles.h @@ -0,0 +1,97 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// audio_kernel_roles.h -- which GEMM shape an encoder's projection is, decided by SHAPE against the hidden width (engine#1783). +// +// Why by shape and not by name: a packed export reaches its weights through generically named initializers (`val_452`) and the +// human name sits on a tensor the MatMul never touches, so a name-based classifier reports every projection as unknown. Shape is +// also what a kernel actually depends on -- one ELF serves every role with the same (K, N), because q, k, v and o are all h x h. +// +// The four roles are the ones the family generator emits (see tools/audio_kernel_shapes.cpp and the private family's +// `full_i8__K_N_b8[_m16].elf` naming): +// +// fused QKV K = h, N = 3h attention's q|k|v packed into one matrix +// O K = h, N = h q, k, v and o all land here -- one ELF serves all four +// GU K = h, N = 4h the MLP's fused gate|up (Whisper's 4x expansion) +// D K = 4h, N = h the MLP's down projection +// +// A shape that is none of these is `unknown`, and that is deliberate: an unrecognised shape is a projection nobody has built a +// kernel for, and saying so is more useful than guessing a role and asking for the wrong ELF. + +#pragma once + +#include + +namespace onebit::audio { + +struct ProjectionShape { + int k = 0; // reduction (K) + int n = 0; // output width (N) +}; + +// The hidden width, from the model's own square projections: q/k/v/o are h x h in every layer, so the most common square width +// IS h. A median over all weights would be skewed by the embedding and the 4h MLP, which is what the first version did. +// The hidden width is the MODE of the square projection widths (h x h is q/k/v/o, and there are four of them per layer). +// A tie is refused rather than broken by whichever width came first: two widths occurring equally often means the pack has +// two candidate hidden widths, and picking one silently would decide every role after it -- the same "confident guess in the +// middle of its own evidence" that a 0.5 threshold on a correlation has. Returns 0 for "no evidence" and for "ambiguous". +inline int hidden_from_square_widths(const int* widths, int count) { + int best_count = 0, hidden = 0, tied = 0; + for (int i = 0; i < count; ++i) { + int occurrences = 0; + for (int j = 0; j < count; ++j) + if (widths[j] == widths[i]) ++occurrences; + if (occurrences > best_count) { + best_count = occurrences; + hidden = widths[i]; + tied = 0; + } else if (occurrences == best_count && widths[i] != hidden) { + ++tied; + } + } + return tied == 0 ? hidden : 0; +} + +inline const char* role_of_shape(const ProjectionShape& s, int hidden) { + if (hidden <= 0) return "unknown"; + if (s.k == hidden && s.n == 3 * hidden) return "QKV"; + if (s.k == hidden && s.n == hidden) return "O"; // q, k, v and o share this shape and this ELF + if (s.k == hidden && s.n == 4 * hidden) return "GU"; + if (s.k == 4 * hidden && s.n == hidden) return "D"; + return "unknown"; +} + +// The ELF the family generator emits for this shape, whether or not that role is real: the caller decides what to do with a +// name for an `unknown` shape (the tool reports it, a gate would refuse it). +// The engine's own staging grid, from npu/forward/src/q4nx_pack.c (`q4nx_pad_k` / `q4nx_pad_n`): a container's K is a +// multiple of 256 and its N a multiple of 512, and the ELF the forward asks for is named at THOSE widths, not at the model's. +// Measured on the synth768 run (engine#1783): QKV 2304 -> the forward asks for N2560, O 768 -> N1024, and a set emitted at the +// unpadded widths is "missing full ELF" while this header said the family was covered. +constexpr int kQ4nxKGrid = 256; +constexpr int kQ4nxNGrid = 512; + +inline int q4nx_pad_k(int k) { return k <= 0 ? k : ((k + kQ4nxKGrid - 1) / kQ4nxKGrid) * kQ4nxKGrid; } +inline int q4nx_pad_n(int n) { return n <= 0 ? n : ((n + kQ4nxNGrid - 1) / kQ4nxNGrid) * kQ4nxNGrid; } + +// The shape as the forward names it: padded on the engine's grid. +inline ProjectionShape staged(const ProjectionShape& s) { return ProjectionShape{q4nx_pad_k(s.k), q4nx_pad_n(s.n)}; } + +inline std::string elf_name_for(const ProjectionShape& s, int hidden, bool m16) { + const ProjectionShape p = staged(s); + return std::string("full_i8_") + role_of_shape(s, hidden) + "_K" + std::to_string(p.k) + "_N" + std::to_string(p.n) + + "_b8" + (m16 ? "_m16" : "") + ".elf"; +} + +} // namespace onebit::audio diff --git a/npu/forward/include/audio_launch_plan.h b/npu/forward/include/audio_launch_plan.h new file mode 100644 index 000000000..de4e16bdc --- /dev/null +++ b/npu/forward/include/audio_launch_plan.h @@ -0,0 +1,117 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// audio_launch_plan.h -- walk an encoder's launch list, device first, host for what the device cannot take (engine#1783). +// +// The route this belongs to (item 2 of the issue's close condition) is not "run the encoder on the NPU": it is a loop over the +// projections in graph order that sends each one to the device when the device can take it and to the host when it cannot, and +// REPORTS WHICH. That report is the measurement the row is scored on -- `device_gemm= s + host_remainder= s` -- so the +// accounting is part of the contract, not a log line: a launch that silently fell back is a launch whose time is being counted +// in the wrong column, and the whole point of measuring the split is to know which column is worth attacking next. +// +// It is deliberately not a planner: the specs come from `audio_kernel_shapes --write-plan` (already emitted, 72 launches for +// whisper-small, dense order, staged widths), and the host path is a CALLBACK so this file neither duplicates the executor's +// GEMM nor decides what "host" means. +// +// Why a callback for the activations too: each launch reads the previous one's output, so the caller owns the buffers. A +// function that took a single big X would be a function that only works for a stack of independent projections. + +#pragma once + +#include +#include +#include +#include + +#include "audio_device_gemm.h" + +namespace onebit::audio { + +struct LaunchSpec { + int index = -1; // the plan's own `order`, so a report can be compared with the plan line for line + int role = 0; + int layer = -1; + int k = 0; + int n = 0; + int rows = 0; + const char* tag = nullptr; +}; + +struct LaunchRun { + bool device = false; // ran on the device + bool host = false; // ran on the host (either because there is no target, or after a device refusal) + std::string why; // why it did not run on the device, when it did not +}; + +struct LaunchReport { + int device_launches = 0; + int host_launches = 0; + int failed = 0; + std::vector per_launch; // parallel to the specs, so a caller can print them in order + std::vector failures; // " : / " + bool ok() const { return failed == 0; } +}; + +// A host path for one launch: writes the caller's width (n), returns false and fills why when it cannot. +using HostLaunchFn = std::function*, std::string*)>; +// The activations for one launch, supplied by the caller because each launch reads the previous one's result. +using LaunchInputFn = std::function; + +// Runs every spec in order. A launch goes to the device when `target` is not null AND `device_projection` accepts it; otherwise +// the host callback runs it and the refusal reason is kept. A launch both paths refuse is a failure and does not silently +// count as either column. +inline LaunchReport run_launches(::npu::target::NpuTarget* target, const std::vector& specs, + const LaunchInputFn& input_of, const HostLaunchFn& host) { + LaunchReport rep; + rep.per_launch.resize(specs.size()); + for (size_t i = 0; i < specs.size(); ++i) { + const LaunchSpec& s = specs[i]; + LaunchRun& r = rep.per_launch[i]; + if (target != nullptr) { + DeviceProjection p; + p.role = s.role; + p.layer = s.layer; + p.k = s.k; + p.n = s.n; + p.rows = s.rows; + p.x = input_of != nullptr ? input_of(s) : nullptr; + p.tag = s.tag; + std::vector out; + std::string why; + if (device_projection(*target, p, &out, &why)) { + r.device = true; + ++rep.device_launches; + continue; + } + r.why = why; + } else { + r.why = "no target"; + } + std::vector out; + std::string hwhy; + if (host != nullptr && host(s, &out, &hwhy)) { + r.host = true; + ++rep.host_launches; + continue; + } + ++rep.failed; + rep.failures.push_back("launch " + std::to_string(s.index) + " " + (s.tag != nullptr ? s.tag : "?") + + ": device: " + (r.why.empty() ? std::string("not attempted") : r.why) + + " / host: " + (hwhy.empty() ? std::string("no host path") : hwhy)); + } + return rep; +} + +// The bar this route is scored on, formatted from a run rather than assembled by hand at the end of a session: the issue's close +// condition asks for `device_gemm= s + host_remainder= s per 30 s window`, and a number typed from memory is not evidence. +// A launch that is in NEITHER column (failed) is printed explicitly -- a split with a silent third category is a split that adds +// up to less than the run. +inline std::string format_split_run(const LaunchReport& r, double device_ms, double host_ms) { + char buf[220]; + std::snprintf(buf, sizeof buf, + "device_gemm=%.3f s + host_remainder=%.3f s per window (%d device launch(es), %d host, %d failed)", + device_ms / 1000.0, host_ms / 1000.0, r.device_launches, r.host_launches, r.failed); + return std::string(buf); +} + +} // namespace onebit::audio diff --git a/npu/forward/include/q4nx_pack.h b/npu/forward/include/q4nx_pack.h index f1f97cb88..441a226b2 100644 --- a/npu/forward/include/q4nx_pack.h +++ b/npu/forward/include/q4nx_pack.h @@ -67,8 +67,18 @@ extern "C" { * groups still looped inside the ELF (so one launch per role stays the * contract). Measured cost of an M=16 launch is ~0.34 ms against 3.4-4.9 ms * for the M=128 path. */ -#define Q4NX_DESIGN_M_PREFILL 128 +// Overridable at build time, because a wider prefill tier turns out to be a KNOB and not a hardware floor (engine#1783, +// measured on the Strix Halo array): M is rows x m, m is a generator parameter, and the four compute rows this part has are +// unchanged by M. Emitting the encoder's shapes at M=1536 (rows=4, m=16) and building with -DQ4NX_DESIGN_M_PREFILL=1536 takes +// whisper-small's projection GEMM from 2.05 s to 0.163 s per 30 s window -- 12x the rows for the same launch time: +// M=128 : QKV 4.468 O 4.351 GU 5.460 D 4.418 ms/launch +// M=1536: QKV 4.740 O 4.373 GU 4.662 D 4.588 ms/launch +// What keeps an override safe: whatever a build sets here must equal the M the adopted ELFs were generated with, and the target +// CHECKS that against the loaded ELFs' own MD rather than assuming it ("plan: prefill M=1536 but the adopted kernels run M=128", +// observed before the guard existed). Absent -D, both constants are exactly what they were. +#ifndef Q4NX_DESIGN_M_DECODE #define Q4NX_DESIGN_M_DECODE 16 +#endif typedef enum { Q4NX_DESIGN_QKV = 0, @@ -299,12 +309,6 @@ const char* q4nx_design_log_tag(Q4nxDesign role); // side of the split: a duplicate would be two contracts that drift silently. const char* q4nx_pb16i_instance(Q4nxDesign role); -// The same for a SEALED int8 b8 ELF (engine#2208): one hw_context for the decode roles of a b8 container whose -// full-attention QKV is one fused role (QKV, with the output gate) and whose gate/up is one fused role (GU). -// The split G and U roles have no instance: the sealed GU instance is the fused width, so they keep their own -// kernels. NULL means the sealed ELF does not cover the role. -const char* q4nx_i8_sealed_instance(Q4nxDesign role); - /* Look up the entry for a role in a derived set; NULL if absent. */ const Q4nxDesignGeom* q4nx_find_design(const Q4nxDesignGeom* designs, int count, Q4nxDesign role); diff --git a/npu/forward/include/whisper_ids.h b/npu/forward/include/whisper_ids.h new file mode 100644 index 000000000..0ca1227ca --- /dev/null +++ b/npu/forward/include/whisper_ids.h @@ -0,0 +1,46 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// whisper_ids.h -- the special-token ids a Whisper decoder's prompt needs, selected by VOCABULARY WIDTH (engine#1783). +// +// Why by width and not by a constant: whisper-small exports 51865 logits and large-v3 exports 51866, and the extra token shifts +// <|translate|>/<|transcribe|>/<|nospeech|>/<|notimestamps|> by one. A hard-coded 50359/50363 decodes whisper-small correctly -- +// because English audio makes <|translate|> and <|transcribe|> agree -- and makes large-v3 answer <|endoftext|> at the first step. +// Measured on both packs against onnxruntime: argmax 50257 with 50359/50363 on large-v3, 400 with 50360/50364. +// +// Neither pack ships a tokenizer or generation_config, so the width (read from the decoder's declared output shape) is the only +// fact available. A `.en` model (51864) is NOT covered: its prompt differs and nobody has measured it here, so it refuses rather +// than guessing one of the two known layouts. + +#pragma once + +#include + +namespace onebit::audio { + +struct WhisperIds { + int64_t sot = 50258; // <|startoftranscript|> + int64_t lang_en = 50259; // <|en|> + int64_t transcribe = 0; // <|transcribe|> + int64_t no_timestamps = 0; // <|notimestamps|> + int64_t eot = 50257; // <|endoftext|> +}; + +// Returns false, leaving `out` untouched, for a vocabulary this header has no measured layout for. +inline bool whisper_ids_for_vocab(int64_t vocab, WhisperIds* out) { + if (out == nullptr) return false; + WhisperIds ids; + if (vocab >= 51866) { // large-v3 and its derivatives + ids.transcribe = 50360; + ids.no_timestamps = 50364; + } else if (vocab == 51865) { // v1/v2/small/medium, multilingual + ids.transcribe = 50359; + ids.no_timestamps = 50363; + } else { + return false; // a `.en` vocabulary or an unknown family: refuse rather than guess + } + *out = ids; + return true; +} + +} // namespace onebit::audio diff --git a/npu/onnx_graph.h b/npu/onnx_graph.h index 87acc99cd..c4002e8c8 100644 --- a/npu/onnx_graph.h +++ b/npu/onnx_graph.h @@ -57,7 +57,11 @@ struct OnnxGraphInfo { std::string dir; // directory of the .onnx file, for resolving external_data std::vector init_tensors; std::vector inputs; // graph inputs, in declaration order + // The declared shape of each input, parallel to `inputs`. A decoder's token window lives here (whisper-small exports 128, + // large-v3 exports 448), and a reader that assumes one of them builds a mask the other cannot broadcast against. + std::vector> input_dims; std::vector outputs; // graph outputs + std::vector> output_dims; // parallel to `outputs`; a decoder's vocabulary is the last of these std::vector nodes; // graph order, which ONNX guarantees is topological std::vector initializers; // weight/bias names std::vector external; // initializers whose data is in a side file (location set) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index fcff4a8ed..bfd76a0ec 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3325,29 +3325,12 @@ std::string Plan::manifest() const { c["tensors"] = container_tensors; c["payload_bytes"] = container_payload_bytes; c["dtypes"] = container_dtypes; - if (container_role_layer_rows >= 0) - c["role_layer_types"] = {{"keys", container_role_layer_rows}, - {"spans", container_role_layer_spans}, - {"out_of_range", container_role_layer_out_of_range}}; - if (!container_role_layer_span_names.empty()) - c["role_layer_types"]["span_names"] = container_role_layer_span_names; c["note"] = container_note; j["container"] = c; } j["mlp_section"] = {{"gate_up_units_measured", mlp_gu_units}, {"down_groups_measured", mlp_down_groups}, {"note", mlp_note}}; - // engine#1934: the section offsets, in bytes, measured from the fconst's start. They were computed all along and - // never printed, so anyone comparing a container against the pack (the folded-q/k-gains question, an HF diff, a - // hand decode) had to derive them -- and a derived offset reads plausible data from the wrong place, which cost - // three wrong measurements in one night. `info` reports the measurement, not the plan (engine#734's lesson), and a - // byte offset nothing prints is exactly the thing that gets re-derived wrong. - j["sections"] = {{"attn_tiles", attn_tiles}, - {"o_off", o_off}, - {"o_k", o_k}, - {"knorm_off", knorm_off}, - {"qnorm_off", qnorm_off}, - {"mlp_off", mlp_off}}; j["incomplete"] = refused(); const TextArtifacts text = scan_text_artifacts(dir); j["text_artifacts"] = {{"present", text.present}, @@ -3445,402 +3428,6 @@ std::string header_witness_refusal(int vote_hdrs, bool declared_qk) { // engine#1786 (item F1), slice 3a: roles from names. The rule is ordered -- the first match wins -- so `q_proj.bias` // classifies as a bias of q_proj rather than as an unknown tensor, and a name that matches nothing lands in `unassigned` // with its own name rather than being guessed from its shape. -// engine#1788, slice 3c step 1: (role, layer type), the key the writer needs. See the header for the measurement. -// The two GROUP roles (ssm, gemma_local) carry several destinations each, and the destination is in the tensor's own -// suffix. Two of gemma_local's seven stems have a slot in npu/forward/src -- the einsum-free `pre/post_feedforward_ -// layernorm` pair -- and the rest have none, so this table places exactly those two and leaves the others refused. -// engine#1788, 3c: `ssm` is the other group, and unlike gemma_local the engine has a slot for EVERY member -- measured by -// grepping npu/forward/src for the ssm family, whose literal names match the classifier's own needles one for one. This is -// the 35B's largest bucket (210 tensors), so it is the placement's biggest single win. -static const struct { const char* role; const char* needle; const char* slot; bool suffix_only; } kGroupSuffixSlots[] = { - {"gemma_local", "pre_feedforward_layernorm.weight", "pre_feedforward_layernorm.weight", true}, - {"gemma_local", "post_feedforward_layernorm.weight", "post_feedforward_layernorm.weight", true}, - {"ssm", "ssm_alpha_proj", "linear_attn.ssm_alpha_proj.weight", false}, - {"ssm", "ssm_beta_proj", "linear_attn.ssm_beta_proj.weight", false}, - {"ssm", "ssm_conv1d", "linear_attn.ssm_conv1d.weight", false}, - {"ssm", "ssm_dt", "linear_attn.ssm_dt.bias", false}, - {"ssm", "ssm_norm", "linear_attn.ssm_norm.weight", false}, - {"ssm", "ssm_out_proj", "linear_attn.ssm_out_proj.weight", false}, - // the `a` member has no `.weight` on either side, and its name is a PREFIX of `ssm_alpha_proj`, so it is matched as a - // suffix only -- a `find()` here would shadow alpha and beta, which is the substring defect in its exact form. - {"ssm", "ssm_a", "linear_attn.ssm_a", true}, - {"ssm", ".ssm.a", "linear_attn.ssm_a", true}, -}; - -static std::string group_suffix_slot(const std::string& role, const std::string& tensor) { - for (const auto& kv : kGroupSuffixSlots) { - if (role != kv.role) continue; - const std::string suffix = kv.needle; - if (kv.suffix_only) { - if (tensor.size() >= suffix.size() && - tensor.compare(tensor.size() - suffix.size(), suffix.size(), suffix) == 0) - return kv.slot; - } else if (tensor.find(suffix) != std::string::npos) { - return kv.slot; - } - } - return std::string(); -} - -// engine#1788, 3c: WHY THIS IS A BYTE COPY AND MUST STAY ONE. npu/q4nx.h records that the Q8 chunk (8704 B) comes in TWO -// code layouts and that nothing in the chunk or the container says which: the forward's decoder picks per tensor from the -// codes (q4nx_q8_split16 -- in the right layout every group's absmax code is 127). So a writer that re-encoded or -// re-quantised would have to make a choice the container does not make, and getting it wrong is a pack that decodes to -// wrong weights with no error anywhere. Copying the payload's bytes verbatim cannot make that mistake: the layout is the -// container's, and the forward already knows how to read it. -WritePackResult container_write_pack(const std::string& payload_path, const ImportPlan& plan, - const std::string& out_dir, bool allow_unplaced) { - WritePackResult r; - r.payload = out_dir + "/model.q4nx"; - r.header = out_dir + "/pack.json"; - if (!plan.unplaced.empty() && !allow_unplaced) { - r.why = std::to_string(plan.unplaced.size()) + " tensor(s) have no destination (e.g. " + plan.unplaced.front() + - "); refusing to write a partial pack"; - return r; - } - std::ifstream in(payload_path, std::ios::binary); - if (!in) { r.why = "cannot open the payload: " + payload_path; return r; } - std::error_code ec; - std::filesystem::create_directories(out_dir, ec); - if (ec) { r.why = "cannot create " + out_dir + ": " + ec.message(); return r; } - std::ofstream out(r.payload, std::ios::binary | std::ios::trunc); - if (!out) { r.why = "cannot write " + r.payload; return r; } - json tensors = json::object(); - int64_t off = 0; - for (const auto& item : plan.items) { - in.seekg(plan.payload_base + item.begin); - std::string bytes(size_t(item.end - item.begin), '\0'); - if (!in.read(&bytes[0], std::streamsize(bytes.size()))) { - r.why = "short read at " + std::to_string(plan.payload_base + item.begin) + " for " + item.source; return r; - } - out.write(bytes.data(), std::streamsize(bytes.size())); - if (!out) { r.why = "short write to " + r.payload; return r; } - // the header records where each slot's bytes sit in the contiguous payload: the offsets ARE the format - tensors[item.slot] = {{"offset", off}, {"bytes", int64_t(bytes.size())}, - {"shape", item.shape}, {"dtype", item.dtype}, {"source", item.source}}; - off += int64_t(bytes.size()); - ++r.tensors; - } - out.close(); - if (!out) { r.why = "short write to " + r.payload; return r; } - json h; - h["tensors"] = tensors; - h["bytes"] = off; - h["payload_bytes"] = plan.payload_bytes; - h["unplaced"] = plan.unplaced; - std::ofstream hout(r.header, std::ios::trunc); - if (!hout) { r.why = "cannot write " + r.header; return r; } - hout << h.dump(2) << "\n"; - if (!hout) { r.why = "short write to " + r.header; return r; } - r.bytes = off; - r.ok = true; - return r; -} - -WriteLayoutResult container_write_layout(const std::string& payload_path, const ImportPlan& plan, - const std::string& out_dir, bool allow_unplaced) { - WriteLayoutResult r; - if (!plan.unplaced.empty() && !allow_unplaced) { - r.why = std::to_string(plan.unplaced.size()) + " tensor(s) have no destination (e.g. " + plan.unplaced.front() + - "); refusing to write a partial pack"; - return r; - } - std::ifstream in(payload_path, std::ios::binary); - if (!in) { r.why = "cannot open the payload: " + payload_path; return r; } - json manifest = json::object(); - std::error_code ec; - std::filesystem::create_directories(out_dir, ec); - if (ec) { r.why = "cannot create " + out_dir + ": " + ec.message(); return r; } - for (const auto& item : plan.items) { - in.seekg(plan.payload_base + item.begin); - std::string bytes(size_t(item.end - item.begin), '\0'); - if (!in.read(&bytes[0], std::streamsize(bytes.size()))) { - r.why = "short read at " + std::to_string(plan.payload_base + item.begin) + " for " + item.source; - return r; - } - const std::string path = out_dir + "/" + item.slot; - std::ofstream out(path, std::ios::binary | std::ios::trunc); - if (!out) { r.why = "cannot write " + path; return r; } - out.write(bytes.data(), std::streamsize(bytes.size())); - if (!out) { r.why = "short write to " + path; return r; } - ++r.files; - r.bytes += int64_t(bytes.size()); - manifest[item.slot] = {{"source", item.source}, - {"bytes", int64_t(bytes.size())}, - {"shape", item.shape}, - {"dtype", item.dtype}}; - } - // The layout is self-describing: a directory of files whose names are engine slots is not enough, because the reader - // also needs to know which container tensor each came from and what shape/dtype it is. - { - const std::string mpath = out_dir + "/manifest.json"; - std::ofstream mout(mpath, std::ios::trunc); - if (!mout) { r.why = "cannot write " + mpath; return r; } - json j; - j["tensors"] = manifest; - j["files"] = r.files; - j["bytes"] = r.bytes; - j["payload_bytes"] = plan.payload_bytes; - j["unplaced"] = plan.unplaced; - mout << j.dump(2) << "\n"; - if (!mout) { r.why = "short write to " + mpath; return r; } - } - r.ok = true; - return r; -} - -ImportPlan container_import_plan(const Q4nxContainer& container, const std::vector& layer_types) { - ImportPlan out; - out.payload_bytes = container.payload_bytes; - // `header_bytes` is the JSON length alone; the file prefixes it with an 8-byte little-endian length, so the payload - // starts at 8 + header_bytes. Getting this by 8 wrong is what made the first end-to-end run write header bytes. - // ... and there is no prefix when the container declares no header at all (a bare payload has header_bytes 0), so the - // +8 only applies to a real safetensors-shaped container. - out.payload_base = container.header_bytes > 0 ? container.header_bytes + 8 : 0; - const PlacementReport rep = container_placement(container.tensors, layer_types); - std::map by_name; - for (const auto& t : container.tensors) by_name[t.name] = &t; - std::set seen; - for (const auto& row : rep.rows) { - if (row.slot.empty()) { - out.unplaced.push_back((row.role.empty() ? std::string("(no role)") : row.role) + ": " + row.tensor); - continue; - } - if (!seen.insert(row.slot).second) { // a second write would silently replace the first - out.unplaced.push_back(row.role + ": " + row.tensor + " (destination already taken)"); - continue; - } - ImportItem item; - item.slot = row.slot; item.source = row.tensor; item.shape = row.shape; item.dtype = row.dtype; - const auto it = by_name.find(row.tensor); - if (it != by_name.end()) { item.begin = it->second->begin; item.end = it->second->end; } - out.bytes += item.end - item.begin; - out.items.push_back(item); - } - // engine#1788, 3c (review): a SUM is only coverage if the ranges are DISJOINT. Uniqueness is enforced on the slot, but - // two tensors with different slots and overlapping [begin,end) would double-count one region and skip another, and the - // totals would still add up. Sorted by offset, so an overlap is caught once and named. - std::vector> spans; - for (const auto& it : out.items) spans.push_back({it.begin, it.source}); - std::sort(spans.begin(), spans.end()); - std::set overlapping; - for (size_t i = 1; i < spans.size(); ++i) { - const auto prev = by_name.find(spans[i - 1].second); - if (prev != by_name.end() && prev->second->end > spans[i].first) { - overlapping.insert(spans[i - 1].second); - overlapping.insert(spans[i].second); - out.unplaced.push_back("overlapping payload ranges: " + spans[i - 1].second + " and " + spans[i].second + - " (a sum over these would count one region twice and skip another)"); - } - } - if (!overlapping.empty()) { - // the totals must not be quoted when they can cancel: drop the offending items and rebuild the sum - std::vector kept; - out.bytes = 0; - for (const auto& it : out.items) { - if (overlapping.count(it.source)) continue; - kept.push_back(it); - out.bytes += it.end - it.begin; - } - out.items.swap(kept); - } - return out; -} - -PlacementReport container_placement(const std::vector& tensors, - const std::vector& layer_types) { - PlacementReport out; - const ContainerRoles roles = container_roles(tensors); - const RoleSlotReport table = container_role_slots(roles); - for (const auto& t : tensors) { - PlacementRow row; - row.tensor = t.name; - std::string layer_prefix; - row.shape = t.shape; - row.dtype = t.dtype; - const auto it = roles.role_of.find(t.name); - if (it == roles.role_of.end()) { row.why = "no role recognised"; ++out.no_role; out.rows.push_back(row); continue; } - row.role = it->second; - std::string::size_type pos = t.name.find(".layers."); - std::string::size_type off = 8; - if (pos == std::string::npos) { pos = t.name.find(".layer."); off = 7; } - if (pos != std::string::npos) { - const std::string::size_type start = pos + off; - std::string::size_type end = start; - while (end < t.name.size() && t.name[end] >= '0' && t.name[end] <= '9') ++end; - if (end > start) { - const int64_t layer = std::atoll(t.name.substr(start, end - start).c_str()); - row.layer = std::to_string(layer); - layer_prefix = t.name.substr(0, end) + "."; - if (layer >= 0 && layer < int64_t(layer_types.size())) row.layer_type = layer_types[size_t(layer)]; - } - } - row.slot = container_role_slot(row.role, row.layer_type); - // a group role carries its destination in the tensor's own suffix -- the fourth key, applied where the engine - // actually has a slot for one of the group's members - if (row.slot.empty() && !row.layer_type.empty()) row.slot = group_suffix_slot(row.role, t.name); - // engine#1788, 3c: a per-layer slot must be LAYER-QUALIFIED or every layer collides on one destination (the - // collision check found 733 tensors pointing at 26 names). The prefix is the CONTAINER's own spelling through - // the layer digits, because model.c probes both -- "model.layers.%d.…" and "model.layer.%d.…" -- and MiniCPM - // uses the singular. Using the source's own prefix keeps the two agree without this code choosing a dialect. - if (row.role == "weight_scale") row.slot = t.name; // a scale companion goes to its OWN name - else if (!row.slot.empty() && !row.layer.empty()) row.slot = layer_prefix + row.slot; - if (row.slot.empty()) { - const auto why = table.why.find(row.role); - row.why = why != table.why.end() ? why->second - : "the layer type does not give this role a destination"; - ++out.unplaced; - } else { - ++out.placed; - ++out.slot_counts[row.slot]; - } - out.rows.push_back(row); - } - return out; -} - -RoleLayerReport container_role_layer_types(const std::vector& tensors, - const std::vector& layer_types) { - RoleLayerReport out; - const ContainerRoles roles = container_roles(tensors); - std::map, std::map> by_key; - std::map role_of_stem; - std::set seen_in_layer; - for (const auto& t : tensors) { - const auto it = roles.role_of.find(t.name); - if (it == roles.role_of.end()) continue; - std::string::size_type pos = t.name.find(".layers."); - std::string::size_type off = 8; - if (pos == std::string::npos) { pos = t.name.find(".layer."); off = 7; } - if (pos == std::string::npos) { out.no_layer.push_back(it->second + " (" + t.name + ")"); continue; } - const std::string::size_type start = pos + off; - std::string::size_type end = start; - while (end < t.name.size() && t.name[end] >= '0' && t.name[end] <= '9') ++end; - if (end == start) continue; - const int64_t layer = std::atoll(t.name.substr(start, end - start).c_str()); - if (layer < 0 || layer >= int64_t(layer_types.size())) { ++out.out_of_range; continue; } - std::string shape = "["; - for (size_t i = 0; i < t.shape.size(); ++i) shape += (i ? "," : "") + std::to_string(t.shape[i]); - shape += "] " + t.dtype; - // The key is the TENSOR STEM, not the role: `ssm` covers dt.bias, conv1d, norm and a, which are four different - // tensors with four shapes -- a span there is my role vocabulary being coarse, not an ambiguity. Two shapes for - // the SAME stem in ONE layer type is the case a third key would be needed for (gemma4: gate/up/down_proj are - // 2304 wide on 20 layers and 1152 on 15, and both kinds sit in both layer types). - std::string stem = t.name.substr(0, start) + "N" + t.name.substr(end); // `model.layers.7.x` -> `model.layers.N.x` - by_key[{stem, layer_types[size_t(layer)]}][shape]++; - role_of_stem.emplace(stem, it->second); - seen_in_layer.insert(it->second); - } - for (const auto& kv : by_key) { - if (kv.second.size() > 1) { - out.spans.push_back(kv.first.first + " in " + kv.first.second + " appears in " + - std::to_string(kv.second.size()) + " shapes"); - continue; - } - int64_t n = 0; - for (const auto& sh : kv.second) n += sh.second; - out.rows.push_back({kv.first.first, role_of_stem[kv.first.first], kv.first.second, kv.second.begin()->first, n}); - } - for (const auto& kv : roles.counts) { - if (seen_in_layer.count(kv.first)) continue; - bool already = false; - for (const auto& x : out.no_layer) - if (x.rfind(kv.first + " ", 0) == 0) { already = true; break; } - if (!already) out.no_layer.push_back(kv.first + " (no tensor inside a layer)"); - } - return out; -} - -// engine#1788, slice 3c: the roles a layer type does not change, and the engine slot each becomes. Every right-hand -// side below is a literal that exists in npu/forward/src (grep the strings, do not infer them from HF names). -static const struct { const char* role; const char* slot; } kRoleSlots[] = { - {"embed", "model.embed_tokens.weight"}, - {"final_norm", "model.norm.weight"}, - {"input_norm", "input_layernorm.weight"}, - {"post_norm", "post_attention_layernorm.weight"}, - {"lm_head", "lm_head.weight"}, - {"q_proj", "self_attn.q_proj.weight"}, - {"k_proj", "self_attn.k_proj.weight"}, - {"v_proj", "self_attn.v_proj.weight"}, - {"o_proj", "self_attn.o_proj.weight"}, - {"self_attn_gate_proj","self_attn.gate_proj.weight"}, - {"gate_proj", "mlp.gate_proj.weight"}, - {"up_proj", "mlp.up_proj.weight"}, - {"down_proj", "mlp.down_proj.weight"}, - {"gate_exps", "mlp.gate_exps_proj.weight"}, - {"up_exps", "mlp.up_exps_proj.weight"}, - {"down_exps", "mlp.down_exps_proj.weight"}, - {"shortconv_in_proj", "shortconv.in_proj.weight"}, - {"per_layer_proj_norm","model.per_layer_proj_norm.weight"}, - // engine#1788, 3c: I refused these two for two PRs on the strength of a grep that found only the rmsnorm debug LABEL - // ("q_norm"), and called them config fields. They are weight slots, and model.c names them explicitly: - // "model.layer{s}.%d.self_attn.q_norm.weight" / "… .k_norm.weight" (singular and plural both probed) - {"q_norm", "self_attn.q_norm.weight"}, - {"k_norm", "self_attn.k_norm.weight"}, - // engine#1788, 3c: the MoE names are FORMAT-DEPENDENT, and moe_container.h documents both conventions. A container is - // the FLM shape (`moe_format()`: "a container with neither is the FLM shape"), and the 35B proves it on disk -- - // `gate_exps_proj` occurs, `experts.gate_proj` does NOT. So the destinations below are FLM spellings; the - // EngineConverter spellings (mlp.experts.*, mlp.gate.weight, mlp.shared_expert.*) are a different pack format. - {"moe_router", "moe_router.weight"}, - {"share_gate_exps", "mlp.share_gate_exps_proj.weight"}, - {"share_up_exps", "mlp.share_up_exps_proj.weight"}, - {"share_down_exps", "mlp.share_down_exps_proj.weight"}, - {"shared_expert_gate", "mlp.share_router.weight"}, -}; - -// The roles this slice does NOT place, each with the reason. A number here is honest; a guessed slot is not. -static const struct { const char* role; const char* why; } kRoleNoSlot[] = { - {"qkv_proj", "the slot prefix is the LAYER TYPE's (`linear_attn.` or `self_attn.`), so it needs the second key"}, - {"gate_up_proj", "fused in some containers, two slots in the engine; needs a measured split"}, - // Measured on the real containers (35B: 210 tensors): `ssm` is not one destination but FOUR -- dt.bias, conv1d, norm - // and a, with four different shapes. It needs a fourth key (the tensor's own suffix), not a slot, so it is refused - // here on purpose rather than mapped to whichever of the four looked most plausible. - {"ssm", "the members this table does not match: the six that do map one for one onto linear_attn.ssm_* slots, so an unmatched one is a name not yet seen, not a destination that does not exist"}, - // Measured on gemma3/gemma4/bolt. `gemma_local` is likewise a GROUP: pre/post_feedforward_layernorm (both of which DO - // have engine slots), inp_gate, layer_output_scale, per_layer_projection, post_layernorm and per_layer_model_proj. - // Refusing the group is honest; the three with slots need the suffix key to be told apart from the four without. - {"gemma_local", "a group: pre/post_feedforward_layernorm have engine slots, inp_gate / layer_output_scale / per_layer_projection / post_layernorm / per_layer_model_proj do not -- separating them needs the suffix key"}, - // Measured on gemma4/bolt. These ARE recognised and deliberately have no destination: `rope_freqs.weight` is a file the - // forward only FLAGS (model_config.h: has_rope_freqs_file) and the per-layer embeddings have no slot of their own. - {"dtype_variant", "a dtype-tagged (`.bf16.`) copy of a weight the container also ships untagged; the engine has no `.bf16.` literal, so this copy has no destination and must not reach the untagged slot"}, - {"container_extra", "recognised so it is not a gap, and deliberately not written: a table the forward only flags (rope_freqs) or a per-layer embedding with no slot"}, -}; - -// The layer types that are linear attention, as an EXACT set rather than a substring: the defect family this stack -// exists to fix is substring matching where an identity was meant (#1840, #1841, #2277's needle order). A value that -// merely contains "linear" is not linear attention, and a new family (gdn, mamba-style) must be added here on purpose. -static const char* const kLinearAttentionLayerTypes[] = {"linear_attention"}; - -static bool is_linear_attention(const std::string& layer_type) { - for (const char* t : kLinearAttentionLayerTypes) if (layer_type == t) return true; - return false; -} - -std::string container_role_slot(const std::string& role, const std::string& layer_type) { - if (role == "qkv_proj") { - // The engine's only fused-qkv slot is the linear-attention one; `self_attn.qkv_proj` is not a name it reads. - return is_linear_attention(layer_type) ? "linear_attn.qkv_proj.weight" : std::string(); - } - for (const auto& kv : kRoleSlots) if (role == kv.role) return kv.slot; - return std::string(); -} - -RoleSlotReport container_role_slots(const ContainerRoles& roles) { - RoleSlotReport r; - for (const auto& kv : kRoleSlots) r.slot_of[kv.role] = kv.slot; - for (const auto& kv : kRoleNoSlot) r.why[kv.role] = kv.why; - // Every role the caller actually met must be either placed or explained: a role that is in neither is a hole in - // this table, and saying so is the point of the report. - for (const auto& c : roles.counts) { - if (r.slot_of.count(c.first)) r.placed.push_back(c.first); - else if (r.why.count(c.first)) r.no_slot.push_back(c.first); - else { r.no_slot.push_back(c.first); r.why[c.first] = "a role this table does not know: it needs a decision"; } - } - std::sort(r.placed.begin(), r.placed.end()); - std::sort(r.no_slot.begin(), r.no_slot.end()); - return r; -} - ContainerRoles container_roles(const std::vector& tensors) { struct Rule { const char* role; const char* needle; }; // Longest/most specific spellings first: `qkv_proj` before `q_proj`, `gate_up_proj` before `gate_proj`. @@ -3861,28 +3448,9 @@ ContainerRoles container_roles(const std::vector& tensors) { {"k_proj", "k_proj"}, {"k_proj", "attn_k"}, {"k_proj", "wk."}, {"v_proj", "v_proj"}, {"v_proj", "attn_v"}, {"v_proj", "wv."}, {"o_proj", "o_proj"}, {"o_proj", "attn_output"}, {"o_proj", "wo."}, - // engine#1788: an ATTENTION output gate is not the FFN gate. MiniCPM-V 4.7 carries `attn_output_gate: true`, - // so every one of its 30 `self_attn.gate_proj.weight` tensors was filed under `gate_proj` -- the dense-MLP role - // -- and a consumer would have wired an attention gate into the FFN path. The forward already keeps them apart - // (model.h: `gate_proj_weight` vs `self_attn_gate_proj_weight`), so this uses its name and must precede the - // `gate_proj` needle. - {"self_attn_gate_proj", "self_attn.gate_proj"}, {"gate_proj", "gate_proj"}, {"gate_proj", "ffn_gate"}, {"up_proj", "up_proj"}, {"up_proj", "ffn_up"}, - {"down_proj", "down_proj"}, {"down_proj", "ffn_down"}, // engine#1788 acceptance (F1 slice 3a gap): the MoE families MiniCPM-V 4.7 names, mapped to the roles the NPU - // forward ALREADY reads -- npu/forward/include/model.h carries up_exps_weight / gate_exps_weight / - // down_exps_weight / share_{up,gate,down}_exps_weight / moe_router_weight / shared_expert_gate_weight with - // those exact container names in their comments. So this is translation to an existing vocabulary, not a new - // one. The `share_*` spellings come first because `share_gate_exps_proj` contains `gate_exps_proj`. - {"share_gate_exps", "share_gate_exps_proj"}, - {"share_up_exps", "share_up_exps_proj"}, - {"share_down_exps", "share_down_exps_proj"}, - {"gate_exps", "gate_exps_proj"}, - {"up_exps", "up_exps_proj"}, - {"down_exps", "down_exps_proj"}, - {"moe_router", "moe_router"}, - {"shared_expert_gate","shared_expert_gate"}, - + {"down_proj", "down_proj"}, {"down_proj", "ffn_down"}, {"input_norm", "input_layernorm"},{"input_norm", "attn_norm"}, {"input_norm", "attention_norm"}, {"post_norm", "post_attention_layernorm"}, {"post_norm", "ffn_norm"}, {"final_norm", "final_layernorm"}, @@ -3892,24 +3460,6 @@ ContainerRoles container_roles(const std::vector& tensors) { // and `model.mtp.*` is a draft head rather than a model. static const char* kSsm[] = {"linear_attn.", "ssm_alpha_proj", "ssm_beta_proj", "ssm_conv1d", "ssm_dt", "ssm_norm", "ssm_out_proj"}; - // engine#1788: gemma3/gemma4/bolt carry five tensor families this engine's forward does NOT read -- sandwich norms - // (`pre_/post_feedforward_layernorm`), an input gate, a per-layer projection (and its `_prefill` sibling) and a layer - // output scale. Nothing here has a field in npu/forward/include/model.h, so they are RECOGNISED rather than mapped: - // `unassigned` has to keep meaning "nothing knows this", and these are known -- measured on gemma4-e2b (319 of 709) - // and bolt-e4b (382 of 849), which were the two packs the classifier could not finish. - // `post_layernorm` and `per_layer_model_proj` are here for a measured reason, not a guess: gemma4-e2b carries - // BOTH `post_attention_layernorm` (35, classified `post_norm`) and `post_layernorm` (35), so the second cannot be - // the forward's post-attention norm -- they are different tensors in the same model. gemma3-270m has no - // `post_layernorm` at all, which is why only gemma4/bolt were left with it. - static const char* kGemmaLocal[] = {"pre_feedforward_layernorm", "post_feedforward_layernorm", "inp_gate", - "layer_output_scale", "per_layer_projection", "post_layernorm", - "per_layer_model_proj"}; - // engine#1788: the last three on gemma4/bolt are recognised for a different reason than the gemma ones, so they get - // their own role. `rope_freqs.weight` is a file the forward only FLAGS (`model_config.h`: has_rope_freqs_file), and - // `per_layer_token_embd` (with its `.scale` sibling) is the per-layer token embedding that our ONNX path reads as - // `embed_tokens_per_layer` (engine#1725) but the container path has no field for. Recognised, not mapped: neither - // has a field to fill, and calling them "unassigned" would say nothing knows them when the opposite is true. - static const char* kContainerExtras[] = {"rope_freqs", "per_layer_token_embd"}; static const char* kMtp[] = {"model.mtp."}; ContainerRoles out; for (const auto& t : tensors) { @@ -3919,16 +3469,6 @@ ContainerRoles container_roles(const std::vector& tensors) { role = "vision"; else if (n.find("audio.") != std::string::npos) role = "audio"; - // engine#1788, 3c: FIRST, before any needle table -- a `.scale` companion must never be filed as the weight it - // accompanies, and the `embed` rule matches `model.embed_tokens.weight.scale` by substring. This is the same - // ordering defect as `attn_q` vs `attn_q_norm` (engine#2097): the general rule ran before the specific one. - else if (n.size() > 6 && n.compare(n.size() - 6, 6, ".scale") == 0) - role = "weight_scale"; - // engine#1788, 3c: a dtype-TAGGED companion (a `.bf16.` copy of a weight the engine also ships untagged). Same - // first-position rule as `.scale`, and the opposite outcome: the engine has no `.bf16.` literal at all, so this - // copy has no destination and must not reach the untagged slot -- which is what the 1B collision showed. - else if (n.find(".bf16.") != std::string::npos) - role = "dtype_variant"; if (role.empty()) { bool matched = false; for (const auto& r : kRules) { @@ -3937,27 +3477,8 @@ ContainerRoles container_roles(const std::vector& tensors) { matched = true; break; } - // "never inside a layer" has to be spelled positively enough to survive the container's own naming: - // MiniCPM-V 4.7 spells its blocks `model.layer.N.` (singular), so a guard against "layers." alone filed - // 30 per-layer linear_attn.ssm_norm tensors as final_norm -- ahead of the kSsm loop below (engine#1788). - // engine#1788, 3c: a `.scale` companion is NOT the weight it accompanies, and the engine reads it by its own - // name -- "model.embed_tokens.weight.scale" is a literal in npu/forward. Filed as `embed` it would be written - // over the embedding, or its scales where the embedding belongs, with no error. Matched first. - // and `per_layer_proj_norm` has its own top-level slot; it is not the model's final norm. - if (!matched && role.empty() && n.find("per_layer_proj_norm") != std::string::npos) { - role = "per_layer_proj_norm"; matched = true; - } - if (!matched && n.find("norm.weight") != std::string::npos && n.find("layers.") == std::string::npos && - n.find("layer.") == std::string::npos) + if (!matched && n.find("norm.weight") != std::string::npos && n.find("layers.") == std::string::npos) role = "final_norm"; // `model.norm.weight`: the top-level norm, never inside a layer - if (!matched && role.empty()) { - for (const char* e : kContainerExtras) - if (n.find(e) != std::string::npos) { role = "container_extra"; matched = true; break; } - } - if (!matched && role.empty()) { - for (const char* g : kGemmaLocal) - if (n.find(g) != std::string::npos) { role = "gemma_local"; matched = true; break; } - } if (!matched && role.empty()) { for (const char* s : kSsm) if (n.find(s) != std::string::npos) { role = "ssm"; matched = true; break; } @@ -4069,42 +3590,11 @@ Plan plan(const std::string& pack_dir) { // wins) -- the container's own `dims` still takes precedence in container_geometry(). std::string cfg_source; const json cfg = load_geometry(pack_dir, &cfg_source); - // engine#1788, slice 3c: the (role, layer type) key, when the pack says what its layers are. - if (const Q4nxContainer& cc = c; true) { - std::vector lt; - const json* src = cfg.contains("layer_types") ? &cfg : nullptr; - if (!src && cfg.contains("text_config") && cfg["text_config"].is_object() && - cfg["text_config"].contains("layer_types")) - src = &cfg["text_config"]; - if (src && (*src)["layer_types"].is_array()) - for (const auto& x : (*src)["layer_types"]) - if (x.is_string()) lt.push_back(x.get()); - if (!lt.empty()) { - const RoleLayerReport rep = container_role_layer_types(cc.tensors, lt); - p.container_role_layer_rows = int64_t(rep.rows.size()); - p.container_role_layer_spans = int64_t(rep.spans.size()); - p.container_role_layer_out_of_range = rep.out_of_range; - // the pairs a third key would be needed for, named: this is the writer's remaining question, not a number - for (size_t i = 0; i < rep.spans.size() && i < 8; ++i) p.container_role_layer_span_names.push_back(rep.spans[i]); - } - } const ContainerGeometry g = container_geometry(c.meta, cfg); p.dims = g.dims; p.container_geometry_source = g.source; p.container_geometry_missing = g.missing; p.container_geometry_inferred = g.inferred; - // slice 3b: the roles, so the note states what the pack carries rather than only its size. - const ContainerRoles roles = container_roles(c.tensors); - p.container_role_counts = roles.counts; - p.container_roles_unassigned = roles.unassigned; - std::string role_note; - if (roles.unassigned.empty()) { - role_note = "every tensor has a known role"; - } else { - role_note = std::to_string(roles.unassigned.size()) + " of " + - std::to_string(int64_t(c.tensors.size())) + " tensors have no known role (first: " + - roles.unassigned.front() + ")"; - } std::string geom = g.source.empty() ? "the container declares no `dims` and no config.json answered, so its geometry is not known here" : "geometry from " + g.source; @@ -4120,7 +3610,6 @@ Plan plan(const std::string& pack_dir) { p.container_note = c.why.empty() ? "the pack is a safetensors container (model.q4nx): " + std::to_string(c.tensors.size()) + " tensors, " + std::to_string(c.payload_bytes) + " B of payload, no graph; " + geom + - "; " + role_note + ". Reading it as a weights source needs the container-genre plan (engine#1786, item F1); until" " then this pack is recognised and reported, not importable." : "model.q4nx is present but unreadable: " + c.why; @@ -5189,37 +4678,6 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // Census yes, writer no (see the note where graph_weights is decided): this genre's weights are 1-D external-data // blocks, and until the repacker exists an import would write a container nothing can read. Refused here rather // than in plan(), because `info` must keep reporting the genre it can now see. - // engine#1786 (item F1) / engine#1788: a container-genre pack has no `model.onnx` by design, so without a branch here - // the first byte fails with "cannot open /model.onnx" -- a missing-file message about a pack that has no graph - // at all. Refused by name, the same way the external-data genre below is, until the container-genre writer exists. - if (p.container_tensors >= 0) { - // engine#1788: the writer exists now. The payload is copied, never re-quantised -- the container's bytes ARE the - // q4nx chunks (a tensor shaped [4096,8,4736] I8 is 4096 rows of the 4736-byte Q4_K chunk), and copying is the only - // safe answer to q4nx.h's two-Q8-layouts problem: the bytes never change, so the choice the container does not make - // never has to be made. - const std::string multi = pack_dir + "/model.q4nx"; - const Q4nxContainer c = read_q4nx_container(multi); - if (!c.why.empty()) throw std::runtime_error("container: " + c.why); - // the layer types live in config.json beside the container, not in its safetensors metadata - std::string cfg_source; - const json cfg = load_geometry(pack_dir, &cfg_source); - std::vector lt; - const json* src = cfg.contains("layer_types") ? &cfg : nullptr; - if (!src && cfg.contains("text_config") && cfg["text_config"].is_object() && cfg["text_config"].contains("layer_types")) - src = &cfg["text_config"]; - if (src && (*src)["layer_types"].is_array()) - for (const auto& x : (*src)["layer_types"]) if (x.is_string()) lt.push_back(x.get()); - const ImportPlan ip = container_import_plan(c, lt); - const WritePackResult w = container_write_pack(multi, ip, out_dir, opt.allow_incomplete); - if (!w.ok) throw std::runtime_error("container: " + w.why); - Report rep; - rep.out_dir = out_dir; - rep.tensors = int(w.tensors); - rep.tiles = w.bytes; - // the unplaced list is a REFUSAL list, so it goes where the caller already looks for refusals - for (const auto& u : ip.unplaced) rep.refused.push_back(u); - return rep; - } if (p.graph_packed_siblings > 0) { // The genre has a writer now (engine#1503): decode -> convert -> split by role -> the same I8 tile chunks, // with its own function because import_pack is built around the .fconst envelope. @@ -7099,11 +6557,40 @@ OnnxGraphInfo parse_onnx_graph(const std::string& path) { out.dir = (slash == std::string::npos) ? std::string(".") : path.substr(0, slash); for (const auto* f : fields(g, 11)) { // graph.input const auto vi = pb(f->p, f->n); + std::vector dims; + // ValueInfoProto: 1 = name, 2 = type (TypeProto: 1 = tensor_type, whose 2 = shape, whose 1 = the dims, each with 1 = value) + if (const auto* tp = field1(vi, 2)) { + for (const auto* tt : fields(pb(tp->p, tp->n), 1)) { + const auto ttb = pb(tt->p, tt->n); + for (const auto* sh : fields(ttb, 2)) { + for (const auto* d : fields(pb(sh->p, sh->n), 1)) { + const auto db = pb(d->p, d->n); + if (const auto* v = field1(db, 1)) dims.push_back(int64_t(v->v)); + else dims.push_back(-1); // a symbolic dimension + } + } + } + } if (const auto* nm = field1(vi, 1)) out.inputs.push_back(pb_str(*nm)); + out.input_dims.push_back(std::move(dims)); } for (const auto* f : fields(g, 12)) { // graph.output const auto vi = pb(f->p, f->n); + std::vector dims; + if (const auto* tp = field1(vi, 2)) { + for (const auto* tt : fields(pb(tp->p, tp->n), 1)) { + const auto ttb = pb(tt->p, tt->n); + for (const auto* sh : fields(ttb, 2)) { + for (const auto* d : fields(pb(sh->p, sh->n), 1)) { + const auto db = pb(d->p, d->n); + if (const auto* v = field1(db, 1)) dims.push_back(int64_t(v->v)); + else dims.push_back(-1); + } + } + } + } if (const auto* nm = field1(vi, 1)) out.outputs.push_back(pb_str(*nm)); + out.output_dims.push_back(std::move(dims)); } for (const auto* f : fields(g, 5)) { // graph.initializer const auto t = pb(f->p, f->n); diff --git a/npu/pack_import.h b/npu/pack_import.h index f577b6f35..eaee101bc 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -277,12 +277,6 @@ struct Plan { std::string container_geometry_source; // "dims", "config.json", or empty when neither answered std::vector container_geometry_missing; // fields no source could fill, by name std::vector container_geometry_inferred; // fields derived from another field, by name - // engine#1786 (item F1), slice 3b: what the pack's names say it carries, and what nothing recognised. `unassigned` - // is the list that matters -- it means "nothing knows this tensor", so an empty one is the statement that the role - // map covers the pack. The SSM and MTP families are recognised (roles `ssm` / `mtp`), so a hybrid pack and a draft - // head do not masquerade as unknown tensors. - std::map container_role_counts; - std::vector container_roles_unassigned; int64_t graph_matmuls = -1; // MatMulNBits nodes in the main graph int64_t graph_real_weights = 0; // of those, how many point at an initializer with dims (i.e. real weights) int64_t graph_external = 0; // initializers that declare external data (location + offset) @@ -392,103 +386,6 @@ struct ContainerRoles { }; ContainerRoles container_roles(const std::vector& tensors); -// engine#1788, slice 3c, the writer's destination side. A role says WHAT a tensor is; the engine reads weights by SLOT -// name (the string literals in npu/forward/src), so an import needs role -> slot. Only the roles whose destination does -// not depend on the LAYER TYPE are placed here. The rest are named in `no_slot` with a reason instead of guessed: -// a wrong slot does not fail loudly, it loads a tensor into a field the engine reads with another shape, and the -// symptom is a wrong number rather than an error. -struct RoleSlotReport { - std::map slot_of; // role -> engine slot; absent when this slice does not place it - std::vector placed; // placed roles, sorted - std::vector no_slot; // deliberately unplaced roles, sorted - std::map why; // role -> why it is not placed here -}; -RoleSlotReport container_role_slots(const ContainerRoles& roles); - -// engine#1788, slice 3c: the one role whose destination the LAYER TYPE decides, and the reason it decides it. The engine -// carries `linear_attn.qkv_proj.weight` and has NO `self_attn.qkv_proj` slot at all (grep the literals in npu/forward/src), -// so the fused qkv has a destination on a linear layer and nowhere else. Empty string = no destination; the caller must -// not treat that as "unchanged", which is why this returns a string and the report above returns the reasons. -std::string container_role_slot(const std::string& role, const std::string& layer_type); - -// engine#1788, 3c: the last pure step before bytes are written -- where each tensor's bytes go and which bytes they are. -// Separated from the I/O so it can be tested without a filesystem, and so a caller can refuse a container BEFORE writing -// anything into an output directory. `layer_types` comes from config.json beside the container (NOT from its safetensors -// metadata: that is where the measured ones live, and reading meta instead silently refused every layer-typed role). -struct ImportItem { - std::string slot; // the engine slot the bytes are written to (layer-qualified) - std::string source; // the container tensor they come from - int64_t begin = 0; // data_offsets[0] within the payload - int64_t end = 0; - std::vector shape; - std::string dtype; -}; -struct ImportPlan { - std::vector items; // placed tensors only; an empty slot is never written - std::vector unplaced; // ": " -- a refusal, not a silent skip - int64_t bytes = 0; // payload bytes the items carry - int64_t payload_bytes = 0; // what the container says it has, so a writer can prove coverage - // engine#1788, 3c: where the payload STARTS in the container file. `Q4nxTensor::begin/end` are PAYLOAD-relative, so a - // writer that seeks to `begin` in the file reads the safetensors header instead of the tensor -- which is exactly what - // happened: the bytes came out right in total (1024+128) and wrong in content. - int64_t payload_base = 0; -}; -ImportPlan container_import_plan(const Q4nxContainer& container, const std::vector& layer_types); - -// engine#1788, 3c: the I/O half -- copy each planned byte range out of the container's payload into one file per -// destination slot, plus a manifest. It REFUSES to write when the container has tensors the plan could not place, unless -// `allow_unplaced` is set: a pack that silently omits a fifth of the model is worse than no pack, and the caller is the -// only one who can decide that an omission is intended. -struct WriteLayoutResult { - bool ok = false; - int64_t files = 0; - int64_t bytes = 0; - std::string why; // why it refused, or what failed, when ok is false -}; -WriteLayoutResult container_write_layout(const std::string& payload_path, const ImportPlan& plan, - const std::string& out_dir, bool allow_unplaced); -// The PACK form: one contiguous payload plus a header, which is the shape the forward opens (a container of the same genre). -// Same refusal as the layout form -- nothing is written when the plan has unplaced tensors and allow_unplaced is false -- -// because a header that omits a fifth of the model is exactly as silent as a missing file. -struct WritePackResult { - bool ok = false; - int64_t tensors = 0; - int64_t bytes = 0; - std::string payload; // the file written, when ok - std::string header; - std::string why; -}; -WritePackResult container_write_pack(const std::string& payload_path, const ImportPlan& plan, - const std::string& out_dir, bool allow_unplaced); -// engine#1788, 3c: what the import should EMIT, learned from the source rather than invented. A container tensor's shape is -// already the chunk: `[4096,8,4736] I8` is N=4096, 8 groups, 4736 bytes per row, and 4736 is exactly the Q4_K chunk size. -// So the payload IS q4nx chunk data and the import needs no re-quantisation -- it emits a container OF THE SAME GENRE: -// one contiguous payload (the planned byte ranges in order) plus a header naming each destination slot with its dtype, -// shape and offset. That is also the only safe answer to q4nx.h's two-Q8-layouts problem: the bytes never change. -// `container_write_layout` above is the debugging form (one file per slot); the pack form concatenates instead. - -// engine#1788, slice 3c: the writer's per-TENSOR decision, which is what an import actually iterates. Reads the tensor's -// own layer, takes that layer's declared type, and asks the table above for a destination. A tensor with no role and a -// tensor with a role but no destination are counted apart, because the first is a vocabulary gap and the second is a -// decision that was made on purpose. -struct PlacementRow { - std::string tensor; - std::string layer; // "" when the tensor belongs to no layer (embed, final_norm, lm_head) - std::string layer_type; // "" for the same reason - std::string role; - std::string slot; // "" when there is no destination - std::string why; // the reason, when slot is empty - std::vector shape; // carried so a placement can be checked against the shape the engine documents - std::string dtype; -}; -struct PlacementReport { - std::vector rows; - std::map slot_counts; - int64_t placed = 0, unplaced = 0, no_role = 0; -}; -PlacementReport container_placement(const std::vector& tensors, - const std::vector& layer_types); - // engine#1788, slice 3c: a container's roles need a SECOND key, and the second key is the layer type. // Measured: gemma4-e2b's `q_proj` is N=384 on all 28 `sliding_attention` layers and N=768 on all 7 `full_attention` ones; // bolt-e4b's is (640,5120) on all 35 sliding and (1280,5120) on all 7 full. A perfect partition, structural in origin diff --git a/scripts/npu-kernel-bitx.sh b/scripts/npu-kernel-bitx.sh new file mode 100755 index 000000000..9c61f17de --- /dev/null +++ b/scripts/npu-kernel-bitx.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# Copyright 2026 bong-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# npu-kernel-bitx.sh: is a kernel family BIT-EXACT against the host, on the device? (engine#1783, engine#1808) +# +# npu-kernel-bitx.sh [--repeats N] [prompt ...] +# +# A kernel family's contract is not "it runs": it is bit-exact against the host reference, on the device, over more than one +# input. The engine carries the reference the device can be exact against -- NPU_INFER_HOST_GEMM=i8, whose int32 accumulation +# is the only thing an int8 kernel family can match by construction (engine#1808) -- so the check is: serve the same container +# twice, once on the NPU and once through that host arm, dump every position's raw logits (--dump-logits), and compare bytes. +# +# Why this is a script and not a ctest row: it needs the NPU, a container and a ticket. It therefore REFUSES to run outside an +# npu-hold ticket (no ~/NPU-TOKEN) rather than starting a bare serve on the device, and it releases what it starts. +# +# --repeats N runs the DEVICE arm N times and requires every run's dumps to be byte-identical to each other as well as to the host: +# the family contract is "bit-exact against the host reference, and 3 repeats identical", and doing the repeats by hand in a second +# script is how the second half gets skipped. +# +# Measured with it (whisper's 768 family, 2026-11-12): four prompts -- a single token, a three-token, a five-token and an +# eight-token one, so position-0 and repeated-token edges are covered -- all IDENTICAL, 4096 B each, and three repeats identical. +set -u +bin=${ONEBIT_BIN:-} +if [ -z "$bin" ]; then + for c in "$(dirname "$0")/../build/1bit" "$HOME/.local/share/1bit/1bit" "$(command -v 1bit 2>/dev/null)"; do + [ -n "$c" ] && [ -x "$c" ] && { bin=$c; break; } + done +fi +[ -n "$bin" ] && [ -x "$bin" ] || { echo "npu-kernel-bitx: no 1bit binary (set ONEBIT_BIN=)" >&2; exit 2; } +if [ ! -e "$HOME/NPU-TOKEN" ]; then + echo "npu-kernel-bitx: not inside an npu-hold ticket (no ~/NPU-TOKEN); refusing to start a serve on the device" >&2 + exit 2 +fi +container=${1:-}; kernels=${2:-} +[ -n "$container" ] && [ -n "$kernels" ] || { + echo "usage: $0 [prompt ...]" >&2; exit 2; } +shift 2 +repeats=1 +args=() +while [ $# -gt 0 ]; do + case "$1" in + --repeats) repeats=${2:-1}; shift 2 ;; + *) args+=("$1"); shift ;; + esac +done +if ! [ "$repeats" -ge 1 ] 2>/dev/null; then echo "npu-kernel-bitx: --repeats needs a count >= 1" >&2; exit 2; fi +prompts=(${args[@]+"${args[@]}"}) +[ ${#prompts[@]} -gt 0 ] || prompts=('<1>' '<7> <9> <11>' '<300> <301> <302> <303> <304>' '<1000> <1000> <2> <3> <4> <5> <6> <7>') + +out=${BITX_OUT:-$HOME/.cache/scratch-bitx/$(basename "$container" .q4nx)} +rm -rf "$out"; mkdir -p "$out/dev" "$out/host" + +serve() { # [env ...] + local dir=$1 port=$2; shift 2 + env "$@" "$bin" forward-serve -m "$container" --kernels "$kernels" --prefix-slots 0 \ + -p "$port" --dump-logits "$dir" > "$dir.log" 2>&1 & + local pid=$! + local up=0 + for _ in $(seq 1 60); do + curl -sf "http://127.0.0.1:$port/health" >/dev/null 2>&1 && { up=1; break; } + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + if [ "$up" = 0 ]; then + echo " $dir: serve did not come up -- $(grep -E 'init_elf failed|missing|error' "$dir.log" | tail -1)" >&2 + kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; return 1 + fi + local i=0 rc=0 + for p in "${prompts[@]}"; do + i=$((i + 1)) + code=$(curl -s -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:$port/v1/completions" \ + -H 'Content-Type: application/json' \ + -d "{\"prompt\":\"$p\",\"max_tokens\":1,\"temperature\":0}" -m 240) + [ "$code" = 200 ] || { echo " $dir: prompt $i -> http $code" >&2; rc=1; } + done + kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null + return $rc +} + +echo "device :" +for r in $(seq 1 "$repeats"); do + d=$( [ "$repeats" = 1 ] && printf '%s' "$out/dev" || printf '%s/dev%s' "$out" "$r" ) + mkdir -p "$d" + serve "$d" $((18150 + r - 1)) || exit 1 +done +echo "host :"; serve "$out/host" 18151 NPU_INFER_HOST_GEMM=i8 || exit 1 + +bad=0 n=0 +# Every device run must match the host; then the repeats must match each other. Comparison loops over the run dirs actually made. +devdirs="" +for r in $(seq 1 "$repeats"); do devdirs="$devdirs $( [ "$repeats" = 1 ] && printf '%s' "$out/dev" || printf '%s/dev%s' "$out" "$r" )"; done +first=$(printf '%s' "$devdirs" | awk '{print $1}') +for d in $devdirs; do + for f in "$d"/step_*_0.bin; do + [ -e "$f" ] || continue + b=$(basename "$f"); n=$((n + 1)) + if [ ! -f "$out/host/$b" ]; then echo " $d/$b: the host run wrote no counterpart"; bad=$((bad + 1)); continue; fi + if cmp -s "$f" "$out/host/$b"; then echo " $d/$b IDENTICAL to host ($(stat -c%s "$f") B)" + else echo " $d/$b DIFFERS from host ($(cmp -l "$f" "$out/host/$b" | wc -l) bytes)"; bad=$((bad + 1)); fi + done +done +if [ "$repeats" -gt 1 ]; then + drift=0 total=0 + for f in "$first"/step_*_0.bin; do + [ -e "$f" ] || continue + b=$(basename "$f"); total=$((total + 1)) + for d in $devdirs; do + [ "$d" = "$first" ] && continue + if ! cmp -s "$f" "$d/$b"; then + echo " $b: $first and $d differ -- the family is not deterministic on device" >&2 + drift=$((drift + 1)); break + fi + done + done + [ "$drift" = 0 ] && echo " all $total position(s) identical across $repeats device run(s)" \ + || { echo "npu-kernel-bitx: $drift of $total position(s) differ between repeats" >&2; bad=$((bad + drift)); } +fi +[ "$n" -gt 0 ] || { echo "npu-kernel-bitx: no logits were dumped -- the run proved nothing" >&2; exit 1; } +if [ "$bad" = 0 ]; then echo "npu-kernel-bitx: $n/$n prompt(s) bit-exact against the host integer arm"; exit 0; fi +echo "npu-kernel-bitx: $bad of $n prompt(s) differ from the host" >&2; exit 1 diff --git a/tests/audio_device_gemm_test.cpp b/tests/audio_device_gemm_test.cpp new file mode 100644 index 000000000..fc5ea6f56 --- /dev/null +++ b/tests/audio_device_gemm_test.cpp @@ -0,0 +1,97 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// tests/audio_device_gemm_test.cpp -- the decision layer around one encoder projection on the device (npu/forward/include/ +// audio_device_gemm.h, engine#1783), driven against a FAKE target: no device, no container, no kernels. +// +// The three things it pins are the three the real path gets wrong silently: the target must be asked for the STAGED width +// (N768 is named N1024), the caller must get its OWN width back (de-padded per row, not a staged stride), and every refusal +// (wrong output size, too many rows, no gemv, an unnamed projection, a target failure) must say why by name. +#include +#include +#include +#include "audio_device_gemm.h" + +using namespace onebit::audio; +namespace t = ::npu::target; + +static int failures = 0; +static void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++failures; +} + +struct FakeTarget : t::NpuTarget { + t::Caps c; + bool supports_gemv = true; + bool fail = false; + size_t wrote = 0; // what the fake claims to have written; 0 = "rows x staged_n" + t::GemvArgs seen{}; + const char* name() const override { return "fake"; } + bool open(const onebit::npu::plan::Plan&, const std::string&, const char*, std::string&) override { return true; } + bool load_block(int, int, std::string&) override { return true; } + t::Caps caps() const override { return c; } + bool supports(t::Op op) const override { return op == t::Op::gemv && supports_gemv; } + bool gemv(const t::GemvArgs& a, std::vector& out, std::string& err) override { + seen = a; + if (fail) { err = "kernel not resident"; return false; } + const size_t n = wrote ? wrote : size_t(a.M) * size_t(a.N); + out.assign(n, 0.0f); + for (size_t i = 0; i < n; ++i) out[i] = float(i); // recognisable pattern + return true; + } +}; + +int main() { + FakeTarget ft; + ft.c.max_rows = 16; + ft.c.prefill_rows = 128; + std::vector x(128 * 768, 1.0f); + std::vector out; + std::string why; + + // 1. the de-pad: O at k768 n768 is named N1024, so the fake writes 128x1024 and the caller must get 128x768 + DeviceProjection p; + p.role = 1; p.layer = 3; p.rows = 128; p.k = 768; p.n = 768; p.x = x.data(); p.tag = "O"; + check(device_projection(ft, p, &out, &why), "a staged O projection runs"); + check(ft.seen.N == 1024 && ft.seen.K == 768, "the target is asked for the STAGED width (N1024), not the model's 768"); + check(ft.seen.M == 128 && ft.seen.rows, "a 128-row launch is the wide body"); + check(out.size() == size_t(128) * 768, "the caller gets its own width back (128 x 768), de-padded"); + check(out[1] == 1.0f && out[767] == 767.0f && out[768] == 1024.0f, + "the de-pad drops the padding column: row 1 starts at staged column 1024"); + + // 2. a target that writes the wrong size is refused, not repaired + ft.wrote = 128 * 768; + check(!device_projection(ft, p, &out, &why) && why.find("the target and the plan disagree") != std::string::npos, + "an output that is not rows x staged_n is refused by name"); + ft.wrote = 0; + + // 3. more rows than the target launches in one call is refused with both numbers + p.rows = 129; + check(!device_projection(ft, p, &out, &why) && why.find("prefill_rows=128") != std::string::npos, + "rows beyond the prefill launch is refused and names the limit"); + p.rows = 128; + + // 4. a target without gemv is refused, naming the target + ft.supports_gemv = false; + check(!device_projection(ft, p, &out, &why) && why.find("\"fake\"") != std::string::npos, + "a target without gemv is refused, naming it"); + ft.supports_gemv = true; + + // 5. a target failure keeps the target's own words + ft.fail = true; + check(!device_projection(ft, p, &out, &why) && why == "kernel not resident", "a target failure keeps its reason"); + ft.fail = false; + + // 6. an unnamed projection with no tag is refused (the log must be answerable) + p.tag = nullptr; p.k = 777; p.n = 5; + check(!device_projection(ft, p, &out, &why) && why.find("needs a tag") != std::string::npos, + "an unnamed shape with no tag is refused"); + + // 7. the decode tier: 16 rows is NOT the wide body + p.k = 768; p.n = 768; p.rows = 16; p.tag = "O"; + check(device_projection(ft, p, &out, &why) && !ft.seen.rows, "a 16-row launch is the per-row body"); + + std::printf(failures ? "audio device gemm: %d FAILED\n" : "audio device gemm: all cases ok\n", failures); + return failures ? 1 : 0; +} diff --git a/tests/audio_exec_test.cpp b/tests/audio_exec_test.cpp index 0b923244a..48255fac4 100644 --- a/tests/audio_exec_test.cpp +++ b/tests/audio_exec_test.cpp @@ -1,18 +1,6 @@ // Copyright 2026 bong-water-water-bong // SPDX-License-Identifier: Apache-2.0 // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// // audio_exec_test.cpp -- run an audio pack's encoder through the engine's own host executor and compare it against // onnxruntime, on an input both sides can build from a formula (no binary fixture in the repo). // @@ -26,6 +14,7 @@ #include #include "audio_exec.h" +#include "audio_pack.h" #include "onnx_graph.h" namespace { @@ -50,9 +39,26 @@ void expect_close(float got, double want, double tol, const std::string& what) { } // namespace int main(int argc, char** argv) { + // Two forms, like the tool: a pack directory (npu/audio_pack.h) or the two graph paths. The pack form is what the ctest row + // uses, so the container is exercised by the same run that checks the maths (engine#1783). + std::string pack_encoder, pack_decoder; const char* path = (argc > 1) ? argv[1] : std::getenv("ONEBIT_TEST_WHISPER"); + const char* dpath_arg = (argc > 2) ? argv[2] : nullptr; + if (path != nullptr && *path != '\0' && std::string(path).find(".onnx") == std::string::npos) { + onebit::audio::AudioPack pack; + std::string why; + if (!onebit::audio::discover_audio_pack(path, &pack, &why)) { + std::printf("FAIL pack: %s\n", why.c_str()); + return 1; + } + pack_encoder = pack.encoder; + pack_decoder = pack.decoder; + std::printf("pack: %s (mel bins %d from the %s)\n", path, pack.mel_bins, pack.mel_bins_source.c_str()); + path = pack_encoder.c_str(); + dpath_arg = pack_decoder.c_str(); + } if (path == nullptr || *path == '\0') { - std::printf("SKIP: pass the encoder .onnx, or set ONEBIT_TEST_WHISPER to it\n"); + std::printf("SKIP: pass a pack directory or the encoder .onnx, or set ONEBIT_TEST_WHISPER\n"); return 77; } using namespace onebit::npu::pack_import; @@ -127,7 +133,7 @@ int main(int argc, char** argv) { std::printf("audio exec: whole encoder on the host matches onnxruntime\n"); // ---- the decoder, on the same pack: 128-position token window x, the encoder output xa ---------------------------- - const char* dpath = (argc > 2) ? argv[2] : std::getenv("ONEBIT_TEST_WHISPER_DECODER"); + const char* dpath = (dpath_arg != nullptr) ? dpath_arg : std::getenv("ONEBIT_TEST_WHISPER_DECODER"); if (dpath == nullptr || *dpath == '\0') { std::printf("SKIP decoder: pass the decoder .onnx as the second argument\n"); return 0; diff --git a/tests/audio_kernel_plan.sh b/tests/audio_kernel_plan.sh new file mode 100755 index 000000000..572d8e967 --- /dev/null +++ b/tests/audio_kernel_plan.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Copyright 2026 bong-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# tests/audio_kernel_plan.sh -- the launch plan a device route reads (engine#1783). +# +# The plan is the artifact the encoder's device arm consumes: one entry per projection, in the order the graph runs them, naming the +# ELF each one needs at the width the ENGINE stages it to. Three properties make it usable, and each has been wrong somewhere today: +# the list is not empty (an empty plan must not read as a servable pack), `order` is dense from 0 (a route indexes by it), and every +# staged width is on the engine's grid -- K a multiple of 256, N of 512 (`npu/forward/src/q4nx_pack.c`) -- because the forward asks +# for the staged name and a plan naming the model's own width sends a route after a file that is not there. +# +# Usage: audio_kernel_plan.sh +set -u +tool=${1:?audio_kernel_shapes binary} +pack=${2:-} +[ -n "$pack" ] || pack=${ONEBIT_TEST_WHISPER_PACK:-} +[ -n "$pack" ] || { echo "SKIP: no pack (pass one, or set ONEBIT_TEST_WHISPER_PACK)"; exit 77; } +[ -d "$pack" ] || { echo "SKIP: $pack is not a directory"; exit 77; } + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +plan=$tmp/plan.json +if ! "$tool" "$pack" --write-plan "$plan" > "$tmp/out.txt" 2>&1; then + echo "FAIL the plan could not be written:"; sed 's/^/ /' "$tmp/out.txt"; exit 1 +fi +fail=0 +ok() { printf 'ok %s\n' "$1"; } +bad() { printf 'FAIL %s\n' "$1"; fail=1; } + +n=$(grep -c '"order":' "$plan" || true) +[ "$n" -gt 0 ] && ok "the plan lists $n projection(s)" || bad "the plan lists no projection -- nothing would launch" + +orders=$(grep -o '"order": [0-9]*' "$plan" | grep -o '[0-9]*' | tr '\n' ' ') +want=$(seq 0 $((n - 1)) | tr '\n' ' ') +[ "$orders" = "$want" ] && ok "...ordered 0..$((n - 1)) with no gaps" || bad "order is not dense from 0: $orders" + +offk=$(grep -o '"staged_k": [0-9]*' "$plan" | grep -o '[0-9]*' | awk '$1 % 256 != 0' | wc -l) +offn=$(grep -o '"staged_n": [0-9]*' "$plan" | grep -o '[0-9]*' | awk '$1 % 512 != 0' | wc -l) +[ "$offk" -eq 0 ] && ok "every staged K is a multiple of 256" || bad "$offk staged K value(s) off the grid" +[ "$offn" -eq 0 ] && ok "every staged N is a multiple of 512" || bad "$offn staged N value(s) off the grid" + +# The plan's ELF names are the family's, and they differ from the model's own widths exactly where staging bites. +bad_elf=$(grep -o '"elf_b8": "[^"]*"' "$plan" | grep -vc 'full_i8_[A-Z]*_K[0-9]*_N[0-9]*_b8\.elf' || true) +[ "$bad_elf" -eq 0 ] && ok "every elf_b8 is a full_i8__K_N_b8.elf name" \ + || bad "$bad_elf elf_b8 name(s) do not look like a family member" +if grep -q '"staged_n": 1024' "$plan"; then + grep -q '_N1024_b8.elf' "$plan" && ok "a staged N1024 is named N1024 (768 -> 1024), not the model's 768" \ + || bad "a launch is staged to 1024 but its ELF name is not" +fi + +[ "$fail" -eq 0 ] && { echo "audio kernel plan: ok"; exit 0; } +echo "audio kernel plan: FAILED"; exit 1 diff --git a/tests/audio_kernel_roles_test.cpp b/tests/audio_kernel_roles_test.cpp new file mode 100644 index 000000000..d9bfc06b9 --- /dev/null +++ b/tests/audio_kernel_roles_test.cpp @@ -0,0 +1,102 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// tests/audio_kernel_roles_test.cpp -- the shape-to-role rule of tools/audio_kernel_shapes.cpp, as a pure function +// (npu/forward/include/audio_kernel_roles.h, engine#1783). Hand-computed cases; no graph, no device. +// +// The rule was wrong twice before it was right, both times in ways a compiler cannot see: a name-based classifier reported +// `unknown` for every projection (packed exports name their weights `val_N`), and a version that classified BEFORE assigning +// the shape read (0, 0, hidden) and said `unknown` for everything. These arms are the ones that would have caught both. + +#include +#include + +#include "audio_kernel_roles.h" + +namespace { + +int failures = 0; + +void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++failures; +} + +onebit::audio::ProjectionShape sh(int k, int n) { return onebit::audio::ProjectionShape{k, n}; } + +} // namespace + +using onebit::audio::elf_name_for; +using onebit::audio::q4nx_pad_k; +using onebit::audio::q4nx_pad_n; + +int main() { + using onebit::audio::elf_name_for; + using onebit::audio::hidden_from_square_widths; + using onebit::audio::role_of_shape; + + // 1. whisper-small's four shapes at h = 768 + check(std::string(role_of_shape(sh(768, 768), 768)) == "O", "h x h is O (q, k, v and o share it)"); + check(std::string(role_of_shape(sh(768, 3072), 768)) == "GU", "h -> 4h is GU"); + check(std::string(role_of_shape(sh(3072, 768), 768)) == "D", "4h -> h is D"); + check(std::string(role_of_shape(sh(768, 2304), 768)) == "QKV", "h -> 3h is fused QKV"); + + // 2. the ELF the family emits, both M tiers + // The emitted name is on the engine's staging grid (768 -> 1024), not the model's own width -- the defect the synth768 + // device run exposed: the forward asked for O_K768_N1024 while this said N768 and the set "covered". + check(elf_name_for(sh(768, 768), 768, false) == "full_i8_O_K768_N1024_b8.elf", "b8 ELF name, staged"); + check(elf_name_for(sh(768, 3072), 768, true) == "full_i8_GU_K768_N3072_b8_m16.elf", "m16 ELF name"); + + // 3. a shape that is none of the four is `unknown`, never a guess -- asking for the wrong ELF wastes a ticket + check(std::string(role_of_shape(sh(1000, 2000), 768)) == "unknown", "an unrecognised shape is unknown"); + check(std::string(role_of_shape(sh(768, 768), 0)) == "unknown", "no hidden width means no roles at all"); + + // 4. the hidden width comes from the most common SQUARE width, not the median of everything: embeddings and the 4h MLP + // would drag a median away from h, which is the bug the first version had. + { + const int widths[] = {768, 768, 768, 768, 3072, 3072, 51865, 1500, 128}; + check(hidden_from_square_widths(widths, 9) == 768, "the mode of the square widths is the hidden width"); + const int one[] = {1280}; + check(hidden_from_square_widths(one, 1) == 1280, "a single width is that width"); + // A tie is refused, not broken: two widths occurring equally often (here 768 and 896, four times each) mean the pack + // has two candidate hidden widths, and whichever one a loop happened to see first used to decide every role after it. + { + const int tied_widths[] = {768, 768, 768, 768, 896, 896, 896, 896}; + check(hidden_from_square_widths(tied_widths, 8) == 0, "an equal-mode tie is refused, not broken by order"); + const int clear[] = {768, 768, 768, 896, 896}; + check(hidden_from_square_widths(clear, 5) == 768, "a clear mode is still taken"); + check(hidden_from_square_widths(nullptr, 0) == 0, "no evidence is 0, not a guess"); + } + check(hidden_from_square_widths(nullptr, 0) == 0, "no widths means no hidden width, not a crash"); + } + + // 5. large-v3's shapes at h = 1280, so the rule is not tuned to one pack + check(std::string(role_of_shape(sh(1280, 1280), 1280)) == "O", "lv3: h x h is O"); + check(std::string(role_of_shape(sh(1280, 5120), 1280)) == "GU", "lv3: h -> 4h is GU"); + check(std::string(role_of_shape(sh(5120, 1280), 1280)) == "D", "lv3: 4h -> h is D"); + + // The engine stages every container on its own grid -- K a multiple of 256, N of 512 (npu/forward/src/q4nx_pack.c) -- and + // the forward asks for the ELF at THOSE widths. Measured on the synth768 device run (engine#1783): with the set emitted at + // the model's own widths the forward said "missing full ELF ... QKV_K768_N2560_b8" (2304 rounded up) and then "...O_K768_N1024" + // (768 rounded up), while this header was reporting the family covered. + check(q4nx_pad_n(768) == 1024, "N 768 stages to 1024"); + check(q4nx_pad_n(2304) == 2560, "N 2304 (a fused QKV at hidden 768) stages to 2560"); + check(q4nx_pad_n(3072) == 3072, "N 3072 is already on the 512 grid"); + check(q4nx_pad_k(768) == 768 && q4nx_pad_k(3000) == 3072, "K pads to 256: 768 in place, 3000 -> 3072"); + check(q4nx_pad_n(0) == 0 && q4nx_pad_k(-5) == -5, "a non-positive width is left alone, not rounded into a shape"); + check(elf_name_for({768, 2304}, 768, false) == "full_i8_QKV_K768_N2560_b8.elf", + "the QKV name the forward asks for at hidden 768 (2304 staged to 2560)"); + check(elf_name_for({768, 768}, 768, true) == "full_i8_O_K768_N1024_b8_m16.elf", + "the O name, m16 tier: the hidden width stages too (768 to 1024)"); + check(elf_name_for({3072, 768}, 768, false) == "full_i8_D_K3072_N1024_b8.elf", + "the D name: K already on its grid, N staged"); + check(elf_name_for({768, 3072}, 768, false) == "full_i8_GU_K768_N3072_b8.elf", + "the MLP name: 4h is on the 512 grid, so whisper-small's own width is the name"); + + if (failures != 0) { + std::printf("audio kernel roles: %d failure(s)\n", failures); + return 1; + } + std::printf("audio kernel roles: all cases ok\n"); + return 0; +} diff --git a/tests/audio_launch_plan_test.cpp b/tests/audio_launch_plan_test.cpp new file mode 100644 index 000000000..4b9efb8a8 --- /dev/null +++ b/tests/audio_launch_plan_test.cpp @@ -0,0 +1,83 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// tests/audio_launch_plan_test.cpp -- the launch loop's routing and its ACCOUNTING (npu/forward/include/audio_launch_plan.h, +// engine#1783), driven against a fake target and a fake host: no device, no container, no kernels. +// +// The report is the measurement the row is scored on (`device_gemm= s + host_remainder= s`), so the arms here are about +// where a launch is COUNTED, not only whether it ran: a launch that falls back must land in the host column with the device's +// own reason beside it, and a launch both paths refuse must land in neither column -- counting it as host would quietly move an +// unexplained projection into the number the route is judged by. + +#include +#include +#include + +#include "audio_launch_plan.h" +using namespace onebit::audio; +namespace t = ::npu::target; +static int failures = 0; +static void check(bool ok, const std::string& what) { std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); if (!ok) ++failures; } +struct FakeTarget : t::NpuTarget { + t::Caps c; int calls = 0; bool refuse_all = false; + const char* name() const override { return "fake"; } + bool open(const onebit::npu::plan::Plan&, const std::string&, const char*, std::string&) override { return true; } + bool load_block(int, int, std::string&) override { return true; } + t::Caps caps() const override { return c; } + bool supports(t::Op op) const override { return op == t::Op::gemv; } + bool gemv(const t::GemvArgs& a, std::vector& out, std::string& err) override { + ++calls; + if (refuse_all) { err = "no kernel for this shape"; return false; } + out.assign(size_t(a.M) * size_t(a.N), 1.0f); + return true; + } +}; +int main() { + std::vector x(128 * 768, 1.0f); + std::vector specs = { + {0, 1, 0, 768, 768, 128, "O"}, + {1, 0, 0, 768, 3072, 128, "GU"}, + {2, 2, 0, 3072, 768, 128, "D"}, + }; + auto input = [&](const LaunchSpec&) { return x.data(); }; + int host_calls = 0; + auto host = [&](const LaunchSpec&, std::vector*, std::string*) { ++host_calls; return true; }; + + FakeTarget ft; ft.c.max_rows = 16; ft.c.prefill_rows = 128; + auto r = run_launches(&ft, specs, input, host); + check(r.ok() && r.device_launches == 3 && r.host_launches == 0, "with a target, every launch goes to the device"); + check(ft.calls == 3 && host_calls == 0, "...the device was called three times and the host not at all"); + check(r.per_launch[0].device && r.per_launch[2].device, "...and each launch records which column it went in"); + + ft.refuse_all = true; host_calls = 0; + auto r2 = run_launches(&ft, specs, input, host); + check(r2.ok() && r2.host_launches == 3 && r2.device_launches == 0, "a target that refuses every shape falls back to the host"); + check(r2.per_launch[1].why == "no kernel for this shape", "...keeping the target's own reason per launch"); + check(host_calls == 3, "...the host ran all three"); + + ft.refuse_all = false; host_calls = 0; + auto r3 = run_launches(nullptr, specs, input, host); + check(r3.ok() && r3.host_launches == 3, "no target at all is the host path, counted as such"); + check(r3.per_launch[0].why == "no target", "...and says why it is not on the device"); + + auto bad_host = [&](const LaunchSpec&, std::vector*, std::string* why) { *why = "host GEMM refused"; return false; }; + auto r4 = run_launches(nullptr, specs, input, bad_host); + check(!r4.ok() && r4.failed == 3 && r4.device_launches == 0 && r4.host_launches == 0, + "a launch both paths refuse counts in NEITHER column"); + check(r4.failures.size() == 3 && r4.failures[0].find("launch 0 O") == 0 && r4.failures[0].find("host GEMM refused") != std::string::npos, + "...and the failure names the launch, its tag and both reasons"); + // The close condition's own line, built from a report rather than typed. A split with a silent third category (failed) is a + // split that adds up to less than the run, so the failure count is part of the string. + { + ft.refuse_all = true; // every launch falls back to the host + auto r5 = run_launches(&ft, specs, input, host); + const std::string line = format_split_run(r5, 163.0, 2050.0); + check(line == "device_gemm=0.163 s + host_remainder=2.050 s per window (0 device launch(es), 3 host, 0 failed)", + "the split line reads exactly as the close condition asks: " + line); + auto r6 = run_launches(nullptr, specs, input, bad_host); + const std::string bad = format_split_run(r6, 0.0, 0.0); + check(bad.find("3 failed") != std::string::npos, "...and a run whose launches all failed says so: " + bad); + } + std::printf(failures ? "audio launch plan: %d FAILED\n" : "audio launch plan: all cases ok\n", failures); + return failures ? 1 : 0; +} diff --git a/tests/audio_pack_test.cpp b/tests/audio_pack_test.cpp new file mode 100644 index 000000000..8912aba27 --- /dev/null +++ b/tests/audio_pack_test.cpp @@ -0,0 +1,143 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// tests/audio_pack_test.cpp -- audio pack discovery (npu/audio_pack.h, engine#1783). Fixture directories only: no model, no +// device, no network. The graph-derived arms need a real pack and skip (77) without ONEBIT_TEST_WHISPER. + +#include +#include +#include +#include +#include + +#include "audio_pack.h" + +namespace fs = std::filesystem; +namespace { + +int failures = 0; + +void check(bool ok, const std::string& what) { + if (ok) { + std::printf("ok %s\n", what.c_str()); + } else { + std::printf("FAIL %s\n", what.c_str()); + ++failures; + } +} + +void write(const fs::path& p, const std::string& body) { + fs::create_directories(p.parent_path()); + std::ofstream(p) << body; +} + +} // namespace + +int main() { + const fs::path root = fs::temp_directory_path() / "audio-pack-test"; + + // 7. a pack whose graphs are named after the model (large_v3_encoder.onnx): found by scan, and the choice is REPORTED -- + // and a `decoder_with_past` sibling is never picked, because it is a different graph with a different input list + { + const fs::path d = root / "model-named"; + write(d / "large_v3_encoder.onnx", "x"); + write(d / "large_v3_decoder_with_past.onnx", "x"); + write(d / "large_v3_decoder.onnx", "x"); + onebit::audio::AudioPack p; + std::string why; + // no bins anywhere: the graphs are not real, so discovery refuses on the bin count -- but the NAME choice must already + // have been made, which is what the two checks below read out of the pack struct. + onebit::audio::discover_audio_pack(d.string(), &p, &why); + check(p.decoder.find("large_v3_decoder.onnx") != std::string::npos, + "model-named: the plain decoder is chosen, not the with_past one"); + std::printf(" [debug] concern=<%s>\n", p.concern.c_str()); + check(p.concern.find("taken from its name") != std::string::npos, + "model-named: the name-based choice is reported, not silent"); + } + + fs::remove_all(root); + + // 1. a complete pack, whisper-* spelling, with a config.json that states the bins + { + const fs::path d = root / "complete"; + write(d / "encoder_model.onnx", "not really a graph"); + write(d / "decoder_model.onnx", "not really a graph"); + write(d / "config.json", R"({"num_mel_bins": 80, "sampling_rate": 16000, "n_fft": 400, "hop_length": 160})"); + onebit::audio::AudioPack p; + std::string why; + const bool ok = onebit::audio::discover_audio_pack(d.string(), &p, &why); + check(ok, "complete pack: discovered"); + check(p.encoder.find("encoder_model.onnx") != std::string::npos && + p.decoder.find("decoder_model.onnx") != std::string::npos, + "complete pack: both graphs found by their whisper names"); + check(p.mel_bins == 80 && p.mel_bins_source == "config.json", + "complete pack: the bins come from config.json when the graph cannot be read"); + check(!p.concern.empty(), "complete pack: the unreadable graph is reported as a concern, not a refusal"); + } + + // 2. our own spelling (encoder.onnx / decoder.onnx) is found too + { + const fs::path d = root / "own-names"; + write(d / "encoder.onnx", "x"); + write(d / "decoder.onnx", "x"); + write(d / "config.json", R"({"num_mel_bins": 128})"); + onebit::audio::AudioPack p; + std::string why; + check(onebit::audio::discover_audio_pack(d.string(), &p, &why), "own names: discovered"); + check(p.mel_bins == 128, "own names: 128 bins read from config.json"); + } + + // 3. a missing graph is refused, by name, and says which one + { + const fs::path d = root / "no-decoder"; + write(d / "encoder_model.onnx", "x"); + write(d / "config.json", R"({"num_mel_bins": 80})"); + onebit::audio::AudioPack p; + std::string why; + check(!onebit::audio::discover_audio_pack(d.string(), &p, &why), "no decoder: refused"); + check(p.missing.size() == 1 && p.missing[0].find("decoder") != std::string::npos, + "no decoder: the missing piece is named"); + check(why.find("missing") != std::string::npos, "no decoder: the reason says missing"); + } + + // 4. no mel-bin count anywhere: refused rather than guessed -- this is the run-that-answers-wrongly case + { + const fs::path d = root / "no-bins"; + write(d / "encoder_model.onnx", "x"); + write(d / "decoder_model.onnx", "x"); + onebit::audio::AudioPack p; + std::string why; + check(!onebit::audio::discover_audio_pack(d.string(), &p, &why), "no bin count: refused"); + check(why.find("guessing") != std::string::npos, "no bin count: the refusal says why guessing is not done"); + } + + // 5. not a directory at all + { + onebit::audio::AudioPack p; + std::string why; + check(!onebit::audio::discover_audio_pack((root / "nope").string(), &p, &why), "absent directory: refused"); + check(why.find("not a directory") != std::string::npos, "absent directory: said plainly"); + } + + // 6. a real pack, when one is named: the bins come from the encoder's own Conv + { + const char* pack = std::getenv("ONEBIT_TEST_WHISPER_PACK"); + if (pack == nullptr || *pack == '\0') { + std::printf("skip a real pack: set ONEBIT_TEST_WHISPER_PACK to a pack directory\n"); + } else { + onebit::audio::AudioPack p; + std::string why; + check(onebit::audio::discover_audio_pack(pack, &p, &why), std::string("real pack: discovered (") + why + ")"); + check(p.mel_bins > 0 && p.mel_bins_source == "encoder graph", + "real pack: the mel bins come from the encoder graph, not from a config field"); + } + } + + fs::remove_all(root); + if (failures != 0) { + std::printf("audio pack: %d failure(s)\n", failures); + return 1; + } + std::printf("audio pack: all cases ok\n"); + return 0; +} diff --git a/tests/data/whisper-jfk-ids.json b/tests/data/whisper-jfk-ids.json new file mode 100644 index 000000000..2a8554483 --- /dev/null +++ b/tests/data/whisper-jfk-ids.json @@ -0,0 +1,31 @@ +{ + "what": "the reference decode of jfk.wav through whisper-small (encoder_model.onnx + decoder_model.onnx)", + "source": "produced by tools/whisper_transcribe and checked against onnxruntime and the reference decode; see engine#1783", + "greedy_ids": [ + 400, + 370, + 452, + 7177, + 6280, + 11, + 1029, + 406, + 437, + 428, + 1941, + 393, + 360, + 337, + 291, + 11, + 1029, + 437, + 291, + 393, + 360, + 337, + 428, + 1941, + 13 + ] +} diff --git a/tests/onnx_graph_test.cpp b/tests/onnx_graph_test.cpp index 2d6234dc5..7bf5b3ecc 100644 --- a/tests/onnx_graph_test.cpp +++ b/tests/onnx_graph_test.cpp @@ -1,18 +1,6 @@ // Copyright 2026 bong-water-water-bong // SPDX-License-Identifier: Apache-2.0 // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// // onnx_graph_test.cpp -- the audio pack's graph, read through the engine and checked for the property the executor will // depend on: every node's inputs are graph inputs, initializers, or outputs of an *earlier* node. Skips (77) without // ONEBIT_TEST_WHISPER naming the encoder's .onnx. @@ -24,6 +12,7 @@ #include #include +#include "audio_pack.h" #include "onnx_graph.h" namespace { @@ -40,7 +29,20 @@ void check(bool cond, const std::string& what) { } // namespace int main(int argc, char** argv) { + // A pack directory (the ctest row's form) or a graph path: the pack form is what a lane has, and it also proves the mel-bin + // count comes from the encoder's own Conv (engine#1783). + std::string from_pack; const char* path = (argc > 1) ? argv[1] : std::getenv("ONEBIT_TEST_WHISPER"); + if (path != nullptr && *path != '\0' && std::string(path).find(".onnx") == std::string::npos) { + onebit::audio::AudioPack pack; + std::string why; + if (!onebit::audio::discover_audio_pack(path, &pack, &why)) { + std::printf("FAIL pack: %s\n", why.c_str()); + return 1; + } + from_pack = pack.encoder; + path = from_pack.c_str(); + } if (path == nullptr || *path == '\0') { std::printf("SKIP: pass the encoder .onnx, or set ONEBIT_TEST_WHISPER to it\n"); return 77; diff --git a/tests/whisper_ids_test.cpp b/tests/whisper_ids_test.cpp new file mode 100644 index 000000000..f56e75e83 --- /dev/null +++ b/tests/whisper_ids_test.cpp @@ -0,0 +1,62 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// tests/whisper_ids_test.cpp -- the special-token layout selection (npu/forward/include/whisper_ids.h, engine#1783). +// +// These are the two verdicts the tool got wrong before it read a pack: a hard-coded 50359/50363 decodes whisper-small and makes +// large-v3 answer <|endoftext|> at the first step. Both values here were measured against onnxruntime on the real packs. + +#include +#include + +#include "whisper_ids.h" + +namespace { + +int failures = 0; + +void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++failures; +} + +} // namespace + +int main() { + using onebit::audio::whisper_ids_for_vocab; + using onebit::audio::WhisperIds; + + WhisperIds ids; + check(whisper_ids_for_vocab(51865, &ids), "51865 (v1/v2/small/medium): a layout exists"); + check(ids.sot == 50258 && ids.lang_en == 50259 && ids.eot == 50257, "51865: sot/en/eot are unchanged across families"); + check(ids.transcribe == 50359 && ids.no_timestamps == 50363, "51865: transcribe 50359, notimestamps 50363"); + + check(whisper_ids_for_vocab(51866, &ids), "51866 (large-v3): a layout exists"); + check(ids.transcribe == 50360 && ids.no_timestamps == 50364, "51866: both shift by one"); + + // A larger derivative of the v3 line keeps v3's layout; the whole point is that the ids are offsets into the vocabulary. + check(whisper_ids_for_vocab(51890, &ids) && ids.transcribe == 50360, "a wider v3 derivative keeps v3's layout"); + + // A `.en` model (51864) is refused: its prompt differs and nobody measured it here. Guessing one of the two known layouts is + // how a wrong prompt looks right on English audio. + WhisperIds untouched; + untouched.transcribe = -1; + check(!whisper_ids_for_vocab(51864, &untouched), "51864 (.en): refused rather than guessed"); + check(untouched.transcribe == -1, "... and the caller's struct is left untouched"); + check(!whisper_ids_for_vocab(0, &untouched), "vocabulary 0: refused"); + check(!whisper_ids_for_vocab(51865, nullptr), "a null output is refused, not dereferenced"); + + // The two families must not agree on transcribe: that difference is the defect this header exists to prevent. + WhisperIds small, v3; + whisper_ids_for_vocab(51865, &small); + whisper_ids_for_vocab(51866, &v3); + check(small.transcribe != v3.transcribe && small.no_timestamps != v3.no_timestamps, + "the two families differ in both ids (a shared constant is the bug)"); + + if (failures != 0) { + std::printf("whisper ids: %d failure(s)\n", failures); + return 1; + } + std::printf("whisper ids: all cases ok\n"); + return 0; +} diff --git a/tests/whisper_lv3_row.sh b/tests/whisper_lv3_row.sh new file mode 100755 index 000000000..760ff9feb --- /dev/null +++ b/tests/whisper_lv3_row.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Copyright 2026 bong-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# tests/whisper_lv3_row.sh -- the whisper-large-v3 half of the transcription row (engine#1783). +# +# Why a second pack and not just the first: every constant that fit whisper-small was wrong for large-v3 -- the token window +# (128 vs 448), the vocabulary width (51865 vs 51866) and the <|transcribe|>/<|notimestamps|> ids that follow from it +# (50359/50363 vs 50360/50364). Each one was invisible until the tool was pointed at the second pack, and each produced a +# plausible-looking wrong answer rather than a crash. So the row runs BOTH families and requires the same ids from both. +# +# Usage: whisper_lv3_row.sh +# ONEBIT_TEST_WHISPER_PACK_LV3 and ONEBIT_TEST_WHISPER_WAV_LV3 name the pack and the wav; unset means skip (77), because a +# pack is a multi-GB download and cannot live in the tree. +set -u +bin=${1:?whisper_transcribe binary} +ids=${2:?expected ids json} +[ -n "${ONEBIT_TEST_WHISPER_PACK_LV3:-}" ] || { + echo "SKIP: set ONEBIT_TEST_WHISPER_PACK_LV3 (and ONEBIT_TEST_WHISPER_WAV_LV3) to run the large-v3 arm" + exit 77 +} +[ -n "${ONEBIT_TEST_WHISPER_WAV_LV3:-}" ] || { + echo "SKIP: ONEBIT_TEST_WHISPER_PACK_LV3 is set but ONEBIT_TEST_WHISPER_WAV_LV3 is not" + exit 77 +} +exec "$bin" "$ONEBIT_TEST_WHISPER_PACK_LV3" "$ONEBIT_TEST_WHISPER_WAV_LV3" "$ids" diff --git a/tools/audio_kernel_shapes.cpp b/tools/audio_kernel_shapes.cpp new file mode 100644 index 000000000..408155c60 --- /dev/null +++ b/tools/audio_kernel_shapes.cpp @@ -0,0 +1,249 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// audio_kernel_shapes: which GEMM shapes an audio pack's encoder needs, and whether a kernel directory covers them (engine#1783). +// +// audio_kernel_shapes # the shapes the encoder's projections ask for +// audio_kernel_shapes --write-plan # ... as a launch plan (JSON): node order, staged shapes, ELF names +// audio_kernel_shapes --kernels # ... and whether each has a full ELF in that directory +// +// Why a tool and not a test: what it reports depends on which pack and which kernel directory are given, and the useful output is a +// list a lane reads before taking a device ticket. The shape derivation is the part worth trusting -- it reads the pack's own ONNX +// graph through the engine's reader, so a pack whose projections differ (768 for small, 1280 for large-v3) reports its own numbers +// instead of a table someone maintained. +// +// The role a projection plays is decided by the tensor it multiplies, not by node order: a packed export fuses q/k/v into one +// 3N-wide matrix, and the ELF it needs is the fused one (N = 3 x hidden). A name this tool does not recognise is reported as +// `unknown`, because a shape with no role is a shape nobody has built a kernel for. + +#include +#include +#include +#include +#include +#include +#include + +#include "audio_pack.h" +#include "audio_kernel_roles.h" +#include "onnx_graph.h" + +namespace fs = std::filesystem; +using onebit::npu::pack_import::OnnxGraphInfo; + +namespace { + +struct Need { + std::string role; // QKV | O | GU | D, or "unknown" + int kindex = 0; // the reduction (K) + int n = 0; // the output width (N) + std::string weight; // the tensor it multiplies, for the message +}; + +std::string role_of_unused(const std::string& weight) { + const auto has = [&](const char* s) { return weight.find(s) != std::string::npos; }; + if (has("qkv") || (has("q_proj") && has("k_proj"))) return "QKV"; + if (has("gate_up") || (has("gate_proj") && has("up_proj"))) return "GU"; + if (has("q_proj") || has("query")) return "QKV"; + if (has("k_proj") || has("key") || has("v_proj") || has("value")) return "QKV"; + if (has("out_proj") || has("o_proj") || has("dense")) return "O"; + if (has("down_proj")) return "D"; + if (has("fc1")) return "GU"; + if (has("fc2")) return "D"; + return "unknown"; +} + +// A packed export names its weights generically (`val_452`) and reaches them through a Transpose; the name that says what the +// projection IS (`blocks.0.attn.query`) sits on the Transpose's input. So the role comes from following the producer chain back to +// a named tensor -- without this every shape reports `unknown`, which is what the first run of this tool did. +const std::string* source_name(const std::map& producer, std::string name) { + for (int hop = 0; hop < 4; ++hop) { + const auto it = producer.find(name); + if (it == producer.end()) return nullptr; + name = it->second; + // a name that looks like a parameter path rather than a val_N intermediate is the one to report + if (name.find("val_") == std::string::npos && name.find("weight") != std::string::npos) { + static thread_local std::string keep; + keep = name; + return &keep; + } + } + static thread_local std::string keep; + keep = name; + return &keep; +} + +std::vector shapes_of(const OnnxGraphInfo& g, int hidden) { + std::map by_weight; // one entry per weight, so a shared weight is not counted twice + std::map> dims; + for (const auto& t : g.init_tensors) dims[t.name] = t.dims; + std::map producer; // output -> its single non-initializer input, for the role lookup + for (const auto& node : g.nodes) { + if (node.outs.empty() || node.ins.empty()) continue; + if (node.op != "Transpose" && node.op != "Reshape" && node.op != "Cast" && node.op != "Identity") continue; + producer[node.outs[0]] = node.ins[0]; + } + for (const auto& node : g.nodes) { + if (node.op != "MatMul" || node.ins.size() < 2) continue; + const auto it = dims.find(node.ins[1]); + if (it == dims.end() || it->second.size() != 2) continue; + Need need; + const std::string* named = source_name(producer, node.ins[1]); + need.weight = (named != nullptr && !named->empty()) ? *named : node.ins[1]; + need.kindex = int(it->second[0]); + need.n = int(it->second[1]); + need.role = onebit::audio::role_of_shape(onebit::audio::ProjectionShape{need.kindex, need.n}, hidden); // after the shape is set: this read 0, 0 on the first run + by_weight[need.weight] = need; + } + std::vector out; + for (auto& kv : by_weight) out.push_back(kv.second); + std::sort(out.begin(), out.end(), [](const Need& a, const Need& b) { + return a.role == b.role ? a.n < b.n : a.role < b.role; + }); + return out; +} + +// The ELF names the family generator produces: full_i8__K_N_b8[_m16].elf +std::string elf_name(const Need& n, bool m16) { + return onebit::audio::elf_name_for(onebit::audio::ProjectionShape{n.kindex, n.n}, 0, m16) == "" ? "" : + "full_i8_" + n.role + "_K" + std::to_string(onebit::audio::q4nx_pad_k(n.kindex)) + "_N" + + std::to_string(onebit::audio::q4nx_pad_n(n.n)) + "_b8" + + (m16 ? "_m16" : "") + ".elf"; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 2) { + std::fprintf(stderr, "usage: %s [--kernels ]\n", argv[0]); + return 2; + } + onebit::audio::AudioPack pack; + std::string why; + if (!onebit::audio::discover_audio_pack(argv[1], &pack, &why)) { + std::fprintf(stderr, "pack: %s\n", why.c_str()); + return 1; + } + OnnxGraphInfo g; + try { + g = onebit::npu::pack_import::parse_onnx_graph(pack.encoder); + } catch (const std::exception& e) { + std::fprintf(stderr, "encoder: %s\n", e.what()); + return 1; + } + // The hidden width comes from the graph's own biggest projection: the widest N among the distinct shapes. + int hidden = 0; + { + // The hidden width is the most common SQUARE weight width: q/k/v/o are h x h in every layer, so the mode of that width is + // h, where a median over all weights is skewed by the embeddings and the 4h MLP. + // ONE implementation of the rule, in the header the tests drive: the mode of the square widths, and 0 when two widths + // tie. A second copy here is the drift this file exists to prevent -- and it had one: this loop took whichever + // width came first out of a std::map, i.e. the SMALLEST of a tie, silently. + std::vector square; + for (const auto& t : g.init_tensors) + if (t.dims.size() == 2 && t.dims[0] == t.dims[1] && t.dims[0] > 256) square.push_back(int(t.dims[0])); + hidden = onebit::audio::hidden_from_square_widths(square.data(), int(square.size())); + if (hidden <= 0) { + std::fprintf(stderr, + "audio_kernel_shapes: the hidden width is ambiguous or absent in %s (%zu square weight width(s), no " + "single most common one). Every role would be reported `unknown`, and a role this tool cannot name is " + "one whose kernel cannot be checked -- refusing rather than picking one.\n", + pack.encoder.c_str(), square.size()); + return 2; + } + } + const std::vector needs = shapes_of(g, hidden); + std::printf("pack %s: %zu projection shape(s) in %s (hidden %d)\n", argv[1], needs.size(), pack.encoder.c_str(), hidden); + std::map counts; + for (const auto& n : needs) { + ++counts[n.role + " K" + std::to_string(n.kindex) + " N" + std::to_string(n.n)]; + } + for (const auto& kv : counts) std::printf(" %-18s x%d\n", kv.first.c_str(), kv.second); + + std::string kernels, plan_path; + for (int i = 2; i + 1 < argc; ++i) { + if (std::strcmp(argv[i], "--kernels") == 0) kernels = argv[i + 1]; + else if (std::strcmp(argv[i], "--write-plan") == 0) plan_path = argv[i + 1]; + } + if (!plan_path.empty()) { + // The launch plan a device route would read: every projection IN TOPOLOGICAL ORDER (the graph's own order, because a + // route executes them in it), with the shape the model declares beside the shape the kernel is staged at and the ELF + // name that staging produces. It is written here rather than derived by the route so that the one place which knows the + // staging rule is the one place that names the kernel -- the defect this file exists to prevent (engine#1783). + std::FILE* f = std::fopen(plan_path.c_str(), "wb"); + if (f == nullptr) { + std::fprintf(stderr, "audio_kernel_shapes: cannot write %s\n", plan_path.c_str()); + return 1; + } + std::fprintf(f, "{\n \"pack\": \"%s\",\n \"encoder\": \"%s\",\n \"hidden\": %d,\n \"launches\": [\n", + argv[1], pack.encoder.c_str(), hidden); + int idx = 0; + std::vector dims; + std::vector mats; + for (const auto& t : g.init_tensors) + if (t.dims.size() == 2 && t.dims[1] > 1) mats.push_back(t.name); + for (const auto& node : g.nodes) { + if (node.op != "MatMul" || node.ins.size() < 2) continue; + const auto need = std::find_if(needs.begin(), needs.end(), [&](const Need& n) { + return n.weight == node.ins[1] || n.weight == node.ins[0]; + }); + if (need == needs.end()) continue; + const auto staged = onebit::audio::staged(onebit::audio::ProjectionShape{need->kindex, need->n}); + std::fprintf(f, + "%s {\"order\": %d, \"op\": \"MatMul\", \"weight\": \"%s\", \"role\": \"%s\", " + "\"k\": %d, \"n\": %d, \"staged_k\": %d, \"staged_n\": %d, \"elf_b8\": \"%s\", " + "\"elf_m16\": \"%s\"}", + idx == 0 ? "" : ",\n", idx, need->weight.c_str(), need->role.c_str(), need->kindex, need->n, + staged.k, staged.n, elf_name(*need, false).c_str(), elf_name(*need, true).c_str()); + ++idx; + } + std::fprintf(f, "\n ],\n \"projections\": %d,\n \"note\": \"staged_* are the widths the ELF is named at " + "(K%%256, N%%512); the forward asks for those names, not the model's\"\n}\n", idx); + std::fclose(f); + std::printf("plan: %d projection(s) in topological order -> %s\n", idx, plan_path.c_str()); + if (idx == 0) { + std::fprintf(stderr, "audio_kernel_shapes: the plan is empty -- the pack's graph has no 2-D MatMul on a " + "recognised weight, so nothing would launch\n"); + return 1; + } + } + + if (kernels.empty()) return 0; + + // One ELF per distinct (role, K, N), in both M tiers: the family the encoder needs is the same parameterised generator's output. + std::map distinct; + for (const auto& n : needs) distinct[n.role + ":" + std::to_string(n.kindex) + ":" + std::to_string(n.n)] = 1; + int missing = 0; + for (const auto& kv : distinct) { + const std::string key = kv.first; + const size_t a = key.find(':'), b = key.find(':', a + 1); + Need n; + n.role = key.substr(0, a); + n.kindex = std::stoi(key.substr(a + 1, b - a - 1)); + n.n = std::stoi(key.substr(b + 1)); + for (bool m16 : {false, true}) { + const std::string name = elf_name(n, m16); + const bool present = fs::is_regular_file(fs::path(kernels) / name); + if (!present) ++missing; + std::printf(" %-6s %5s %s\n", present ? "have" : "MISSING", m16 ? "m16" : "b8", name.c_str()); + } + } + std::printf("kernels %s: %d missing of %zu x 2\n", kernels.c_str(), missing, distinct.size()); + if (missing != 0) { + std::printf("note: a role this tool does not recognise is reported as `unknown` above, so an %s* ELF will always be MISSING\n", + "unknown"); + } + return missing == 0 ? 0 : 1; +} diff --git a/tools/whisper_transcribe.cpp b/tools/whisper_transcribe.cpp index 119a17131..b8285e49e 100644 --- a/tools/whisper_transcribe.cpp +++ b/tools/whisper_transcribe.cpp @@ -1,22 +1,14 @@ // Copyright 2026 bong-water-water-bong // SPDX-License-Identifier: Apache-2.0 // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// // whisper_transcribe.cpp -- the whole host pipeline: wav -> log-mel -> encoder -> greedy decode -> token ids. // +// ./whisper_transcribe [expected-ids-csv ...] // ./whisper_transcribe [expected-ids-csv ...] // +// The pack form reads both graphs, and the mel-bin count, from the directory (npu/audio_pack.h): 80 bins for whisper-small and 128 +// for large-v3, and passing the wrong one produces a graph that runs and answers wrongly, so the count comes from the encoder. +// // A tool rather than a test because one forward pass of each graph is ~26 s at these shapes and a sentence is 25 of them // (see PR #1931); `ctest` would time out for a reason that has nothing to do with correctness. The numbers it prints are // the ones a test would assert, so `expected-ids-csv` turns it into a check when run by hand. @@ -25,6 +17,7 @@ // stricter check than comparing text (two ids can share a detokenisation, and neither can share an id). #include +#include #include #include #include @@ -33,6 +26,9 @@ #include #include "audio_exec.h" +#include "audio_pack.h" +#include "whisper_ids.h" +#include "audio_parallel.h" #include "audio_mel.h" #include "onnx_graph.h" @@ -100,26 +96,132 @@ std::vector read_wav_16k_mono(const char* path, std::string* why) { } // namespace + +// The expected ids as a file (tests/data/whisper-jfk-ids.json), so a lane does not have to retype 25 numbers from a PR body. The +// parser is deliberately narrow: it finds "greedy_ids" and reads the integers that follow, and refuses a file with none. +static bool read_ids_file(const char* path, std::vector* out, std::string* why) { + FILE* f = std::fopen(path, "rb"); + if (f == nullptr) { + *why = std::string("cannot open ") + path; + return false; + } + std::string text; + char buf[4096]; + size_t n; + while ((n = std::fread(buf, 1, sizeof buf, f)) > 0) text.append(buf, n); + std::fclose(f); + const size_t key = text.find("\"greedy_ids\""); + if (key == std::string::npos) { + *why = std::string(path) + " has no \"greedy_ids\" list"; + return false; + } + // The scan stops at the array's closing bracket. Running to the END of the file (as this did) reads every digit after + // the key, including numbers in later keys and any mention of the key in a prose note. + size_t end = text.find('[', key); + if (end == std::string::npos) { + *why = std::string(path) + ": \"greedy_ids\" is not followed by a list"; + return false; + } + int depth = 0; + for (size_t i = end; i < text.size(); ++i) { + if (text[i] == '[') ++depth; + else if (text[i] == ']' && --depth == 0) { end = i; break; } + } + for (size_t i = key; i < end;) { + if (std::isdigit(static_cast(text[i])) || (text[i] == '-' && i + 1 < text.size())) { + const size_t start = i; + ++i; + while (i < text.size() && std::isdigit(static_cast(text[i]))) ++i; + out->emplace_back(text, start, i - start); + } else { + ++i; + } + } + if (out->empty()) { + *why = std::string(path) + ": the greedy_ids list is empty"; + return false; + } + return true; +} + int main(int argc, char** argv) { - if (argc < 4) { - std::fprintf(stderr, "usage: %s [expected-id ...]\n", argv[0]); + // ctest form: no arguments means "run the reference sentence from the environment", so the chain can be checked by CI or by + // a lane without remembering the 25 ids. Missing inputs skip with 77, the same convention as every other gated test here. + if (argc == 1) { + static std::vector from_env; + const char* pack = std::getenv("ONEBIT_TEST_WHISPER_PACK"); + const char* wav = std::getenv("ONEBIT_TEST_WHISPER_WAV"); + const char* ids = std::getenv("ONEBIT_TEST_WHISPER_IDS"); + const char* ids_file = std::getenv("ONEBIT_TEST_WHISPER_IDS_FILE"); + if (pack == nullptr || wav == nullptr || (ids == nullptr && ids_file == nullptr)) { + std::printf("SKIP: set ONEBIT_TEST_WHISPER_PACK and _WAV, and either _IDS or _IDS_FILE " + "(tests/data/whisper-jfk-ids.json)\n"); + return 77; + } + from_env.push_back(argv[0]); + from_env.push_back(pack); + from_env.push_back(wav); + if (ids == nullptr) { + std::string ids_why; + if (!read_ids_file(ids_file, &from_env, &ids_why)) { + std::fprintf(stderr, "ids: %s\n", ids_why.c_str()); + return 1; + } + ids = ""; + } + for (size_t i = 0; ids != nullptr && i < std::strlen(ids);) { // split on whitespace + while (i < std::strlen(ids) && std::isspace(static_cast(ids[i]))) ++i; + const size_t start = i; + while (i < std::strlen(ids) && !std::isspace(static_cast(ids[i]))) ++i; + if (i > start) from_env.emplace_back(ids + start, i - start); + } + std::vector argv_env; + for (auto& s : from_env) argv_env.push_back(const_cast(s.c_str())); + return main(static_cast(argv_env.size()), argv_env.data()); + } + if (argc < 3) { + std::fprintf(stderr, + "usage: %s [expected-id ...]\n" + " %s [expected-id ...]\n" + " The pack form reads the graphs, and the mel-bin count, from the directory (engine#1783).\n", + argv[0], argv[0]); return 2; } using namespace onebit::npu::pack_import; + std::string enc_path = (argc > 1) ? argv[1] : ""; + std::string dec_path = (argc > 2) ? argv[2] : ""; std::string why; - // The spans the last line reports, each named for what it contains and nothing else: wav (reading the file), mel - // (the log-mel front-end), load (parsing both graphs), encoder, decode. `wall` is their sum, from before the file - // is opened. - const auto t_begin = std::chrono::steady_clock::now(); - std::vector audio = read_wav_16k_mono(argv[3], &why); + // `1` means the two-graph form is in use and the wav is argv[3]; the pack form sets this to 2 after it discovers the graphs. + int audio_arg_index = 1; + onebit::audio::MelConfig cfg; + // The pack form is decided by the first argument: a directory (or anything without `.onnx`) is a pack, and it also carries the + // mel-bin count, which has to be known before the wav is turned into a spectrogram. + if (argc >= 3 && std::string(argv[1]).find(".onnx") == std::string::npos) audio_arg_index = 2; + if (audio_arg_index == 2) { + const std::string pack_dir = argv[1]; + onebit::audio::AudioPack pack; + if (!onebit::audio::discover_audio_pack(pack_dir, &pack, &why)) { + std::fprintf(stderr, "pack: %s\n", why.c_str()); + return 1; + } + if (!pack.concern.empty()) std::fprintf(stderr, "pack note: %s\n", pack.concern.c_str()); + std::fprintf(stderr, "pack: %s (mel bins %d from the %s)\n", pack_dir.c_str(), pack.mel_bins, + pack.mel_bins_source.c_str()); + cfg.bins = pack.mel_bins; + cfg.sample_rate = pack.sample_rate; + cfg.n_fft = pack.n_fft; + cfg.hop = pack.hop; + cfg.frames = pack.frames; + enc_path = pack.encoder; + dec_path = pack.decoder; + } + const size_t want = size_t(cfg.sample_rate) * 30; + std::vector audio = read_wav_16k_mono(argv[audio_arg_index], &why); if (audio.empty()) { std::fprintf(stderr, "wav: %s\n", why.c_str()); return 1; } - // Whisper's window is a fixed 30 s: pad with silence, do not scale. - onebit::audio::MelConfig cfg; - const size_t want = size_t(cfg.sample_rate) * size_t(cfg.frames) * size_t(cfg.hop) / size_t(cfg.frames); audio.resize(std::max(audio.size(), want), 0.0f); const auto t_start = std::chrono::steady_clock::now(); @@ -131,13 +233,12 @@ int main(int argc, char** argv) { const auto t_mel = std::chrono::steady_clock::now(); OnnxGraphInfo enc, dec; try { - enc = parse_onnx_graph(argv[1]); - dec = parse_onnx_graph(argv[2]); + enc = parse_onnx_graph(enc_path); + dec = parse_onnx_graph(dec_path); } catch (const std::exception& e) { std::fprintf(stderr, "graph: %s\n", e.what()); return 1; } - const auto t_load = std::chrono::steady_clock::now(); AudioRunResult xa; if (!run_audio_graph(enc, mel.data(), cfg.bins, cfg.frames, "", &xa, &why)) { std::fprintf(stderr, "encoder: %s\n", why.c_str()); @@ -150,8 +251,40 @@ int main(int argc, char** argv) { // <|startoftranscript|>, <|en|>, <|transcribe|>, <|notimestamps|> -- then greedy, reading the logits of the last real // prompt position (HF pads the 448/128-token window with <|endoftext|>, so the prompt is a prefix, not a suffix). - const int64_t kSot = 50258, kEn = 50259, kTranscribe = 50359, kNoTimestamps = 50363, kEot = 50257; - const int window = 128; + // The token window is the DECODER's declared input length, not a constant: whisper-small exports 128, large-v3 448, + // and a 128-wide window against a 448-wide graph builds a mask the scores cannot broadcast against (measured). + int window = 128; + if (!dec.input_dims.empty() && dec.input_dims[0].size() == 2 && dec.input_dims[0][1] > 0) { + window = int(dec.input_dims[0][1]); + std::printf("decoder window: %d positions (from the graph's input shape)\n", window); + } + + // Whisper's special ids are a property of the VOCABULARY, and the two packs here do not share one: whisper-small exports + // 51865 logits, large-v3 exports 51866 (v3 added a token), which shifts <|translate|>/<|transcribe|>/<|nospeech|>/ + // <|notimestamps|> by one. Hard-coding either set is what produced the first version of this tool: 50359/50363 decode + // whisper-small correctly *because English audio makes <|translate|> and <|transcribe|> agree*, and produce <|endoftext|> + // at the first step on large-v3 (measured: argmax 50257 with 50359/50363, 400 with 50360/50364, same pack and input). + // Neither pack ships a tokenizer or generation_config, so the vocabulary width is the only fact available -- and it is the + // right one, because the ids are offsets into it. + // The decoder's declared output shape is [1, window, vocab]; the vocabulary is its last dimension. (A scan over the MatMul + // weights cannot find it: the export applies the head through a Transpose, so the widest weight is the MLP's 4h.) + int64_t vocab = 0; + if (!dec.output_dims.empty() && !dec.output_dims[0].empty()) vocab = dec.output_dims[0].back(); + + // The ids come from the pack's own vocabulary width (npu/forward/include/whisper_ids.h), for the reason recorded there. + onebit::audio::WhisperIds slot; + if (!onebit::audio::whisper_ids_for_vocab(vocab, &slot)) { + std::fprintf(stderr, "special ids: no measured layout for a vocabulary of %lld\n", (long long)vocab); + return 1; + } + // Say what the split is: a rate quoted without it is unreadable, because the default follows the CPUs this process is + // ALLOWED (a thermal-run scope allows far fewer than the machine has) and the executor is only faster than serial + // when the threads actually fit (engine#1783). + std::printf("threads: %zu\n", onebit::npu::pack_import::audio_thread_count()); + std::printf("special ids: vocab %lld -> transcribe %lld, notimestamps %lld\n", (long long)vocab, + (long long)slot.transcribe, (long long)slot.no_timestamps); + const int64_t kEot = slot.eot, kSot = slot.sot, kEn = slot.lang_en; + const int64_t kTranscribe = slot.transcribe, kNoTimestamps = slot.no_timestamps; std::vector seq = {kSot, kEn, kTranscribe, kNoTimestamps}; std::vector ids; for (int step = 0; step < 60; ++step) { @@ -162,7 +295,8 @@ int main(int argc, char** argv) { std::fprintf(stderr, "decoder at step %d: %s\n", step, why.c_str()); return 1; } - const int64_t vocab = y.dims.empty() ? 0 : y.dims.back(); + const int64_t out_vocab = y.dims.empty() ? 0 : y.dims.back(); + const int64_t& vocab = out_vocab; const size_t row = (seq.size() - 1) * size_t(vocab); int64_t best = 0; for (int64_t v = 1; v < vocab; ++v) @@ -175,24 +309,43 @@ int main(int argc, char** argv) { const auto secs = [](std::chrono::steady_clock::time_point a, std::chrono::steady_clock::time_point b) { return std::chrono::duration(b - a).count(); }; - std::printf("time: wav %.3f s, mel %.3f s, load %.3f s, encoder %.3f s, decode %.3f s for %zu token(s) -> %.3f s/token, " - "wall %.3f s\n", - secs(t_begin, t_start), secs(t_start, t_mel), secs(t_mel, t_load), secs(t_load, t_enc), secs(t_enc, t_dec), - ids.size(), ids.empty() ? 0.0 : secs(t_enc, t_dec) / double(ids.size()), secs(t_begin, t_dec)); + std::printf("time: mel+load %.3f s, encoder %.3f s, decode %.3f s for %zu token(s) -> %.3f s/token, wall %.3f s\n", + secs(t_start, t_mel), secs(t_mel, t_enc), secs(t_enc, t_dec), ids.size(), + ids.empty() ? 0.0 : secs(t_enc, t_dec) / double(ids.size()), secs(t_start, t_dec)); + // The row's completion asks for "s/window AND time to first token", and TTFT for this chain is everything before the first + // decoded token: the mel front-end, the graph load, and the whole encoder pass. Named rather than left to be derived from the + // line above, because "which of those three is the TTFT" is exactly the question a reader gets wrong at 2 a.m. + std::printf("ttft: %.3f s (mel %.3f + encoder %.3f; the first token needs no extra forward)\n", + secs(t_start, t_enc), secs(t_start, t_mel), secs(t_mel, t_enc)); std::printf("ids:"); for (int64_t v : ids) std::printf(" %lld", (long long)v); std::printf("\n"); - if (argc > 4) { + // The expectation is either the ids themselves (one per argument) or, when a single .json path is given, that file's + // `greedy_ids`. A path used to be compared as a literal id, which reported "length: got 25 want 1" -- true, and useless: + // it said nothing about whether the file was readable or held the list. + std::vector want_ids; + if (argc > audio_arg_index + 1) { + const std::string first = argv[audio_arg_index + 1]; + if (argc == audio_arg_index + 2 && first.size() > 5 && first.compare(first.size() - 5, 5, ".json") == 0) { + std::string ids_why; + if (!read_ids_file(first.c_str(), &want_ids, &ids_why)) { + std::fprintf(stderr, "ids: %s\n", ids_why.c_str()); + return 1; + } + } else { + for (int i = audio_arg_index + 1; i < argc; ++i) want_ids.emplace_back(argv[i]); + } + } + if (!want_ids.empty()) { int bad = 0; - const int n = argc - 4; - if (int(ids.size()) != n) { - std::printf("FAIL length: got %zu want %d\n", ids.size(), n); + if (ids.size() != want_ids.size()) { + std::printf("FAIL length: got %zu want %zu\n", ids.size(), want_ids.size()); ++bad; } else { - for (int i = 0; i < n; ++i) - if (ids[size_t(i)] != std::atoll(argv[4 + i])) { - std::printf("FAIL id[%d]: got %lld want %s\n", i, (long long)ids[size_t(i)], argv[4 + i]); + for (size_t i = 0; i < want_ids.size(); ++i) + if (ids[i] != std::atoll(want_ids[i].c_str())) { + std::printf("FAIL id[%zu]: got %lld want %s\n", i, (long long)ids[i], want_ids[i].c_str()); ++bad; } }