From 0ae65c8d3a7dd9d1c16af408ddcee5cfec585e42 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:59:02 -0400 Subject: [PATCH 01/98] bump-hrx: say when AMD rebased its line, instead of calling AMD's commit ours (Refs #1945) (#1964) A mirror commit on no AMD branch was always reported as a commit of ours to move to 1bit/hrx-vulkan-patched. On 2026-11-10 it was AMD's own #155: AMD had re-committed its series on a newer base and dropped the old line. The script now checks AMD's pin for a commit with the same author, author date and subject, and names the two cases apart. Test with local repos. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 3 + docs/hrx.md | 8 ++- scripts/bump-hrx-sync.sh | 38 +++++++++++-- tests/bump_hrx_sync_test.sh | 106 ++++++++++++++++++++++++++++++++++++ 4 files changed, 150 insertions(+), 5 deletions(-) create mode 100755 tests/bump_hrx_sync_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 71b63011b..381553e54 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -197,6 +197,9 @@ add_test(NAME embed_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/embed_serve.sh $) # serve's GET /metrics (docs/serve.md, "Metrics"): a fake backend, no GPU. add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.sh $) +# engine#1945: scripts/bump-hrx-sync.sh tells AMD's own commit from a line AMD rebased apart from a commit that +# is not AMD's. Local git repositories only: no network, no GPU. +add_test(NAME bump_hrx_sync COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_sync_test.sh ${CMAKE_SOURCE_DIR}/scripts/bump-hrx-sync.sh) add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $) # serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU. add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $) diff --git a/docs/hrx.md b/docs/hrx.md index 8a8b687ba..e4d661a58 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -86,7 +86,13 @@ fails a pin that leaves a gate model's node off HRX. `DRY_RUN=1` and pushes nothing): - syncs the fork (`master` to ggml-org, `1bit/hrx-vulkan` to AMD's pin). When AMD rewrites its history, the old `1bit/hrx-vulkan` tip is tagged `hrx-vulkan-` first, - and the run fails if that branch held a commit that is on no AMD branch; + and the run fails if that branch held a commit that is on no AMD branch. The failure + says which of two things that commit is, because they need opposite actions: + **rebased** (AMD's pin holds a commit with the same author, author date and subject: + AMD rebased its line and dropped the old one, so the commit is AMD's and the pin is merged + by hand) or **foreign** (no such commit: it is not AMD's and moves to + `1bit/hrx-vulkan-patched`). `scripts/bump-hrx-sync.sh off-branch ` + prints the classification alone, from a commits-only clone; - merges AMD's pin into `1bit/hrx-vulkan-patched`, after tagging the previous tip `patched-` so every commit the engine has pinned stays reachable. Conflicts under `ggml/src/ggml-hrx` keep our side; a conflict anywhere else fails the run and names the files; diff --git a/scripts/bump-hrx-sync.sh b/scripts/bump-hrx-sync.sh index 760077bfe..c8beae0a9 100755 --- a/scripts/bump-hrx-sync.sh +++ b/scripts/bump-hrx-sync.sh @@ -31,7 +31,9 @@ # an ancestor of the new pin. If every commit on the mirror is still on an AMD branch, # the old tip is tagged and the mirror moves; the merge treats the squash commit as a # child of its pre-squash equivalent, so only AMD's genuinely new changes are merged. -# A mirror commit that is on no AMD branch fails the run and is named. +# A mirror commit that is on no AMD branch fails the run and is named, as one of two +# things (off_branch_commits): AMD's own commit from a line AMD rebased and dropped, to be +# merged by hand, or a commit that is not AMD's, to be moved to our line. # - hrx-system is held (config/hrx-system-hold.config). AMD's ggml-hrx is written against # AMD's hrx-system (e.g. #139 moved the kernel corpus to the `multiple_of` predicate that # our held hrx-system cannot parse), so only AMD's changes OUTSIDE the HRX backend paths @@ -83,6 +85,27 @@ hold_sha() { grep -v '^[[:space:]]*#' "$ENGINE_DIR/$HOLD_FILE" | grep -oE '^[0-9a-f]{40}' | head -n1 || true } +# Why a mirror commit is on no AMD branch. Reads repo $1 (commits only are enough), whose refs/amd/* +# are AMD's branches; $2 is the mirror tip and $3 AMD's new pin. One line per such commit: +# rebased the new pin holds a commit with the same author, author date and +# subject. A rebase keeps exactly those and changes the hash, so this +# is AMD's commit from before AMD rebased its line and dropped the old one. +# foreign no such commit: it is not AMD's, so it does not belong on the mirror. +# 2026-11-10 (engine#1945): AMD re-committed its HRX series on a base 1777 commits newer and left the +# old line on no branch. The run called the old tip, AMD's own #155, a commit of ours to move to +# 1bit/hrx-vulkan-patched, and a PR that did so was opened before anyone compared authors. +off_branch_commits() { + local repo=$1 mirror=$2 pin=$3 h ae at s + declare -A twin=() + while IFS=$'\t' read -r h ae at s; do + [ -n "${twin["$ae|$at|$s"]:-}" ] || twin["$ae|$at|$s"]=$h + done < <(git -C "$repo" log --format='%h%x09%ae%x09%at%x09%s' "$pin" --not "$mirror") + while IFS=$'\t' read -r h ae at s; do + if [ -n "${twin["$ae|$at|$s"]:-}" ]; then echo "rebased $h ${twin["$ae|$at|$s"]} $s" + else echo "foreign $h $s"; fi + done < <(git -C "$repo" log --format='%h%x09%ae%x09%at%x09%s' "$mirror" --not "$pin" --glob='refs/amd/*') +} + sync_llama() { : "${AMD_LLAMA:?}" "${OURS_HRX:?}" local hold refresh; hold=$(hold_sha) @@ -114,10 +137,15 @@ sync_llama() { git -C ../amdrefs fetch -q --filter=tree:0 https://github.com/AMD-Ecosystem/llama.cpp.git \ '+refs/heads/*:refs/amd/*' "$pin" git -C ../amdrefs fetch -q --filter=tree:0 "$(remote_url 1bit-MONSTER/llama.cpp)" "+$mirror:refs/old-mirror" - local foreign - foreign=$(git -C ../amdrefs log --format='%h %s' "$mirror" --not "$pin" --glob='refs/amd/*') + local off foreign rebased + off=$(off_branch_commits ../amdrefs "$mirror" "$pin") + foreign=$(sed -n 's/^foreign //p' <<< "$off") + rebased=$(sed -n 's/^rebased //p' <<< "$off") if [ -n "$foreign" ]; then - die "1bit/hrx-vulkan has commits that are on no branch of AMD-Ecosystem/llama.cpp; the mirror must hold AMD's commits only. Move these to 1bit/hrx-vulkan-patched by hand: $(printf '%s; ' "$foreign")" + die "1bit/hrx-vulkan has commits that are on no branch of AMD-Ecosystem/llama.cpp and have no same-author, same-date, same-subject commit in AMD's pin; the mirror must hold AMD's commits only. Move these to 1bit/hrx-vulkan-patched by hand: $(tr '\n' ';' <<< "$foreign")" + fi + if [ -n "$rebased" ]; then + die "AMD rebased its line and dropped the old one: $(wc -l <<< "$rebased" | tr -d ' ') commit(s) of 1bit/hrx-vulkan are on no AMD branch, and each has a commit in AMD's pin ${pin:0:12} with the same author, author date and subject (old new subject: $(tr '\n' ';' <<< "$rebased")). They are AMD's commits, NOT ours: do not move them to 1bit/hrx-vulkan-patched. Merge AMD's pin into 1bit/hrx-vulkan-patched by hand, then move the mirror to the pin." fi # AMD rewrote history. Find the newest commit of the new pin whose tree equals a commit that # descends from the old mirror on some AMD branch (its pre-rewrite equivalent). @@ -263,5 +291,7 @@ sync_hrx() { case "$cmd" in llama) sync_llama ;; hrx) sync_hrx ;; + # For tests/bump_hrx_sync_test.sh: the classification alone, on a local repository, no network. + off-branch) off_branch_commits "${2:?repo}" "${3:?mirror}" "${4:?pin}" ;; *) echo "usage: $0 llama|hrx" >&2; exit 2 ;; esac diff --git a/tests/bump_hrx_sync_test.sh b/tests/bump_hrx_sync_test.sh new file mode 100755 index 000000000..4108db8ce --- /dev/null +++ b/tests/bump_hrx_sync_test.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# scripts/bump-hrx-sync.sh, the part that says WHY a commit of our AMD mirror is on no AMD branch +# (engine#1945). Two answers, and they need opposite actions: +# rebased AMD rebased its line and dropped the old one; the commit is AMD's and its twin is in the pin +# foreign the commit is not AMD's and has to leave the mirror +# Local repositories only: no network, no GPU, no token. +set -euo pipefail + +script=${1:?usage: bump_hrx_sync_test.sh } +script=$(cd "$(dirname "$script")" && pwd)/$(basename "$script") +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +fails=0 +ok() { echo "ok $*"; } +fail() { echo "FAIL $*"; fails=$((fails + 1)); } +expect() { # expect + if [ "$2" = "$3" ]; then ok "$1"; else fail "$1"; printf ' got: %s\n wanted: %s\n' "$2" "$3"; fi +} + +export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null +repo=$work/r +git init -q "$repo" +g() { git -C "$repo" "$@"; } +# commit +commit() { + echo "$5 $RANDOM$RANDOM" >> "$repo/f" + g add f + GIT_AUTHOR_NAME=$1 GIT_AUTHOR_EMAIL=$2 GIT_AUTHOR_DATE=$3 \ + GIT_COMMITTER_NAME=c GIT_COMMITTER_EMAIL=c@example.invalid GIT_COMMITTER_DATE=$4 \ + git -C "$repo" commit -q -m "$5" +} +amd='amd dev'; amd_mail=dev@amd.example.invalid + +# The old line: upstream base, then two AMD commits. The mirror is its tip. +commit up up@example.invalid 2026-01-01T00:00:00Z 2026-01-01T00:00:00Z "upstream base" +base=$(g rev-parse HEAD) +commit "$amd" "$amd_mail" 2026-08-17T03:02:45Z 2026-08-17T03:09:08Z "hrx: add the backend" +commit "$amd" "$amd_mail" 2026-10-28T03:17:36Z 2026-10-28T03:17:36Z "hrx: fix concat bounds (#155)" +mirror=$(g rev-parse HEAD) +old1=$(g rev-parse --short HEAD~1); old2=$(g rev-parse --short HEAD) + +# The new line: a newer upstream base, the same two commits re-committed (same author, same author date, +# same subject, a later committer date -- what a rebase leaves), then one new commit. The pin is its tip. +g checkout -q -b newline "$base" +commit up up@example.invalid 2026-10-27T00:00:00Z 2026-10-27T00:00:00Z "upstream moved on" +commit "$amd" "$amd_mail" 2026-08-17T03:02:45Z 2026-10-28T09:00:00Z "hrx: add the backend" +new1=$(g rev-parse --short HEAD) +commit "$amd" "$amd_mail" 2026-10-28T03:17:36Z 2026-10-28T09:00:01Z "hrx: fix concat bounds (#155)" +new2=$(g rev-parse --short HEAD) +commit "$amd" "$amd_mail" 2026-11-09T15:29:43Z 2026-11-09T15:29:43Z "hrx: something new" +pin=$(g rev-parse HEAD) +g update-ref refs/amd/main "$pin" # AMD's only branch: the old line is on none + +run() { bash "$script" off-branch "$repo" "$1" "$2"; } + +got=$(run "$mirror" "$pin") +expect "a rebased line: both old commits are named with their twins in the pin, newest first" "$got" \ +"rebased $old2 $new2 hrx: fix concat bounds (#155) +rebased $old1 $new1 hrx: add the backend" +if grep -q '^foreign ' <<< "$got"; then fail "a rebased AMD commit is not called foreign"; else ok "a rebased AMD commit is not called foreign"; fi + +# While AMD still has the old line on a branch, nothing is off-branch at all. +g update-ref refs/amd/backup "$mirror" +expect "an old line AMD still keeps on a branch reports nothing" "$(run "$mirror" "$pin")" "" +g update-ref -d refs/amd/backup + +# A commit of ours on top of the mirror: no twin in the pin, so it is foreign, and the AMD ones stay rebased. +g checkout -q --detach "$mirror" +commit "a lane" lane@example.invalid 2026-11-01T00:00:00Z 2026-11-01T00:00:00Z "hrx: a fix of ours" +ours=$(g rev-parse HEAD); ours_s=$(g rev-parse --short HEAD) +got=$(run "$ours" "$pin") +expect "a commit that is not AMD's is foreign, beside the rebased ones" "$got" \ +"foreign $ours_s hrx: a fix of ours +rebased $old2 $new2 hrx: fix concat bounds (#155) +rebased $old1 $new1 hrx: add the backend" + +# The subject alone is not identity: the same subject from another author, or at another author date, is foreign. +g checkout -q --detach "$mirror" +commit "a lane" lane@example.invalid 2026-11-09T15:29:43Z 2026-11-09T15:29:43Z "hrx: something new" +other=$(g rev-parse HEAD) +expect "the same subject from another author is foreign" "$(run "$other" "$pin" | head -n1)" \ + "foreign $(g rev-parse --short "$other") hrx: something new" +g checkout -q --detach "$mirror" +commit "$amd" "$amd_mail" 2026-11-09T15:29:44Z 2026-11-09T15:29:44Z "hrx: something new" +other=$(g rev-parse HEAD) +expect "the same subject and author at another author date is foreign" "$(run "$other" "$pin" | head -n1)" \ + "foreign $(g rev-parse --short "$other") hrx: something new" + +if [ "$fails" -ne 0 ]; then echo "$fails check(s) failed"; exit 1; fi +echo "bump_hrx_sync: ok" From abd2cb1e07018fa710d4de055f6ff36085f3ac83 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:10:00 -0400 Subject: [PATCH 02/98] bump llama.cpp to a5dc597b8: one-token decode at any head size, and a clang build has its HRX0 again (fixes #1926, #1942) (#1960) Four fork commits past 2d62cc7af: - 7fb31383b (#282) the fused Q decode kernel compiles at every head size, not only a power of two (#1926) - c47eae4a7 (#279) the host term is read before it is classified, so a clang build registers HRX0 (#1942) - 69ee6f50e (#281) IQ3_XXS routed experts take the workgroup-table decode by default (#572) - a5dc597b8 (#278, carries #277) a pool refusal at compute time is ALLOC_FAILED, with device-free tally arms (#1618, #1823) Measured at a5dc597b8 on gfx1151 (amdclang build, gpu-hold): --list-devices prints HRX0; test-hrx-attention-q-decode-head-size 15 cases, 0 failures; Sarashina2.2-3B bf16 tg128 26.49 / 25.92 tok/s, stories15M Q8_0 tg128 317.77 / 338.54 tok/s. registry/architectures.json: the recorded source pin only. docs/hrx.md: the #1926 fix. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/hrx.md | 17 +++++++++++++++++ registry/architectures.json | 2 +- third_party/llama.cpp | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/hrx.md b/docs/hrx.md index e4d661a58..f6c7c3313 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -252,6 +252,23 @@ decode-path logits (`llama-perplexity -c 2049 -b 1 --save-all-logits`) are byte- to the previous pin, the buried code word is exact at 4700 tokens, and no GPU faults were observed. +### Fixed: one-token decode at a head size that is not a power of two ([engine#1926](https://github.com/1bit-MONSTER/engine/issues/1926)) + +A model whose attention head size is not a power of two (stories15M: 48; Sarashina2.2-3B: 160) and whose +Q weights are not Q4_K or Q6_K loaded and prefilled on `HRX0`, then failed at the first decode step: +`HRX Loom JIT Loom compilation failed`, `llama_decode: failed to decode, ret = -3`. The one-token Q +projection kernel wrote each rotated pair at `channel rem head_size`, and the kernel compiler can only +prove that two-wide store in bounds when the remainder is a mask. The kernel now indexes by rotation +pair, which is in bounds at every head size (llama.cpp #282). Q4_K and Q6_K models took a different +kernel at one token and were not affected. + +Measured at pin `a5dc597b8` on gfx1151 (2026-11-10, power mode `performance`): +`test-hrx-attention-q-decode-head-size` compares the fused projection with the CPU backend at head +sizes 48, 96 and 160 (controls 64 and 128) on BF16, F16 and Q8_0 weights: 9 of 15 cases failed before +the fix, 0 of 15 after. `llama-bench -p 512 -n 128 -r 3`, two runs each: Sarashina2.2-3B bf16 decodes +at 26.49 and 25.92 tok/s, stories15M Q8_0 at 317.77 and 338.54 tok/s (the box was shared; the same +binary gave 545.39 tok/s on a quieter run). + ### MoE decode and the kernel cache (llama.cpp #32, #33) - **MoE decode.** With one to four tokens, a `MUL_MAT_ID` (a MoE expert matmul) runs as a GEMV over diff --git a/registry/architectures.json b/registry/architectures.json index b3bf4f23a..b2fb3f4e9 100644 --- a/registry/architectures.json +++ b/registry/architectures.json @@ -1,7 +1,7 @@ { "about": "HF architecture -> GGUF architecture and the backends whose code accepts it. Generated by tools/registry_build.py from the pinned sources; do not edit.", "sources": { - "llama.cpp (hrx)": "2d62cc7afdae961bce83abcdb54dd95a17db7b50" + "llama.cpp (hrx)": "a5dc597b81f10c1fd29356018e56da074380ce60" }, "counts": { "hrx": 298, diff --git a/third_party/llama.cpp b/third_party/llama.cpp index 2d62cc7af..a5dc597b8 160000 --- a/third_party/llama.cpp +++ b/third_party/llama.cpp @@ -1 +1 @@ -Subproject commit 2d62cc7afdae961bce83abcdb54dd95a17db7b50 +Subproject commit a5dc597b81f10c1fd29356018e56da074380ce60 From a8e77454444690bd3b224b8ea03521166cd91217 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:21:00 -0400 Subject: [PATCH 03/98] npu/mtp: the distribution-following acceptance rule, as a tested header (engine#1781 gate 2) (#1958) The NPU draft loop accepts a draft when it equals the base model's argmax. That is exact for greedy decode and wrong for a sampler: it always emits the argmax, 0.55 away from p on p=[.45,.35,.20], q=[.10,.80,.10]. #1781's contract (decided on the issue) is that the output follows the target's distribution, with greedy identity as the temperature-0 case. npu/mtp/spec_accept.h is that rule, pure and header-only: accept a drafted token x ~ q with probability min(1, p(x)/q(x)); at the first rejection emit a token from normalise(max(0, p - q)) and stop the chain. The caller supplies the uniforms, so a run is reproducible from its seed. tests/spec_accept_test.cpp (ctest spec_accept, 21 arms, fixed seed): the temperature-0 identity; a deliberately wrong drafter whose output is within 0.0016 of p over 200k draws at acceptance 0.5497 (sum(min(p,q)) = 0.55); the greedy rule failing the same arm at 0.55; good, identical, disjoint, one-hot and 257-token drafters; the zero-mass and out-of-range edges; the chain's stop-at-first-rejection and two-draws-per-position properties. An always-accept mutant fails 7 arms. Not wired into a serve: that is the rest of #1781. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 7 ++ docs/npu.md | 13 ++ npu/mtp/spec_accept.h | 122 +++++++++++++++++++ tests/spec_accept_test.cpp | 243 +++++++++++++++++++++++++++++++++++++ 4 files changed, 385 insertions(+) create mode 100644 npu/mtp/spec_accept.h create mode 100644 tests/spec_accept_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 381553e54..12c4803fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -444,6 +444,13 @@ if(TARGET OpenMP::OpenMP_CXX) target_link_libraries(mtp_draft_test PRIVATE OpenMP::OpenMP_CXX) endif() add_test(NAME mtp_draft COMMAND mtp_draft_test) +# engine#1781 gate 2: the distribution-following acceptance rule (accept with min(1, p/q), resample the residual at the +# first rejection), with the temperature-0 identity and a deliberately wrong drafter whose output must still follow p. +# Header-only and device-free; pulled into CI through npu_sampling_test for the same reason as mtp_draft_test below. +add_executable(spec_accept_test tests/spec_accept_test.cpp) +target_include_directories(spec_accept_test PRIVATE npu) +add_test(NAME spec_accept COMMAND spec_accept_test) +add_dependencies(npu_sampling_test spec_accept_test) # ci.yml's build step lists every ctest target explicitly, and this branch cannot edit that file (the push # token has no `workflow` scope), so the binary is pulled in as a dependency of a listed test target. When a # workflow-scoped push is available, add `mtp_draft_test` to ci.yml's build list and delete this line. diff --git a/docs/npu.md b/docs/npu.md index aa30b1795..ab1a76b58 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2163,6 +2163,19 @@ is worth more than the whole bar. Those depths keep their PASS/FAIL (the bar is `(thin: N live chain(s), one chain = X pp > the 2 pp bar)` and are counted out of the summary line, because a bare FAIL on 2 chains reads as a drafter regression when it is arithmetic. +**Which drafts a sampler may keep** is a separate rule from how they are committed, and it lives in +`npu/mtp/spec_accept.h` (engine#1781, gate 2). The loop above accepts a draft when it equals the base model's argmax, +which is exact for greedy decode only. With a sampler the contract is that the output follows the **target's** +distribution, and the rule that guarantees it is: accept a drafted token `x` (drawn from the drafter's distribution `q`) +with probability `min(1, p(x)/q(x))`, and at the first rejection emit a token from `normalise(max(0, p - q))` and stop +the chain. `p` and `q` are the final sampling distributions (after temperature, top-k, top-p), and an argmax drafter +states a one-hot `q`. At temperature 0 both rows are one-hot and the rule reduces to the greedy contract, so that +contract is its degenerate case. `tests/spec_accept_test.cpp` (ctest `spec_accept`) pins both: the temperature-0 +identity, and a deliberately wrong drafter (`p = [.45, .35, .20]`, `q = [.10, .80, .10]`) whose emitted tokens are +within 0.0016 of `p` over 200,000 draws at an acceptance rate of 0.55, where the greedy rule on the same rows misses +`p` by 0.55. The header is not called by a serve yet: wiring a speculative loop into `forward-serve` is the rest of +#1781, and that loop must print its acceptance rate beside its tok/s. + **The one thing that must be right**: the drafter attends over its own states, so its cache has to be the base model's *committed stream, row by row*. Feeding it only the current chain is silent — drafts get worse, the text stays correct — and it cost this project 69 % → 14 % first-draft acceptance until the cursor was made a checked diff --git a/npu/mtp/spec_accept.h b/npu/mtp/spec_accept.h new file mode 100644 index 000000000..1eeeadc34 --- /dev/null +++ b/npu/mtp/spec_accept.h @@ -0,0 +1,122 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// spec_accept.h -- the acceptance rule of a speculative decode whose output follows the TARGET's distribution +// (engine#1781, gate 2). +// +// draft.h's loop accepts a drafted token when it equals the base model's argmax. That is exact for greedy decode and +// for nothing else: with a sampler, "accept iff the draft is the target's most likely token, otherwise emit that +// token" always emits the argmax, which is a different model's output (0.55 away from p on p = [.45,.35,.20] with +// q = [.10,.80,.10] -- the arm in tests/spec_accept_test.cpp). The rule that preserves the distribution is the +// standard one, for a drafted token x drawn from the drafter's distribution q at a position where the target's +// distribution is p: +// +// accept x with probability min(1, p(x) / q(x)); +// on the first rejection, emit a token drawn from normalise(max(0, p - q)) and stop the chain. +// +// The emitted token is then distributed exactly as p, whatever q is; q only decides how often a draft survives +// (the acceptance probability is sum_i min(p_i, q_i)). At temperature 0 both rows are one-hot, the ratio is 0 or 1, +// and the rule IS "accept iff the drafter's token is the target's argmax, else emit the argmax" -- so draft.h's +// greedy contract is this rule's degenerate case and not a second rule. +// +// What the caller owes: +// * p is the target's FINAL sampling distribution at that position -- after temperature, top-k, top-p and any +// penalty -- and q is the drafter's, in the same sense. A deterministic drafter (an argmax head) has a one-hot q. +// * x was drawn from q. If q(x) is 0 the premise is false; the step then rejects and resamples, so the output stays +// inside p's support, but nothing is promised about a drafter that lies about its own distribution. +// * the uniforms are independent draws in [0, 1). The functions take them as arguments and own no generator, so a +// run is reproducible from the caller's seed and a test can drive every branch. +// +// Pure and header-only: no model, no device, no allocation. It decides WHICH rows are kept; begin_verify() / +// commit_verify_rows() decide how the kept rows are committed. +#pragma once + +#include + +namespace onebit { +namespace npu { +namespace mtp { + +struct SpecStep { + bool accepted; // the drafted token survives + int token; // the token to emit at this position: the draft when accepted, the residual sample otherwise +}; + +// A token from normalise(max(0, p - q)), by inverse CDF on u in [0, 1). When the residual has no mass (p == q up to +// rounding, where a rejection has probability 0 and can only come from float error) it samples p itself. Never +// returns an index whose weight is zero, and returns -1 only for V <= 0 or a row with no positive mass at all. +inline int spec_residual_sample(const float* p, const float* q, int V, double u) { + if (!p || V <= 0) return -1; + double mass = 0.0; + if (q) + for (int i = 0; i < V; ++i) { + const double r = double(p[i]) - double(q[i]); + if (r > 0.0) mass += r; + } + const bool use_p = !(mass > 0.0); + if (use_p) { + mass = 0.0; + for (int i = 0; i < V; ++i) + if (p[i] > 0.0f) mass += double(p[i]); + if (!(mass > 0.0)) return -1; + } + if (u < 0.0) u = 0.0; + const double target = u * mass; + double acc = 0.0; + int last = -1; + for (int i = 0; i < V; ++i) { + const double w = use_p ? (p[i] > 0.0f ? double(p[i]) : 0.0) : (double(p[i]) - double(q[i])); + if (!(w > 0.0)) continue; + acc += w; + last = i; + if (target < acc) return i; + } + return last; // u so close to 1 that rounding left `target` at or past the total: the last weighted index +} + +// One drafted position. `x` is the token the drafter drew from q. +inline SpecStep spec_accept_step(const float* p, const float* q, int V, int x, double u_accept, double u_resample) { + if (p && q && x >= 0 && x < V && q[x] > 0.0f) { + // u < min(1, p/q) written without the division: u * q < p. With p >= q it holds for every u in [0, 1). + if (u_accept * double(q[x]) < double(p[x])) return {true, x}; + } + return {false, spec_residual_sample(p, q, V, u_resample)}; +} + +struct SpecResult { + int accepted; // how many leading drafts survive: 0..k + int next_token; // the residual sample at the first rejection; -1 when all k survive (the caller then samples + // position k from the target's own row there, which is the pass's k+1-th row) +}; + +// The rule along a chain of k drafted positions. p_rows and q_rows are k rows of V probabilities, row d being the +// distributions at drafted position d; `uniform()` returns the next independent draw in [0, 1). Exactly two draws are +// consumed per position examined, accepted or not, so the number of draws does not leak the outcome into the stream. +template +inline SpecResult spec_accept_prefix(const float* p_rows, const float* q_rows, int V, const int* drafts, int k, + Uniform&& uniform) { + for (int d = 0; d < k; ++d) { + const double ua = uniform(); + const double ur = uniform(); + const SpecStep s = spec_accept_step(p_rows + std::size_t(d) * std::size_t(V), + q_rows + std::size_t(d) * std::size_t(V), V, drafts[d], ua, ur); + if (!s.accepted) return {d, s.token}; + } + return {k, -1}; +} + +} // namespace mtp +} // namespace npu +} // namespace onebit diff --git a/tests/spec_accept_test.cpp b/tests/spec_accept_test.cpp new file mode 100644 index 000000000..c2644af7e --- /dev/null +++ b/tests/spec_accept_test.cpp @@ -0,0 +1,243 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// spec_accept_test -- npu/mtp/spec_accept.h, the distribution-following acceptance rule (engine#1781, gate 2). +// +// Two properties make the rule what the issue asks for, and each has an arm that a wrong rule fails: +// +// 1. at temperature 0 (one-hot rows) it is exactly "accept iff the draft is the target's argmax, else emit the +// argmax" -- the contract tests/mtp_draft_test.cpp already pins for the greedy loop; +// 2. with a sampler, the emitted token follows the TARGET's distribution p whatever the drafter's q is -- checked +// with a drafter that is deliberately wrong, and against the greedy rule, which misses p by 0.55 on the same rows. +// +// No model and no device: a fixed-seed generator, so the counts below are the same on every run and the tolerances +// are bounds on a known sample, not on luck. The sampling error of a frequency over N = 200,000 draws is at most +// sqrt(0.25 / N) = 0.0011 (one sigma), so 0.006 is more than five sigma and 0.3 is nowhere near it. +#include "mtp/spec_accept.h" + +#include +#include +#include +#include + +using onebit::npu::mtp::spec_accept_prefix; +using onebit::npu::mtp::spec_accept_step; +using onebit::npu::mtp::spec_residual_sample; +using onebit::npu::mtp::SpecResult; +using onebit::npu::mtp::SpecStep; + +static int g_fail = 0; +static void check(bool ok, const char* what) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", what); + if (!ok) g_fail++; +} + +struct Rng { // splitmix64: small, fixed, and good enough to make 200k draws look uniform + uint64_t s; + double operator()() { + uint64_t z = (s += 0x9E3779B97F4A7C15ull); + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + z ^= z >> 31; + return double(z >> 11) * (1.0 / 9007199254740992.0); // [0, 1) + } +}; + +static int draw(const std::vector& d, double u) { + double acc = 0.0; + for (size_t i = 0; i < d.size(); ++i) { acc += d[i]; if (u < acc) return int(i); } + return int(d.size()) - 1; +} + +// Emitted-token frequencies and the acceptance rate of one position over n draws, with the drafter sampling q. +static double max_dev_from_p(const std::vector& p, const std::vector& q, int n, uint64_t seed, + double* accept_rate, bool greedy_rule = false) { + const int V = int(p.size()); + std::vector cnt(size_t(V), 0); + long acc = 0; + Rng r{seed}; + int amax = 0; + for (int i = 1; i < V; ++i) if (p[size_t(i)] > p[size_t(amax)]) amax = i; + for (int i = 0; i < n; ++i) { + const int x = draw(q, r()); + int out; + if (greedy_rule) { // the rule this header replaces: accept iff x is the target's argmax + const double ua = r(), ur = r(); (void)ua; (void)ur; + if (x == amax) acc++; + out = amax; + } else { + const SpecStep s = spec_accept_step(p.data(), q.data(), V, x, r(), r()); + if (s.accepted) acc++; + out = s.token; + } + if (out >= 0 && out < V) cnt[size_t(out)]++; + } + double worst = 0.0; + for (int i = 0; i < V; ++i) worst = std::fmax(worst, std::fabs(double(cnt[size_t(i)]) / n - double(p[size_t(i)]))); + if (accept_rate) *accept_rate = double(acc) / n; + return worst; +} + +int main() { + const int N = 200000; + char msg[256]; + + // ---- 1. temperature 0: one-hot rows, the greedy contract ------------------------------------------------------ + { + const std::vector pa = {0, 1, 0, 0}, qa = {0, 1, 0, 0}, qb = {0, 0, 1, 0}; + bool all_acc = true, none_acc = true, emit_ok = true; + Rng r{1}; + for (int i = 0; i < 1000; ++i) { + const SpecStep s = spec_accept_step(pa.data(), qa.data(), 4, 1, r(), r()); + all_acc &= s.accepted && s.token == 1; + const SpecStep t = spec_accept_step(pa.data(), qb.data(), 4, 2, r(), r()); + none_acc &= !t.accepted; + emit_ok &= t.token == 1; + } + check(all_acc, "temp 0: a draft equal to the target's argmax is accepted on every draw"); + check(none_acc, "temp 0: a draft that is not the target's argmax is rejected on every draw"); + check(emit_ok, "temp 0: the token emitted at a rejection is the target's argmax"); + } + + // ---- 2. a sampler, with a deliberately wrong drafter: the output still follows p ------------------------------ + { + const std::vector p = {0.45f, 0.35f, 0.20f}, q = {0.10f, 0.80f, 0.10f}; + double rate = 0.0; + const double dev = max_dev_from_p(p, q, N, 42, &rate); + std::snprintf(msg, sizeof msg, "poor drafter: emitted frequencies are within 0.006 of p (max deviation %.4f)", dev); + check(dev < 0.006, msg); + std::snprintf(msg, sizeof msg, "poor drafter: acceptance rate is sum(min(p, q)) = 0.55 (measured %.4f)", rate); + check(std::fabs(rate - 0.55) < 0.006, msg); + double grate = 0.0; + const double gdev = max_dev_from_p(p, q, N, 42, &grate, true); + std::snprintf(msg, sizeof msg, "the GREEDY rule on the same rows misses p by more than 0.3 (max deviation %.4f): " + "this arm tells the two rules apart", gdev); + check(gdev > 0.3, msg); + } + { + const std::vector p = {0.45f, 0.35f, 0.20f}, q = {0.40f, 0.38f, 0.22f}; + double rate = 0.0; + const double dev = max_dev_from_p(p, q, N, 7, &rate); + std::snprintf(msg, sizeof msg, "good drafter: follows p (max deviation %.4f) and accepts 0.95 (measured %.4f)", dev, rate); + check(dev < 0.006 && std::fabs(rate - 0.95) < 0.006, msg); + } + { + const std::vector p = {0.5f, 0.3f, 0.2f}; + double rate = 0.0; + const double dev = max_dev_from_p(p, p, N, 9, &rate); + std::snprintf(msg, sizeof msg, "identical drafter: every draft is accepted (rate %.6f) and the output follows p (%.4f)", rate, dev); + check(rate == 1.0 && dev < 0.006, msg); + } + { + const std::vector p = {0.6f, 0.4f, 0.0f, 0.0f}, q = {0.0f, 0.0f, 0.5f, 0.5f}; + double rate = 1.0; + const double dev = max_dev_from_p(p, q, N, 11, &rate); + std::snprintf(msg, sizeof msg, "disjoint drafter: nothing is accepted (rate %.6f) and the output still follows p (%.4f)", rate, dev); + check(rate == 0.0 && dev < 0.006, msg); + } + { // a deterministic (argmax) drafter states a one-hot q: accepted with probability p(x), residual is p without x + const std::vector p = {0.2f, 0.5f, 0.3f}, q = {0.0f, 1.0f, 0.0f}; + double rate = 0.0; + const double dev = max_dev_from_p(p, q, N, 13, &rate); + std::snprintf(msg, sizeof msg, "one-hot drafter: accepted with probability p(x) = 0.5 (measured %.4f), output follows p (%.4f)", rate, dev); + check(std::fabs(rate - 0.5) < 0.006 && dev < 0.006, msg); + } + { // a wide vocabulary with random rows: the property is not an artefact of three tokens + const int V = 257; + Rng g{2024}; + std::vector p(V), q(V); + double sp = 0, sq = 0; + for (int i = 0; i < V; ++i) { p[size_t(i)] = float(g() * g()); q[size_t(i)] = float(g() * g()); sp += p[size_t(i)]; sq += q[size_t(i)]; } + double want = 0; + for (int i = 0; i < V; ++i) { p[size_t(i)] = float(p[size_t(i)] / sp); q[size_t(i)] = float(q[size_t(i)] / sq); } + for (int i = 0; i < V; ++i) want += std::fmin(p[size_t(i)], q[size_t(i)]); + double rate = 0.0; + const double dev = max_dev_from_p(p, q, 2 * N, 5, &rate); + std::snprintf(msg, sizeof msg, "257 tokens, random rows: output within 0.003 of p (%.4f), acceptance %.4f against sum(min) %.4f", dev, rate, want); + check(dev < 0.003 && std::fabs(rate - want) < 0.006, msg); + } + + // ---- 3. the edges a wiring will hit --------------------------------------------------------------------------- + { + const std::vector p = {0.5f, 0.5f, 0.0f}, q = {1.0f, 0.0f, 0.0f}; + bool ok = true; + Rng r{3}; + for (int i = 0; i < 2000; ++i) { + const SpecStep s = spec_accept_step(p.data(), q.data(), 3, 1, r(), r()); // x has q(x) == 0 + ok &= !s.accepted && (s.token == 0 || s.token == 1); + } + check(ok, "a draft the drafter gives no mass to is rejected, and the emitted token stays in p's support"); + const SpecStep bad = spec_accept_step(p.data(), q.data(), 3, 7, 0.0, 0.5); + check(!bad.accepted && bad.token >= 0 && bad.token < 2, "an out-of-range draft id is rejected and resampled, not read"); + } + { + const std::vector p = {0.25f, 0.0f, 0.75f, 0.0f}, q = {0.25f, 0.5f, 0.25f, 0.0f}; + check(spec_residual_sample(p.data(), q.data(), 4, 0.0) == 2 && spec_residual_sample(p.data(), q.data(), 4, 0.999999999) == 2, + "the residual max(0, p - q) is sampled: only token 2 has residual mass here, at both ends of u"); + const std::vector same = {0.0f, 0.3f, 0.7f, 0.0f}; + check(spec_residual_sample(same.data(), same.data(), 4, 0.0) == 1 && spec_residual_sample(same.data(), same.data(), 4, 0.9999999999) == 2, + "an empty residual (p == q) falls back to p and never returns a zero-probability token"); + const std::vector zero = {0, 0, 0}; + check(spec_residual_sample(zero.data(), zero.data(), 3, 0.5) == -1 && spec_residual_sample(nullptr, nullptr, 3, 0.5) == -1, + "a row with no mass, or no row, returns -1 instead of a token"); + } + + // ---- 4. the chain: stop at the first rejection, two draws per position examined ------------------------------ + { + const int V = 3, k = 3; + const std::vector P = {0, 1, 0, 0, 0, 1, 1, 0, 0}; // target argmaxes 1, 2, 0 + const std::vector Q = {0, 1, 0, 0, 0, 1, 0, 1, 0}; // drafter agrees twice, then says 1 + const int drafts[3] = {1, 2, 1}; + int draws = 0; + Rng r{21}; + const SpecResult res = spec_accept_prefix(P.data(), Q.data(), V, drafts, k, [&] { draws++; return r(); }); + check(res.accepted == 2 && res.next_token == 0, "chain: two drafts accepted, the third rejected, and the target's token emitted in its place"); + check(draws == 6, "chain: two uniforms are consumed per position examined"); + const int good[3] = {1, 2, 0}; + const std::vector Q2 = P; + const SpecResult all = spec_accept_prefix(P.data(), Q2.data(), V, good, k, [&] { return r(); }); + check(all.accepted == 3 && all.next_token == -1, "chain: when all k survive, next_token is -1 (the caller samples the pass's last row)"); + const int first_bad[3] = {0, 2, 0}; + const std::vector Q3 = {1, 0, 0, 0, 0, 1, 1, 0, 0}; + draws = 0; + const SpecResult none = spec_accept_prefix(P.data(), Q3.data(), V, first_bad, k, [&] { draws++; return r(); }); + check(none.accepted == 0 && none.next_token == 1 && draws == 2, "chain: a rejection at the first position stops there, later rows are not examined"); + } + { // the chain preserves the distribution at the position where it stops, too: two sampled positions + const int V = 3; + const std::vector P = {0.45f, 0.35f, 0.20f, 0.10f, 0.60f, 0.30f}; + const std::vector Q = {0.10f, 0.80f, 0.10f, 0.30f, 0.30f, 0.40f}; + std::vector c0(3, 0), c1(3, 0); + long n1 = 0; + Rng r{77}; + for (int i = 0; i < N; ++i) { + const int d[2] = {draw({Q[0], Q[1], Q[2]}, r()), draw({Q[3], Q[4], Q[5]}, r())}; + const SpecResult res = spec_accept_prefix(P.data(), Q.data(), V, d, 2, [&] { return r(); }); + const int t0 = res.accepted >= 1 ? d[0] : res.next_token; + c0[size_t(t0)]++; + if (res.accepted >= 1) { n1++; c1[size_t(res.accepted == 2 ? d[1] : res.next_token)]++; } + } + double dev0 = 0, dev1 = 0; + for (int i = 0; i < 3; ++i) { + dev0 = std::fmax(dev0, std::fabs(double(c0[size_t(i)]) / N - double(P[size_t(i)]))); + dev1 = std::fmax(dev1, std::fabs(double(c1[size_t(i)]) / double(n1) - double(P[size_t(3 + i)]))); + } + std::snprintf(msg, sizeof msg, "chain of two sampled positions: each emitted token follows its own row of p (%.4f, %.4f)", dev0, dev1); + check(dev0 < 0.006 && dev1 < 0.008, msg); + } + + std::printf("%s\n", g_fail ? "spec_accept_test: FAILED" : "spec_accept_test: all arms passed"); + return g_fail ? 1 : 0; +} From a0f17ac00359007f3abf60c518aa89550685f443 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:32:39 -0400 Subject: [PATCH 04/98] tools: dflash_draft_info, the gate before a DFlash draft loader: the package is in forms we already read (engine#1780) (#1957) A DFlash draft package (FastFlowLM 1.0.12's two-package tags) is config.json (model_type dflash) plus one Q4NX container. The thread on #1780 stopped at two byte-identical readings of the projections (q4_1 4096x2560 against a half-float 1280x2560) and at a claim that the BF16 labels are really F16. Both are settled from the bytes: - the I8 [tiles, 5120] projections are the ordinary q4_1 tile grid: all 35 have bytes == (N/32)(K/256) x 5120 for the config's N x K, 10,240,000 groups have a finite positive scale, and minimum/scale is -7.59 .. -7.33 per tensor (a main-model container our forward serves reads -7.41 the same way). The half-float reading has 25.7 % of values above 100 in magnitude. So head_dim is 128, as the config says. - the BF16 labels are bf16: the 'tight as half-float' pattern is also present in a main model that runs correctly with the bf16 reading, because half-float folds bf16's exponent range into one octave. tools/dflash_draft_info.py checks a package against its own config (tile-grid sizes by role, tile-header sanity, the container's scalars, fc and norm sizes, no unexplained tensor) and fails closed; 19 of 19 on the Bolt draft in 3.4 s. --selftest builds a synthetic package with eight negative controls (ctest tool_selftest_dflash_draft_info). Stdlib only. docs/amd-drafters.md records the answers and what is actually missing: the draft's forward, not a format. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 3 + docs/amd-drafters.md | 28 +++ tools/dflash_draft_info.py | 359 +++++++++++++++++++++++++++++++++++++ 3 files changed, 390 insertions(+) create mode 100755 tools/dflash_draft_info.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 12c4803fb..94ebb8def 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -320,6 +320,9 @@ add_python_test(tool_selftest_logprob_agree ${CMAKE_SOURCE_DIR}/tools/logprob_ag add_python_test(tool_selftest_mtp_head_ref ${CMAKE_SOURCE_DIR}/tools/mtp_head_ref.py --selftest) add_python_test(tool_selftest_spec_decode_acceptance ${CMAKE_SOURCE_DIR}/tools/spec_decode_acceptance.py --selftest) add_python_test(tool_selftest_whisper_encoder_baseline ${CMAKE_SOURCE_DIR}/tools/whisper_encoder_baseline.py --self-test) +# engine#1780: the DFlash draft package's gate before a loader -- tile-grid sizes from the config, tile-header sanity and +# the container's scalars against config.json, on a synthetic package with eight negative controls. Stdlib only. +add_python_test(tool_selftest_dflash_draft_info ${CMAKE_SOURCE_DIR}/tools/dflash_draft_info.py --selftest) # engine#1777: the release row's own arm, which merged unregistered -- this sweep is what found it (engine#1820). add_test(NAME release_check_hrx_init_reason COMMAND bash ${CMAKE_SOURCE_DIR}/tests/release_check_hrx_init_reason_test.sh ${CMAKE_SOURCE_DIR}/os/mini/release-check.sh) # engine#1615: every process holding the GPU/NPU device must belong to a live gpu-hold/npu-hold ticket. Three diff --git a/docs/amd-drafters.md b/docs/amd-drafters.md index ee89be046..699edf009 100644 --- a/docs/amd-drafters.md +++ b/docs/amd-drafters.md @@ -88,6 +88,34 @@ So the Gumiho row is *"fails as published, because our verifier is the wrong sha quality (its published numbers are the best in its class: 2.97× mean over Eagle-2's 2.74× on vicuna-7B). Running it would mean implementing a tree verifier first, which is its own project. +## DFlash draft packages on the NPU route: **the container is in forms we already read; the forward is what is missing** (engine#1780) + +FastFlowLM 1.0.12 ships speculative decode as two-package tags: a small DFlash draft beside the main model's package. +A draft package is `config.json` (`model_type: dflash`) plus one Q4NX container. `tools/dflash_draft_info.py ` +checks such a package against its own config and prints what it declares; on +`Nemotron-3.5-Bolt-E4B-DFlash-NPU2` (0.27 GB, sha256 matching the repo's `MANIFEST.sha256`) it passes 19 of 19 +checks in 3.4 s. What that settles, each with the evidence the tool prints: + +| question | answer | evidence | +|---|---|---| +| are the `I8 [tiles, 5120]` projections a new packing? | **No.** They are the ordinary Q4NX q4_1 tile grid (32 x 256 weights per 5120-byte tile), the form every main-model container uses. | all 35 projections have `bytes == (N/32)(K/256) x 5120` for the N x K the config gives the role; 10,240,000 groups with every scale finite and positive; minimum/scale median -7.59 .. -7.33 per tensor (a known-good main model reads -7.41 the same way) | +| is `q_proj` 4096 x 2560 or a half-float 1280 x 2560? | **4096 x 2560**, so `head_dim` is 128 as the config says. | both readings are 6,553,600 bytes, but the half-float one has 25.7 % of its values above 100 in magnitude and 1.8 % non-finite, and 1280 x 2560 as a tile grid would be 400 tiles, not 1280 | +| are the `BF16` labels wrong (really F16)? | **No, they are bf16.** | a norm gain read as half-float looks tight (1.6-2.4) and as bf16 looks wide (0.22-6.3), which reads like a mislabel. The same pattern appears in a main-model container our forward serves correctly with the bf16 reading (1.49-1.88 against 0.12-1.05): half-float folds bf16's exponent range into one octave, so "tight" proves nothing | +| does the container agree with `config.json`? | **Yes.** | `dflash.block_size`, `target_layer_ids`, `sliding_window`, `rope_theta`, `rms_eps`, `embedding_scale` all equal | +| is anything in it unexplained? | **No.** | 64 tensors: 35 projections, 22 norms, `model.fc.weight` (2560 x 12800, the fusion of five target layers' features) and six scalars | + +So importing a DFlash draft is a config-schema translation, not a format project. What our runtime does not have is the +draft's **forward**: fuse the target's hidden states at `target_layer_ids` through `fc`, run five non-causal layers +over a block of `block_size` positions filled with `mask_token_id`, and score the block with the target's own +embedding table and head. `forward-serve` stops on such a package at the first key it expects +(`key 'model' not found`), which is the schema, before any of that. + +The tool is the gate before a loader, and it fails closed: a projection that is a tile short, a `head_dim` that +disagrees with the stored tiles, a NaN scale, a plain half-float matrix of the same size, a scalar that differs from +the config, or a tensor it cannot explain (the second draft family, `DFlash2DraftModel`, has selector and +convolution tensors this tool does not know) each fail by name. `--selftest` runs those controls on a synthetic +package (ctest `tool_selftest_dflash_draft_info`). + ## PARD-2's repos are weights-only — borrow the target's tokenizer `amd/PARD2-*` ship **`README.md`, `config.json`, `model.safetensors`, `warp_model.bin`** and nothing else: **no tokenizer diff --git a/tools/dflash_draft_info.py b/tools/dflash_draft_info.py new file mode 100755 index 000000000..09090ab2b --- /dev/null +++ b/tools/dflash_draft_info.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""dflash_draft_info.py -- what a DFlash draft package declares, checked against its own config (engine#1780). + +A DFlash draft is a two-file model source: `config.json` (model_type "dflash": block size, mask token, the target +layers it taps, the draft's own geometry) and a Q4NX container with the draft's weights. Before a forward for it is +written, three questions have to be answered from the bytes, not from the names: + + 1. is every projection in a form our reader already decodes? The container labels them `I8 [tiles, 5120]`, which + reads as a row x column int8 matrix and is not one: it is the ordinary Q4NX q4_1 tile grid (32 rows x 256 + columns per 5120-byte tile: 512 B of bf16 scales, 512 B of bf16 minimums, 4096 B of 4-bit codes), so the tile + count must equal (N/32) * (K/256) for the N x K the config gives that role. + 2. do the container's own scalars agree with config.json? (block size, taps, rope theta, eps, window, scale.) + 3. are the tile headers sane? Every scale finite and positive, every minimum finite. A wrong layout fails this at + once; the right one also shows min/scale near -7.5, the q4_1 signature of a symmetric 16-level group. + + tools/dflash_draft_info.py # PASS/FAIL lines, exit 1 on any FAIL + tools/dflash_draft_info.py --json # the census as JSON + tools/dflash_draft_info.py --selftest # a synthetic package and its negative controls; no model + +Standard library only, and it reads tile headers rather than whole tensors, so the 0.27 GB package takes seconds. +It decodes nothing into a model and runs nothing: it is the gate before a loader, not a loader. +""" +import argparse +import json +import math +import os +import struct +import sys +import tempfile + +TILE_ROWS, TILE_COLS, TILE_BYTES, HDR_BYTES = 32, 256, 5120, 1024 +ROLES = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", + "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj") +REQUIRED = ("hidden_size", "intermediate_size", "num_hidden_layers", "num_attention_heads", "num_key_value_heads", + "head_dim", "block_size", "target_layer_ids", "draft_weight") + + +def bf16(u16): + """A bf16 value: the top 16 bits of an IEEE float32.""" + return struct.unpack(" size - 8: + raise ValueError(f"header length {n} does not fit a {size}-byte file") + return json.loads(f.read(n)), 8 + n, size + + +def tile_header_stats(f, base, lo, ntiles): + """Scale/minimum sanity over every tile header of one tensor. Reads 1024 of each 5120 bytes.""" + bad_scale = bad_min = 0 + ratios = [] + for t in range(ntiles): + f.seek(base + lo + t * TILE_BYTES) + h = f.read(HDR_BYTES) + if len(h) != HDR_BYTES: + return {"short_read_at_tile": t} + vals = struct.unpack("<512H", h) + for i in range(256): + s, m = bf16(vals[i]), bf16(vals[256 + i]) + if not math.isfinite(s) or s <= 0.0: + bad_scale += 1 + elif not math.isfinite(m): + bad_min += 1 + elif t % 16 == 0 and i % 16 == 0: # a sample is enough for the median + ratios.append(m / s) + ratios.sort() + return {"groups": ntiles * 256, "bad_scale": bad_scale, "bad_min": bad_min, + "min_over_scale_median": round(ratios[len(ratios) // 2], 3) if ratios else None} + + +def inspect(pkg): + """Returns (checks, census). `checks` is a list of (ok, text); the census is JSON-serialisable.""" + checks, census = [], {"package": pkg} + + def check(ok, text): + checks.append((bool(ok), text)) + return bool(ok) + + cfg_path = os.path.join(pkg, "config.json") + if not check(os.path.isfile(cfg_path), f"config.json is present in {pkg}"): + return checks, census + c = json.load(open(cfg_path)) + if not check(c.get("model_type") == "dflash", f"config model_type is dflash (got {c.get('model_type')!r})"): + return checks, census + missing = [k for k in REQUIRED if k not in c] + if not check(not missing, f"config carries the geometry and the interface keys (missing: {missing or 'none'})"): + return checks, census + census["config"] = {k: c[k] for k in REQUIRED + ("mask_token_id", "sliding_window", "causal") if k in c} + wpath = os.path.join(pkg, c["draft_weight"]) + if not check(os.path.isfile(wpath), f"the draft weight {c['draft_weight']} is present"): + return checks, census + try: + hdr, base, size = read_header(wpath) + except (ValueError, json.JSONDecodeError) as e: + check(False, f"the weight file parses as a Q4NX table: {e}") + return checks, census + names = [k for k in hdr if k != "__metadata__"] + check(True, f"the weight file parses as a Q4NX table: {len(names)} tensors, {base}-byte header, {size} bytes") + end = max((hdr[k]["data_offsets"][1] for k in names), default=0) + check(base + end <= size, f"every tensor lies inside the file (payload ends at {base + end} of {size})") + + def span(name): + lo, hi = hdr[name]["data_offsets"] + return lo, hi - lo + + # 1. the projections: an ordinary q4_1 tile grid, sized by the config + L = c["num_hidden_layers"] + proj, wrong, absent = [], [], [] + with open(wpath, "rb") as f: + for layer in range(L): + for role in ROLES: + name = f"model.layers.{layer}.{role}.weight" + if name not in hdr: + absent.append(name) + continue + n, k = role_shape(role, c) + want = tiles_for(n, k) + lo, nbytes = span(name) + row = {"name": name, "rows": n, "cols": k, "tiles_expected": want, + "bytes": nbytes, "declared": [hdr[name]["dtype"], hdr[name]["shape"]]} + if want is None or nbytes != want * TILE_BYTES: + wrong.append(f"{name}: {nbytes} B, expected {want} tiles x {TILE_BYTES}") + else: + row.update(tile_header_stats(f, base, lo, want)) + proj.append(row) + check(not absent, f"all {L} x {len(ROLES)} projections are in the container (absent: {absent[:3] or 'none'})") + check(not wrong and proj, + f"each projection is a whole q4_1 tile grid of the config's N x K: bytes == (N/32)(K/256) x 5120 " + f"({len(proj) - len(wrong)} of {len(proj)}{'; ' + wrong[0] if wrong else ''})") + bad = [r["name"] for r in proj if r.get("bad_scale") or r.get("bad_min") or "short_read_at_tile" in r] + groups = sum(r.get("groups", 0) for r in proj) + check(not bad and groups, f"every tile header is sane: scale finite and positive, minimum finite, over {groups} " + f"32-weight groups (first bad: {bad[0] if bad else 'none'})") + ratios = sorted(r["min_over_scale_median"] for r in proj if r.get("min_over_scale_median") is not None) + if ratios: + check(-8.5 <= ratios[0] and ratios[-1] <= -6.5, + f"minimum/scale sits at the q4_1 signature of a symmetric 16-level group: median per tensor " + f"{ratios[0]:.3f} .. {ratios[-1]:.3f} (expected near -7.5)") + census["projections"] = proj + + # 2. the container's own scalars against config.json + def scalars(name, fmt): + if name not in hdr: + return None + lo, nbytes = span(name) + with open(wpath, "rb") as f: + f.seek(base + lo) + return list(struct.unpack("<" + fmt * (nbytes // 4), f.read(nbytes))) + + pairs = (("dflash.block_size", "i", "block_size", None), ("dflash.target_layer_ids", "i", "target_layer_ids", None), + ("dflash.sliding_window", "i", "sliding_window", None), ("dflash.rope_theta", "f", "rope_theta", 1e-6), + ("dflash.rms_eps", "f", "rms_norm_eps", 1e-6), ("dflash.embedding_scale", "f", "embedding_scale", 1e-6)) + stated = {} + for tname, fmt, key, rel in pairs: + got = scalars(tname, fmt) + if got is None or key not in c: + continue + want = c[key] if isinstance(c[key], list) else [c[key]] + same = len(got) == len(want) and all( + (g == w) if rel is None else abs(g - w) <= rel * max(1.0, abs(w)) for g, w in zip(got, want)) + stated[tname] = got + check(same, f"{tname} in the container equals config.json's {key} ({got} vs {want})") + census["container_scalars"] = stated + + # 3. the feature projection and the norms: 2-byte tensors with the sizes the config implies + h, taps = c["hidden_size"], len(c["target_layer_ids"]) + if "model.fc.weight" in hdr: + _, nbytes = span("model.fc.weight") + check(nbytes == h * taps * h * 2, + f"model.fc.weight is hidden x (taps x hidden) at 2 bytes: {nbytes} B == {h} x {taps * h} x 2 " + f"(declared {hdr['model.fc.weight']['dtype']} {hdr['model.fc.weight']['shape']})") + else: + check(False, "model.fc.weight (the tap-fusing projection) is in the container") + norms = [k for k in names if k.endswith("norm.weight")] + off = [k for k in norms if span(k)[1] not in (2 * h, 2 * c["head_dim"])] + check(norms and not off, f"{len(norms)} norm weights are 2 bytes x hidden or x head_dim (off-size: {off[:2] or 'none'})") + known = {f"model.layers.{layer}.{r}.weight" for layer in range(L) for r in ROLES} | set(norms) | \ + {"model.fc.weight"} | {p[0] for p in pairs} + extra = sorted(set(names) - known) + check(not extra, f"no tensor is left unexplained (unexplained: {extra[:4] or 'none'})") + census["tensors"] = len(names) + census["unexplained"] = extra + return checks, census + + +# ---- self-test: a synthetic package small enough to write in milliseconds ---------------------------------------- + +def _write_package(d, mutate=None): + c = {"model_type": "dflash", "architectures": ["DFlashDraftModel"], "draft_weight": "dflash_weight.q4nx", + "block_size": 8, "mask_token_id": 4, "target_layer_ids": [1, 3], "num_hidden_layers": 1, + "hidden_size": 256, "intermediate_size": 512, "num_attention_heads": 2, "num_key_value_heads": 1, + "head_dim": 128, "rms_norm_eps": 1e-06, "rope_theta": 1000000.0, "sliding_window": 1024, + "embedding_scale": 16.0, "causal": False} + tensors = {} + + def tile(): + s = struct.pack("> 16) * 256 + m = struct.pack("> 16) * 256 + return s + m + bytes(4096) + + for role in ROLES: + n, k = role_shape(role, c) + nt = tiles_for(n, k) + tensors[f"model.layers.0.{role}.weight"] = ("I8", [nt, TILE_BYTES], tile() * nt) + for nm, w in (("model.norm.weight", 256), ("model.hidden_norm.weight", 256), + ("model.layers.0.input_layernorm.weight", 256), ("model.layers.0.post_attention_layernorm.weight", 256), + ("model.layers.0.self_attn.q_norm.weight", 128), ("model.layers.0.self_attn.k_norm.weight", 128)): + tensors[nm] = ("BF16", [w], struct.pack("= 14, f"a well-formed synthetic package passes every check ({len(checks)} checks, failing: {bad})") + expect(len(census.get("projections", [])) == 7 and all(r["min_over_scale_median"] == -7.5 for r in census["projections"]), + "its census lists the 7 projections with minimum/scale -7.5") + + def short_q(c, t): + dt, sh, data = t["model.layers.0.self_attn.q_proj.weight"] + t["model.layers.0.self_attn.q_proj.weight"] = (dt, sh, data[:-TILE_BYTES]) + bad, _, _ = run(short_q) + expect(any("whole q4_1 tile grid" in b for b in bad), "a projection one tile short of its config shape FAILS by name") + + def head_dim_40(c, t): + c["head_dim"] = 40 + bad, _, _ = run(head_dim_40) + expect(any("whole q4_1 tile grid" in b for b in bad), + "a config whose head_dim disagrees with the stored tiles FAILS (the geometry is checked, not assumed)") + + def nan_scale(c, t): + dt, sh, data = t["model.layers.0.mlp.up_proj.weight"] + t["model.layers.0.mlp.up_proj.weight"] = (dt, sh, struct.pack(" [--json] | --selftest", file=sys.stderr) + return 2 + checks, census = inspect(a.package) + failed = [t for ok, t in checks if not ok] + if a.json: + census["checks"] = [{"ok": ok, "what": t} for ok, t in checks] + print(json.dumps(census, indent=1)) + else: + for ok, t in checks: + print(f"{'PASS' if ok else 'FAIL'} {t}") + print(f"{len(checks) - len(failed)} of {len(checks)} checks passed") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 74f6db672aa7fc9584eb54a49a7b504673d8bcdd Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:42:12 -0400 Subject: [PATCH 05/98] npu-pack: read the 1.9 block-32 head from the .fconst tail (Fixes #1740) (#1948) A pack that declares /lm_head/MatMul_Q4 at block_size 32 (the Ryzen AI 1.9 Qwen-1.5B packs) keeps the head after the tail envelope in 64-row units of 2,560 + 2*(K/32)*640 bytes, not in the block-128 tile. read_block32_head() reads that layout; import_pack() takes it when the layout checks hold and, on a tied pack, the result correlates above 0.99 with model.embed_tokens. Otherwise a tied pack keeps the synthesis (and the note says why), and an untied one is refused with the reason. Measured on Qwen-2.5_1.5B_Instruct_rai_1.9.0_npu: 2,374 units of 64,000 B, corr 0.996185 against the embedding over all 151,936 rows, row-shifted control 0.0554. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/npu.md | 19 +++ npu/pack_import.cpp | 178 ++++++++++++++++++++++- npu/pack_import.h | 38 +++++ tests/npu_pack_import_test.cpp | 257 +++++++++++++++++++++++++++++++++ 4 files changed, 489 insertions(+), 3 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index ab1a76b58..42f539a38 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1183,6 +1183,25 @@ stored bytes equal the upstream checkpoint's `model.norm.weight` exactly. (The e "the tail envelope is `[eps][pad][448 f32]`, so the final norm's position is unknown" — was the last 1,856 bytes of the *file*, which in the Token_Token program are tile padding.) +**A head the pack declares at `block_size 32` is not in that tile stream** (engine#1740). The three +Ryzen AI 1.9 Qwen-1.5B packs quantise every layer at block 128 and `/lm_head/MatMul_Q4` at block 32, +and their tail after the envelope is 64-row *units* instead: 2,560 zero bytes, then `2 x (K/32)` +strips of 640 B. Strip `s` holds K block `s/2` for rows `32 x (s%2) .. +31` of the unit: 512 B of +4-bit codes (low nibble first; position `p` is row `(p/256) x 8 + p%8`, column `(p/8) % 32`), 16 B +of zero-point nibbles, 48 zero bytes, and 32 **signed** bf16 scales, one per row; +`w = (code - zero_point) x scale`. After the last unit the file is zero to its end (110,208,000 B on +the measured pack, which is why the block-128 arithmetic never landed there). `npu-pack import` +reads it (`read_block32_head`), folds a negative scale into its codes so the container's scales are +positive, and writes the same q4_1f tiles as any other head. The layout checks are part of the read +— unit count `ceil(vocab/64)`, the zero fields, finite scales, zeros to the end of the file — and +each failure is a refusal that says which. On a pack that declares tied embeddings the read head +also has to correlate above 0.99 with `model.embed_tokens`; if it does not, the head is made from +the embedding as before and `q4nx_head_note` says why the read was not taken. On an untied pack the +layout checks are all there is, and the note says the values are not verified. Measured on +`Qwen-2.5_1.5B_Instruct_rai_1.9.0_npu` (tied, 151,936 x 1,536, 2,374 units of 64,000 B): every scale +equals `bf16(-m/8)` of its 32-column embedding block, `m` being the block's largest-magnitude +weight, and the read head correlates 0.996 with the embedding. + The norms that *are* imported sit at the documented offsets, each confirmed by the epsilon that opens its layer block: **48/48 anchors found**. `tests/npu_pack_import_test` (registered as `npu_pack_import`, and as `npu_pack_import_oga` when `-DONEBIT_OGA_PACK=` is given) checks diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index ad46af92a..02fe8c996 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2451,6 +2451,88 @@ static TailHeadLocate locate_tail_head(const Fconst& fc, int64_t tail_off, int64 return r; } +// engine#1740: the block-32 tail head. Layout and its measurement are in pack_import.h. +Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K) { + Block32Head h; + if (V <= 0 || K <= 0 || K % 32 != 0) { + h.why = "a block-32 head needs K to be a multiple of 32 (K " + std::to_string(K) + ")"; + return h; + } + const int64_t kb = K / 32, strips = 2 * kb, unit = block32_unit_bytes(K); + h.units = (V + kB32UnitRows - 1) / kB32UnitRows; + h.bytes = h.units * unit; + if (avail < h.bytes) { + h.why = "the .fconst tail has " + std::to_string(avail) + " B after its envelope, and a block-32 head of " + + std::to_string(h.units) + " units x " + std::to_string(unit) + " B needs " + std::to_string(h.bytes); + return h; + } + for (int64_t i = h.bytes; i < avail; ++i) + if (p[i] != 0) { + h.why = "the .fconst tail is not zero after the block-32 head's " + std::to_string(h.units) + + " units (first non-zero byte " + std::to_string(i - h.bytes) + " B past them), so the head is not " + "where this layout puts it"; + return h; + } + h.padding = avail - h.bytes; + const auto all_zero = [](const uint8_t* b, int64_t n) { + for (int64_t i = 0; i < n; ++i) + if (b[i] != 0) return false; + return true; + }; + Q4& q = h.q; + q.N = int(h.units * kB32UnitRows); + q.K = int(K); + q.block = 32; + q.codes.assign(size_t(q.N) * size_t(K), 0); + q.scale.assign(size_t(q.N) * size_t(kb), 1.0f); + q.zero.assign(size_t(q.N) * size_t(kb), 0.0f); + for (int64_t u = 0; u < h.units; ++u) { + const uint8_t* up = p + u * unit; + if (!all_zero(up, kB32UnitHeader)) { + h.why = "block-32 head unit " + std::to_string(u) + " does not open with " + + std::to_string(kB32UnitHeader) + " zero bytes, so this is not the layout measured"; + return h; + } + for (int64_t s = 0; s < strips; ++s) { + const uint8_t* sp = up + kB32UnitHeader + s * kB32StripBytes; + if (!all_zero(sp + 528, 48)) { + h.why = "block-32 head unit " + std::to_string(u) + " strip " + std::to_string(s) + + " has non-zero bytes between its zero points and its scales, so this is not the layout measured"; + return h; + } + const int64_t col0 = (s / 2) * 32, row0 = u * kB32UnitRows + 32 * (s % 2); + for (int r = 0; r < 32; ++r) { + const uint16_t bits = uint16_t(sp[576 + 2 * r]) | uint16_t(uint16_t(sp[577 + 2 * r]) << 8); + const uint32_t wide = uint32_t(bits) << 16; + float sc; + std::memcpy(&sc, &wide, 4); + if (!std::isfinite(sc)) { + h.why = "block-32 head unit " + std::to_string(u) + " strip " + std::to_string(s) + " row " + + std::to_string(r) + " has a scale that is not finite"; + return h; + } + const int zp = (sp[512 + r / 2] >> (4 * (r & 1))) & 15; + const bool flip = sc < 0; + const size_t g = size_t(row0 + r) * size_t(kb) + size_t(s / 2); + if (sc == 0.0f) continue; // an all-zero block: codes 0, scale 1, zero 0 decode to 0 + const float mag = flip ? -sc : sc; + q.scale[g] = mag; + q.zero[g] = -float(flip ? 15 - zp : zp) * mag; + // position p = band*256 + col*8 + (r%8) holds row band*8 + r%8, column col + const int base = (r / 8) * 256 + (r % 8); + uint8_t* dst = q.codes.data() + size_t(row0 + r) * size_t(K) + size_t(col0); + for (int c = 0; c < 32; ++c) { + const int pos = base + c * 8; + const int code = (sp[pos / 2] >> (4 * (pos & 1))) & 15; + dst[c] = uint8_t(flip ? 15 - code : code); + } + } + } + } + h.ok = true; + return h; +} + // engine#1740: a self-test for the SEARCH, not the arithmetic. `plan_tail_head` is pinned by the host test; this pins // the part that can go silently wrong -- the candidate loop, best/runner-up tracking, the kTieCorr cut, the ambiguity // refusal and the untied-with-padding refusal -- using a synthetic head built through encode_tile, so it needs no pack, @@ -4397,11 +4479,99 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // writes -- and the self-check then measures the synthesis against the embedding rather than being // skipped. An untied pack is refused by name: its head is a matrix nothing else in the pack can // stand in for. - if (!(tie_declared && tied)) { + // + // engine#1740: a block-32 head is READ when the tail holds it in the measured layout (pack_import.h, + // read_block32_head). On a tied pack the read has to correlate with the embedding above kTieCorr to be + // taken -- otherwise the synthesis below stands, and says why the read did not. On an untied pack the + // structural checks are all there is (nothing in the pack can vouch for the values), which the note + // states; it is the same standing the block-128 tail head has on an untied pack. + bool head32_read = false; + std::string head32_why; + if (p.head_block == 32) { + const int64_t tail_off = int64_t(p.dims.layers) * g.stride; + const int64_t env = attn_payload_off(H); + if (tail_off + env > int64_t(fc.size)) { + head32_why = "the .fconst ends before the tail's envelope"; + } else { + Block32Head h32 = read_block32_head(fc.map + tail_off + env, int64_t(fc.size) - tail_off - env, + V, H); + const bool is_tied = tie_declared && tied; + double c32 = 0.0; + if (h32.ok && embed_tbl.ok()) c32 = corr(h32.q, embed_tbl, int(V)); + if (!h32.ok) { + head32_why = h32.why; + } else if (is_tied && !(c32 > kTieCorr)) { + char b[160]; + std::snprintf(b, sizeof b, + "the tail's block-32 head decodes but correlates %.6f with the embedding of " + "a pack that declares them tied", c32); + head32_why = b; + } else { + Q4& q = h32.q; + std::fprintf(stderr, + " lm head: block 32, %lld unit(s) of %lld B at %lld B, %lld B of zeros after " + "them%s\n", + (long long)h32.units, (long long)block32_unit_bytes(H), + (long long)(tail_off + env), (long long)h32.padding, + is_tied ? " (tied: verified by correlation with the pack's embedding)" + : " (untied: layout checked, values not verified)"); + if (embed_tbl.ok()) { + r.lmhead_corr = c32; + { + double sum_abs = 0, sum_ref = 0; + for (int n = 0; n < int(V); ++n) + for (int k = 0; k < q.K; ++k) { + const size_t gg = size_t(n) * (q.K / 32) + size_t(k) / 32; + const double x = double(q.codes[size_t(n) * q.K + k]) * q.scale[gg] + q.zero[gg]; + const double e = embed_at(embed_tbl, n, k); + sum_abs += std::fabs(x - e); + sum_ref += std::fabs(e); + } + if (sum_ref > 0) r.lmhead_rel = sum_abs / sum_ref; + } + r.selfcheck_reference = true; + r.selfcheck_tied = c32 > kTieCorr; + double sxy = 0, sx = 0, sy = 0, sxx = 0, syy = 0; + const int nsh = std::min(2048, int(V) - 1); + for (int rr = 0; rr < nsh; ++rr) + for (int k = 0; k < q.K; ++k) { + const size_t gg = size_t(rr) * (q.K / 32) + size_t(k) / 32; + const double x = double(q.codes[size_t(rr) * q.K + k]) * q.scale[gg] + q.zero[gg]; + const double y = embed_at(embed_tbl, rr + 1, k); + sxy += x * y; sx += x; sy += y; sxx += x * x; syy += y * y; + } + const double m = double(nsh) * q.K; + const double cx = sxx - sx * sx / m, cy = syy - sy * sy / m; + if (m > 0 && cx > 1e-12 && cy > 1e-12) + r.ctrl_corr_rowshift = (sxy - sx * sy / m) / std::sqrt(cx * cy); + } + std::vector tiled; + long long pinexact = 0; + tile_q4_1_f32(q, tiled, &pinexact); + r.codes_total = q.codes.size(); + r.zero_exact_pct = 100.0 * double(q.scale.size() - pinexact) / double(q.scale.size()); + const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes); + r.tiles += head_tiles; + w.add("lm_head.weight", "I8", {head_tiles, kImportTileBytes}, std::move(tiled)); + ++r.tensors; + head_written = true; + head32_read = true; + r.head_read_note = + std::string("lm_head was read from the .fconst tail's block 32 head (") + + std::to_string(h32.units) + " units of " + std::to_string(block32_unit_bytes(H)) + + " B)" + (is_tied ? ", and it reproduces model.embed_tokens" + : "; the pack is untied, so its values are not verified against " + "anything in the pack"); + } + } + } + if (head32_read) { + // nothing more to do: the pack's own head is in the container + } else if (!(tie_declared && tied)) { late.push_back("lm_head.weight: this pack quantises its head at block " + std::to_string(p.head_block) + " along K, which this importer's tile geometry cannot " "walk, and the pack does not declare tied embeddings, so the embedding cannot stand " - "in for it"); + "in for it" + (head32_why.empty() ? "" : " (" + head32_why + ")")); } else { const size_t rows_v = size_t(V), kk = size_t(H); Q4 q; @@ -4461,7 +4631,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons r.head_synthesized_note = "the pack quantises its head at block " + std::to_string(p.head_block) + " along K, so the tail's head tiles are not read; the pack declares tied embeddings, so " - "lm_head is the 4-bit quantisation of model.embed_tokens"; + "lm_head is the 4-bit quantisation of model.embed_tokens" + + (head32_why.empty() ? "" : " (" + head32_why + ")"); std::fprintf(stderr, "note: this pack's head is block %lld along K, which this importer cannot walk; the " "pack is tied, so the head was taken from the embedding (engine#1503)\n", @@ -5028,6 +5199,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons c["vocab_size"] = V; c["attention_bias"] = r.attn_bias_tensors > 0; // what the container actually carries if (!r.head_synthesized_note.empty()) c["q4nx_head_note"] = r.head_synthesized_note; + if (!r.head_read_note.empty()) c["q4nx_head_note"] = r.head_read_note; // The forward picks between "logits from the token embedding" and "logits from a separate head tensor" on this // key, and defaults to TIED when it is absent (npu/forward/src/q4nx_semantics.c). An untied pack that ships no // config.json -- every genre-A hybrid, e.g. Phi-3-mini -- therefore used to look tied, and the forward decoded diff --git a/npu/pack_import.h b/npu/pack_import.h index dcd0aa86a..dbed2c93a 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -371,6 +371,8 @@ struct Report { // walk and the pack is tied). Empty means it was read, as usual. Carried on the Report because the config JSON is // built after the .fconst block that discovers this. std::string head_synthesized_note; + // engine#1740: set when a block-32 head was READ from the .fconst tail (never together with the note above) + std::string head_read_note; // engine#1725: the per-layer embedding block the container carries (0 = none): row width per layer, and rows int64_t per_layer_P = 0, per_layer_V = 0; bool prefill_tail_seen = false; // a second copy was readable in the prefill .fconst @@ -403,6 +405,42 @@ struct Q4 { std::vector scale; // [N, K/32] std::vector zero; // [N, K/32] }; +// The 1.9 block-32 tail head (engine#1740). A pack whose `/lm_head/MatMul_Q4` is declared `block_size 32` keeps the +// head in the Token .fconst tail, after the same `64 + 2*hidden` envelope every other 1.9 tail opens with, but NOT in +// the block-128 tile: it is written in 64-row units of `2,560 + 2*(K/32)*640` bytes. +// +// Measured on `Qwen-2.5_1.5B_Instruct_rai_1.9.0_npu` (tied, V 151,936, K 1,536; unit 64,000 B, 2,374 units, then +// 110,208,000 B of zeros to EOF) against the pack's own `model.embed_tokens`: +// unit : 2,560 B of zeros, then 2*(K/32) strips of 640 B +// strip s: K block `s/2` (32 columns), rows `32*(s%2) .. +31` of the unit +// +0 512 B of 4-bit codes, low nibble first; code position p (0..1023) is row `(p/256)*8 + p%8`, column +// `(p/8)%32` -- four 8-row bands, each written column by column +// +512 16 B of zero points, one nibble per row (0x8 everywhere on the measured pack) +// +528 48 B of zeros +// +576 32 bf16 scales, one per row; SIGNED -- the measured writer stores `-m/8` where m is the block's +// largest-magnitude weight, so about half of them are negative +// w = (code - zero_point) * scale +// Every one of the 3,072 scales of a unit equals `bf16(-m/8)` of its embedding block exactly, and the decoded rows +// correlate 0.990 .. 0.997 with the embedding (4-bit), which is what fixed the field positions. The result is returned +// with positive scales (a negative one is folded into its codes: `15 - code`, `15 - zp`), as a `Q4` at block 32, rows +// padded to whole units the way the block-128 tail head is. +// +// What is checked, because a wrong region read with confidence is the failure this path exists to avoid: the unit +// count is `ceil(V/64)` and has to fit; every unit's 2,560 B and every strip's 48 B are zero; every scale is finite; +// and everything after the last unit is zero. Any of those failing is a refusal with the offset in it. +constexpr int64_t kB32UnitRows = 64, kB32UnitHeader = 2560, kB32StripBytes = 640; +inline int64_t block32_unit_bytes(int64_t K) { return kB32UnitHeader + 2 * (K / 32) * kB32StripBytes; } +struct Block32Head { + bool ok = false; + Q4 q; // N = units x 64 (>= V), K, block 32; value = code*scale + zero, scale > 0 + int64_t units = 0; + int64_t bytes = 0; // units x block32_unit_bytes(K) + int64_t padding = 0; // zero bytes after the last unit + std::string why; // set when !ok +}; +// `p` points at the first unit (tail_off + envelope), `avail` is the bytes from there to the end of the file. +Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K); + // The 1.9 external-data tile (engine#1503): the `.fconst` tile's subvolume written densely per tensor. // // Measured on `amd/phi-4_rai_1.9.0_npu` (`eager-mm.onnx.data`, 9.74 GB): every `MatMulNBitsBf` weight is a 1-D byte diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 1a91f2cf9..d6dbe17c6 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -1749,6 +1749,262 @@ void test_tied_block32_head(const std::string& root) { } } +// ── engine#1740: the block-32 head READ from the .fconst tail ────────────────────────────────────────────────────── +// The layout was measured on Qwen-2.5_1.5B_Instruct 1.9 (pack_import.h, above read_block32_head). This writes it from +// that description -- NOT through the reader -- so a reader that disagrees with the description fails here: 64-row +// units of 2,560 zero bytes + 2*(K/32) strips of 640 B; strip s is K block s/2, rows 32*(s%2)..+31; code position +// `(r/8)*256 + c*8 + r%8`, low nibble first; 16 B of zero-point nibbles (8); 48 zero bytes; 32 signed bf16 scales, +// `-m/8` with m the block's largest-magnitude weight. +static std::string encode_block32_units(const std::vector& W, int rows, int K) { + const int kb = K / 32, units = rows / 64; + const size_t unit = size_t(2560 + 2 * kb * 640); + std::string out(size_t(units) * unit, '\0'); + for (int u = 0; u < units; ++u) + for (int s = 0; s < 2 * kb; ++s) { + char* sp = out.data() + size_t(u) * unit + 2560 + size_t(s) * 640; + for (int r = 0; r < 32; ++r) { + const float* w = W.data() + size_t(u * 64 + 32 * (s % 2) + r) * size_t(K) + size_t((s / 2) * 32); + float m = 0; + for (int c = 0; c < 32; ++c) + if (std::fabs(w[c]) > std::fabs(m)) m = w[c]; + const uint16_t bits = pi::bf16_bits_of(-m / 8.0f); + const uint32_t wide = uint32_t(bits) << 16; + float sc; + std::memcpy(&sc, &wide, 4); + sp[576 + 2 * r] = char(bits & 0xFF); + sp[577 + 2 * r] = char(bits >> 8); + sp[512 + r / 2] = char(uint8_t(sp[512 + r / 2]) | uint8_t(8 << (4 * (r & 1)))); + for (int c = 0; c < 32; ++c) { + const int code = sc == 0 ? 8 : std::min(15, std::max(0, int(std::lround(w[c] / sc)) + 8)); + const int pos = (r / 8) * 256 + c * 8 + r % 8; + sp[pos / 2] = char(uint8_t(sp[pos / 2]) | uint8_t(code << (4 * (pos & 1)))); + } + } + } + return out; +} + +void test_block32_tail_head(const std::string& root) { + const int H = 128, L = 8, V = 64; + const int R = onebit::npu::q4nx::kRows; // the rows of container tile 0, which is what the arms decode + std::vector emb(size_t(V) * H), other(size_t(V) * H); + for (int r = 0; r < V; ++r) + for (int c = 0; c < H; ++c) { + emb[size_t(r) * H + c] = 0.05f * float((r * 7 + c * 13) % 97) - 2.4f; + other[size_t(r) * H + c] = 0.04f * float((r * 31 + c * 5 + (r * c) % 11) % 89) - 1.7f; // an UNRELATED head + } + // Pearson over the first `a.size()` values of both + const auto ncorr = [](const std::vector& a, const std::vector& b) { + double num = 0, da = 0, db = 0, sa = 0, sb = 0; + const double n = double(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + num += double(a[i]) * b[i], da += double(a[i]) * a[i], db += double(b[i]) * b[i]; + sa += a[i], sb += b[i]; + } + const double den = (n * da - sa * sa) * (n * db - sb * sb); + return den > 0 ? (n * num - sa * sb) / std::sqrt(den) : 0.0; + }; + // The fixture's own premise, asserted: the two matrices must not resemble each other, or "reads the tail" and + // "falls back to the embedding" could not be told apart by a correlation. + check(std::fabs(ncorr(emb, other)) < 0.3, + "block-32 tail: the fixture's two heads are unrelated (corr " + std::to_string(ncorr(emb, other)) + ")"); + + const auto build = [&](bool tied, const char* tag, const std::string& tail_after_env) { + const fs::path dir = fs::path(root) / tag; + fs::remove_all(dir); + fs::create_directories(dir / "cache"); + const int64_t stride = 1 << 20, second = 262144; + std::string fc(size_t(int64_t(L) * stride + 64 + size_t(H) * 2), '\0'); + auto put_eps = [&](int64_t off) { + fc[size_t(off)] = char(0x86); + fc[size_t(off) + 1] = char(0x35); + }; + for (int l = 0; l < L; ++l) { + put_eps(int64_t(l) * stride); + put_eps(int64_t(l) * stride + second); + } + put_eps(int64_t(L) * stride); + for (int i = 0; i < H; ++i) { + const uint16_t bits = pi::bf16_bits_of(float(i + 1)); + fc[size_t(int64_t(L) * stride + 64 + 2 * i)] = char(bits & 0xFF); + fc[size_t(int64_t(L) * stride + 64 + 2 * i) + 1] = char(bits >> 8); + } + fc += tail_after_env; + put_file((dir / "dd_metastate_Llm_Token_Token_crafted.fconst").string(), fc); + put_file((dir / "config.json").string(), + std::string(R"({"model_type":"qwen2","hidden_size":128,"intermediate_size":256,"num_hidden_layers":8,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":64,)" + R"("rope_theta":10000.0,"tie_word_embeddings":)") + + (tied ? "true" : "false") + R"(,"rms_norm_eps":1e-06})"); + std::string data(size_t(V) * size_t(H) * 2, '\0'); + for (size_t i = 0; i < emb.size(); ++i) { + const uint16_t bits = pi::bf16_bits_of(emb[i]); + data[2 * i] = char(bits & 0xFF); + data[2 * i + 1] = char(bits >> 8); + } + put_file((dir / "model.onnx.data").string(), data); + // the declaration that sets `p.head_block` (see test_tied_block32_head for why it lives in cache/) + put_file((dir / "cache" / "lm_head_meta.json").string(), + R"({"op_list":[{"type":"MladfMatMul","name":"lm_head.MatMulNBits",)" + R"("attrs":{"N":{"value":[64]},"K":{"value":[128]},"block_size":{"value":[32]},"bits":{"value":[4]}}}],)" + R"("tensor_map":{}})"); + put_file((dir / "model.onnx").string(), + pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "model.onnx.data", 0, + uint64_t(V) * uint64_t(H) * 2, 16), // 16 = BFLOAT16: the bytes ARE bf16 + pb::placeholder("lm_head.weight.MatMulNBits.qweight"), + pb::placeholder("lm_head.weight.MatMulNBits.scales"), + pb::placeholder("lm_head.weight.MatMulNBits.zeros")}, + {})); + return dir; + }; + struct Run { + pi::Report r; + std::string said, note; + bool has_head = false; + std::vector head; // the container's lm_head tile 0, decoded from its own bytes: [kRows][H] + }; + const auto run = [&](const fs::path& dir, const char* tag) { + Run x; + const std::string out = + (fs::temp_directory_path() / + (std::string("oga_pack_import_b32tail_") + tag + "." + std::to_string(::getpid()))).string(); + fs::remove_all(out); + pi::Options allow; + allow.allow_incomplete = true; // the fixture's layer payload is empty; the head gate is what is under test + try { + x.r = pi::import_pack(dir.string(), out, allow); + } catch (const std::exception& e) { + x.said = e.what(); + } + for (const auto& w : x.r.refused) + if (w.rfind("lm_head.weight", 0) == 0) x.said += " | " + w; + if (fs::exists(out + "/model.q4nx")) { + const json tbl = container_table(out + "/model.q4nx"); + x.has_head = tbl.contains("lm_head.weight"); + if (x.has_head) { + const std::vector blob = container_payload(out + "/model.q4nx"); + const auto offs = tbl["lm_head.weight"]["data_offsets"].get>(); + std::vector w(size_t(R) * onebit::npu::q4nx::kCols, 0.f); + onebit::npu::q4nx::dequant_tile(onebit::npu::q4nx::Chunk::Q4_1F, blob.data() + offs[0], w.data(), + onebit::npu::q4nx::kCols); + x.head.assign(size_t(R) * H, 0.f); + for (int r = 0; r < R; ++r) + for (int c = 0; c < H; ++c) x.head[size_t(r) * H + c] = w[size_t(r) * onebit::npu::q4nx::kCols + c]; + } + std::ifstream cf(out + "/config.json"); + if (cf.good()) { + const json cfg = json::parse(cf); + if (cfg.contains("q4nx_head_note")) x.note = cfg["q4nx_head_note"].get(); + } + } + fs::remove_all(out); + fs::remove_all(dir); + return x; + }; + + { // arm 1: UNTIED, the tail holds a head that is NOT the embedding, then padding that is not a whole unit + const Run x = run(build(false, "b32-untied-read", encode_block32_units(other, V, H) + std::string(10000, '\0')), + "untied"); + check(x.said.empty(), + "block-32 tail: an untied pack whose tail holds the head imports it (" + x.said.substr(0, 160) + ")"); + check(x.has_head && !x.head.empty(), "block-32 tail: the container carries the head"); + if (!x.head.empty()) { + const double c_tail = ncorr(x.head, other), c_emb = ncorr(x.head, emb); + check(c_tail > 0.99, + "block-32 tail: the container's head is the matrix the TAIL holds (corr " + std::to_string(c_tail) + ")"); + check(std::fabs(c_emb) < 0.3, "block-32 tail: and it is not the embedding (corr " + std::to_string(c_emb) + ")"); + } + check(x.r.head_synthesized_note.empty() && x.r.head_read_note.find("block 32") != std::string::npos && + x.r.head_read_note.find("untied") != std::string::npos, + "block-32 tail: the report says the head was read, and that an untied one is not verified (" + + x.r.head_read_note + ")"); + check(x.note == x.r.head_read_note && !x.note.empty(), "block-32 tail: config.json carries the same note"); + } + { // arm 2: TIED, the tail holds the embedding's own quantisation -> read (not synthesised), verified + const Run x = run(build(true, "b32-tied-read", encode_block32_units(emb, V, H)), "tied"); + check(x.said.empty(), "block-32 tail, tied: imports (" + x.said.substr(0, 160) + ")"); + check(x.r.head_synthesized_note.empty() && x.r.head_read_note.find("reproduces") != std::string::npos, + "block-32 tail, tied: the head is READ and the note says it reproduces the embedding (" + + x.r.head_read_note + ")"); + check(x.r.selfcheck_tied && x.r.lmhead_corr > 0.99, + "block-32 tail, tied: the self-check measures the read head (corr " + std::to_string(x.r.lmhead_corr) + ")"); + check(!x.head.empty() && ncorr(x.head, emb) > 0.99, + "block-32 tail, tied: the container's head is the embedding (corr " + + std::to_string(x.head.empty() ? 0.0 : ncorr(x.head, emb)) + ")"); + } + { // arm 3: TIED, but the tail's head is some other matrix -> the read is NOT taken, the synthesis stands, says why + const Run x = run(build(true, "b32-tied-mismatch", encode_block32_units(other, V, H)), "mismatch"); + check(x.r.head_read_note.empty() && x.r.head_synthesized_note.find("correlates") != std::string::npos, + "block-32 tail, tied mismatch: a tail head that is not the embedding is not taken (" + + x.r.head_synthesized_note.substr(0, 260) + ")"); + const double c_emb = x.head.empty() ? 0.0 : ncorr(x.head, emb), c_oth = x.head.empty() ? 1.0 : ncorr(x.head, other); + check(c_emb > 0.99 && std::fabs(c_oth) < 0.3, + "block-32 tail, tied mismatch: the container's head is the embedding's synthesis, not the tail's matrix " + "(corr " + std::to_string(c_emb) + " / " + std::to_string(c_oth) + ")"); + } + { // arm 4: UNTIED, a non-zero byte after the units -> refused by name with the reason, nothing written + std::string tail = encode_block32_units(other, V, H) + std::string(4096, '\0'); + tail[tail.size() - 100] = 1; + const Run x = run(build(false, "b32-untied-junk", tail), "junk"); + check(!x.has_head, "block-32 tail, junk after the head: no head is written"); + check(x.said.find("block 32") != std::string::npos && x.said.find("does not declare tied") != std::string::npos && + x.said.find("not zero after") != std::string::npos, + "block-32 tail, junk after the head: the refusal names the block, the missing tie and the reason (" + + x.said.substr(0, 300) + ")"); + } + { // arm 5: known answers on the reader itself, typed byte by byte (K 64: one unit is 2,560 + 4 x 640 = 5,120 B) + std::vector u(5120, 0); + uint8_t* sp = u.data() + 2560 + 3 * 640; // strip 3: K block 1, rows 32..63 + const auto put_scale = [&](int r, float v) { + const uint16_t bits = pi::bf16_bits_of(v); + sp[576 + 2 * r] = uint8_t(bits & 0xFF); + sp[577 + 2 * r] = uint8_t(bits >> 8); + }; + put_scale(9, 0.5f); + put_scale(10, -0.25f); + sp[512 + 4] = 0x80; // row 9's zero point (high nibble of byte 4) = 8 + sp[512 + 5] = 0x06; // row 10's zero point (low nibble of byte 5) = 6 + sp[(256 + 5 * 8 + 1) / 2] |= 0xB0; // row 9 (band 1, r%8 1), column 5: position 297, high nibble + sp[(256 + 31 * 8 + 2) / 2] |= 0x03; // row 10 (band 1, r%8 2), column 31: position 506, low nibble + pi::Block32Head h = pi::read_block32_head(u.data(), int64_t(u.size()), 64, 64); + check(h.ok && h.units == 1 && h.q.N == 64 && h.q.K == 64 && h.q.block == 32, + "block-32 reader: one unit of K 64 reads (" + h.why + ")"); + if (h.ok) { + const auto val = [&](int row, int col) { + const size_t g = size_t(row) * 2 + size_t(col / 32); + return double(h.q.codes[size_t(row) * 64 + col]) * h.q.scale[g] + h.q.zero[g]; + }; + check(val(41, 37) == 1.5, + "block-32 reader: (row 41, col 37) = (11 - 8) x 0.5 (" + std::to_string(val(41, 37)) + ")"); + check(val(41, 36) == -4.0, + "block-32 reader: its neighbour is code 0, (0 - 8) x 0.5 (" + std::to_string(val(41, 36)) + ")"); + check(val(42, 63) == 0.75, + "block-32 reader: a NEGATIVE scale, (3 - 6) x -0.25 (" + std::to_string(val(42, 63)) + ")"); + check(val(42, 32) == 1.5, + "block-32 reader: and its code 0, (0 - 6) x -0.25 (" + std::to_string(val(42, 32)) + ")"); + check(h.q.scale[size_t(42) * 2 + 1] == 0.25f, "block-32 reader: the stored scale is positive"); + check(val(41, 5) == 0.0 && val(0, 0) == 0.0, "block-32 reader: a block with a zero scale decodes to zero"); + } + check(!pi::read_block32_head(u.data(), 5119, 64, 64).ok, "block-32 reader: a tail one byte short is refused"); + std::vector two = u; + two.resize(5120 + 300, 0); + check(pi::read_block32_head(two.data(), int64_t(two.size()), 64, 64).padding == 300, + "block-32 reader: zeros after the last unit are padding, and counted"); + two.back() = 7; + check(!pi::read_block32_head(two.data(), int64_t(two.size()), 64, 64).ok, + "block-32 reader: a non-zero byte after the last unit is refused"); + std::vector hdr = u; + hdr[100] = 1; + check(!pi::read_block32_head(hdr.data(), int64_t(hdr.size()), 64, 64).ok, + "block-32 reader: a non-zero unit header is refused"); + std::vector gap = u; + gap[2560 + 640 + 540] = 1; + check(!pi::read_block32_head(gap.data(), int64_t(gap.size()), 64, 64).ok, + "block-32 reader: a non-zero byte between the zero points and the scales is refused"); + check(!pi::read_block32_head(u.data(), int64_t(u.size()), 64, 48).ok, "block-32 reader: K 48 is refused"); + } +} + // ── the final norm out of the tail envelope, on a crafted pack (engine#844) ────────────────── // The real-pack leg only covers this where a pack is supplied, and three of the four packs on the // box ship no prefill program, so the import's tail-envelope read had no coverage in the default @@ -4324,6 +4580,7 @@ int main(int argc, char** argv) { test_qwen3_attn_sections(); test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir); test_tied_block32_head(golden_dir.empty() ? "." : golden_dir); + test_block32_tail_head(golden_dir.empty() ? "." : golden_dir); test_final_norm_envelope(golden_dir.empty() ? "." : golden_dir); test_import_fp32_params(golden_dir.empty() ? "." : golden_dir); test_text_artifacts(golden_dir.empty() ? "." : golden_dir); From 0c852e89632388f4a797ece33a6116763d2b200d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:16:20 -0400 Subject: [PATCH 06/98] npu: the integer host arm packs an unaligned geometry row-major, so it stays inside its K*N image (Fixes #1994) (#1999) host_gemm_i8 and the i8 LM head pack UNPADDED images (K*N bytes) with blocked=1 and read them back with blocked_index(), which addresses whole 64x128 tiles. On a geometry that is not whole tiles, both the packer and the MAC run past the vector. Phi-3-mini's last vocabulary block (K 3072, N 7488) overruns by 196,096 B; a body width of N 8512 overruns by 130,560 B. Both sites now pick blocked = q4nx_layout::blocked_fits(K, N), record it in HostI8Tensor::blocked, and read through the new q4nx_layout::image_index(). Row-major holds the same codes and scales, so the int32 sums are unchanged. Every whole-tile geometry (the measured 11008 body and 2688 head block) keeps the blocked fast path. tests/npu_blocked_layout_test.cpp: test_unpadded_image packs into K*N plus 256 KiB of guard bytes at 3072x7488, 2048x8512 and 100x128 (row-major) and 4096x2688 (blocked). It checks that the guard is intact, every code reads back inside K*N, and forced blocked really leaves the buffer. Mutation: forcing blocked=true fails 3 checks on each unaligned shape. docs/npu.md describes the rule. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Opus 5.5 --- docs/npu.md | 8 ++++- npu/forward/include/q4nx_forward.h | 3 +- npu/forward/include/q4nx_weight_layout.h | 8 +++++ npu/forward/src/q4nx_forward.cpp | 21 ++++++++----- tests/npu_blocked_layout_test.cpp | 38 ++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 9 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 42f539a38..bff42342d 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1859,7 +1859,7 @@ none of which is a kernel fault. A family gated against f32 can therefore never `NPU_INFER_HOST_GEMM=i8` is the reference that can. It builds each body GEMM's weight image with the engine's own packer (`q4nx_pack_B_int8_blocked()`: one multiplier per column, int8 codes, blocked microtile order), walks -it with `q4nx_layout::blocked_index()`, quantises the activation row to int8 with `amax/127`, accumulates in +it with `q4nx_layout::image_index()`, quantises the activation row to int8 with `amax/127`, accumulates in int32 and applies `d[n] * (ascale * acc)`. **Same packer, same image, same order** -- that is the whole point: a reference that re-quantises by a different rule computes a different model, and will disagree with a correct device (measured: per-128-K int4 codes answer `ikt` to a question four other packs answer correctly on). @@ -1868,6 +1868,12 @@ The LM head is included, in vocabulary blocks (a whole head image is H x V bytes Without it the body is exact and the head is not, and the residual shows up as `max|d| ~ 0.3` on logits that no amount of body-side agreement can remove. +The images are unpadded (K x N bytes), and the blocked order only exists for whole 64 x 128 tiles, so a geometry +that is not whole tiles is packed and read **row-major** instead: the same codes and scales, so the same int32 +sums. The common case is the head's last vocabulary block (phi-3-mini: `32064 % 8192 = 7488`, which is 58.5 +tiles). Before engine#1994 such a shape was forced blocked, so the packer wrote past the vector and the MAC read +back whatever was there. `tests/npu_blocked_layout_test.cpp` pins the rule with guard bytes behind the image. + **What it measures.** Run the device arm and the `=i8` arm on the same request with `--dump-logits`, then `scripts/families/compare-logits.py` (npu-kernels). On `kernels-h4096-b8` with Mistral-7B-Instruct-v0.2 1.9: diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index c64be7992..ffc67b3fd 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -243,7 +243,8 @@ class Q4nxNpuForward { // quantisation, a wrong 128-K group boundary, an int8 intermediate or a saturated accumulator. bool host_gemm_i8_ = false; // Weight codes + per-128-K multipliers for the integer host arm, quantised once per tensor (engine#1809). - struct HostI8Tensor { int groups = 0; int N = 0; std::vector codes; std::vector d; }; + // `blocked` is the layout `codes` was packed in: blocked only when K x N is whole tiles (engine#1994). + struct HostI8Tensor { int groups = 0; int N = 0; bool blocked = true; std::vector codes; std::vector d; }; // Keyed on (LAYER, wkey, K, N): `wkey` alone is a per-role slot that repeats in every layer and `Bmat` is the one // scratch buffer each layer is decoded into, so keying on it served layer 0's weights to every later layer // (engine#1830 review). Routed experts pass wkey < 0 and are never cached: their weights change per token. diff --git a/npu/forward/include/q4nx_weight_layout.h b/npu/forward/include/q4nx_weight_layout.h index 722407646..291e943b7 100644 --- a/npu/forward/include/q4nx_weight_layout.h +++ b/npu/forward/include/q4nx_weight_layout.h @@ -66,6 +66,14 @@ inline bool blocked_fits(int KD, int ND) { return KD > 0 && ND > 0 && KD % 64 == 0 && ND % 128 == 0; } +// Element (i,j) in whichever layout an image was packed in. A caller that packs UNPADDED (KD == K, ND == N), like +// the integer host arm, must pick `blocked = blocked_fits(K, N)`: blocked_index() addresses whole tiles, so on a +// geometry that is not whole tiles it walks past a K*N buffer -- phi-3-mini's last vocabulary block (K 3072, +// N 7488) by 196,608 B (engine#1994). The row-major image holds the same codes, so the int32 result is unchanged. +inline size_t image_index(size_t i, size_t j, int KD, int ND, bool blocked) { + return blocked ? blocked_index(i, j, KD, ND) : rowmajor_index(i, j, KD, ND); +} + // Copy an int8 K x N weight (row-major, K rows of N bytes) into a KD x ND image // in the given layout; bytes outside the logical K x N matrix come out zero. // diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 09cba1ef7..d8ccbc971 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3157,10 +3157,14 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, // 2026-11-10, the arm answered "ikt" to "The capital of France is" and then locked onto one token, while the // per-GEMM A/B read cos 0.9999 against f32 on the same tensors. Same packer, same bytes, same model. double ssum = 0.0; - // blocked=1: the image is written in the blocked microtile order, which is the fast path AND the layout the - // device actually reads, so the MAC below walks it with q4nx_layout::blocked_index(). (blocked=0 was measured - // at 174.5 s of warm-up on this container versus 32.2 s blocked -- the row-major scatter is the slow one.) - q4nx_pack_B_int8_blocked(t.codes.data(), Bmat, K, N, K, N, /*blocked=*/1, t.d.data(), &ssum); + // Blocked: the image is written in the blocked microtile order, which is the fast path AND the layout the + // device actually reads, so the MAC below walks it with the same index. (blocked=0 was measured at 174.5 s of + // warm-up on this container versus 32.2 s blocked -- the row-major scatter is the slow one.) But the image is + // UNPADDED, K*N bytes, and blocked_index() addresses whole 64x128 tiles: an N that is not a multiple of 128 (or + // a K not of 64) packs and reads past the vector (engine#1994). Those geometries take row-major, which holds + // the same codes, so the result is unchanged and only an unaligned shape pays the slower pack. + t.blocked = q4nx_layout::blocked_fits(K, N); + q4nx_pack_B_int8_blocked(t.codes.data(), Bmat, K, N, K, N, t.blocked ? 1 : 0, t.d.data(), &ssum); if (!cacheable) { scratch = std::move(t); cached = &scratch; @@ -3200,7 +3204,7 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, for (int n = 0; n < N; n++) { int32_t acc = 0; for (int k = 0; k < K; k++) - acc += (int32_t)((int)t.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, K, N)] * + acc += (int32_t)((int)t.codes[q4nx_layout::image_index((size_t)k, (size_t)n, K, N, t.blocked)] * (int)a8[(size_t)k]); dst[(size_t)n] = t.d[(size_t)n] * (ascale * (float)acc); } @@ -5362,12 +5366,15 @@ bool Q4nxNpuForward::run_lm_head(const std::vector& h, std::vector c.codes.assign((size_t)H * (size_t)Tn, 0); c.d.assign((size_t)Tn, 0.0f); double ssum = 0.0; - q4nx_pack_B_int8_blocked(c.codes.data(), bt.data(), H, Tn, H, Tn, /*blocked=*/1, c.d.data(), &ssum); + // The last vocabulary block is rarely whole tiles (phi-3-mini: 32064 % 8192 = 7488, not a multiple of + // 128), and this image is unpadded, so it takes the layout its geometry fits (engine#1994). + c.blocked = q4nx_layout::blocked_fits(H, Tn); + q4nx_pack_B_int8_blocked(c.codes.data(), bt.data(), H, Tn, H, Tn, c.blocked ? 1 : 0, c.d.data(), &ssum); } for (int n = 0; n < Tn; n++) { int32_t acc = 0; for (int k = 0; k < H; k++) - acc += (int32_t)((int)c.codes[q4nx_layout::blocked_index((size_t)k, (size_t)n, H, Tn)] * + acc += (int32_t)((int)c.codes[q4nx_layout::image_index((size_t)k, (size_t)n, H, Tn, c.blocked)] * (int)a8i[(size_t)k]); float v = c.d[(size_t)n] * (ascale * (float)acc); if (sem_.logit_softcap > 0) v = sem_.logit_softcap * std::tanh(v / sem_.logit_softcap); diff --git a/tests/npu_blocked_layout_test.cpp b/tests/npu_blocked_layout_test.cpp index 4f092884e..c2d9278dd 100644 --- a/tests/npu_blocked_layout_test.cpp +++ b/tests/npu_blocked_layout_test.cpp @@ -208,6 +208,40 @@ void test_production_packer() { std::printf(" production packer: %zu codes, %d scales, blocked == bpos(rowmajor)\n", (size_t)K * N, N); } +// engine#1994: the integer host arm packs UNPADDED images (KD == K, ND == N) into K*N bytes. blocked_index() addresses +// whole 64x128 tiles, so a geometry that is not whole tiles has to be packed and read row-major, or both the packer and +// the MAC walk past the buffer. This pins the rule the arm uses -- blocked = blocked_fits(K, N), read through +// image_index() -- on the shapes that broke it, with guard bytes after the image to catch a write past K*N. +void test_unpadded_image(int K, int N, bool expect_blocked) { + const bool blocked = q4nx_layout::blocked_fits(K, N); + check(blocked == expect_blocked, "unpadded image: blocked only when K x N is whole tiles", K, N); + std::vector w((size_t)K * N); + for (size_t i = 0; i < w.size(); ++i) w[i] = float((i * 2654435761u) % 2000) * 1e-3f - 1.0f; + const size_t guard = size_t(1) << 18, img = (size_t)K * N; + std::vector buf(img + guard, int8_t(0x5A)), rowmaj(img); + std::vector sc((size_t)N), sc_ref((size_t)N); + q4nx_pack_B_int8(w.data(), K, N, rowmaj.data(), N, sc_ref.data()); + double ssum = 0; + const int rc = q4nx_pack_B_int8_blocked(buf.data(), w.data(), K, N, K, N, blocked ? 1 : 0, sc.data(), &ssum); + bool guard_ok = true; + for (size_t g = img; g < img + guard; ++g) guard_ok &= buf[g] == int8_t(0x5A); + size_t past = 0, mismatched = 0, forced_max = 0; + for (int i = 0; i < K; ++i) + for (int j = 0; j < N; ++j) { + const size_t at = q4nx_layout::image_index((size_t)i, (size_t)j, K, N, blocked); + if (at >= img) past++; + else if (buf[at] != rowmaj[(size_t)i * N + (size_t)j]) mismatched++; + forced_max = std::max(forced_max, q4nx_layout::blocked_index((size_t)i, (size_t)j, K, N)); + } + check(rc == 0 && guard_ok, "unpadded image: the packer writes nothing past K*N", K, N); + check(past == 0 && mismatched == 0, "unpadded image: every code reads back at image_index, inside K*N", K, N); + // The mechanism, so the arm above cannot pass for the wrong reason: on an unaligned shape the blocked index the + // arm used to force really does leave the buffer. + if (!expect_blocked) check(forced_max >= img, "unpadded image: forced blocked_index would run past K*N", K, N); + std::printf(" unpadded %dx%d: %s, forced blocked reaches %zu of %zu B\n", K, N, blocked ? "blocked" : "row-major", + forced_max + 1, img); +} + int main() { for (Geom g : kGeoms) { test_bijection(g); @@ -218,6 +252,10 @@ int main() { test_production_packer(); test_row_major_unchanged(); test_pack_geometries(); + test_unpadded_image(3072, 7488, false); // phi-3-mini's last vocabulary block (32064 % 8192) + test_unpadded_image(2048, 8512, false); // a body width of 66.5 tiles + test_unpadded_image(100, 128, false); // K not a whole number of 64-row tiles + test_unpadded_image(4096, 2688, true); // the measured head block (150144 % 8192 = 21 tiles): stays blocked if (failures) { std::printf("npu_blocked_layout_test: %d failure(s)\n", failures); return 1; From bc67179e81e0b822438847da3cae654e6cd92e6d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:16:27 -0400 Subject: [PATCH 07/98] npu-pack: refuse a one-header vote on a pack that declares per-head q/k gains, naming both witnesses (engine#1985) (#1995) A regression, not a new bug. `Qwen3-1.7B` imported with **0 gaps** on yesterday's main and refuses with **280** on today's, on the same complete pack: yesterday attention_payload {o_proj_k: 2048, qk_norm: true, tiles_measured: 64} today attention_payload {o_proj_k: 0, qk_norm: false, tiles_measured: 0} first refusal: model.layers.0.self_attn.q_proj.weight: attention payload holds 64 tiles; q/k/v/o at 64 ... rows each need 96 `qk_norm: true -> false` is the tell: the pack has the two per-head q/k sections between q/k/v and o_proj, so its block is a 4-header one -- and it is now read as 2-header, which takes the 2-header arithmetic and refuses every attention projection. The cause is #1838's early return: `if (!g.hdr.empty()) return g;` trusts the epsilon-anchor vote's list, which for this pack recorded ONE in-block header. That guard was right for a vote that found the right list and wrong for a vote that found only one on a block that has three -- the #1831 defect coming back through the vote instead of through the residue step. Neither witness wins, so the reader refuses and names both: the epsilon-anchor vote recorded ONE in-block header, and this pack declares use_qk_norm, whose per-head q/k gains live in the two sections a 2-header block does not have (engine#1985). The two witnesses disagree and neither is being preferred `header_witness_refusal(vote_hdrs, declared_qk)` is the whole decision in one testable place, and the three cases it must NOT fire on are as important as the one it must: a three-header vote (a correct 4-header pack), a one-header vote on a pack that declares nothing (a genuine 2-header family), and **no vote at all** (0 -- that is the residue step's case, not a disagreement, and refusing there would take `Llama-3.2-1B` out). Arms: 8, host-only. Proved they can fail -- making the predicate always return "" right-fails 5 arms. Suite 191/191. (The first run after the rebase showed 3 "Not Run" -- `serve_degenerate`, `q4nx_cache`, `npu_answer_parser` -- targets an incremental build had not built; building them gives 100%. That is the fourth time tonight a suite result was about the build rather than the code, and the reason "1 failed" must always be read as "which target, and was it built".) This does not *fix* Qwen3-1.7B: it stops the silent misclassification and says both witnesses. The 4-header read itself is #1934/#1969's measurement (what the 640 B `qk_rms_packed` gap actually holds), and the Qwen3-4B/8B rows should be re-run against this head rather than trusted from before the window. Refs #1985, #1831, #1838, #1889. Co-authored-by: t --- npu/pack_import.cpp | 19 ++++++++++++++++++- npu/pack_import.h | 13 +++++++++++++ tests/npu_pack_import_test.cpp | 24 ++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 02fe8c996..2fb204870 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1650,7 +1650,17 @@ FconstGeom locate_fconst_with_header(const std::string& tok, const std::string& // engine#1831: only when the vote did NOT record the block's headers. When it did, its list is the measurement // (`plan()` reads 2-header vs 4-header from `hdr.size()`, and `post_attn_hdr()` is `hdr.back()`), and rewriting it // to one entry turned every Qwen3 4-header 1.9 block into a 2-header one whose "post-attention" header was k_norm's. - if (!g.hdr.empty()) return g; + if (!g.hdr.empty()) { + /* engine#1985: the vote's list is a measurement, but it is not the only witness. A pack that declares per-head q/k + * gains cannot have a 2-header block, so a one-header vote on such a pack is the #1831 defect coming back through + * the vote instead of through the residue step -- measured: Qwen3-1.7B read as 2-header, qk_norm true -> false, 280 + * refused projections, where main imported it with 0 the day before. Refuse naming both rather than pick. */ + if (const std::string why = header_witness_refusal(int(g.hdr.size()), dims.use_qk_norm); !why.empty()) { + g.why = why; + g.stride = 0; + } + return g; + } const std::vector pos = eps_header_positions(tok, size_t(1) << 30, dims.eps_bits); // The scan window is the first GiB (about 8 blocks at h3584), so a real section boundary repeats ~8 times against // 1-2 for a payload coincidence; 3 separates them (requiring `layers/2` was too strict and dropped the real one). @@ -2861,6 +2871,13 @@ QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_b return o; } +std::string header_witness_refusal(int vote_hdrs, bool declared_qk) { + if (!declared_qk || vote_hdrs != 1) return std::string(); + return "the epsilon-anchor vote recorded ONE in-block header, and this pack declares use_qk_norm, whose per-head q/k " + "gains live in the two sections a 2-header block does not have (engine#1985). The two witnesses disagree and " + "neither is being preferred"; +} + std::string qk_norm_refusal(bool declared, bool found) { if (declared && !found) return "this pack declares use_qk_norm and its per-head q/k gains are not where the block layout says " diff --git a/npu/pack_import.h b/npu/pack_import.h index dbed2c93a..7ec92ac36 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -96,6 +96,19 @@ struct QkGainOffsets { QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_bytes); std::string qk_norm_refusal(bool declared, bool found); +// Why a pack's two layout witnesses cannot be reconciled, or "" when they can (engine#1985). +// +// `vote_hdrs` is the number of in-block headers the epsilon-anchor vote recorded (0 when it could not win); `declared_qk` +// is config.json's `use_qk_norm`. A pack that states per-head q/k gains keeps them in the two sections between q/k/v and +// o_proj, which a 2-header block does not have -- so a ONE-header vote on such a pack is self-contradictory, and engine#1985 +// is what trusting it costs: `Qwen3-1.7B` read as a 2-header block, `qk_norm` flipped true -> false, and every attention +// projection was refused (280 gaps) where yesterday's main imported the same pack with 0. +// +// Neither witness wins: the vote measured where anchors repeat, and the declaration states what the block must contain. +// The reader refuses and names both, because picking either one silently is the defect. Exposed so the three cases can be +// pinned without a 2.6 GB pack. +std::string header_witness_refusal(int vote_hdrs, bool declared_qk); + // A hybrid pack's embedding table lives in its own graph (`embedding.onnx` + `.data`, gathered with // GatherBlockQuantized). Returns [V, H] f32 row-major, empty when the pack has no such component, with `why` naming the // reason. Exposed so the dequantisation can be tested against known codes and scales instead of only on a 3 GB pack. diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index d6dbe17c6..3d1c246b0 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -4523,7 +4523,31 @@ void test_qk_gain_offsets() { check(n4.k_norm == 0 && n4.q_norm == 0, "qk gains: the byte count alone is not enough, head_dim has to agree"); } +/* engine#1985: a REGRESSION, not a new bug -- `Qwen3-1.7B` imported with 0 gaps on yesterday's main and refused with 280 + * on today's, because #1838's early return trusted an epsilon-anchor vote that had recorded ONE in-block header on a block + * that has three. The tell in the report is `qk_norm: true -> false`: a pack that declares per-head q/k gains keeps them in + * the two sections a 2-header block does not have, so the two witnesses disagree and neither may be preferred silently. + * + * The arms below pin the refusal and, just as importantly, the three cases it must NOT fire on: a vote that found three + * headers (a correct 4-header pack), a genuine 2-header pack that declares nothing, and no vote at all (0), which is the + * residue step's case and not a disagreement. */ +void test_header_witness_refusal() { + const std::string a = pi::header_witness_refusal(1, true); + check(!a.empty(), "header witness: a one-header vote on a pack declaring use_qk_norm is refused"); + check(a.find("engine#1985") != std::string::npos, "header witness: the refusal carries its issue"); + check(a.find("use_qk_norm") != std::string::npos, "header witness: it names the declaration it contradicts"); + check(a.find("ONE in-block header") != std::string::npos, "header witness: and what the vote measured"); + check(a.find("neither is being preferred") != std::string::npos, + "header witness: and that no witness is being chosen, which is the point"); + check(pi::header_witness_refusal(3, true).empty(), "header witness: a three-header vote is consistent with the declaration"); + check(pi::header_witness_refusal(1, false).empty(), "header witness: a 2-header pack that declares nothing is fine"); + check(pi::header_witness_refusal(0, true).empty(), + "header witness: NO vote is not a disagreement (the residue step decides it)"); + std::printf(" header witness: 8 arms (one-header vote vs declaration refuses; 3/1-without-declaration/0 allowed)\n"); +} + int main(int argc, char** argv) { + test_header_witness_refusal(); test_encode_tile_code_range(); test_qk_norm_refusal(); test_qk_gain_offsets(); From f7ecac400f481c455040ac12d4a57bb179ab3d54 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:16:35 -0400 Subject: [PATCH 08/98] serve: refuse --npu-opt on a model-generic container, where the per-op forward cannot use it (refs #1740, #1868) (#1991) engine#1868 refuses --npu-opt when the build links no private route. With an add-on linked there IS one, so the option was accepted -- and then bypassed by the CONTAINER TYPE: a model-generic Q4NX container is routed to the per-op forward before the private route is consulted, so the option was ignored again, one level in. Found by running W6: 'serve -m --npu-opt dx=' asked for the fused route, got per-op, and produced *identical* numbers to the per-op baseline (12,010 vs 12,169 ms for 24 tokens, same reply) -- which reads as 'no gain' rather than 'wrong route'. The engine's own log line says which forward it chose, but nothing said the option had been dropped. Now the per-op branch refuses by name when the caller asked for a private-route option: --npu-opt dx was given, but is a model-generic container, which serves through the per-op forward: the option has no effect here (it applies to a private route's own model directory). Drop it, or serve the model directory that route expects. Verified both directions on the real container with an add-on-enabled build: with the option -> exit 2 and that message; without it -> the per-op forward still starts and prints 'serving it through the per-op forward' as before. Co-authored-by: pi-inferzo --- app/serve.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/app/serve.cpp b/app/serve.cpp index 2ea0d3da3..146ef794e 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -3417,6 +3417,21 @@ int run_serve(int argc, char** argv) { fs::exists(fs::path(o.model) / "config.json") && fs::exists(fs::path(o.model) / "tokenizer.json"); if (model_generic && !kernels.empty() && fs::is_directory(kernels)) { + // engine#1740 (W6): --npu-opt is for a PRIVATE route, and this branch is the per-op forward, which + // cannot consume one. #1868 made the option refuse when no add-on is linked; with an add-on linked the + // option was still accepted here and then bypassed by the CONTAINER TYPE, silently -- a W6 run asked for + // the fused route, got per-op, and produced *identical* numbers, which reads as "no gain" rather than + // "wrong route". Refusing is the only answer that cannot be misread. + if (!o.npu_opts.empty()) { + std::string keys; + for (const auto& kv : o.npu_opts) keys += (keys.empty() ? "" : ", ") + kv.substr(0, kv.find('=')); + std::fprintf(stderr, + "1bit serve: --npu-opt %s was given, but %s is a model-generic container, which serves " + "through the per-op forward: the option has no effect here (it applies to a private " + "route's own model directory). Drop it, or serve the model directory that route " + "expects.\n", keys.c_str(), o.model.c_str()); + return 2; + } std::fprintf(stderr, "1bit serve: %s is a model-generic container: serving it through the per-op forward " "(kernels %s)\n", o.model.c_str(), kernels.c_str()); From f1ad35e01cf25ac0ebaff1f1bb7a01bdad72c0ab Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:16:42 -0400 Subject: [PATCH 09/98] bump llama.cpp to fa533bd21: a large expert activation no longer ends in all-NaN logits (fixes #1539) (#1987) Carries fork #284 and #285 (the routed-expert and dense matmul kernels keep their activations inside the half-precision range) and #283. Adds the docs/hrx.md section for the fix; the first part of #1962. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/hrx.md | 35 +++++++++++++++++++++++++++++++++++ registry/architectures.json | 2 +- third_party/llama.cpp | 2 +- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/docs/hrx.md b/docs/hrx.md index f6c7c3313..f8ad5eea2 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -269,6 +269,41 @@ the fix, 0 of 15 after. `llama-bench -p 512 -n 128 -r 3`, two runs each: Sarashi at 26.49 and 25.92 tok/s, stories15M Q8_0 at 317.77 and 338.54 tok/s (the box was shared; the same binary gave 545.39 tok/s on a quieter run). +### Fixed: all-NaN logits after a large expert activation ([engine#1539](https://github.com/1bit-MONSTER/engine/issues/1539), [#1962](https://github.com/1bit-MONSTER/engine/issues/1962)) + +`flint-40b-a2.5b` on `HRX0` answered some requests and then failed a later one with +`HRX returned NaN logits: count=150144 of 150144`. The matmul kernels multiply in half precision, and a +half-precision value ends at 65504. An FFN down-projection reads the SwiGLU output, which nothing +normalises: flint's layer 46 reaches about 9.1e4 on an ordinary 22-token prompt. That value became +`inf`, the sum over the routed experts `inf - inf = NaN`, and the NaN was stored as those tokens' +keys and values. The request that overflowed still returned finite logits. The next, shorter request +on the same slot found the stale NaN row inside its masked attention window and returned NaN for the +whole vocabulary. So the failure depended on the prompt (which tokens drive an activation that large) +and on the order of requests, not on memory, load or timing, and the CPU backend, which multiplies in +single precision, was never affected. + +The routed-expert kernel (llama.cpp #284) and the dense kernel for single-precision activations +(llama.cpp #285) now scale the activations by 2^-6 before the half-precision stage and scale the +result back by 2^6. Both factors are powers of two, so values that fit before are rounded as before; +the usable range moves from [6.1e-5, 6.55e4] to [3.9e-3, 4.19e6]. + +Measured on gfx1151 (2026-11-10): + +- flint, the sequence that failed on 9 of 9 fresh loads (a 22-token request, then an 8-token one on + the same slot): 150144 of 150144 NaN before, 0 after; a read-back of all 3056 graph nodes finds no + non-finite value, and the first short request now returns the same logits as its repeat. +- `test-backend-ops -o MUL_MAT_ID`: the new case at flint's shape with activations of 9.1e4 returned + NaN against the CPU's 56214.23 before, and passes after; the whole op is 845 of 845. `-o MUL_MAT`: + the 47 new large-activation cases went from 23 to 47 passing; the whole op is 589 of 589. +- The range the shift gives up at the low end is not used by the models measured: the smallest + activation column maximum is 3.2e-2 on flint's expert down-projections (37278 columns) and 2.4e-2 on + the dense inputs of Qwen3-0.6B, Qwen3-4B, ZAYA1-8B and flint. Cases at those two values are in the + suite, so the shift has a test on each side. + +Still open under #1962: two prefill kernels that re-quantise the activations into 8-bit blocks +(Q5_K and Q6_K weights at 512 tokens) keep a per-block term in half precision and still return NaN +above 65504. And the limit is moved, not removed: an activation above 4.19e6 would overflow again. + ### MoE decode and the kernel cache (llama.cpp #32, #33) - **MoE decode.** With one to four tokens, a `MUL_MAT_ID` (a MoE expert matmul) runs as a GEMV over diff --git a/registry/architectures.json b/registry/architectures.json index b2fb3f4e9..705031c6d 100644 --- a/registry/architectures.json +++ b/registry/architectures.json @@ -1,7 +1,7 @@ { "about": "HF architecture -> GGUF architecture and the backends whose code accepts it. Generated by tools/registry_build.py from the pinned sources; do not edit.", "sources": { - "llama.cpp (hrx)": "a5dc597b81f10c1fd29356018e56da074380ce60" + "llama.cpp (hrx)": "fa533bd21053eeb7eab5354aef749c44cde86987" }, "counts": { "hrx": 298, diff --git a/third_party/llama.cpp b/third_party/llama.cpp index a5dc597b8..fa533bd21 160000 --- a/third_party/llama.cpp +++ b/third_party/llama.cpp @@ -1 +1 @@ -Subproject commit a5dc597b81f10c1fd29356018e56da074380ce60 +Subproject commit fa533bd21053eeb7eab5354aef749c44cde86987 From e6f93acdb1d159154ab63e92aea0507bb355e529 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:16:51 -0400 Subject: [PATCH 10/98] build: every cmake/tests-*.cmake is included by a glob, so a new test's registration is a new file (Refs #1979) (#1983) * build: every cmake/tests-*.cmake is included by a glob, so a new test's registration is a new file (Refs #1979) A test-adding PR appends a row to CMakeLists.txt, and that file changed 37 times in the 24 h before #1979 was filed, so the row is the first part of the PR to stop applying. The family-file include from #1696 becomes a glob over cmake/tests-*.cmake at the end of the file, after every target and after find_package(Python3). A registration added as a new file there touches no shared line. No existing row moves: ctest --show-only=json-v1 gives the same 186 tests, commands and properties before and after. tools/ci/ci_targets_check.py's glob reader wanted /*.cmake and matched nothing for a prefix glob; it is widened, with an arm that fails on the old pattern. Co-Authored-By: Claude Fable 5.2 * build: the glob skips a tests-*.cmake file CMakeLists.txt already includes by name So a family file moved out with its own include() line (#1981's shape) and the glob can land in either order: without the skip, the second include stops the configure with 'add_test given test NAME ... which already exists'. ci_targets_check reads such a file once. Co-Authored-By: Claude Fable 5.2 * build: the glob's skip matches an include() line, not the same text in a comment Review of #1983 at 8dc5a26 (two CHANGES): the skip was a substring search over CMakeLists.txt, so a comment such as "# was include(cmake/tests-gpu-hold.cmake)" made the glob skip the file while nothing included it. The seven gpu_hold_* tests left the configuration (186 -> 179) and ci_targets_check stayed green. The skip is now a line-anchored match, the rule ci_targets_check.py's INCLUDE reads by, with the dots in the name escaped and the CMAKE_SOURCE_DIR and quoted spellings recognised. tests/registration_glob_test.sh cuts the block out of the real CMakeLists.txt and configures it in scratch projects (9 cases). It is registered from cmake/tests-registration-glob.cmake, a file nothing names, so it is also the first registration that arrives through the glob. Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 41 +++++++++++++-- CONTRIBUTING.md | 11 +++++ cmake/tests-registration-glob.cmake | 23 +++++++++ tests/registration_glob_test.sh | 77 +++++++++++++++++++++++++++++ tools/ci/ci_targets_check.py | 33 ++++++++++--- 5 files changed, 175 insertions(+), 10 deletions(-) create mode 100644 cmake/tests-registration-glob.cmake create mode 100755 tests/registration_glob_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 94ebb8def..a1daffc0e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -266,9 +266,8 @@ add_test(NAME copyright_gate COMMAND ${CMAKE_SOURCE_DIR}/tests/copyright_gate_te # engine#1701: the NPU hold's queue/token follow ONEBIT_HOLD_DIR, and its queue settings never reach the payload. # Shell only (a stub gpu-hold plus the tree's own), no device, ~1 s. add_test(NAME npu_hold COMMAND bash ${CMAKE_SOURCE_DIR}/tests/npu_hold_test.sh) -# engine#1696: the gpu_hold_* registrations live in their own file, so adding one does not conflict with every -# other test-adding PR through the one contiguous add_test block this used to be part of (see that file's header). -include(cmake/tests-gpu-hold.cmake) +# engine#1696: the gpu_hold_* registrations live in cmake/tests-gpu-hold.cmake. It is no longer named here: every +# cmake/tests-*.cmake is included by the glob near the end of this file (engine#1979). # engine#1485: the two port read-backs the route tests use (tests/serve_port.sh). Script-based, no device, no model -- # it feeds the parsers the exact lines `1bit serve` and `llama-server` print, because a read-back that silently returns # empty turns into "the server did not come up" and is blamed on whatever the row under test checks. @@ -1723,6 +1722,42 @@ if(ONEBIT_NPU_PRIVATE) endif() endif() +# ── Test registrations that live in their own file (engine#1696, engine#1979) ─ +# Every cmake/tests-*.cmake is included here, in name order. A test added as a NEW file there touches no line any +# other pull request touches, so it cannot conflict and its registration cannot go stale while the PR waits: 37 commits +# changed this file in the 24 h before engine#1979 was filed, and three open PRs had a registration hunk that no longer +# applied. The rows above stay where they are; nothing has to move. +# Here, after every target and after find_package(Python3), so a file may use if(TARGET ...), add_python_test() and +# ${Python3_EXECUTABLE}; and before the RESOURCE_LOCK loop below, which asks if(TEST ...). +# CONFIGURE_DEPENDS re-runs the glob at build time, so a file added after the first configure is picked up. +# Readers that follow this shape: tools/ci/ci_targets_check.py (the glob), tools/ci/hrx_build_scope.py and +# .github/workflows/hrx-build.yml (the cmake/tests-*.cmake path), tools/ci/tests_referenced_check.py (all of cmake/). +# The lines between the two markers are run as they stand by tests/registration_glob_test.sh, in a scratch project. +# >>> registration glob +file(GLOB ONEBIT_TEST_REGISTRATION_FILES CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/cmake/tests-*.cmake) +list(SORT ONEBIT_TEST_REGISTRATION_FILES) +# A file this CMakeLists.txt already include()s by name is skipped: that include fixes the file's position (some rows +# need one), and including it twice stops the configure with `add_test given test NAME ... which already exists`. So a +# family file moved out with its own include() line and this glob can land in either order. +# "By name" means a line CMake would execute -- include( at the start of a line, the same rule +# tools/ci/ci_targets_check.py reads by -- and not the same text in a comment: a substring search took +# `# was include(cmake/tests-gpu-hold.cmake)` for an include, and the seven gpu_hold_* tests left the configuration with +# every gate green (review of engine#1983). An include this pattern does not recognise is the loud miss: both includes +# run and the configure stops. +file(READ ${CMAKE_CURRENT_LIST_FILE} _onebit_top_cmake_text) +set(_onebit_include_prefix "(^|\n)[ \t]*include\\([ \t]*\"?(\\$\\{CMAKE_(CURRENT_)?SOURCE_DIR\\}/)?cmake/") +foreach(_onebit_test_registration_file IN LISTS ONEBIT_TEST_REGISTRATION_FILES) + get_filename_component(_onebit_test_registration_name ${_onebit_test_registration_file} NAME) + string(REGEX REPLACE "([.+])" "\\\\\\1" _onebit_test_registration_re "${_onebit_test_registration_name}") + string(REGEX MATCH "${_onebit_include_prefix}${_onebit_test_registration_re}\"?[ \t]*\\)" _onebit_named + "${_onebit_top_cmake_text}") + if(_onebit_named STREQUAL "") + include(${_onebit_test_registration_file}) + endif() +endforeach() +unset(_onebit_top_cmake_text) +# <<< registration glob + # ── Loopback ports: route tests run one at a time (engine#1070) ────────────── # Every test named below starts a server on 127.0.0.1. Most pick a port by binding :0, closing the # socket, then letting the child bind that number; three use fixed 43xxx literals. Under `ctest -j` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d5f58f6d..b86b8a268 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,6 +87,17 @@ an RFC or issue. hands the script to the interpreter instead of relying on the file's exec bit; `tests/python_test_order_check.py --check` enforces that it follows `find_package(Python3)`. Two rows may not share a name: CMake stops at configure with `add_test given test NAME "…" which already exists in this directory`. +- **A new test's registration goes in its own file.** Put the `add_test(...)` (or `add_python_test(...)`) row, its + comment, and its `set_tests_properties` in a new `cmake/tests-.cmake` with the Apache notice, and do not touch + `CMakeLists.txt` for it: every `cmake/tests-*.cmake` is included by a glob near the end of `CMakeLists.txt`, after all + targets exist. A new file cannot conflict with another pull request, and a row appended to `CMakeLists.txt` does — + that file changed 37 times in one day and three open pull requests had a registration hunk that no longer applied + (engine [#1979](https://github.com/1bit-MONSTER/engine/issues/1979)). A row that belongs to an existing family goes in + that family's file (`cmake/tests-gpu-hold.cmake`). An `add_executable` for a test binary still goes in + `CMakeLists.txt` next to the sources it builds; only the registration moves. Existing rows stay where they are. + A registration file that needs a fixed position may still be `include()`d by name from `CMakeLists.txt`; the glob + then leaves it alone. `tests/registration_glob_test.sh` (itself registered from `cmake/tests-registration-glob.cmake`) + runs the glob block and holds both rules. - **Pins only move forward.** CI fails a pull request whose submodule pin is behind, or has diverged from, the pin on `main` (`tools/check_pins.py`), because that drops commits the engine already shipped. A rollback on purpose carries the `pin rollback` label. diff --git a/cmake/tests-registration-glob.cmake b/cmake/tests-registration-glob.cmake new file mode 100644 index 000000000..2e4e1b43b --- /dev/null +++ b/cmake/tests-registration-glob.cmake @@ -0,0 +1,23 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The registration glob's own test (engine#1979), registered the way it asks new tests to be: this file is named by +# nothing and is picked up by the cmake/tests-*.cmake glob near the end of CMakeLists.txt. +# +# It configures the glob block of the real CMakeLists.txt in scratch projects: a new file is registered, a file named +# only in a comment is still registered, and a file CMakeLists.txt include()s by name is registered once. Configure +# only (project(... NONE)): no compiler, no device, about a second. +add_test(NAME registration_glob + COMMAND bash ${CMAKE_SOURCE_DIR}/tests/registration_glob_test.sh ${CMAKE_COMMAND} ${CMAKE_SOURCE_DIR}/CMakeLists.txt) diff --git a/tests/registration_glob_test.sh b/tests/registration_glob_test.sh new file mode 100755 index 000000000..5b83cec19 --- /dev/null +++ b/tests/registration_glob_test.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The cmake/tests-*.cmake glob in CMakeLists.txt, run as it stands (engine#1979). +# +# The block between `# >>> registration glob` and `# <<< registration glob` is cut out of the real CMakeLists.txt and +# configured in a scratch project, so what is tested is the text main configures with, not a copy of it. Each case +# says which tests ctest must then list: +# - a new file nothing names -> its test is registered (the point of the glob), +# - the same file named only in a COMMENT -> still registered. A substring search skipped it +# here: the family's tests left the configuration and no gate noticed (review of engine#1983), +# - a file CMakeLists.txt include()s by name, four spellings -> registered once; a second include would stop the +# configure with `add_test given test NAME ... which already exists`, +# - tests-a.b.cmake beside an include of tests-axb.cmake -> both registered: the dot is not a wildcard. +# +# usage: tests/registration_glob_test.sh (configure only: no compiler, no device) +set -uo pipefail +CMAKE=${1:?usage: registration_glob_test.sh } +TOP=${2:?usage: registration_glob_test.sh } +CTEST="$(dirname "$CMAKE")/ctest" +[ -x "$CTEST" ] || CTEST=ctest +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +BLOCK=$(sed -n '/^# >>> registration glob$/,/^# <<< registration glob$/p' "$TOP") +if [ -z "$BLOCK" ] || ! printf '%s\n' "$BLOCK" | grep -q 'file(GLOB'; then + echo "FAIL: no '# >>> registration glob' block with a file(GLOB in $TOP"; exit 1 +fi +fails=0 +# case ... +case_() { + local what=$1 want=$2 prelude=$3; shift 3 + local d="$T/p$((++n))"; mkdir -p "$d/cmake" + local f + for f in "$@"; do + printf 'add_test(NAME %s COMMAND true)\n' "$(printf '%s' "${f%.cmake}" | tr -c 'A-Za-z0-9_\n' '_')" > "$d/cmake/$f" + done + { printf 'cmake_minimum_required(VERSION 3.20)\nproject(regglob NONE)\nenable_testing()\n%s\n%s\n' "$prelude" "$BLOCK" + } > "$d/CMakeLists.txt" + local got + if ! "$CMAKE" -S "$d" -B "$d/b" > "$d/log" 2>&1; then + got="CONFIGURE FAILED: $(grep -m1 -A1 'CMake Error' "$d/log" | tr '\n' ' ')" + else + got=$(cd "$d/b" && "$CTEST" -N | sed -n 's/^ *Test *#[0-9]*: //p' | sort | tr '\n' ' ' | sed 's/ $//') + fi + if [ "$got" = "$want" ]; then echo " ok $what"; else echo " FAIL $what: got '$got', wanted '$want'"; fails=$((fails + 1)); fi +} +n=0 +case_ "a new file nothing names is registered" "tests_one" "" tests-one.cmake +case_ "a file named only in a comment is still registered" "tests_one" \ + "# history: this used to be include(cmake/tests-one.cmake) right here" tests-one.cmake +case_ "a file named in a trailing comment is still registered" "tests_one" \ + "set(X 1) # was include(cmake/tests-one.cmake)" tests-one.cmake +case_ "a file included by name is registered once" "tests_one tests_two" \ + "include(cmake/tests-one.cmake)" tests-one.cmake tests-two.cmake +case_ "an indented include under if() counts as an include" "tests_one" \ + "$(printf 'if(TRUE)\n include(cmake/tests-one.cmake)\nendif()')" tests-one.cmake +case_ 'a ${CMAKE_SOURCE_DIR}/ include counts as an include' "tests_one" \ + 'include(${CMAKE_SOURCE_DIR}/cmake/tests-one.cmake)' tests-one.cmake +case_ "a quoted include counts as an include" "tests_one" \ + 'include("cmake/tests-one.cmake")' tests-one.cmake +case_ "the dot in a file name is not a wildcard" "tests_a_b tests_axb" \ + "include(cmake/tests-axb.cmake)" tests-a.b.cmake tests-axb.cmake +case_ "a file the glob does not match is not registered" "tests_one" "" tests-one.cmake other.cmake +echo "registration_glob_test: $fails failure(s)" +[ "$fails" -eq 0 ] diff --git a/tools/ci/ci_targets_check.py b/tools/ci/ci_targets_check.py index 4ac56efec..4c103ef67 100755 --- a/tools/ci/ci_targets_check.py +++ b/tools/ci/ci_targets_check.py @@ -65,7 +65,9 @@ # shape engine#1696 landed. The include argument is a VARIABLE, so INCLUDE above cannot match it, and a # reader that only followed literal includes would stop seeing every registration at once - the count # would fall to zero and read as "the gap closed". -GLOB = re.compile(r'file\s*\(\s*GLOB\s+(\w+)[^)]*?([^\s")]+/\*\.cmake)', re.IGNORECASE) +# engine#1979: the pattern main uses is `cmake/tests-*.cmake` (a prefix before the star), not `/*.cmake`; a +# reader that wanted the star right after the slash matched nothing there, and every such file went unread. +GLOB = re.compile(r'file\s*\(\s*GLOB\s+(\w+)[^)]*?([^\s")]+/[^\s"/)]*\*[^\s"/)]*\.cmake)', re.IGNORECASE) INCLUDE_VAR = re.compile(r'include\(\s*\$\{(\w+)\}\s*\)') FOREACH_LISTS = re.compile(r'foreach\s*\(\s*(\w+)\s+IN\s+LISTS\s+(\w+)', re.IGNORECASE) DEFINE = re.compile(r"-D([A-Za-z0-9_]+)=([^\s\"]*)") @@ -352,6 +354,7 @@ def family_text(root, cmake_text): tracks a guard stack across the text this returns. """ out = [] + seen = set() # a glob the include loop iterates: every file matching it is configured, so every one is read for gvar, pattern in GLOB.findall(cmake_text): # the loop variable differs from the glob variable: `foreach(v IN LISTS GLOBVAR) include(${v})` @@ -360,12 +363,16 @@ def family_text(root, cmake_text): continue pat = pattern.replace("${CMAKE_SOURCE_DIR}/", "").replace("${PROJECT_SOURCE_DIR}/", "") for path in sorted(glob.glob(os.path.join(root, pat))): - if os.path.isfile(path): + if os.path.isfile(path) and os.path.realpath(path) not in seen: + seen.add(os.path.realpath(path)) out.append(open(path, encoding="utf-8").read()) for m in INCLUDE.finditer(cmake_text): rel = m.group(1).replace("${CMAKE_SOURCE_DIR}/", "").replace("${PROJECT_SOURCE_DIR}/", "") path = os.path.join(root, rel) - if os.path.isfile(path): + # engine#1979: a file both named by a literal include() and matched by the glob is configured once + # (CMakeLists.txt skips it in the glob loop), so it is read once. + if os.path.isfile(path) and os.path.realpath(path) not in seen: + seen.add(os.path.realpath(path)) out.append(open(path, encoding="utf-8").read()) return "\n".join(out) @@ -516,16 +523,16 @@ def self_test(): import tempfile fails = 0 - def case(what, ci, cmake, want, baseline="auto", family=None): + def case(what, ci, cmake, want, baseline="auto", family=None, family_rel="cmake/tests/zz_family.cmake"): nonlocal fails with tempfile.TemporaryDirectory() as d: os.makedirs(os.path.join(d, ".github", "workflows")) open(os.path.join(d, CI), "w", encoding="utf-8").write(ci) open(os.path.join(d, CMAKE), "w", encoding="utf-8").write(cmake) if family is not None: - fam_dir = os.path.join(d, "cmake", "tests") - os.makedirs(fam_dir, exist_ok=True) - open(os.path.join(fam_dir, "zz_family.cmake"), "w", encoding="utf-8").write(family) + fam_path = os.path.join(d, family_rel) + os.makedirs(os.path.dirname(fam_path), exist_ok=True) + open(fam_path, "w", encoding="utf-8").write(family) if baseline == "auto": # The direction-A fixtures below are not about direction B, so the baseline records # whatever they drift - a direction-B case must then say what its baseline is. @@ -600,6 +607,18 @@ def case(what, ci, cmake, want, baseline="auto", family=None): case("an add_test in a glob-included family file is still read", B_CI, GLOB_CMAKE, 1, baseline="", family="add_executable(glob_fam g.cpp)\nadd_test(NAME glob_suite COMMAND glob_fam)\n") + # engine#1979: the shape main has, a prefix before the star (`cmake/tests-*.cmake`). With the GLOB pattern that + # wanted `/*.cmake` this arm returned 0: the file was configured by CMake and read by nobody here. + PREFIX_GLOB_CMAKE = GLOB_CMAKE.replace("cmake/tests/*.cmake", "cmake/tests-*.cmake") + case("an add_test in a file included by a cmake/tests-*.cmake glob is still read", + B_CI, PREFIX_GLOB_CMAKE, 1, baseline="", + family="add_executable(glob_fam g.cpp)\nadd_test(NAME glob_suite COMMAND glob_fam)\n", + family_rel="cmake/tests-zz.cmake") + # ... and a file the glob does not match (one directory down) is not configured, so it is not read. + case("a file the cmake/tests-*.cmake glob does not match is not read", + B_CI, PREFIX_GLOB_CMAKE, 0, baseline="", + family="add_executable(glob_fam g.cpp)\nadd_test(NAME glob_suite COMMAND glob_fam)\n", + family_rel="cmake/tests-zz/inner.cmake") # ... and a baseline name nothing names any more is a FAILURE, not "progress": the two reasons a # count can fall are "now built" (good) and "no longer seen" (the reader narrowed). case("a baseline name no add_test names any more fails instead of reading as progress", From 2f8dc3bd4cfafb69b9e152b9bf0c7fd2b63b4d75 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:28:05 -0400 Subject: [PATCH 11/98] npu: a Q4NX cache hit counts as a use, and the cache bound cannot wrap (Refs #1956) (#1988) The sweep orders entries by the entry directory's mtime, which nothing moved after the entry was written, so the order was by creation: a container served every day went before yesterday's one-off. A hit now touches its entry (repacked GGUF, lane layout, device-form conversion). ONEBIT_Q4NX_CACHE_MAX_GIB was multiplied by 2^30 unchecked: 2^34+1 came out as a 1 GiB bound, and a value that is not a number turned the sweep off. The bound is now clamped, and a non-number gets the default. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- app/q4nx_cache.cpp | 21 +++++++++++++++++++++ app/q4nx_cache.h | 13 +++++++++++++ app/serve.cpp | 12 ++++++++---- docs/npu.md | 10 ++++++++++ npu/q4nx_device.cpp | 3 +++ tests/q4nx_cache_test.cpp | 38 ++++++++++++++++++++++++++++++++++++-- 6 files changed, 91 insertions(+), 6 deletions(-) diff --git a/app/q4nx_cache.cpp b/app/q4nx_cache.cpp index 361974546..3bcb727c6 100644 --- a/app/q4nx_cache.cpp +++ b/app/q4nx_cache.cpp @@ -16,8 +16,10 @@ #include "q4nx_cache.h" #include +#include #include #include +#include #include #include @@ -83,3 +85,22 @@ Q4nxCacheSweep q4nx_cache_evict(const std::string& root, const std::string& keep } return r; } + +int64_t q4nx_cache_bound_bytes(const char* text, int64_t default_gib) { + const int64_t max_gib = INT64_MAX >> 30; // the largest GiB count whose byte count still fits + long long gib = default_gib; + if (text && *text) { + char* end = nullptr; + errno = 0; + const long long v = std::strtoll(text, &end, 10); + if (end != text) gib = v; // a number (ERANGE saturates, which the clamp below handles) + } + if (gib <= 0) return 0; + if (gib > max_gib) gib = max_gib; + return int64_t(gib) << 30; +} + +void q4nx_cache_touch(const std::string& entry) { + std::error_code ec; + if (fs::is_directory(entry, ec)) fs::last_write_time(entry, fs::file_time_type::clock::now(), ec); +} diff --git a/app/q4nx_cache.h b/app/q4nx_cache.h index 53783bac3..f5d987511 100644 --- a/app/q4nx_cache.h +++ b/app/q4nx_cache.h @@ -44,3 +44,16 @@ Q4nxCacheSweep q4nx_cache_evict(const std::string& root, const std::string& keep // Directory bytes under `path` (0 when it does not exist). int64_t q4nx_dir_bytes(const std::string& path); + +// The byte bound for ONEBIT_Q4NX_CACHE_MAX_GIB's value (`text` may be null). Unset, empty or not a number gives the default +// (`default_gib`); zero or negative gives 0, which q4nx_cache_evict() reads as "no sweep". The result never exceeds +// INT64_MAX: the plain `gib << 30` this replaces overflowed from 2^33 GiB up, so a very large number written to mean "no +// limit" could come out as a 1 GiB bound, and a typo ("sixty") silently turned the sweep off. +int64_t q4nx_cache_bound_bytes(const char* text, int64_t default_gib = 60); + +// Mark `entry` (a directory in the cache) as used now. The sweep orders by the entry directory's mtime, which the filesystem +// only moves when a file is created or removed inside it -- so without this the order is by CREATION, and a container +// imported last week and served every day is evicted before yesterday's one-off. Call it on every cache hit. It also puts a +// just-opened entry inside the sweep's age window, which is what makes "a serve may be reading it" true for a reused entry +// and not only for a new one. Failure is ignored (a read-only cache still serves). +void q4nx_cache_touch(const std::string& entry); diff --git a/app/serve.cpp b/app/serve.cpp index 146ef794e..9556af962 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -2958,12 +2958,12 @@ std::string q4nx_cache_root() { // engine#1956: the cache is regenerable scratch, not user data, and nothing ever removed an old entry -- measured at 104 G in // 29 entries on one box, which closed two lanes' build gates the same day. The bound is a size, not an age, because a busy // day imports more packs than any age window expires; ONEBIT_Q4NX_CACHE_MAX_GIB=0 restores the old unbounded behaviour. +// The order is least-recently-USED: a cache hit touches its entry (q4nx_cache_touch), here and in npu/q4nx_device.cpp. // The entry just written is never a candidate, and neither is one younger than the sweep's age window (a serve may be reading // it) -- see app/q4nx_cache.h for why that is a heuristic rather than a guarantee. static void q4nx_cache_sweep(const std::string& cache, const std::string& keep) { - const char* env = std::getenv("ONEBIT_Q4NX_CACHE_MAX_GIB"); - long long gib = env && *env ? std::atoll(env) : 60; - q4nx_cache_evict(cache, keep, gib > 0 ? gib * (1LL << 30) : 0); + q4nx_cache_touch(keep); // a reused entry counts as used now: the sweep is LRU by this, not by creation + q4nx_cache_evict(cache, keep, q4nx_cache_bound_bytes(std::getenv("ONEBIT_Q4NX_CACHE_MAX_GIB"))); } std::string repack_gguf(const std::string& gguf) { @@ -3041,9 +3041,13 @@ std::string repack_gguf(const std::string& gguf) { "1bit serve: ============================================================\n\n", checker.empty() ? "(unresolved)" : checker.c_str()); } + q4nx_cache_touch(out); // a hit is a use: without this the sweep's order is by creation + return out; + } + if (run_program({ python, checker, gguf, out }, true) == 0) { + q4nx_cache_touch(out); return out; } - if (run_program({ python, checker, gguf, out }, true) == 0) return out; std::fprintf(stderr, "1bit serve: cached Q4NX dir %s FAILED the declared-scale scan; " "discarding and re-repacking\n", out.c_str()); fs::remove_all(out); diff --git a/docs/npu.md b/docs/npu.md index bff42342d..727a20037 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -262,6 +262,16 @@ cache (`$ONEBIT_Q4NX_CACHE`, else `~/.cache/1bit/q4nx`) and hands the loader tha device streams its 5,120-byte bf16 q4_1 while the container on disk stays exact. The repack is where the bf16 rounding reappears for the device, never in the container. +**The cache is bounded, and the bound is least-recently-used (engine#1956).** After writing an +entry, `1bit serve` removes the least-recently-used entries until the cache is within +`ONEBIT_Q4NX_CACHE_MAX_GIB` (default 60; `0` or a negative number turns the sweep off). The entry +just written and any entry used in the last ten minutes are never removed. "Used" means reused, +not only written: a cache hit — a repacked GGUF found in the cache, or a device-form conversion +reused at load — marks the entry's directory as used now, so a container imported once and served +every day outlives a one-off imported yesterday. The value is read as a whole number of GiB: a +number too large to be a byte count is the largest bound that is one (it does not wrap to a small +one), and a value that is not a number gets the default rather than an unbounded cache. + **The packed lm-head is cached beside it (engine#1716).** A model whose head runs as int8 tiles packs those tiles on the host — gather `kpad × T` weights per tile, quantise each column to int8 with its own scale — and until this cache that happened on every serve. The image is a function diff --git a/npu/q4nx_device.cpp b/npu/q4nx_device.cpp index b6e014028..794e476a2 100644 --- a/npu/q4nx_device.cpp +++ b/npu/q4nx_device.cpp @@ -185,6 +185,9 @@ DeviceForm device_form(const std::string& model_path, const std::string& cache_d out.model = model; out.converted = true; out.reused = true; + // A reuse is a use. The cache's sweep (app/q4nx_cache.h) orders entries by this directory's mtime, which nothing + // else moves after the conversion is written; the stamp compares the SOURCE's mtime, so this does not disturb it. + fs::last_write_time(dir, fs::file_time_type::clock::now(), ec); std::fprintf(stderr, "1bit q4nx: reusing the device-form conversion of %s at %s\n", model_path.c_str(), dir.c_str()); return out; diff --git a/tests/q4nx_cache_test.cpp b/tests/q4nx_cache_test.cpp index ba5ddbcb8..32cd463a0 100644 --- a/tests/q4nx_cache_test.cpp +++ b/tests/q4nx_cache_test.cpp @@ -87,8 +87,42 @@ int main() { check(r.removed == 0 && fs::exists(root / "keepme-1") && fs::exists(root / "keepme-2"), "a non-positive bound disables the sweep"); + // (f) a hit is a use: the older entry, touched, outlives a younger one nobody opened. Without the touch the order is + // by creation and the entry served every day goes first. fs::remove_all(root, ec); - std::printf("q4nx cache: %d arm(s) over the sweep (bound, LRU order, keep, age window, disabled)\n", - failures ? 0 : 12); + make_entry(root, "served-daily", 4, 7 * 24 * 60); + make_entry(root, "one-off", 4, 24 * 60); + q4nx_cache_touch((root / "served-daily").string()); + r = q4nx_cache_evict(root.string(), "", 6LL << 20, /*min_age_seconds=*/0); + check(r.removed == 1, "one entry has to go (removed " + std::to_string(r.removed) + ")"); + check(fs::exists(root / "served-daily"), "the entry that was just used survives"); + check(!fs::exists(root / "one-off"), "and the unused one is the one removed"); + // ... and a touched entry is inside the age window, so a serve that just opened it is covered like a new one + make_entry(root, "one-off", 4, 24 * 60); + make_entry(root, "opened-now", 4, 7 * 24 * 60); + fs::remove_all(root / "served-daily", ec); + q4nx_cache_touch((root / "opened-now").string()); + r = q4nx_cache_evict(root.string(), "", 1LL << 20); + check(fs::exists(root / "opened-now") && !fs::exists(root / "one-off"), + "a touched entry is protected by the age window"); + q4nx_cache_touch((root / "absent").string()); + check(!fs::exists(root / "absent"), "touching a missing entry creates nothing"); + + // (g) the bound from ONEBIT_Q4NX_CACHE_MAX_GIB's text: never wraps, and a typo is not "unbounded" + const int64_t gib = 1LL << 30, top = (INT64_MAX >> 30) << 30; + check(q4nx_cache_bound_bytes(nullptr) == 60 * gib && q4nx_cache_bound_bytes("") == 60 * gib, "unset is the default"); + check(q4nx_cache_bound_bytes("100") == 100 * gib, "a plain number is that many GiB"); + check(q4nx_cache_bound_bytes("0") == 0 && q4nx_cache_bound_bytes("-5") == 0, "zero or negative disables the sweep"); + check(q4nx_cache_bound_bytes("8589934592") == top, "2^33 GiB clamps instead of overflowing"); + check(q4nx_cache_bound_bytes("17179869184") == top, "2^34 GiB is not a bound of 0"); + check(q4nx_cache_bound_bytes("17179869185") == top, "2^34+1 GiB is not a bound of 1 GiB"); + check(q4nx_cache_bound_bytes("99999999999999999999999") == top, "a number past long long clamps too"); + check(q4nx_cache_bound_bytes("sixty") == 60 * gib, "a value that is not a number is the default, not unbounded"); + check(q4nx_cache_bound_bytes("40G") == 40 * gib, "a number with a suffix keeps its number"); + check(q4nx_cache_bound_bytes("7", 9) == 7 * gib && q4nx_cache_bound_bytes(nullptr, 9) == 9 * gib, "the default is the caller's"); + + fs::remove_all(root, ec); + std::printf("q4nx cache: %d arm(s) over the sweep (bound, LRU order, keep, age window, disabled, touch on a hit, bound text)\n", + failures ? 0 : 26); return failures ? 1 : 0; } From 701c6dd500b11a0fa398fb040773d5a74ad9dfc5 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:37:54 -0400 Subject: [PATCH 12/98] npu: NPU_INFER_DUMP_OPS prints one magnitude line per projection and norm (refs #1503, follows #1822) (#1977) #1822's instrument, re-applied on current main with its open review answered: the variable is read once per process, the mean/absmax arithmetic and the line's text are tested, and the line is printed on the host arms and the device arm of both npu_gemm() and npu_gemm_rows(), so a host run and a device run can be compared line for line. Non-finite values are counted instead of vanishing from absmax. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 8 ++ docs/npu.md | 28 ++++++ npu/forward/include/ops-dump.h | 112 ++++++++++++++++++++++ npu/forward/include/q4nx_forward.h | 3 +- npu/forward/src/q4nx_forward.cpp | 36 ++++++- tests/ops_dump_test.cpp | 148 +++++++++++++++++++++++++++++ 6 files changed, 329 insertions(+), 6 deletions(-) create mode 100644 npu/forward/include/ops-dump.h create mode 100644 tests/ops_dump_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a1daffc0e..52552b533 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1213,6 +1213,14 @@ target_link_libraries(npu_pack_test PRIVATE onebit_npu) # golden fixture that pins them. target_include_directories(npu_pack_test PRIVATE npu/forward/include ${CMAKE_SOURCE_DIR}/tests) +# NPU_INFER_DUMP_OPS (npu/forward/include/ops-dump.h, engine#1503): the on/off rule, the mean|x|/absmax arithmetic and +# the line's text. Header-only, no device. `add_dependencies(onebit ...)` is what gets it built in CI, whose target +# list is explicit (tools/ci/ci_targets_check.py). +add_executable(ops_dump_test tests/ops_dump_test.cpp) +target_include_directories(ops_dump_test PRIVATE npu/forward/include) +add_test(NAME ops_dump COMMAND ops_dump_test) +add_dependencies(onebit ops_dump_test) + # The device weight layout contract (q4nx_weight_layout): pure index math, no device # and no runtime. CI builds an explicit target list, so the check compiles itself with # the build's compiler at test time (tests/npu_blocked_layout_test.sh) rather than being diff --git a/docs/npu.md b/docs/npu.md index 727a20037..e77642369 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2004,6 +2004,34 @@ reaps or signals the holder -- killing another lane's live serve is the box-cras Qwen3-4B, whose kernel sets are on this box): each model solo, then both at once, three reps, every number stamped with the build that produced it. +### Finding where a forward diverges: `NPU_INFER_DUMP_DIR` and `NPU_INFER_DUMP_OPS` + +Two diagnostic switches of the NPU forward. Both are off by default, neither changes a result, and neither +can fail a run. + +| variable | what it writes | answers | +|---|---|---| +| `NPU_INFER_DUMP_DIR=` | `/layer_.f32` per layer: the residual stream after that layer, M x H little-endian f32, row-major. The directory must exist; a file that cannot be written is reported on stderr and the run goes on. | which **layer** two runs part at (per-layer cosine against a reference trace) | +| `NPU_INFER_DUMP_OPS=1` | one stderr line per projection and per weighted RMSNorm: `[op] L rows= \|x\| mean absmax ` | which **op** inside the layer | + +`NPU_INFER_DUMP_OPS` is on for any value other than empty or `0`, and is read once per process. + +``` + [op] L00 rmsnorm rows=1 |x| mean 0.0143171 absmax 3.01935 + [op] L00 ffn_gate_up rows=1 |x| mean 0.0258482 absmax 1.25728 + [op] L00 ffn_down rows=1 |x| mean 0.00315921 absmax 0.673885 +``` + +- The tag is the one the projection's call site already passes (`attn_qkv`, `attn_output`, `ffn_gate_up`, + `ffn_down`, `ssm_out`, ...). Layer norms are `rmsnorm`; on a family with per-head q/k norms those print as + `q_norm` and `k_norm`, one line per head. +- The numbers are over row 0 of the result (its first N values). A batched projection prints one line with + `rows=M`; a norm is called per row, so a batched pass prints one norm line per row. +- A NaN or Inf is counted (`nonfinite 2/4096` at the end of the line) and left out of the mean and the maximum. +- The same lines come out on the host arms (`NPU_INFER_HOST_GEMM=1` or `=i8`) and on the device arm, in the same + order and number, so the two logs of one prompt can be compared with `diff`. Start there: the first line whose + numbers differ names the op. + ### When a load follows an evict On the appliance the NPU model is swapped by evicting it and starting a fresh `1bit` for the next diff --git a/npu/forward/include/ops-dump.h b/npu/forward/include/ops-dump.h new file mode 100644 index 000000000..42b3d0fd6 --- /dev/null +++ b/npu/forward/include/ops-dump.h @@ -0,0 +1,112 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ops-dump.h -- NPU_INFER_DUMP_OPS=1 prints mean|x| and absmax after every projection and every weighted RMSNorm, +// tagged with the call site's own name: +// +// [op] L00 rmsnorm rows=1 |x| mean 0.0105 absmax 0.0598 +// [op] L00 attn_qkv rows=1 |x| mean 1.3877 absmax 31.8909 +// +// Why it exists: NPU_INFER_DUMP_DIR (residual-dump.h) writes the residual stream at LAYER boundaries, which says +// that a model diverges and not which op did it (engine#1503: a residual already 1e4 too large at layer 0 while +// the container matched its pack tensor for tensor). The insertion points are `rmsnorm()`, `npu_gemm()` and +// `npu_gemm_rows()`; the tag each projection already carries is the label, so there is no second name table. +// The line is printed on the host arms and on the device arm alike, so the same run under NPU_INFER_HOST_GEMM=1 +// and without it can be compared line for line: a batched projection prints one line with rows=M on every arm, +// also when the device arm walks it in blocks. The per-head q and k norms of a qk-norm family are tagged +// "q_norm" and "k_norm" (one line per head), so they cannot be mistaken for the layer norms. +// +// Row 0 only (the first N values of a batched result): one line per call, which is what localises a divergence. +// A norm is called per row, so a batched pass prints one norm line per row. +// +// Off unless the variable is set to something other than "" or "0". The variable is read ONCE per process +// (ops_dump_enabled() is called per projection per token, and off has to cost a load and a branch, not a +// getenv). It can never fail a run: no allocation, no file, stderr only. +#pragma once + +#include +#include +#include +#include + +namespace onebit::npu { + +// The rule itself, on a value: what the tests pin (the cached read below cannot be re-armed inside one process). +inline bool ops_dump_value_on(const char* v) { return v && *v && std::strcmp(v, "0") != 0; } + +inline bool ops_dump_enabled() { + static const bool on = ops_dump_value_on(std::getenv("NPU_INFER_DUMP_OPS")); + return on; +} + +// A projection that is carried out as several inner calls (the device arm's blocked launches) holds one of these +// across them, so the inner calls stay quiet and the outer call prints the one line. Per thread. +inline int& ops_dump_mute_depth() { + thread_local int depth = 0; + return depth; +} +struct OpsDumpMute { + OpsDumpMute() { ++ops_dump_mute_depth(); } + ~OpsDumpMute() { --ops_dump_mute_depth(); } + OpsDumpMute(const OpsDumpMute&) = delete; + OpsDumpMute& operator=(const OpsDumpMute&) = delete; +}; +inline bool ops_dump_muted() { return ops_dump_mute_depth() > 0; } + +struct OpsStats { + double mean = 0.0; // mean of |x[i]| + float absmax = 0.0f; // max of |x[i]| + int nonfinite = 0; // NaN/Inf count: they are counted, and left out of mean and absmax +}; + +// The whole content of the diagnostic. A NaN would otherwise vanish from absmax (`a > mx` is false) and turn the +// mean into "nan" with no count, so non-finite values are reported as a number of their own. +inline OpsStats ops_stats(const float* x, int n) { + OpsStats s; + if (!x || n <= 0) return s; + double sum = 0.0; + int finite = 0; + for (int i = 0; i < n; ++i) { + if (!std::isfinite(x[i])) { ++s.nonfinite; continue; } + const float a = std::fabs(x[i]); + sum += a; + if (a > s.absmax) s.absmax = a; + ++finite; + } + if (finite > 0) s.mean = sum / finite; + return s; +} + +// Formats the line into `buf` (always NUL-terminated when cap > 0) and returns snprintf's count; 0 when there is +// nothing to say. Separate from the print so a test can match the text. +inline int ops_dump_line(char* buf, size_t cap, const char* tag, int layer, int rows, const float* x, int n) { + if (!buf || cap == 0) return 0; + buf[0] = '\0'; + if (!x || n <= 0) return 0; + const OpsStats s = ops_stats(x, n); + if (s.nonfinite) + return std::snprintf(buf, cap, " [op] L%02d %-12s rows=%d |x| mean %.6g absmax %.6g nonfinite %d/%d\n", layer, + tag ? tag : "?", rows, s.mean, (double)s.absmax, s.nonfinite, n); + return std::snprintf(buf, cap, " [op] L%02d %-12s rows=%d |x| mean %.6g absmax %.6g\n", layer, tag ? tag : "?", + rows, s.mean, (double)s.absmax); +} + +inline void ops_dump(const char* tag, int layer, int rows, const float* x, int n) { + if (ops_dump_muted()) return; + char buf[192]; + if (ops_dump_line(buf, sizeof buf, tag, layer, rows, x, n) > 0) std::fputs(buf, stderr); +} + +} // namespace onebit::npu diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index ffc67b3fd..2ee8c1da2 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -283,7 +283,8 @@ class Q4nxNpuForward { // The same elementwise gate*up on caller memory, so a batched pass can read its ACT row and write its HH // row where they already are instead of staging both through a vector (a copy on each side). bool mlp(const float* gu, float* h, int im); - float rmsnorm(const float* x, const float* w, int n, float* out) const; + // `tag` is the NPU_INFER_DUMP_OPS label (ops-dump.h); the per-head q/k norms pass their own. + float rmsnorm(const float* x, const float* w, int n, float* out, const char* tag = "rmsnorm") const; float rmsnorm_noweight(const float* x, int n, float* out) const; bool run_lm_head(const std::vector& h, std::vector& logits); // M rows of hidden state -> M x NV logits, row-major. Same head, same packed tiles: the rows ride diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index d8ccbc971..b5334363a 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -34,6 +34,7 @@ #include #include #include "batch_check_compare.h" +#include "ops-dump.h" #include "residual-dump.h" #include #include @@ -3211,6 +3212,14 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, } } +// NPU_INFER_DUMP_OPS (ops-dump.h): one line per projection, tagged with the tag its caller already passes +// ("attn_qkv", "ssm_out", ...). Called where a projection's result is final -- both host arms and the device arm -- +// so a host run and a device run print the same lines. Row 0 of the result; off is one cached bool. +static inline void op_line(const char* tag, int layer, int rows, const std::vector& out, int n) { + if (onebit::npu::ops_dump_enabled() && n > 0 && out.size() >= (size_t)n) + onebit::npu::ops_dump(tag, layer, rows, out.data(), n); +} + void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const float* Bmat, std::vector& out, int wkey, const char* sc, int a_k) { if (host_gemm_) { @@ -3221,9 +3230,11 @@ void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const floa out.assign((size_t)N, 0.0f); if (host_gemm_i8_) { host_gemm_i8(A, a_k, 1, K, N, Bmat, wkey, out.data()); + op_line(sc, cur_layer_, 1, out, N); return; } q4nx_host_gemm_f32_padded(A, a_k, 1, K, N, Bmat, out.data()); + op_line(sc, cur_layer_, 1, out, N); return; } npu::target::GemvArgs a; @@ -3232,7 +3243,8 @@ void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const floa a.a_k = a_k; a.tiles = &Q4nxNpuForward::gemv_tile_cb; a.tiles_self = this; std::string err; - if (!xd_ || !xd_->gemv(a, out, err)) err_ = err.empty() ? "npu_gemm: no target" : err; + if (!xd_ || !xd_->gemv(a, out, err)) { err_ = err.empty() ? "npu_gemm: no target" : err; return; } + op_line(sc, cur_layer_, 1, out, N); } void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N, @@ -3244,9 +3256,11 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N out.assign((size_t)M * N, 0.0f); if (host_gemm_i8_) { host_gemm_i8(A, a_k, M, K, N, Bmat, wkey, out.data()); + op_line(sc, cur_layer_, M, out, N); return; } q4nx_host_gemm_f32_padded(A, a_k, M, K, N, Bmat, out.data()); + op_line(sc, cur_layer_, M, out, N); return; } // The device's activation BO holds exactly max_batch_rows() rows (the M=128 prefill design), and a wider @@ -3266,6 +3280,9 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N if (note && note[0]) { std::fprintf(stderr, " [batch] %s\n", note); note_said = true; } } out.assign((size_t)M * N, 0.0f); + // NPU_INFER_DUMP_OPS: the blocks are one projection, so they print one line (rows=M, row 0), as the host + // arms do; the inner launches are muted for the length of the walk. + onebit::npu::OpsDumpMute ops_mute; for (int b0 = 0; b0 < M && err_.empty(); b0 += blk) { const int rows = std::min(blk, M - b0); std::vector part; @@ -3277,6 +3294,12 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N } std::copy(part.begin(), part.begin() + (size_t)rows * N, out.begin() + (size_t)b0 * N); } + // Printed by hand: op_line() would be muted by this scope's own guard. A nested walk stays quiet. + if (err_.empty() && onebit::npu::ops_dump_enabled() && onebit::npu::ops_dump_mute_depth() == 1) { + char line[192]; + if (onebit::npu::ops_dump_line(line, sizeof line, sc, cur_layer_, M, out.data(), N) > 0) + std::fputs(line, stderr); + } return; } npu::target::GemvArgs a; @@ -3286,7 +3309,8 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N a.rows = true; a.decode_twin = decode_twin; a.tiles = &Q4nxNpuForward::gemv_tile_cb; a.tiles_self = this; std::string err; - if (!xd_ || !xd_->gemv(a, out, err)) err_ = err.empty() ? "npu_gemm_rows: no target" : err; + if (!xd_ || !xd_->gemv(a, out, err)) { err_ = err.empty() ? "npu_gemm_rows: no target" : err; return; } + op_line(sc, cur_layer_, M, out, N); } // The folded container's sign table: magic "HAD1", u32 count, count x (u32 width, u32 offset), @@ -4021,7 +4045,7 @@ int Q4nxNpuForward::max_batch_rows() const { // -------------------------------------------------------------------------- // host helpers (float32, matching the Python reference) // -------------------------------------------------------------------------- -float Q4nxNpuForward::rmsnorm(const float* x, const float* w, int n, float* out) const { +float Q4nxNpuForward::rmsnorm(const float* x, const float* w, int n, float* out, const char* tag) const { LhScope _tlh(LH_RMS); double ss = 0.0; for (int i = 0; i < n; i++) ss += (double)x[i] * (double)x[i]; @@ -4032,6 +4056,8 @@ float Q4nxNpuForward::rmsnorm(const float* x, const float* w, int n, float* out) } else { for (int i = 0; i < n; i++) out[i] = x[i] * inv * w[i]; } + // NPU_INFER_DUMP_OPS: the norm's own output, so a divergence can be put on the norm and not on its layer. + if (onebit::npu::ops_dump_enabled()) onebit::npu::ops_dump(tag, cur_layer_, 1, out, n); return inv; } @@ -4475,12 +4501,12 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int if (sem_.qk_norm) { std::vector t(HD); for (int h = 0; h < NH; h++) { - rmsnorm(&q[(size_t)h * HD], q_norm_[cur_layer_].data(), HD, t.data()); + rmsnorm(&q[(size_t)h * HD], q_norm_[cur_layer_].data(), HD, t.data(), "q_norm"); memcpy(&q[(size_t)h * HD], t.data(), sizeof(float) * HD); } if (append_kv) { for (int h = 0; h < NKV; h++) { - rmsnorm(&k[(size_t)h * HD], k_norm_[cur_layer_].data(), HD, t.data()); + rmsnorm(&k[(size_t)h * HD], k_norm_[cur_layer_].data(), HD, t.data(), "k_norm"); memcpy(&k[(size_t)h * HD], t.data(), sizeof(float) * HD); } } diff --git a/tests/ops_dump_test.cpp b/tests/ops_dump_test.cpp new file mode 100644 index 000000000..71496d7d4 --- /dev/null +++ b/tests/ops_dump_test.cpp @@ -0,0 +1,148 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ops_dump_test -- NPU_INFER_DUMP_OPS (ops-dump.h): the on/off rule, the arithmetic of the line, its text, and +// that the variable is read once per process. No device, no container, no model. +// +// ops_dump_test the checks below +// ops_dump_test --enabled prints 1 or 0: what ops_dump_enabled() says in a process started with this environment +#include "ops-dump.h" + +#include +#include +#include +#include +#include +#include + +#include + +static int failures = 0; +static void check(bool ok, const std::string& what) { + if (ok) return; + ++failures; + std::printf("FAIL %s\n", what.c_str()); +} + +// What a fresh process decides for one environment: the cached read cannot be re-armed in this one. +static int child_says(const char* self, const char* env_prefix) { + const std::string cmd = std::string(env_prefix) + " '" + self + "' --enabled"; + FILE* p = popen(cmd.c_str(), "r"); + if (!p) return -1; + int c = std::fgetc(p); + pclose(p); + return c == '1' ? 1 : c == '0' ? 0 : -1; +} + +int main(int argc, char** argv) { + using namespace onebit::npu; + if (argc > 1 && !std::strcmp(argv[1], "--enabled")) { + std::printf("%d\n", ops_dump_enabled() ? 1 : 0); + return 0; + } + + // The rule. + check(!ops_dump_value_on(nullptr), "unset is off"); + check(!ops_dump_value_on(""), "an empty value is off"); + check(!ops_dump_value_on("0"), "\"0\" is off"); + check(ops_dump_value_on("1"), "\"1\" is on"); + check(ops_dump_value_on("00"), "anything else is on"); + + // The rule as a process sees it. + check(child_says(argv[0], "env -u NPU_INFER_DUMP_OPS") == 0, "process: unset is off"); + check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=") == 0, "process: empty is off"); + check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=0") == 0, "process: 0 is off"); + check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=1") == 1, "process: 1 is on"); + + // Read once: whatever this process decided first, a later change of the variable does not move it. + const bool first = ops_dump_enabled(); + setenv("NPU_INFER_DUMP_OPS", first ? "0" : "1", 1); + check(ops_dump_enabled() == first, "the variable is read once per process"); + + // The arithmetic: mean of |x| over n, and the largest |x|. + const float x[] = {1.0f, -3.0f, 2.0f, 0.0f}; + OpsStats s = ops_stats(x, 4); + check(s.mean == 1.5, "mean|x| of {1,-3,2,0} is 1.5"); + check(s.absmax == 3.0f, "absmax of {1,-3,2,0} is 3 (the negative one)"); + check(s.nonfinite == 0, "no non-finite value in a finite array"); + s = ops_stats(x, 2); + check(s.mean == 2.0 && s.absmax == 3.0f, "n bounds the read: {1,-3} gives mean 2, absmax 3"); + s = ops_stats(x, 0); + check(s.mean == 0.0 && s.absmax == 0.0f, "n = 0 is all zeros, not a division by zero"); + s = ops_stats(nullptr, 4); + check(s.mean == 0.0 && s.absmax == 0.0f, "a null array is all zeros"); + + // Non-finite values are counted and kept out of the two numbers. + const float bad[] = {1.0f, std::numeric_limits::quiet_NaN(), -5.0f, std::numeric_limits::infinity()}; + s = ops_stats(bad, 4); + check(s.nonfinite == 2, "NaN and Inf are counted"); + check(s.mean == 3.0 && s.absmax == 5.0f, "mean and absmax are over the finite values"); + + // The text. + char buf[192]; + int n = ops_dump_line(buf, sizeof buf, "attn_qkv", 7, 16, x, 4); + check(std::string(buf) == " [op] L07 attn_qkv rows=16 |x| mean 1.5 absmax 3\n", std::string("line text, got: ") + buf); + check(n == (int)std::strlen(buf), "the count is the line's length"); + ops_dump_line(buf, sizeof buf, "rmsnorm", 0, 1, bad, 4); + check(std::string(buf) == " [op] L00 rmsnorm rows=1 |x| mean 3 absmax 5 nonfinite 2/4\n", + std::string("non-finite line text, got: ") + buf); + check(ops_dump_line(buf, sizeof buf, "t", 0, 1, nullptr, 4) == 0 && buf[0] == '\0', "a null array prints nothing"); + check(ops_dump_line(buf, sizeof buf, "t", 0, 1, x, 0) == 0 && buf[0] == '\0', "n = 0 prints nothing"); + // The longest tags in the tree are 11 characters; the column has to hold them, or their lines sit one out of step. + ops_dump_line(buf, sizeof buf, "ffn_gate_up", 3, 1, x, 4); + check(std::string(buf) == " [op] L03 ffn_gate_up rows=1 |x| mean 1.5 absmax 3\n", std::string("11-char tag, got: ") + buf); + + // The mute: held across the inner calls of one projection, it nests and it ends with its scope. + check(!ops_dump_muted(), "not muted to begin with"); + { + OpsDumpMute outer; + check(ops_dump_muted(), "muted inside a scope"); + { OpsDumpMute inner; check(ops_dump_muted(), "still muted when nested"); } + check(ops_dump_muted(), "the inner scope ending does not unmute the outer one"); + } + check(!ops_dump_muted(), "unmuted when the scope ends"); + // ops_dump() itself prints nothing while muted: stderr goes to a file for the two calls, and the file holds one line. + { + char path[] = "/tmp/ops_dump_test_XXXXXX"; + const int fd = mkstemp(path); + check(fd >= 0, "mkstemp"); + if (fd >= 0) { + std::fflush(stderr); + const int saved = dup(2); + dup2(fd, 2); + { OpsDumpMute m; ops_dump("muted_tag", 1, 1, x, 4); } + ops_dump("heard_tag", 1, 1, x, 4); + std::fflush(stderr); + dup2(saved, 2); + close(saved); + close(fd); + std::string got; + if (FILE* f = std::fopen(path, "r")) { + char line[256]; + while (std::fgets(line, sizeof line, f)) got += line; + std::fclose(f); + } + std::remove(path); + check(got == " [op] L01 heard_tag rows=1 |x| mean 1.5 absmax 3\n", "muted call silent, next call heard; got: " + got); + } + } + + char small[8]; + ops_dump_line(small, sizeof small, "attn_qkv", 7, 16, x, 4); + check(std::strlen(small) == sizeof small - 1, "a short buffer is truncated and terminated"); + + std::printf("%s\n", failures ? "FAILED" : "ops_dump: ok"); + return failures ? 1 : 0; +} From c6ef5b110d0f5a9812da4abded931db47907c963 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:46:53 -0400 Subject: [PATCH 13/98] npu-pack: read a FastFlowLM NPU2 model source's container, report its genre, and match both layer-name spellings (engine#1786, item F1) (#1974) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tools/flm: report a FastFlowLM NPU2 model source's container without a device (engine#1786, item F1) F1 of the FLM 1.0.12 parity work is 'every FastFlowLM/*-NPU2 repo imports and serves on --device npu'. The first row showed the packs are a genre our importer does not know: model.q4nx is a safetensors container (8-byte LE header length + JSON of name -> {dtype, shape, data_offsets}), the tensor NAMES are the model's own HF names (no graph, no model.onnx), and the pack holds only BF16 (norms/embedding) and I8 (matrices, packed -- down_proj is [640,1280] for a 640-wide FFN). This tool reads the header only: no weights, no device, no runtime. It exists so each of the 59 rows in the board's FLM NPU2 section can be characterised in one command and the importer's work list is a fact rather than a guess. Measured on FastFlowLM/Gemma3-270M-NPU2 (0.54 GB): header 26496 B, 237 tensors, payload 0.503 GB, dtypes {BF16: 110, I8: 127}, 18 layers, metadata keys [] (empty -- this pack has no __metadata__ block). * npu-pack: read a FastFlowLM NPU2 model source's container (engine#1786, item F1) The first row of the F1 sweep showed why the importer refuses every NPU2 pack: its genre is an ONNX graph plus external data, while an NPU2 model source is a safetensors container named model.q4nx (8-byte LE header length, then JSON of name -> {dtype, shape, data_offsets}). Measured on FastFlowLM/Gemma3-270M-NPU2: header 26496 B, 237 tensors, payload 0.503 GB, dtypes {BF16: 110, I8: 127}, 18 layers, no __metadata__. This is the reader for that container, header-only -- no weights, no device -- exported so the half that can be wrong silently is testable without a pack. Each failure names what was wrong, because the caller's next move depends on which: a truncated header must not read as "a pack with no tensors". Arms in the existing registered test (no new target): the real two-tensor shape (BF16 vector + I8 matrix at twice its logical width), __metadata__ skipped rather than counted, a truncated header named, and an implausible header length refused by the number so a mis-typed length cannot make the caller allocate it. Verified by -fsyntax-only of pack_import.cpp and the test against a configured tree's json dep; NOT built or run: disk is at 146 GB, below this repo's 150 GB floor and the 175 GB build threshold. The syntax check found one real defect already (my ContainerTensor collided with the existing ONNX-role type of that name; renamed Q4nxTensor). Next: wire it into plan()/info so an NPU2 pack reports its inventory instead of refusing for model.onnx, then import -> --device npu -> known-answer prompt -> tok/s under npu-hold. * npu-pack: recognise and report a FastFlowLM NPU2 model source instead of refusing at the first byte (engine#1786, item F1) Reading the container header (previous commit) is only half of it: plan() then went on to open a graph the pack does not have, so `npu-pack info` answered a 0.5 GB model source with "cannot open /model.onnx" -- a missing file on a pack that has no graph by design. plan() now detects the genre first (model.q4nx present, no resolvable graph file), reads the container, and records the census on the Plan: tensor count, payload bytes, a dtype map, and a note saying what the genre still needs. manifest() reports it under "container", so info prints it and a test can pin it. Measured on FastFlowLM/Gemma3-270M-NPU2: 237 tensors, 0.503 GB payload, {BF16: 110, I8: 127}. The note is deliberately explicit that the pack is RECOGNISED AND REPORTED, NOT IMPORTABLE: reading a container as a weights source is the container-genre plan this item still owes, and a plan that quietly listed 237 tensors as importable would be worse than the refusal it replaces. Syntax-checked against a configured tree's json dep; still not built or run (disk 149 GB, under the 175 GB build threshold). * npu-pack: pin the container genre's report, including that it is not importable (engine#1786, item F1) The plan() detection had no arm, which for this change is the difference between a report and a claim: a plan that listed a container's tensors as importable would be worse than the refusal it replaces, because it would assert a compute mapping this importer does not have. The arm builds a pack directory that IS a container and has NO graph, then checks plan() reports the tensor count, the payload the offsets imply, both dtypes, and a note containing "not importable" -- and that manifest() carries it under "container", which is what npu-pack info prints. Syntax-checked against a configured tree's json dep; still not built or run (disk 148 GB, under the 175 GB build threshold). * npu-pack: fix two bugs in my own container arms now that they build and run (engine#1786) Built npu_pack_import_test for the first time and ran it: two of my four arms failed, and both were the arm's fault, not the reader's. Worth recording precisely because the reader was right either way: - the payload arm expected 819200 but the reader returned 820480 -- and the reader was correct: the sum of the data_offsets SPANS is 1280 (first tensor) + 819200 (second). I had summed only the second span, i.e. the arm asserted a number I had derived by hand instead of the number the offsets imply. - the implausible-length arm left its std::ofstream alive while the reader opened the same path, so the reader saw an empty file and reported "shorter than the 8-byte header length" -- the arm was testing its own missing flush. Scoped the writer so it closes first. With those fixed: all npu-pack checks pass (pack leg SKIPPED, no pack directory given), which is the whole suite, so the container reader and the genre report are now verified on this box rather than only syntax-checked. Also: the two failures are a small demonstration of the night's recurring rule -- a check that reports a number you cannot re-derive is a check you have to run to trust. * npu: the per-layer prefix has two spellings, so a failed lookup retries the other (engine#1786, item F1) Three of the fifty-eight FastFlowLM NPU2 model sources spell the per-layer prefix `model.layer.` (singular) where this forward addresses `model.layers.` (measured by header census: engine#1786). Every other tensor in those containers matches by name, so the pack would miss its layer tensors by spelling alone and the failure would look like a missing tensor rather than a naming difference. model_tensor_by_name() now retries the lookup with the other spelling AFTER an exact match has failed, so a container that carries both is unaffected -- the exact match still wins. The rewrite itself is model_tensor_name_alt(), a pure static-inline helper in model.h: it swaps the leading prefix once and leaves everything after it byte-identical (a global replace would corrupt a tensor whose name carries the same text further along, and a near-miss prefix like `model.layersX.` must be left alone). Five arms in npu_forward_loader_test (which compiles model.c and includes the header): both round trips, a model-level tensor with no alternative, the near-miss prefix, and a too-small buffer reporting no alternative rather than truncating. Built and run on this box: all cases ok. The arms earned their keep immediately: the first version skipped the SOURCE prefix using the TARGET's length (12 vs 13 characters), which produced `model.layer.s.0.…`; two of the five arms failed and named it. --------- Co-authored-by: bcloud --- npu/forward/include/model.h | 28 +++++++ npu/forward/src/model.c | 12 +++ npu/pack_import.cpp | 114 +++++++++++++++++++++++++++++ npu/pack_import.h | 31 ++++++++ tests/npu_forward_loader_test.c | 20 +++++ tests/npu_pack_import_test.cpp | 102 ++++++++++++++++++++++++++ tools/flm/q4nx_container_report.py | 77 +++++++++++++++++++ 7 files changed, 384 insertions(+) create mode 100755 tools/flm/q4nx_container_report.py diff --git a/npu/forward/include/model.h b/npu/forward/include/model.h index 86b60ab0a..05ddb7346 100644 --- a/npu/forward/include/model.h +++ b/npu/forward/include/model.h @@ -18,6 +18,7 @@ #include #include +#include #include "common.h" #ifdef __cplusplus @@ -130,6 +131,33 @@ void* model_tensor_data(ModelWeights* mw, TensorDesc* desc); // Used for architecture-specific tensors the fixed fields do not cover. // (Distinct from the legacy model_find_tensor(name, mw), which returns an index.) TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name); +// engine#1786 (item F1): three FastFlowLM NPU2 model sources spell the per-layer prefix `model.layer.` (singular) +// where this forward addresses `model.layers.`. Writes the OTHER spelling into `out` and returns 1 when `name` carries +// either prefix, 0 when it does not. Only the leading prefix is swapped, once: a caller that wants a fallback retries +// the lookup with this name AFTER an exact match has failed, so a container carrying both spellings still resolves +// exactly first. Pure string work, exported so a test can pin it without a container. +static inline int model_tensor_name_alt(const char* name, char* out, size_t cap) { + static const char kPlural[] = "model.layers."; + static const char kSingular[] = "model.layer."; + const size_t nl = sizeof(kPlural) - 1, ns = sizeof(kSingular) - 1; + const char* to; + size_t fl; + if (!name || !out || cap == 0) return 0; + if (strncmp(name, kPlural, nl) == 0) { /* model.layers. -> model.layer. */ + to = kSingular; fl = nl; /* fl is the MATCHED prefix's length, not the target's */ + } else if (strncmp(name, kSingular, ns) == 0) { /* model.layer. -> model.layers. */ + to = kPlural; fl = ns; + } else { + return 0; + } + const char* rest = name + fl; /* everything after the prefix, unchanged */ + const size_t rl = strlen(rest); + const size_t tl = (to == kPlural) ? nl : ns; + if (tl + rl + 1 > cap) return 0; + memcpy(out, to, tl); + memcpy(out + tl, rest, rl + 1); + return 1; +} void model_print_info(ModelWeights* mw); int model_find_tensor(const char* name, ModelWeights* mw); diff --git a/npu/forward/src/model.c b/npu/forward/src/model.c index a5983c9c3..21a3d119c 100644 --- a/npu/forward/src/model.c +++ b/npu/forward/src/model.c @@ -1157,12 +1157,24 @@ void model_free(ModelWeights* mw) { free(mw); } +// engine#1786 (item F1): the per-layer prefix, in the two spellings the NPU2 model sources use. `layers.` is this +// forward's own spelling; `layer.` is what three of the fifty-eight containers carry (`model.layer.0.input_layernorm…`), +// where every other tensor matches by name. Swapping one prefix is a spelling fallback, not a rename: it happens only +// after an exact match has failed, so a container that carries both is unaffected. + TensorDesc* model_tensor_by_name(ModelWeights* mw, const char* name) { if (!mw || !mw->all_tensors || !name) return NULL; for (int i = 0; i < mw->all_count; i++) { const char* n = mw->all_tensors[i].name; if (n[0] && strcmp(n, name) == 0) return &mw->all_tensors[i]; } + // Exact match failed: try the other per-layer spelling before reporting absence (engine#1786). + char alt[512]; + if (!model_tensor_name_alt(name, alt, sizeof(alt))) return NULL; + for (int i = 0; i < mw->all_count; i++) { + const char* n = mw->all_tensors[i].name; + if (n[0] && strcmp(n, alt) == 0) return &mw->all_tensors[i]; + } return NULL; } diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 2fb204870..78bc8fa7e 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1704,6 +1704,85 @@ bool fconst_set_block_headers(FconstGeom& g, const std::vector& inner) std::string find_token_fconst_exported(const std::string& pack_dir) { return find_token_fconst(pack_dir); } +// engine#1786 (item F1): read a FastFlowLM NPU2 model source's container header. Safetensors layout: an 8-byte +// little-endian header length, then that many bytes of JSON. `__metadata__` is a string map, not a tensor, and the +// vendor's converter does not always write one -- so its presence is not a genre test and its absence is not an error. +// Every failure names what was wrong, because the caller's next move ("refuse this pack" vs "read it") depends on which. +Q4nxContainer read_q4nx_container(const std::string& path) { + Q4nxContainer out; + std::ifstream f(path, std::ios::binary); + if (!f) { + out.why = "cannot open " + path; + return out; + } + unsigned char len[8] = {0}; + if (!f.read(reinterpret_cast(len), 8)) { + out.why = path + ": shorter than the 8-byte safetensors header length"; + return out; + } + int64_t n = 0; + for (int i = 7; i >= 0; --i) n = (n << 8) | int64_t(len[i]); + // A header this large is not a container: the largest real one seen on the NPU2 sources is 26,496 B (237 tensors). + // Refusing above 64 MiB keeps a mis-typed length from making the caller allocate whatever the file claims. + if (n <= 0 || n > (int64_t(64) << 20)) { + out.why = path + ": implausible header length " + std::to_string(n) + " B"; + return out; + } + std::string hdr(size_t(n), '\0'); + if (!f.read(&hdr[0], n)) { + out.why = path + ": truncated header (" + std::to_string(n) + " B claimed)"; + return out; + } + out.header_bytes = n; + json j; + try { + j = json::parse(hdr); + } catch (const std::exception& e) { + out.why = path + ": header is not JSON (" + e.what() + ")"; + return out; + } + if (!j.is_object()) { + out.why = path + ": header is not a JSON object"; + return out; + } + for (auto it = j.begin(); it != j.end(); ++it) { + if (it.key() == "__metadata__") continue; + if (!it.value().is_object()) { + out.why = path + ": tensor '" + it.key() + "' is not an object"; + out.tensors.clear(); + return out; + } + Q4nxTensor t; + t.name = it.key(); + const json& v = it.value(); + t.dtype = v.value("dtype", std::string()); + if (t.dtype.empty()) { + out.why = path + ": tensor '" + t.name + "' states no dtype"; + out.tensors.clear(); + return out; + } + if (v.contains("shape") && v["shape"].is_array()) + for (const json& d : v["shape"]) t.shape.push_back(d.is_number_integer() ? d.get() : 0); + if (!v.contains("data_offsets") || !v["data_offsets"].is_array() || v["data_offsets"].size() != 2 || + !v["data_offsets"][0].is_number_integer() || !v["data_offsets"][1].is_number_integer()) { + out.why = path + ": tensor '" + t.name + "' has no [begin, end] data_offsets"; + out.tensors.clear(); + return out; + } + t.begin = v["data_offsets"][0].get(); + t.end = v["data_offsets"][1].get(); + if (t.begin < 0 || t.end < t.begin) { + out.why = path + ": tensor '" + t.name + "' has data_offsets [" + std::to_string(t.begin) + ", " + + std::to_string(t.end) + "]"; + out.tensors.clear(); + return out; + } + out.payload_bytes += t.end - t.begin; + out.tensors.push_back(std::move(t)); + } + return out; +} + // engine#1638: the geometry READ (not just its dims), so a test can assert where each number came from -- the // text_config lift and the top-level-wins rule are both properties of load_geometry(), not of read_dims(). json load_geometry_exported(const std::string& pack_dir, std::string* source) { return load_geometry(pack_dir, source); } @@ -2840,6 +2919,18 @@ std::string Plan::manifest() const { {"rule", "length == N * (K/128 + 1) * 68"}}; j["graph_weights"] = g; } + // engine#1786 (item F1): a FastFlowLM NPU2 model source has no graph, so the census above is absent and this is + // what `info` reports instead -- a count and a dtype map taken from the container's own header, plus what the genre + // still needs. Measured on FastFlowLM/Gemma3-270M-NPU2 (0.54 GB): 237 tensors, 0.503 GB, {BF16: 110, I8: 127}. + if (container_tensors >= 0) { + json c; + c["format"] = "safetensors (model.q4nx)"; + c["tensors"] = container_tensors; + c["payload_bytes"] = container_payload_bytes; + c["dtypes"] = container_dtypes; + c["note"] = container_note; + j["container"] = c; + } j["mlp_section"] = {{"gate_up_units_measured", mlp_gu_units}, {"down_groups_measured", mlp_down_groups}, {"note", mlp_note}}; @@ -2891,6 +2982,29 @@ Plan plan(const std::string& pack_dir) { p.dir = pack_dir; p.pack = fs::path(pack_dir).lexically_normal().filename().string(); + // engine#1786 (item F1): the FastFlowLM NPU2 model sources are a safetensors container named model.q4nx with no + // graph at all. Detect that genre before anything looks for a graph, report what the container holds, and say what + // it still needs -- the alternative is the first byte failing with "cannot open /model.onnx", which describes + // a missing file on a pack that has no graph by design. + { + std::error_code ec; + const std::string container = pack_dir + "/model.q4nx"; + const std::string graph_now = graph_filename(pack_dir); + if (fs::exists(container, ec) && !fs::exists(pack_dir + "/" + graph_now, ec)) { + const Q4nxContainer c = read_q4nx_container(container); + p.container_tensors = int64_t(c.tensors.size()); + p.container_payload_bytes = c.payload_bytes; + for (const auto& t : c.tensors) p.container_dtypes[t.dtype]++; + p.container_note = c.why.empty() + ? "the pack is a safetensors container (model.q4nx): " + std::to_string(c.tensors.size()) + + " tensors, " + std::to_string(c.payload_bytes) + " B of payload, no graph. Reading it as a" + " weights source needs the container-genre plan (engine#1786, item F1); until then this pack is" + " recognised and reported, not importable." + : "model.q4nx is present but unreadable: " + c.why; + return p; + } + } + std::string geom_src; json geom = load_geometry(pack_dir, &geom_src); // `intermediate_size` is the one key no 1.6/1.7 pack states: genai_config.json has no such field, and on the diff --git a/npu/pack_import.h b/npu/pack_import.h index 7ec92ac36..1f8263de7 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -208,6 +208,14 @@ struct Plan { // third-party parser: the rai_1.3/1.6/1.7.1 line ships its 4-bit weights as ordinary ONNX initializers, while // 1.8 and R1-Distill ship placeholders and keep the weights in model.bin (pi-inferzo, 2026-11-08). // graph_matmuls < 0 means "no graph was read", which is why it is not just a count. + // engine#1786 (item F1): a FastFlowLM NPU2 model source is a safetensors container (`model.q4nx`) with no graph, + // so the two census fields above stay -1. `container_tensors` is -1 for every pack that is not that genre; when it + // is not, it is the container's tensor count and the dtype map says what the reader found. Reported rather than + // refused at the first byte, because "cannot open model.onnx" says nothing about a pack that has no graph by design. + int64_t container_tensors = -1; + int64_t container_payload_bytes = 0; + std::map container_dtypes; + std::string container_note; // what the container genre still needs, empty for a non-container pack int64_t graph_matmuls = -1; // MatMulNBits nodes in the main graph int64_t graph_real_weights = 0; // of those, how many point at an initializer with dims (i.e. real weights) int64_t graph_external = 0; // initializers that declare external data (location + offset) @@ -251,6 +259,29 @@ int intermediate_size_from_graph(const std::string& model_path, int64_t hidden); // multi-component one (`type: compositemodel`), whose geometry files stay at the root. Exported for the unit test. std::string find_token_fconst_exported(const std::string& pack_dir); +// A FastFlowLM NPU2 model source (engine#1786, item F1) is a safetensors container named `model.q4nx`: an 8-byte +// little-endian header length, that many bytes of JSON mapping tensor name -> {dtype, shape, data_offsets}, then the +// payload. There is no graph and no `model.onnx`, which is why the importer above refuses such a pack before reading a +// byte of it. The tensor NAMES are the model's own HF names, so this reader is also the work list: what it reports is +// what a container-genre plan has to map. +// +// Header only: no weights are read, no device is opened. Exported because it is the half that can be wrong silently +// (an off-by-one header length reports an empty inventory and looks like "a pack with no tensors"). +struct Q4nxTensor { + std::string name; + std::string dtype; + std::vector shape; + int64_t begin = 0; // data_offsets[0] + int64_t end = 0; // data_offsets[1] +}; +struct Q4nxContainer { + int64_t header_bytes = 0; + int64_t payload_bytes = 0; + std::vector tensors; // in header order + std::string why; // non-empty when the container could not be read; tensors is then empty +}; +Q4nxContainer read_q4nx_container(const std::string& path); + // Reads the pack (model.onnx, its external data, both .fconst headers, config.json). // Throws only when the pack is structurally unreadable. Plan plan(const std::string& pack_dir); diff --git a/tests/npu_forward_loader_test.c b/tests/npu_forward_loader_test.c index c32b8513c..fe0b6395e 100644 --- a/tests/npu_forward_loader_test.c +++ b/tests/npu_forward_loader_test.c @@ -165,6 +165,26 @@ int main(void) { model_free(dm); } } + /* engine#1786 (item F1): three of the fifty-eight NPU2 model sources spell the per-layer prefix + * `model.layer.` (singular) where this forward addresses `model.layers.`. model_tensor_by_name() retries the lookup + * with the other spelling AFTER an exact match has failed, and this helper is the whole of that rewrite -- so it + * must swap the leading prefix and nothing else: a global replace would corrupt a tensor that carries the same text + * further along, and a near-miss prefix must be left alone rather than "helpfully" matched. */ + { + char alt[512]; + check(model_tensor_name_alt("model.layers.0.input_layernorm.weight", alt, sizeof alt) == 1 && + strcmp(alt, "model.layer.0.input_layernorm.weight") == 0, + "ALT: model.layers. -> model.layer., the rest untouched"); + check(model_tensor_name_alt("model.layer.0.input_layernorm.weight", alt, sizeof alt) == 1 && + strcmp(alt, "model.layers.0.input_layernorm.weight") == 0, + "ALT: and back, model.layer. -> model.layers."); + check(model_tensor_name_alt("lm_head.weight", alt, sizeof alt) == 0, + "ALT: a model-level tensor has no per-layer spelling to try"); + check(model_tensor_name_alt("model.layersX.0.proj.weight", alt, sizeof alt) == 0, + "ALT: a near-miss prefix is not matched helpfully"); + check(model_tensor_name_alt("model.layers.0.a.weight", alt, 16) == 0, + "ALT: a too-small buffer reports no alternative rather than truncating"); + } char cmd[96]; snprintf(cmd, sizeof cmd, "rm -rf %s", dir); if (system(cmd) != 0) fprintf(stderr, "could not remove %s\n", dir); diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 3d1c246b0..b88cdc0ce 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -4015,6 +4015,107 @@ void test_embedding_f32_component(const std::string& root) { fs::remove_all(b); } +// engine#1786 (item F1): an NPU2 model source is a safetensors container, and this is the reader that decides whether +// a pack is that genre. The failure that matters is the silent one -- a wrong header length yields an empty inventory, +// which reads as "a pack with no tensors" rather than "a malformed header" -- so each arm asserts the reason, not just +// the emptiness. No weights are read and no device is opened: the header is 26,496 B on the smallest real pack. +void test_q4nx_container(const std::string& root) { + namespace fs = std::filesystem; + const fs::path dir = fs::path(root) / "q4nx-container"; + std::error_code ec; + fs::create_directories(dir, ec); + + const auto write_container = [&](const std::string& name, const std::string& header_json, int64_t payload, + bool truncate_header) { + const fs::path p = dir / name; + std::string out(8, '\0'); + int64_t n = int64_t(header_json.size()); + for (int i = 0; i < 8; ++i) out[size_t(i)] = char((n >> (8 * i)) & 0xFF); + out += truncate_header ? header_json.substr(0, header_json.size() / 2) : header_json; + out.append(size_t(payload), '\x22'); + std::ofstream f(p, std::ios::binary); + f.write(out.data(), std::streamsize(out.size())); + return p.string(); + }; + + // (a) the real shape: one BF16 vector and one packed I8 matrix, with the payload the offsets imply. + { + const std::string hdr = + R"({"model.layers.0.input_layernorm.weight":{"dtype":"BF16","shape":[640],"data_offsets":[0,1280]},)" + R"("model.layers.0.mlp.down_proj.weight":{"dtype":"I8","shape":[640,1280],"data_offsets":[1280,820480]}})"; + const auto c = pi::read_q4nx_container(write_container("two.q4nx", hdr, 820480, false)); + check(!c.tensors.empty(), "q4nx container: a two-tensor header reads (" + c.why + ")"); + check(c.tensors.size() == 2, "q4nx container: both tensors are reported"); + check(c.header_bytes == int64_t(hdr.size()), "q4nx container: the 8-byte length is the header's own"); + check(c.payload_bytes == 820480, "q4nx container: the payload is the sum of the data_offsets spans (got " + + std::to_string(c.payload_bytes) + ")"); + bool saw_bf16 = false, saw_i8 = false; + for (const auto& t : c.tensors) { + if (t.dtype == "BF16" && t.shape.size() == 1 && t.shape[0] == 640) saw_bf16 = true; + // The packed matrix: I8 with TWICE the logical element count of a 640-wide FFN is the NPU2 shape. + if (t.dtype == "I8" && t.shape.size() == 2 && t.shape[0] == 640 && t.shape[1] == 1280) saw_i8 = true; + } + check(saw_bf16 && saw_i8, "q4nx container: dtype and shape survive for a vector and a packed matrix"); + } + + // (b) `__metadata__` is a string map, not a tensor, and the vendor's converter does not always write one. + { + const std::string hdr = + R"({"__metadata__":{"group_size":"32"},"t":{"dtype":"I8","shape":[32,256],"data_offsets":[0,5120]}})"; + const auto c = pi::read_q4nx_container(write_container("meta.q4nx", hdr, 5120, false)); + check(c.tensors.size() == 1 && c.tensors[0].name == "t", + "q4nx container: __metadata__ is skipped, not counted as a tensor"); + } + + // (c) a truncated header is named, and reports no tensors rather than a partial inventory. + { + const std::string hdr = R"({"a":{"dtype":"I8","shape":[32,256],"data_offsets":[0,5120]}})"; + const auto c = pi::read_q4nx_container(write_container("short.q4nx", hdr, 5120, true)); + check(c.tensors.empty() && !c.why.empty(), + "q4nx container: a truncated header names the problem instead of reporting no tensors (" + c.why + ")"); + } + + // (e) engine#1786: a pack that IS a container and has NO graph is recognised and reported, not refused at the first + // byte. The alternative -- `npu-pack info` answering a 0.5 GB model source with "cannot open /model.onnx" -- + // describes a missing file on a pack that has no graph by design, and says nothing about what it does contain. + { + const fs::path pdir = dir / "container-genre"; + fs::create_directories(pdir, ec); + const std::string hdr = + R"({"model.embed_tokens.weight":{"dtype":"BF16","shape":[64,8],"data_offsets":[0,1024]},)" + R"("model.layers.0.mlp.down_proj.weight":{"dtype":"I8","shape":[8,16],"data_offsets":[1024,1152]}})"; + write_container("container-genre/model.q4nx", hdr, 1152, false); + const pi::Plan p = pi::plan(pdir.string()); + check(p.container_tensors == 2, "container genre: plan() reports the container's tensors (" + + std::to_string(p.container_tensors) + ")"); + check(p.container_payload_bytes == 1152, "container genre: and the payload the offsets imply"); + check(p.container_dtypes.count("BF16") == 1 && p.container_dtypes.count("I8") == 1, + "container genre: and both dtypes it found"); + // The note must say it is NOT importable: a plan that listed these as importable would be worse than the + // refusal it replaces, because it would claim a compute mapping this importer does not have. + check(p.container_note.find("not importable") != std::string::npos, + "container genre: the note says the pack is recognised and not importable (" + p.container_note + ")"); + const json m = json::parse(p.manifest()); + check(m.contains("container") && m["container"]["tensors"] == 2, + "container genre: and manifest() reports it, so npu-pack info prints it"); + } + + // (d) an implausible length is refused by the number, so a mis-typed length cannot make the caller allocate it. + { + const fs::path p = dir / "huge.q4nx"; + std::string out(8, '\0'); + const int64_t n = int64_t(1) << 40; + for (int i = 0; i < 8; ++i) out[size_t(i)] = char((n >> (8 * i)) & 0xFF); + { + std::ofstream f(p, std::ios::binary); + f.write(out.data(), std::streamsize(out.size())); + } // flushed and closed: the reader must see the bytes, not an empty file + const auto c = pi::read_q4nx_container(p.string()); + check(c.tensors.empty() && c.why.find("implausible") != std::string::npos, + "q4nx container: an implausible header length is refused by the number (" + c.why + ")"); + } +} + // engine#1725: a multi-component embedding carries a PER-LAYER table and its projection norm beside the token table, and // the importer writes both into the container so the forward path can gather them. Measured shapes on @@ -4584,6 +4685,7 @@ int main(int argc, char** argv) { test_fconst_variable_stride(golden_dir.empty() ? "." : golden_dir); test_attn_payload_off(); test_text_config_geometry(golden_dir.empty() ? "." : golden_dir); + test_q4nx_container(golden_dir.empty() ? "." : golden_dir); test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir); test_rope_parameters_theta(golden_dir.empty() ? "." : golden_dir); test_three_input_matmulnbits(golden_dir.empty() ? "." : golden_dir); diff --git a/tools/flm/q4nx_container_report.py b/tools/flm/q4nx_container_report.py new file mode 100755 index 000000000..0e8243983 --- /dev/null +++ b/tools/flm/q4nx_container_report.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Report a FastFlowLM NPU2 model source's container without loading a device (engine#1786, item F1). + +The NPU2 model sources on the Hub are a safetensors container named `model.q4nx`: 8-byte little-endian +header length, then that many bytes of JSON mapping tensor name -> {dtype, shape, data_offsets}, then +the payload. The tensor NAMES are the model's own HF names, so a report from this script says exactly +what the importer has to accept. No device, no model weights are read: only the header. + + tools/flm/q4nx_container_report.py [...] +""" +import json +import os +import struct +import sys +from collections import Counter + + +def read_header(path): + with open(path, "rb") as f: + n = struct.unpack(" 64 << 20: + raise ValueError(f"{path}: implausible header length {n}") + hdr = json.loads(f.read(n)) + meta = hdr.pop("__metadata__", {}) or {} + return hdr, meta, n + + +def report(path): + hdr, meta, n = read_header(path) + dtypes = Counter(v.get("dtype", "?") for v in hdr.values()) + payload = sum(v["data_offsets"][1] - v["data_offsets"][0] for v in hdr.values()) + layers = {k.split(".layers.")[1].split(".")[0] for k in hdr if ".layers." in k} + print(f"{os.path.basename(path)}: header {n} B, {len(hdr)} tensors, payload {payload/1e9:.3f} GB") + print(f" dtypes {dict(dtypes)}") + print(f" layers seen: {len(layers)} metadata keys: {sorted(meta)}") + if meta: + for k, v in list(meta.items())[:8]: + print(f" {k} = {v}") + # The first layer's tensors, because they are the set the importer must map per layer. + for k in sorted(hdr): + if ".layers.0." in k: + v = hdr[k] + print(f" {k:52} {v['dtype']:5} {v['shape']}") + return 0 + + +def main(argv): + if len(argv) < 2: + print(__doc__.strip().splitlines()[-1], file=sys.stderr) + return 2 + rc = 0 + for arg in argv[1:]: + path = os.path.join(arg, "model.q4nx") if os.path.isdir(arg) else arg + if not os.path.isfile(path): + print(f"{arg}: no model.q4nx", file=sys.stderr) + rc = 1 + continue + rc |= report(path) + return rc + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From 2d6c039c6e8a30a00538dd95b6664e114d8e47cd Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:56:27 -0400 Subject: [PATCH 14/98] npu-pack: sweep stale fconst scratch in every candidate directory, never the current one (Fixes #1907) (#1952) * npu-pack: sweep every fconst scratch directory, and stop the test suite leaking into them (Fixes #1907) Since #1801 the canonical copy is written to ~/.cache, and the sweep ran only in the directory being written, so copies older binaries had left in /tmp (tmpfs, i.e. RAM: 17.0 GiB measured) were never reclaimed. The sweep now covers every candidate directory, creating none. With ONEBIT_FCONST_SCRATCH set, only that directory is swept, so a run with an override never removes another lane's files. The test suite was the source of 179 of the 188 /tmp files: test_fconst_scratch_order() left HOME=/home/x with TMPDIR and the override unset, so later crafted imports fell through to /tmp, and the earlier ones wrote into the real ~/.cache. It now restores the four variables, and main() gives the run its own scratch dir, removed at exit. docs/npu.md: where the copy goes, and what the sweep removes. Co-Authored-By: Claude Opus 5.5 * npu-pack: never sweep the current directory, and test the sweep through the importer (#1915 review, engine#1907) - sweep_fconst_scratch_dirs skips the current directory (compared on canonical paths) unless it is the directory being written, and sweeps a directory listed twice once - test_sweep_is_wired runs import_pack() with an abandoned foreign copy in the scratch directory and asserts it is gone; with an override set, the cache directory's copy must still be there - boundary arms: a file that is not fconst19 scratch is kept however old; the cwd under three spellings; the cwd when it is the written directory - the 6 h constant says what it protects Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Opus 5.5 --- docs/npu.md | 20 ++++ npu/pack_import.cpp | 43 +++++++- npu/pack_import.h | 8 ++ tests/npu_pack_import_test.cpp | 175 ++++++++++++++++++++++++++++++++- 4 files changed, 244 insertions(+), 2 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index e77642369..94dffe23b 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1030,6 +1030,26 @@ Two facts about what the container holds, both reported, not hidden: scale 0, zero 0). That is 12.5 % more bytes than the pack; the true K is in the container's manifest and in `config.json`, and nothing reads a padding column as a weight. +### Where a 1.9 pack's rewritten `.fconst` goes + +A Ryzen AI 1.9 pack's Token `.fconst` is rewritten into its 1.8 form before it is read. The copy is as large as the source +(3.6 GB for an h4096 pack), so where it lands matters: on strixhalo `/tmp` is tmpfs, which means RAM. The importer tries these +directories in order: + +1. `$ONEBIT_FCONST_SCRATCH`, if set; +2. `$XDG_CACHE_HOME/1bit/fconst19`, or `~/.cache/1bit/fconst19` (on disk); +3. `$TMPDIR`, then `/tmp`, then the current directory. + +The copy is a cache: it is reused while the source's size and mtime and the rules version match. On each write the importer +sweeps the scratch directories. It removes other identities of the same pack and geometry, the pre-version name, and any +`1bit-fconst19*` copy nobody has written for 6 hours. Without an override, **every** candidate directory is swept, not only the +one being written, so copies older binaries left in `/tmp` get reclaimed (engine#1907). With `ONEBIT_FCONST_SCRATCH` set, only +that directory is swept, and a run with an override never removes another lane's files. The current directory is the +one candidate that is never swept unless the copy is being written there: it is only the last write fallback, and a +working directory's files are not the importer's to clean. The 6 hours are what keep a concurrent run's copy alive in a +shared directory, so the number stays above the longest import; it is not the knob for disk or tmpfs pressure. The test suite sets one per run and +removes it at exit. + ### Linking the kernel set the container will be served with A model directory carries its own NPU kernels in `npu/` (see "A model directory"), and for an imported diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 78bc8fa7e..df92b468e 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -763,6 +763,9 @@ std::string materialise_token_fconst(const std::string& pack_dir, const Dims& di constexpr const char* kScratchStem = "1bit-fconst19"; // A scratch copy older than this is a pack no run is importing: no import holds one for hours, so an mtime this far back // means the pack was abandoned, not that somebody is using it. Kept well above the longest import measured (~minutes). + // This number is what protects a CONCURRENT run's copy in a shared directory (the sweep reaches every candidate + // directory now, engine#1907), so it has to stay above the longest import: do not lower it to relieve disk or tmpfs + // pressure -- remove the files, or point ONEBIT_FCONST_SCRATCH at disk, instead. constexpr int64_t kScratchMaxAgeSeconds = 6 * 3600; // Everything but the source identity: the group this run evicts its stale siblings from (engine#1770). const std::string ident_prefix = "1bit-fconst19v" + std::to_string(kCanonicalFormVersion) + "-" + tag + "-" + @@ -798,7 +801,17 @@ std::string materialise_token_fconst(const std::string& pack_dir, const Dims& di // GiB of tmpfs, had been resident all day). const std::string legacy_name = std::string(kScratchStem) + "-" + tag + "-" + std::to_string(dims.hidden) + "-" + std::to_string(dims.heads) + "-" + std::to_string(dims.layers) + ".fconst"; - evict_stale_canonical_copies(d.string(), name, ident_prefix, legacy_name, kScratchMaxAgeSeconds); + // Sweep EVERY candidate directory, not only the one written (engine#1907). Since #1801 moved the cache to disk, a + // write lands in ~/.cache, so the sweep never visited /tmp again, and the copies older binaries had left there + // (17.0 GiB of tmpfs, i.e. RAM, measured 2026-11-10) stayed until reboot. The other directories are only swept, + // never created, and the rule is the same one as here: the exact name is kept wherever it is. + // An explicit ONEBIT_FCONST_SCRATCH is a lane saying "my scratch is here": then only that directory is swept, so a + // run with an override (the test suite sets one) never removes another lane's files from the shared /tmp or cache. + std::vector dir_names; + const char* override_dir = std::getenv("ONEBIT_FCONST_SCRATCH"); + if (override_dir && *override_dir) dir_names.push_back(d.string()); + else for (const auto& x : dirs) dir_names.push_back(x.string()); + sweep_fconst_scratch_dirs(dir_names, d.string(), name, ident_prefix, legacy_name, kScratchMaxAgeSeconds); // Say where it went whenever that is not the ordinary place: the copy is a few hundred MB and it is scratch. std::error_code ec2; const fs::path tmpdir = [&] { try { return fs::temp_directory_path(); } catch (...) { return fs::path(); } }(); @@ -5623,4 +5636,32 @@ void evict_stale_canonical_copies(const std::string& dir, const std::string& kee } } +void sweep_fconst_scratch_dirs(const std::vector& dirs, const std::string& written_dir, + const std::string& keep_name, const std::string& prefix, const std::string& legacy_name, + int64_t max_age_seconds) { + // engine#1907: every candidate is swept, not only the one being written. A directory that does not exist is skipped, + // never created: only the written one is made ready, by its caller. + // + // The CURRENT DIRECTORY is the one candidate that is not this importer's to clean (#1915 review): it is in the list + // only as the last write fallback, and a lane that runs `npu-pack` from inside its own scratch would otherwise have + // that scratch's old `1bit-fconst19*` files removed. So it is swept only when it is the directory being written -- + // which is what happened before #1907 and is the evict-on-write rule of engine#1770, unchanged. The comparison is + // on canonical paths: `.`, a relative spelling and the absolute path are the same directory. A directory listed + // twice is swept once. + std::error_code cwd_ec; + const fs::path cwd = fs::weakly_canonical(fs::current_path(cwd_ec), cwd_ec); + std::set seen; + for (const auto& dir : dirs) { + if (dir.empty()) continue; + std::error_code ec, cec; + const bool written = dir == written_dir; + const fs::path canon = fs::weakly_canonical(dir, cec); + if (!written && (cwd_ec || cec || canon == cwd)) continue; + if (!cec && !seen.insert(canon.string()).second) continue; + if (written || fs::is_directory(dir, ec)) + evict_stale_canonical_copies(dir, keep_name, prefix, legacy_name, max_age_seconds); + } +} + + } // namespace onebit::npu::pack_import diff --git a/npu/pack_import.h b/npu/pack_import.h index 1f8263de7..775eed018 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -307,6 +307,14 @@ void evict_stale_canonical_copies(const std::string& dir, const std::string& kee const std::string& legacy_name = std::string(), int64_t max_age_seconds = 0); +// engine#1907: apply evict_stale_canonical_copies() to EVERY scratch candidate, not only the one being written. Since #1801 the +// write lands in the on-disk cache, so a sweep of that directory alone never reached /tmp again, and the copies older binaries +// had left in tmpfs (17.0 GiB of RAM, measured 2026-11-10) stayed until reboot. `written_dir` is swept even if it is not yet a +// directory as seen here; every other entry is swept only when it already exists, and is never created. +void sweep_fconst_scratch_dirs(const std::vector& dirs, const std::string& written_dir, + const std::string& keep_name, const std::string& prefix, const std::string& legacy_name, + int64_t max_age_seconds); + // The text artifacts a runnable container needs beside model.q4nx, and which of them the pack // actually carries. A pack fetched weight by weight can be weights-only (engine#1024; AMD's repos // do publish the text, and how much of it varies per pack), and an importer that says nothing about diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index b88cdc0ce..a2656f574 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -926,6 +926,157 @@ void test_evict_stale_canonical_copies() { " geometry/age rules + 3 degenerate)\n"); } +// engine#1907: the sweep ran only in the directory being written. Since #1801 that is the on-disk cache, so /tmp was never swept +// again, and 17.0 GiB of copies left there by older binaries stayed in tmpfs (RAM). Every candidate is swept now; one that does +// not exist is not created. +void test_sweep_fconst_scratch_dirs() { + const fs::path root = fs::temp_directory_path() / ("1bit-sweep-" + std::to_string(::getpid())); + std::error_code ec; + fs::remove_all(root, ec); + const fs::path written = root / "cache", other = root / "tmp", absent = root / "never-made"; + fs::create_directories(written); + fs::create_directories(other); + auto touch = [&](const fs::path& d, const std::string& base) { + std::ofstream o(d / base, std::ios::binary | std::ios::trunc); + o << 'x'; + }; + const std::string prefix = "1bit-fconst19v1-12345-896-14-24-"; + const std::string keep = prefix + "291653952-76263210.fconst"; + const std::string stale = prefix + "291653952-99999.fconst"; + const std::string legacy = "1bit-fconst19-12345-896-14-24.fconst"; + const std::string old_other = "1bit-fconst19v1-77777-1536-12-28-995712640-73345656.fconst"; + const std::string fresh_other = "1bit-fconst19v1-88888-1536-12-28-995712640-73345656.fconst"; + for (const auto& f : {keep, stale, legacy, old_other, fresh_other}) touch(other, f); + touch(written, keep); + const auto seven_hours_ago = fs::file_time_type::clock::now() - std::chrono::hours(7); + fs::last_write_time(other / old_other, seven_hours_ago, ec); + pi::sweep_fconst_scratch_dirs({written.string(), other.string(), absent.string()}, written.string(), keep, prefix, legacy, + 6 * 3600); + const auto in_other = [&](const std::string& b) { return fs::exists(other / b); }; + check(!in_other(old_other), "sweep: an abandoned copy in a directory that was NOT written goes (the /tmp left behind by #1801)"); + check(!in_other(stale) && !in_other(legacy), + "sweep: a stale identity and the legacy name of this pack go from the other directory too"); + check(in_other(keep), "sweep: the exact name is kept in every directory (another environment may reuse it there)"); + check(in_other(fresh_other), "sweep: a fresh copy of another pack stays -- another run may be writing it"); + check(fs::exists(written / keep), "sweep: the copy being written is kept"); + check(!fs::exists(absent), "sweep: a candidate that does not exist is not created"); + // #1915 review, boundary arms. (a) A file that is not fconst19 scratch is kept however old it is. (b) The current + // directory is never swept unless it is the directory being written -- under every spelling of it. + { + const fs::path lane = root / "lane-cwd"; + fs::create_directories(lane); + const std::string notes = "notes-about-1bit-fconst19.txt"; + for (const auto& d : {other, lane}) { + touch(d, notes); + touch(d, old_other); + fs::last_write_time(d / notes, seven_hours_ago, ec); + fs::last_write_time(d / old_other, seven_hours_ago, ec); + } + const fs::path was = fs::current_path(); + fs::current_path(lane); + pi::sweep_fconst_scratch_dirs({written.string(), other.string(), ".", lane.string(), (lane / ".." / "lane-cwd").string()}, + written.string(), keep, prefix, legacy, 6 * 3600); + const bool cwd_kept = fs::exists(lane / old_other); + // and when the cwd IS the directory being written, the evict-on-write rule still applies to it + pi::sweep_fconst_scratch_dirs({lane.string(), other.string()}, lane.string(), keep, prefix, legacy, 6 * 3600); + const bool cwd_written_swept = !fs::exists(lane / old_other); + fs::current_path(was); + check(!in_other(old_other), "sweep: (control) the same old file in a directory that is not the cwd goes"); + check(in_other(notes) && fs::exists(lane / notes), "sweep: a file that is not fconst19 scratch is kept however old"); + check(cwd_kept, "sweep: the current directory is never swept, as '.', as its absolute path or through '..'"); + check(cwd_written_swept, "sweep: unless it is the directory being written (evict-on-write, engine#1770)"); + } + fs::remove_all(root, ec); + std::printf(" scratch sweep: every candidate directory, 10 arms\n"); +} + +// #1915 review: the arms above call the sweep; nothing ran the importer and looked. "Delete the call site and the suite +// stays green" was true. This runs the production path once -- import_pack() on a crafted pack with a Token .fconst -- +// with a scratch directory that already holds an abandoned copy of ANOTHER pack, and asserts that copy is gone. With an +// override set only that directory is swept, so the second directory's copy must still be there. +void test_sweep_is_wired(const std::string& root) { + const fs::path dir = fs::path(root) / "sweep-wired"; + const fs::path scratch = fs::temp_directory_path() / ("1bit-sweep-wired-" + std::to_string(::getpid())); + const fs::path cache_home = fs::temp_directory_path() / ("1bit-sweep-wired-xdg-" + std::to_string(::getpid())); + std::error_code ec; + fs::remove_all(dir, ec); + fs::remove_all(scratch, ec); + fs::remove_all(cache_home, ec); + fs::create_directories(dir / "cache"); + fs::create_directories(scratch); + fs::create_directories(cache_home / "1bit" / "fconst19"); + const int H = 128, L = 8, V = 64; + { + const int64_t stride = 1 << 20, second = 262144; + std::string fc(size_t(int64_t(L) * stride + 64 + size_t(H) * 2), '\0'); + for (int l = 0; l <= L; ++l) { + fc[size_t(int64_t(l) * stride)] = char(0x86); + fc[size_t(int64_t(l) * stride) + 1] = char(0x35); + if (l < L) { + fc[size_t(int64_t(l) * stride + second)] = char(0x86); + fc[size_t(int64_t(l) * stride + second) + 1] = char(0x35); + } + } + put_file((dir / "dd_metastate_Llm_Token_Token_crafted.fconst").string(), fc); + } + put_file((dir / "config.json").string(), + R"({"model_type":"qwen2","hidden_size":128,"intermediate_size":256,"num_hidden_layers":8,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":64,)" + R"("rope_theta":10000.0,"tie_word_embeddings":true,"rms_norm_eps":1e-06})"); + put_file((dir / "model.onnx.data").string(), std::string(size_t(V) * size_t(H) * 2, '\x11')); + put_file((dir / "model.onnx").string(), + pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "model.onnx.data", 0, uint64_t(V) * uint64_t(H) * 2, 10)}, + {})); + const std::string foreign_old = "1bit-fconst19v1-77777-1536-12-28-995712640-73345656.fconst"; + const std::string foreign_fresh = "1bit-fconst19v1-88888-1536-12-28-995712640-73345656.fconst"; + const std::string unrelated = "keep-me.txt"; + const auto seven_hours_ago = fs::file_time_type::clock::now() - std::chrono::hours(7); + for (const fs::path& d : {scratch, cache_home / "1bit" / "fconst19"}) + for (const auto& f : {foreign_old, foreign_fresh, unrelated}) { + put_file((d / f).string(), "x"); + if (f != foreign_fresh) fs::last_write_time(d / f, seven_hours_ago, ec); + } + struct Env { + std::string name, value; + bool had; + }; + std::vector saved; + for (const char* n : {"ONEBIT_FCONST_SCRATCH", "XDG_CACHE_HOME"}) { + const char* v = std::getenv(n); + saved.push_back({n, v ? v : "", v != nullptr}); + } + setenv("ONEBIT_FCONST_SCRATCH", scratch.c_str(), 1); + setenv("XDG_CACHE_HOME", cache_home.c_str(), 1); + const std::string out = (fs::temp_directory_path() / ("oga_pack_import_sweepwired." + std::to_string(::getpid()))).string(); + fs::remove_all(out, ec); + pi::Options allow; + allow.allow_incomplete = true; + try { + (void)pi::import_pack(dir.string(), out, allow); + } catch (const std::exception&) { + // the import's own outcome is not what is under test: the scratch copy is made, and swept around, before any + // refusal about this fixture's empty payload + } + for (const auto& e : saved) { + if (e.had) setenv(e.name.c_str(), e.value.c_str(), 1); + else unsetenv(e.name.c_str()); + } + bool wrote_here = false; + for (const auto& e : fs::directory_iterator(scratch, ec)) + if (e.path().filename().string().rfind("1bit-fconst19v", 0) == 0 && e.path().filename().string() != foreign_fresh) + wrote_here = true; + check(wrote_here, "sweep wiring: (premise) the import put its scratch copy in the override directory"); + check(!fs::exists(scratch / foreign_old), "sweep wiring: an abandoned copy of another pack is GONE after a real import"); + check(fs::exists(scratch / foreign_fresh), "sweep wiring: a fresh copy of another pack is still there"); + check(fs::exists(scratch / unrelated), "sweep wiring: a file that is not fconst19 scratch is still there"); + check(fs::exists(cache_home / "1bit" / "fconst19" / foreign_old), + "sweep wiring: with an override set, the cache directory is NOT swept (another lane's files stay)"); + fs::remove_all(out, ec); + fs::remove_all(dir, ec); + fs::remove_all(scratch, ec); + fs::remove_all(cache_home, ec); +} + void test_modelbin_genre_refusal(const std::string& root) { const fs::path dir = fs::path(root) / "crafted-modelbin"; fs::remove_all(dir); @@ -3342,6 +3493,19 @@ void test_fconst_block_headers() { // cache whose size is the pack's own .fconst, and on this box /tmp is tmpfs, so where it lands is a RAM decision. The bug was a // cache directory that was never created -- every pack then fell through to /tmp and nothing noticed. void test_fconst_scratch_order() { + // engine#1907: this arm rewrites four variables, and it used to leave HOME=/home/x with TMPDIR and the override unset, so + // every crafted import after it fell through to the literal /tmp (179 of the 188 files found there were this suite's). + // Save and restore them, so the rest of the run keeps the per-run scratch directory main() set. + struct Saved { const char* name; bool had; std::string value; }; + std::vector saved; + for (const char* n : {"ONEBIT_FCONST_SCRATCH", "XDG_CACHE_HOME", "HOME", "TMPDIR"}) { + const char* v = std::getenv(n); + saved.push_back({n, v != nullptr, v ? v : ""}); + } + struct Restore { + std::vector& s; + ~Restore() { for (const auto& x : s) { if (x.had) setenv(x.name, x.value.c_str(), 1); else unsetenv(x.name); } } + } restore{saved}; const auto first_with = [](const char* scratch, const char* xdg, const char* home) { if (scratch) setenv("ONEBIT_FCONST_SCRATCH", scratch, 1); else unsetenv("ONEBIT_FCONST_SCRATCH"); if (xdg) setenv("XDG_CACHE_HOME", xdg, 1); else unsetenv("XDG_CACHE_HOME"); @@ -3396,7 +3560,6 @@ void test_fconst_scratch_order() { check(std::count(no_tmpdir.begin(), no_tmpdir.end(), std::string("/tmp")) == 1, "fconst scratch: with TMPDIR unset, /tmp appears exactly once (no duplicate from the resolver)"); std::system(("rm -rf " + tdir).c_str()); - setenv("HOME", "/home/x", 1); // leave the environment as the arms above expect it } std::printf("fconst scratch: order is override -> disk cache -> temp/cwd, pinned\n"); } @@ -4648,6 +4811,14 @@ void test_header_witness_refusal() { } int main(int argc, char** argv) { + // engine#1907: every crafted 1.9 import writes a canonical copy into the scratch cache. Without this the suite left 0-byte + // copies in the user's real ~/.cache/1bit/fconst19 and in /tmp on every run. Give the run its own directory, removed at exit. + const fs::path run_scratch = fs::temp_directory_path() / ("1bit-pack-test-scratch-" + std::to_string(::getpid())); + if (!std::getenv("ONEBIT_FCONST_SCRATCH")) setenv("ONEBIT_FCONST_SCRATCH", run_scratch.c_str(), 1); + struct ScratchCleanup { + fs::path p; + ~ScratchCleanup() { std::error_code ec; fs::remove_all(p, ec); } + } scratch_cleanup{run_scratch}; test_header_witness_refusal(); test_encode_tile_code_range(); test_qk_norm_refusal(); @@ -4673,6 +4844,8 @@ int main(int argc, char** argv) { test_external_packed_genre(golden_dir.empty() ? "." : golden_dir); test_external_import_success(golden_dir.empty() ? "." : golden_dir); test_canonical_copy_reusable(); test_evict_stale_canonical_copies(); + test_sweep_fconst_scratch_dirs(); + test_sweep_is_wired(golden_dir.empty() ? "." : golden_dir); test_crafted_refusals(golden_dir.empty() ? "." : golden_dir); test_modelbin_genre_refusal(golden_dir.empty() ? "." : golden_dir); test_1671_plan_head_arm(golden_dir.empty() ? "." : golden_dir); From 0f305edf6e7d3ae3a061718631afb6c0739634ae Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:18:33 -0400 Subject: [PATCH 15/98] bump llama.cpp to 50d28600a: Qwen3-VL's rotation runs on the device (fixes #1467) (#2011) One fork commit past fa533bd21: 50d28600a (#286), the multi-section rotation in the two generic rotation kernels. With fa533bd21's #283 (already pinned) Qwen3-VL-2B decodes on HRX0 again and its text graph is 2-3 splits instead of 114. Measured at 50d28600a on gfx1151 (amdclang build, gpu-hold): test-backend-ops ROPE 166/166; text identical to the CPU for 12 greedy tokens; llama-bench pp512 6481 / 6532 tok/s, tg64 133.7 / 135.5 tok/s; llama-server, projector on HRX0, first image request 2.99 s (prompt_ms 2782) against the issue's 16.97 s. registry/architectures.json: the recorded source pin only. docs/hrx.md: the #1467 section. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/hrx.md | 26 ++++++++++++++++++++++++++ registry/architectures.json | 2 +- third_party/llama.cpp | 2 +- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/hrx.md b/docs/hrx.md index f8ad5eea2..a1c96fb7c 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -304,6 +304,32 @@ Still open under #1962: two prefill kernels that re-quantise the activations int (Q5_K and Q6_K weights at 512 tokens) keep a per-block term in half precision and still return NaN above 65504. And the limit is moved, not removed: an activation above 4.19e6 would overflow again. +### Fixed: Qwen3-VL on `HRX0`: it decodes again, and its rotation runs on the device ([engine#1467](https://github.com/1bit-MONSTER/engine/issues/1467)) + +Two faults, both in the text model's graph rather than in the projector: + +- **A refused view (llama.cpp #283).** A deepstack model adds a slice of the wide input embedding to each of + its first layers, taken as a view of a view. With the first view's node in another split, the HRX graph + never holds the wide tensor, the slice could not alias its input, and the graph was refused at + `unsupported HRX node 20: VIEW`. So Qwen3-VL-2B could not decode even text, and every image request failed + with it. Such a view is external to the graph and a layout op writes nothing, so the scheduler now skips it. +- **The rotation on the CPU (llama.cpp #286).** The model's rotation is the interleaved multi-section one + with sections that do not collapse to the plain rotation, so 56 `ROPE` nodes per text graph ran on the CPU + and cut the graph into 114 splits. The two generic rotation kernels gained a mode that reads the position + section each pair selects; the text graph is now 2 or 3 splits with `PAD` (and `GET_ROWS` on token + batches) left on the CPU. + +Measured at pin `50d28600a` on gfx1151 (2026-11-10, power mode `performance`, Qwen3VL-2B-Instruct Q4_K_M with +`mmproj-Qwen3VL-2B-Instruct-F16`): `test-backend-ops -o ROPE -b HRX0` 166/166; 12 greedy text tokens +byte-identical to the CPU backend; `llama-bench -p 512 -n 64 -r 3` at 6,481 and 6,532 tok/s prefill, 133.7 +and 135.5 tok/s decode (2,331 and 57.8 before the rotation moved). `llama-server` with the projector on +`HRX0`: the first request for the 448x448 test image takes 2.99 s (`prompt_ms` 2,782 for 227 prompt tokens) +against the 16.97 s this issue was filed on, and answers "A red rectangle and a green circle." + +The projector's own graph still runs 103 of 626 nodes on the CPU for an image of that size (its 2-D rotation, +the attention score matmul and softmax, `IM2COL`, `UPSCALE`); with it offloaded, encoding the image is no +faster than with `--no-mmproj-offload` (0.5 to 1.1 s against 0.4 to 0.6 s). + ### MoE decode and the kernel cache (llama.cpp #32, #33) - **MoE decode.** With one to four tokens, a `MUL_MAT_ID` (a MoE expert matmul) runs as a GEMV over diff --git a/registry/architectures.json b/registry/architectures.json index 705031c6d..031dafa8d 100644 --- a/registry/architectures.json +++ b/registry/architectures.json @@ -1,7 +1,7 @@ { "about": "HF architecture -> GGUF architecture and the backends whose code accepts it. Generated by tools/registry_build.py from the pinned sources; do not edit.", "sources": { - "llama.cpp (hrx)": "fa533bd21053eeb7eab5354aef749c44cde86987" + "llama.cpp (hrx)": "50d28600a5f7a38df390709b94c463df02785a54" }, "counts": { "hrx": 298, diff --git a/third_party/llama.cpp b/third_party/llama.cpp index fa533bd21..50d28600a 160000 --- a/third_party/llama.cpp +++ b/third_party/llama.cpp @@ -1 +1 @@ -Subproject commit fa533bd21053eeb7eab5354aef749c44cde86987 +Subproject commit 50d28600a5f7a38df390709b94c463df02785a54 From 69250678670b6618b538ec1b8da041a88f18626f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:18:39 -0400 Subject: [PATCH 16/98] npu-pack: say what a rule row's `prefix` actually is (a twin-diff byte, not an arithmetic term) (#2010) * npu-pack: say what a rule row's `prefix` actually is (a twin-diff byte, not an arithmetic term) The table's comments described `prefix` as `64 + 2*hidden + 4,352` (or `64 + 2*hidden` on the families with no QK-norm term). It is not: `tools/fconst19_measure.py:104` walks prefix = 0 while b0[prefix] == a0[prefix]: prefix += 1 on the 1.9 block and its 1.8 twin, so `prefix` is the FIRST BYTE AT WHICH THE TWO DIVERGE, and it is a measurement of that pair and nothing else. The arithmetic agrees on every row measured so far, which is exactly why the mistake is easy: it reads as a rule derived from the geometry when it is a coincidence of the packs that happened to be measured. I reviewed three of those rows myself and "verified" the prefix against that formula, which confirmed the coincidence and not the rule. Correcting the comments is the cheap half of the fix; the expensive half is the row someone derives from it next. A stated consequence, in the header: a 1.9 pack with NO 1.8 twin cannot be given a row at all -- there is nothing to diff it against -- so it stays refused by the shape gate rather than being handed an inferred prefix and reported working. `amd/granite-4.1-8b_rai_1.9.0_npu` is the standing example and must stay refused. Two comments that leaned on the formula are reworded to report the observation as an observation (the 2,560 row's agreement with the 1,504... with the 1.7B twin's landing, and the h3584 pair's note that the prefix is *not* separated by that pair). The four commented rows keep their numbers -- a row's prefix was measured; only the explanation of where it comes from changes. `npu_pack_import_test tests/golden`: all npu-pack checks passed. tools/copyright.py --check rc 0. * fconst19_measure: --check-row, so a table row can be held against the pair it claims to describe (engine#2010) The comment fix says what `prefix` is; this makes it checkable. `--check-row npu/pack_import.cpp` parses `kFconst19Rules`, finds the row for the measured geometry, and FAILS when any field differs from what this pair measures -- so a row derived from `64 + 2*hidden + 4,352`, or copied from a neighbour's shape, is caught by the tool that would have measured it. A geometry with no row reports that too: that is the shape gate's answer for a pack with no 1.8 twin, and it should be a named refusal rather than a row someone inferred. Field order follows the struct (prefix, n, rot, tile), not the order the numbers appear in a comment -- I got that wrong twice while writing this, once in the parser and once at the call site, and the arm caught both because a swapped pair still parses. The arm drives the real tool on the synthetic pair the other three cases already build: a row that matches must pass, and the same row with `prefix + 64` must fail naming both numbers (`8320 ... but this pair measures 8256`). tests/fconst19_measure_test.py: all 4 cases pass. --------- Co-authored-by: bong-water-water-bong --- npu/pack_import.cpp | 25 ++++++++++++++------- tests/fconst19_measure_test.py | 33 +++++++++++++++++++++++++++- tools/fconst19_measure.py | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 9 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index df92b468e..2c06dfcf2 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -380,6 +380,13 @@ constexpr double kTieCorr = 0.99; // Qwen3's per-head QK norm gains, but Llama-3.2-1B -- which has no QK norms at all -- carries the same +4,352 at hidden // 2,048, so the extra is a property of the geometry, not of the architecture. Measured anyway, because a plausible // explanation that survives one model is how this table would get a wrong row. +// WHAT `prefix` IS, because it has been misunderstood twice (engine#2010): it is the FIRST BYTE AT WHICH THE 1.9 LAYER +// BLOCK DIVERGES FROM ITS 1.8 TWIN -- `tools/fconst19_measure.py:104` walks `while b0[prefix] == a0[prefix]: prefix += 1` +// on the pair, so the number is a measurement of that pair and nothing else. The frequent agreement with +// `64 + 2*hidden + 4,352` (or `64 + 2*hidden` on the families with no QK-norm term) is an OBSERVATION about the rows +// measured so far, not a rule: do not derive a row from it. A 1.9 pack with NO 1.8 twin cannot be given a row at all -- +// there is nothing to diff it against, so it stays refused by the shape gate (amd/granite-4.1-8b_rai_1.9.0_npu is the +// standing example) rather than being given an inferred prefix and reported working. constexpr Fconst19Rule kFconst19Rules[] = { // Qwen2.5-0.5B-Instruct: normalises to its 1.8 twin byte for byte, whole file, all 24 blocks. {896, 14, 24, 1856, 16, 2, 34816}, @@ -401,9 +408,10 @@ constexpr Fconst19Rule kFconst19Rules[] = { // Qwen-2.5_1.5B_Instruct measured to the SAME bundle and rotation as the Coder pack -- byte for byte over all 28 // layer blocks -- so the two packs share one row now that the key is geometry, not the pack name. {1536, 12, 28, 3136, 7, 1, 226304}, - // Qwen3-4B: 2,933 of 2,208,061,248 bytes differ (1.3e-6), again content not layout. The prefix corroborates the - // Qwen3 offset seen at 1.7B -- 64 + 2*hidden + 4,352, so 8,512 at hidden 2,048 and 9,536 at 2,560 -- and the tile - // count differs from 1.7B's 3, which is why this is measured per pack and never derived. + // Qwen3-4B: 2,933 of 2,208,061,248 bytes differ (1.3e-6), again content not layout. The prefix lands where 1.7B's + // twin-diff landed for its hidden size (8,512 at 2,048 here, 9,536 at 2,560) -- an agreement worth noticing, since the + // two packs are different models -- and the tile count differs from 1.7B's 3, which is why this is measured per pack and + // never derived. {2560, 32, 36, 9536, 5, 1, 1462272}, // Llama-3.2-1B-Instruct: hidden 2,048 with 32/16 heads and no QK norms -- the first non-Qwen family here. Same // prefix as Qwen3-1.7B at this hidden size (8,512), a different bundle (5 tiles of 591,872, not 3 of 1,183,744), @@ -427,9 +435,10 @@ constexpr Fconst19Rule kFconst19Rules[] = { // 28,271. // The prefix is the ONE field this pair does not separate: `prefix+64` differs in 33,291 bytes, which is under the tool's // 38,306-byte tolerance -- so `prefix = 7,232` here rests on the two models agreeing and on the plain Qwen2.5 arithmetic - // (64 + 2*hidden, with no QK-norm term, unlike the Qwen3 line's +4,352), not on the pair alone. Said plainly because a row - // whose comment implies all seven fields were equally pinned would be the same kind of overstatement this table exists to - // avoid. + // (64 + 2*hidden, with no QK-norm term, unlike the Qwen3 line's +4,352), not on the pair alone -- and note that even that + // arithmetic is an observation about measured rows, not the definition: `prefix` is where the twins first diverge. Said + // plainly because a row whose comment implies all seven fields were equally pinned would be the same kind of + // overstatement this table exists to avoid. // The block's arithmetic, from the measurement rather than from a formula: prefix 7,232 + 9 x tile 1,009,664 = 9,094,208, // which is exactly the `header_off` the tool reports for this pair -- so the payload after the prefix is NINE tile slots, // of which the rule rotates eight and the ninth is the gap the post-attention header sits behind. (An earlier version of @@ -445,7 +454,7 @@ constexpr Fconst19Rule kFconst19Rules[] = { {2048, 16, 36, 4160, 9, 1, 295936}, // Mistral-7B-Instruct-v0.2 at hidden 4,096 / 32 heads / 32 layers: measured with tools/fconst19_measure.py against // amd/Mistral-7B-Instruct-v0.2_rai_1.8.0_npu_4K, 1,954 of 3,843,419,200 bytes differ (5.08e-05) -- content, not - // layout. The prefix follows the Qwen3 line (64 + 2*hidden + 4,352 = 12,608), the bundle is 5 tiles of 2,297,856 + // layout. The prefix is 12,608 (the first twin-diff byte), the bundle is 5 tiles of 2,297,856 // (11,489,280 B, which is NOT header_off minus the prefix: the header sits after the prefix and the bundle follows // it, so this row is one the arithmetic would have got wrong), and the header stride 117,862,720 is uniform over // all 32 blocks. This is the largest family on #1503 (31 packs). Measured by another lane on main, whose @@ -453,7 +462,7 @@ constexpr Fconst19Rule kFconst19Rules[] = { {4096, 32, 32, 12608, 5, 1, 2297856}, // Llama-3.2-3B (and -Instruct, same geometry) at hidden 3,072 / 24 heads / 28 layers: measured with // tools/fconst19_measure.py against amd/Llama-3.2-3B_rai_1.8.0_npu_4K, 1,090 of 1,756,930,368 bytes differ - // (6.20e-07 of the file, which the tool prints as 6.20e-05 %) -- content, not layout. The prefix is the Qwen3 line's 64 + 2*hidden + 4,352 = 10,560 at this hidden + // (6.20e-07 of the file, which the tool prints as 6.20e-05 %) -- content, not layout. The prefix is 10,560 here (the first twin-diff byte at this hidden // size, but the bundle is FOUR tiles of 1,740,800 (6,963,200 B) where the Phi/h3072 row has two of 5,222,400 -- // which is the whole reason a row is measured per geometry and never derived from the hidden size. {3072, 24, 28, 10560, 4, 1, 1740800}, diff --git a/tests/fconst19_measure_test.py b/tests/fconst19_measure_test.py index beb7a4f88..2fa505bae 100755 --- a/tests/fconst19_measure_test.py +++ b/tests/fconst19_measure_test.py @@ -139,12 +139,43 @@ def case_refuse(root): return ok, f"rc={rc} err={err.strip().splitlines()[-1][:90] if err.strip() else ''!r}" + +def case_check_row(root): + """engine#2010: a row must BE the measurement. `prefix` is the first byte where the twins diverge, so a row + whose prefix came from `64 + 2*hidden + 4,352` (or from a neighbour's shape) is wrong in a way no geometry + check sees. The guard is `--check-row `; here the source file is a two-line stand-in.""" + a = b"".join(block_18(L) for L in range(LAYERS)) + b"TAIL" * 16 + b = b"".join(block_19(L) for L in range(LAYERS)) + b"TAIL" * 16 + p18 = write_pack(root, "p18", a) + p19 = write_pack(root, "p19", b, declared_18=False) + rc, out, err = run((p18, p19), ("--json",)) + if rc != 0: + return False, f"the pair itself did not measure: rc={rc} {err.strip()[:80]}" + row = json.loads([l for l in out.splitlines() if l.startswith("{")][0]) + good = (f" {{{row['hidden']}, {row['heads']}, {row['layers']}, {row['prefix']}, {row['n_tiles']}, " + f"{row['rot']}, {row['tile']}}},") # the struct's order: prefix, n, rot, tile + src = os.path.join(root, "good.cpp") + with open(src, "w") as f: + f.write("constexpr Fconst19Rule kFconst19Rules[] = {\n" + good + "\n};\n") + rc1, out1, err1 = run((p18, p19), ("--check-row", src)) + # and a row whose prefix is off by one tile-length, which is what a derived prefix looks like + bad = good.replace(f", {row['prefix']},", f", {row['prefix'] + 64},", 1) + src2 = os.path.join(root, "bad.cpp") + with open(src2, "w") as f: + f.write("constexpr Fconst19Rule kFconst19Rules[] = {\n" + bad + "\n};\n") + rc2, out2, err2 = run((p18, p19), ("--check-row", src2)) + ok = rc1 == 0 and rc2 == 1 and "is exactly what this pair measures" in (out1 + err1) \ + and "MISMATCH" in (out2 + err2) and f"prefix {row['prefix'] + 64} vs measured" in (out2 + err2) + return ok, f"match rc={rc1}; mismatch rc={rc2}; detail={(out2 + err2).strip()[-90:]!r}" + + def main(): root = tempfile.mkdtemp(prefix="fconst19-test.") try: results = [("exact pair (and the refusal path executes)", case_exact(root)), ("a few hundred codes differ: accepted with residual", case_tolerance(root)), - ("a scrambled tile: refused", case_refuse(root))] + ("a scrambled tile: refused", case_refuse(root)), + ("--check-row: the table's row must be this pair's measurement", case_check_row(root))] finally: shutil.rmtree(root, ignore_errors=True) bad = [n for n, (ok, _) in results if not ok] diff --git a/tools/fconst19_measure.py b/tools/fconst19_measure.py index 386b1b14f..49c6c5658 100755 --- a/tools/fconst19_measure.py +++ b/tools/fconst19_measure.py @@ -37,6 +37,7 @@ """ import argparse import json +import re import os import sys @@ -139,11 +140,45 @@ def normalise(blob, layers, p, hdr_len): return bytes(res) +ROW_RE = re.compile(r"^\s*\{(\d+), (\d+), (\d+), (\d+), (\d+), (\d+), (\d+)\},", re.M) + + +def table_rows(source_text): + """The `kFconst19Rules` rows in a pack_import.cpp, as tuples in the tool's own field order.""" + out = {} + for m in ROW_RE.finditer(source_text): + hidden, heads, layers, prefix, n, rot, tile = (int(g) for g in m.groups()) + out[(hidden, heads, layers)] = (prefix, n, rot, tile) + return out + + +def check_row(source_text, measured): + """engine#2010: a row must BE the measurement. `prefix` is the first byte where the twins diverge, so a row + derived from `64 + 2*hidden + 4,352` (or from a neighbour's shape) is wrong in a way no geometry check sees -- + and a 1.9 pack with no 1.8 twin cannot be given a row at all, because there is nothing to diff. Returns + (ok, message).""" + hidden, heads, layers, prefix, n, rot, tile = measured + rows = table_rows(source_text) + key = (hidden, heads, layers) + if key not in rows: + return False, f"no kFconst19Rules row for {hidden}/{heads}/{layers}: the table cannot describe this pack" + got = rows[key] + want = (prefix, n, rot, tile) # the struct's field order + if got != want: + return False, (f"row {hidden}/{heads}/{layers} is {got} but this pair measures {want} " + f"(prefix {got[0]} vs measured {prefix})") + return True, f"row {hidden}/{heads}/{layers} is exactly what this pair measures" + + def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--pack-1-8", required=True, help="the 1.8 pack of the same model") ap.add_argument("--pack-1-9", required=True, help="the 1.9 pack to measure") ap.add_argument("--json", action="store_true", help="print the result as JSON instead of the table line") + ap.add_argument("--check-row", metavar="PACK_IMPORT_CPP", + help="after measuring, find this geometry's row in PACK_IMPORT_CPP and FAIL if any field differs " + "from what this pair measures. The guard for a row derived from arithmetic (or from a " + "neighbour's shape) instead of from the twin-diff this tool computes: engine#2010") ap.add_argument("--probe-wrong", action="store_true", help="having found the rule, perturb it one field at a time and print each residual, so the " "tolerance has a measurement behind it instead of an argument") @@ -196,6 +231,11 @@ def main(): else: print(f"// verified: {note} ({len(a)} B)") print(f"{{{hidden}, {heads}, {layers}, {prefix}, {n}, {rot}, {tile}}},") + if args.check_row: + ok, why = check_row(open(args.check_row).read(), (hidden, heads, layers, prefix, n, rot, tile)) + print(f"# {'ok' if ok else 'MISMATCH'}: {why}", file=sys.stderr if not ok else sys.stdout) + if not ok: + return 1 if args.probe_wrong: # What a WRONG rule costs, in the same units as the tolerance: perturb one field of the accepted # rule at a time, normalise again, and report the residual. A tolerance is only defensible when the From b0f550dc20d8eb4c837c8e3018b1036c9dbe1a55 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:18:47 -0400 Subject: [PATCH 17/98] npu: a launch's result is dequantised with one multiply per element, not one ldexp() (Refs #1990) (#2008) 2^e_ref depends only on the column, and it was a libm call per element after readback, where no launch phase column counted it: 137 ms of a 630 ms prefill block on the served 27B pack. It is now computed once per column. Same bits (exact power-of-two multiply; out-of-range exponents keep the per-element form): NPU_DEQ_LDEXP=2 counts differing elements, and npu_pb16i_layout_test holds the two forms equal. The [bl] line names the term as deq. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/npu.md | 17 +++++++ npu/forward/engine-src/npu_engine_i8ctx_inc.h | 22 +++++++- npu/forward/include/q4nx_lh.h | 3 +- npu/forward/include/q4nx_pb16i.h | 43 ++++++++++++++++ npu/forward/src/q4nx_forward.cpp | 4 +- npu/target/xdna/xdna_target.cpp | 9 +++- tests/npu_pb16i_layout_test.cpp | 50 +++++++++++++++++++ 7 files changed, 141 insertions(+), 7 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 94dffe23b..5337e7d9c 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -518,6 +518,23 @@ checks keep the contract honest without a device: `tests/npu_t2s_layout_test.cpp against the numbers the tool measured on a real image (gate/up: 50,135,040 B of pb16i → 23,674,880 B of stream), and `tests/q4nx_t2s_test.py` pins the code order byte for byte. +**That single rounding costs one multiply per element, not one `ldexp()` (#1990).** A launch's result is +`float32(S * 2^e_ref) * ascale` for every element, and `2^e_ref` depends only on the column. It was computed +with a libm `ldexp()` call per element, after readback, where no launch phase column counted it: a 16-row +prefill block on `bonsai2-27b-t2s2` is 416 launches of up to 16 x 17,408 elements, and that call was the +block's largest host term. `q4nx_pb16i::dequant_rows()` computes the power once per column and multiplies. +The value is the same, not close: a double times a normal power of two is exact while the product is a +normal double, which holds for every exponent from -1022 to 960, and any column outside that range takes +the per-element form as before. Measured in one `npu-hold` ticket, one binary, `NPU_DEQ_LDEXP=1` (the old +form) against the default, 103 blocks per arm: the dequant term **137.4 -> 32.3 ms per block**, the block +median **630.4 -> 515.0 ms** (25.4 -> 31.1 rows/s), and three 510-536-token prompts reached their first +token at 24.4-25.1 tok/s before and **29.8-30.6 tok/s** after. That is the first pair of arms; the box +slowed for both forms in the second pair (the old form measured 357 ms of dequant per block there), so the +second pair is not quoted as a speedup. `NPU_DEQ_LDEXP=2` runs both forms and counts differing elements: +0 of 568,483,840 on the device, and the greedy text is identical in all five arms. +`tests/npu_pb16i_layout_test.cpp` holds the two forms to the same bits for one and two planes, the edges +of the exponent range and exponents outside it. The `[bl]` line names the term as `deq`. + **A t2s container wants a sealed set for the same reason a pb16i one does.** The layout is not what makes a phase change expensive: the **hw_context** is. Measured on this box in-loop on 2026-11-02, a t2s decode family shipped as eight per-role `_t2s2_m16` ELFs paid **+2.0-2.3 ms on every launch, in every role** — more diff --git a/npu/forward/engine-src/npu_engine_i8ctx_inc.h b/npu/forward/engine-src/npu_engine_i8ctx_inc.h index 356526609..7755935f5 100644 --- a/npu/forward/engine-src/npu_engine_i8ctx_inc.h +++ b/npu/forward/engine-src/npu_engine_i8ctx_inc.h @@ -462,17 +462,35 @@ struct I8Ctx { // row is dequantised with its own activation scale. void dequant_pb16i(float* out, int am, int an, const float* ascales, const std::vector& eref, int planes) { + // engine#1990: one 2^e_ref per column instead of one ldexp() per element; same bits (q4nx_pb16i.h). + // NPU_DEQ_LDEXP=1 keeps the per-element form for an A/B; =2 runs both and counts differing elements. + static const int legacy = [] { + const char* e = getenv("NPU_DEQ_LDEXP"); + return e ? atoi(e) : 0; + }(); + if (legacy != 1) + q4nx_pb16i::dequant_rows(out, Cm, ND, am, an, ascales, eref.data(), planes, deq_pow2_); + if (legacy == 0) return; const int row = planes > 1 ? 2 * ND : ND; + static long calls = 0, elems = 0, differing = 0; for (int m = 0; m < am; m++) { const int32_t* lo = Cm + (size_t)m * (size_t)row; const float asc = ascales[m]; for (int n = 0; n < an; n++) { const int64_t s = q4nx_pb16i::sum_of(lo + n, lo + ND + n, planes > 1 ? 1 : 0); - out[(size_t)m * (size_t)an + n] = - q4nx_pb16i::scale_i64(s, eref[(size_t)n]) * asc; + const float v = q4nx_pb16i::scale_i64(s, eref[(size_t)n]) * asc; + float& o = out[(size_t)m * (size_t)an + n]; + if (legacy == 2 && memcmp(&o, &v, sizeof v) != 0) differing++; + o = v; } } + if (legacy == 2) { + elems += (long)am * an; + if (++calls % 2000 == 0) + fprintf(stderr, "[deqcheck] calls=%ld elements=%ld differing=%ld\n", calls, elems, differing); + } } + std::vector deq_pow2_; // dequant_pb16i's per-column scratch // Cache-friendly per-column int8 quantization into a [KD][ND] padded buffer. // The original column-at-a-time loop reads w[i*N + j] with a stride of N diff --git a/npu/forward/include/q4nx_lh.h b/npu/forward/include/q4nx_lh.h index e2c0301fb..32f04638d 100644 --- a/npu/forward/include/q4nx_lh.h +++ b/npu/forward/include/q4nx_lh.h @@ -55,6 +55,7 @@ enum LhSlot { LH_RQ_RESP, // response serialise + write LH_PACK_UPLOAD, // weight BO create + pack + upload (wcache_ miss only) LH_HEAD_ROWS, // run_lm_head_rows(): the head for a batched verify pass + LH_DEQ_OUT, // a layer launch's result -> f32 (after readback, so in no launch phase column) LH_N }; inline bool lh_on() { @@ -96,7 +97,7 @@ inline const char* lh_name(int s) { "decode (step loop)", " of-which: sample+grammar", " of-which: detokenise token", "response write", " of-which: weight pack+upload (wcache miss)", - "lm head (batch rows)"}; + "lm head (batch rows)", "launch output dequant"}; return (s >= 0 && s < LH_N) ? kn[s] : "?"; } } // namespace q4nx_lh diff --git a/npu/forward/include/q4nx_pb16i.h b/npu/forward/include/q4nx_pb16i.h index f511a1930..28d1e5abb 100644 --- a/npu/forward/include/q4nx_pb16i.h +++ b/npu/forward/include/q4nx_pb16i.h @@ -152,6 +152,49 @@ inline int64_t sum_of(const int32_t* lo, const int32_t* hi, int mode) { return ((int64_t)*hi << 21) + (int64_t)*lo; } +// A launch's whole result, dequantised: out[m][n] = scale_i64(S[m][n], e_ref[n]) * ascales[m], bit for bit. +// +// scale_i64() is one libm call per element, and a prefill block is 416 launches of up to 16 x 17408 elements, so +// that call was a block's largest host term outside the device wait (engine#1990: 0.15-0.50 ms per 16-row launch, +// measured on the host). 2^e_ref depends only on the column, so it is computed once per column here and each element +// is a multiply. That is the same value, not an approximation: for a normal power of two p, (double)s * p is exact +// whenever the product is a normal double, which holds for every nonzero s when e_ref is in [kExpLo, kExpHi] +// (|s| < 2^63, so the product stays under 2^1023, and it is at least 2^-1022). ldexp() is exact under the same +// condition, so both forms hand the same double to the one float rounding. Outside that range -- no container +// has such a column, but the header does not get to assume it -- every element takes scale_i64() as before. +// +// `Cm` is the result as the kernel wrote it: per row, one int32 plane of ND values, or [lo | hi] when planes > 1. +// `pow2` is caller-owned scratch so a launch does not allocate. +constexpr int kExpLo = -1022, kExpHi = 960; +inline void dequant_rows(float* out, const int32_t* Cm, int ND, int am, int an, const float* ascales, + const int16_t* eref, int planes, std::vector& pow2) { + const int row = planes > 1 ? 2 * ND : ND; + const int mode = planes > 1 ? 1 : 0; + bool table = true; + pow2.resize((size_t)(an > 0 ? an : 0)); + for (int n = 0; n < an; n++) { + const int e = (int)eref[n]; + if (e < kExpLo || e > kExpHi) { table = false; break; } + pow2[(size_t)n] = std::ldexp(1.0, e); + } + for (int m = 0; m < am; m++) { + const int32_t* lo = Cm + (size_t)m * (size_t)row; + const int32_t* hi = lo + ND; + const float asc = ascales[m]; + float* o = out + (size_t)m * (size_t)an; + if (!table) { + for (int n = 0; n < an; n++) o[n] = scale_i64(sum_of(lo + n, hi + n, mode), eref[n]) * asc; + } else if (mode) { + const double* p = pow2.data(); + for (int n = 0; n < an; n++) + o[n] = (float)((double)(((int64_t)hi[n] << 21) + (int64_t)lo[n]) * p[n]) * asc; + } else { + const double* p = pow2.data(); + for (int n = 0; n < an; n++) o[n] = (float)((double)lo[n] * p[n]) * asc; + } + } +} + // The sidecar files a container's pb16i set is keyed on: blk___weight. inline std::string sidecar_stem(int layer, const char* tag) { return std::string("blk_") + std::to_string(layer) + "_" + tag + "_weight"; diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index b5334363a..3bbf2df77 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -2484,10 +2484,10 @@ struct BatchPhases { const double row = M > 0 ? (double)M : 1.0; std::fprintf(stderr, "[bl] M=%2d gdn %7.1f ab %7.1f core %7.1f attn %6.1f mlp %6.1f rms %5.1f " - "res %5.1f gemm %7.1f head %6.1f | per row: gdn %6.1f attn %5.1f " + "res %5.1f gemm %7.1f head %6.1f deq %6.1f | per row: gdn %6.1f attn %5.1f " "gemm %6.1f head %5.1f\n", M, ms(LH_GDN), ms(LH_GDN_AB), ms(LH_GDN_CORE), ms(LH_ATTN), ms(LH_MLP), - ms(LH_RMS), ms(LH_RES), gemm, ms(LH_HEAD_ROWS), + ms(LH_RMS), ms(LH_RES), gemm, ms(LH_HEAD_ROWS), ms(LH_DEQ_OUT), ms(LH_GDN) / row, ms(LH_ATTN) / row, gemm / row, ms(LH_HEAD_ROWS) / row); } }; diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 3c6dcdc45..5864d12e2 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -1080,8 +1080,13 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: const auto tw = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; kctx.readback(); const auto tr = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; - out.assign((size_t)M * (size_t)N, 0.0f); - kctx.dequant_pb16i(out.data(), M, N, ascales.data(), slot->eref, kctx.pb16i_planes); + { + // `out` is already M x N (sized before the refusal paths above) and every element is written here, so it + // is not zero-filled a second time. Timed on its own: this runs after `tr`, so no launch phase column + // ever counted it, and the batched [lh] table had it under UNATTRIBUTED (engine#1990). + q4nx_lh::LhScope _t_deq(LH_DEQ_OUT); + kctx.dequant_pb16i(out.data(), M, N, ascales.data(), slot->eref, kctx.pb16i_planes); + } // The batched (prefill) twin of the trace above: a short prompt runs step_batch, so this // is the path a 1-token request actually takes. if (const char* tl = getenv("NPU_PB16I_TRACE")) { diff --git a/tests/npu_pb16i_layout_test.cpp b/tests/npu_pb16i_layout_test.cpp index 2336c1413..9c0511d12 100644 --- a/tests/npu_pb16i_layout_test.cpp +++ b/tests/npu_pb16i_layout_test.cpp @@ -206,6 +206,56 @@ int main() { for (const geom& g : bad) check(!q4nx_pb16i::fits(g.K, g.N), "a non-whole-tile geometry must be refused", g.K, g.N); + // 7: dequant_rows() (engine#1990) is scale_i64() per element, bit for bit -- one plane and two, the exponents a + // container uses, the edges of the table's range, and exponents outside it (which must take the per-element form + // and still agree). The sums include 0, +-1, the int32 extremes and two-plane values past 2^53, where + // (double)s itself rounds: both forms round it the same way first. + { + struct Case { int planes, M, N, e_lo, e_hi; const char* what; }; + const Case cases[] = { + {1, 16, 1024, -40, -8, "one plane, container-like exponents"}, + {2, 16, 1024, -40, -8, "two planes, container-like exponents"}, + {1, 3, 257, q4nx_pb16i::kExpLo, q4nx_pb16i::kExpLo + 3, "one plane, lowest table exponents"}, + {2, 3, 257, q4nx_pb16i::kExpHi - 3, q4nx_pb16i::kExpHi, "two planes, highest table exponents"}, + {2, 2, 64, -1100, -1000, "exponents below the table range (per-element path)"}, + {1, 2, 64, 950, 1100, "exponents above the table range (per-element path)"}, + {1, 1, 1, 0, 0, "a single element"}, + }; + uint32_t x = 0x9e3779b9u; + auto next = [&]() { x ^= x << 13; x ^= x >> 17; x ^= x << 5; return x; }; + size_t compared = 0, nonzero = 0; + for (const Case& c : cases) { + const int ND = c.N + 5; // the kernel's row is wider than the columns read + const int row = c.planes > 1 ? 2 * ND : ND; + std::vector Cm((size_t)c.M * (size_t)row); + for (int32_t& v : Cm) v = (int32_t)next(); + const int32_t edge[] = {0, 1, -1, INT32_MAX, INT32_MIN}; + for (size_t i = 0; i < Cm.size() && i < 40; i++) Cm[i] = edge[i % 5]; + std::vector eref((size_t)c.N); + for (int n = 0; n < c.N; n++) eref[(size_t)n] = (int16_t)(c.e_lo + (int)(next() % (uint32_t)(c.e_hi - c.e_lo + 1))); + std::vector asc((size_t)c.M); + for (float& a : asc) a = 1e-4f * (float)(1 + next() % 5000); + std::vector want((size_t)c.M * (size_t)c.N), got(want.size(), -7.0f); + for (int m = 0; m < c.M; m++) { + const int32_t* lo = Cm.data() + (size_t)m * (size_t)row; + for (int n = 0; n < c.N; n++) + want[(size_t)m * (size_t)c.N + n] = + q4nx_pb16i::scale_i64(q4nx_pb16i::sum_of(lo + n, lo + ND + n, c.planes > 1 ? 1 : 0), + eref[(size_t)n]) * asc[(size_t)m]; + } + std::vector scratch; + q4nx_pb16i::dequant_rows(got.data(), Cm.data(), ND, c.M, c.N, asc.data(), eref.data(), c.planes, scratch); + size_t diff = 0; + for (size_t i = 0; i < want.size(); i++) { + if (std::memcmp(&want[i], &got[i], sizeof(float)) != 0) diff++; + if (want[i] != 0.0f) nonzero++; + } + compared += want.size(); + if (diff) { std::printf("FAIL dequant_rows != scale_i64 per element: %s (%zu of %zu differ)\n", c.what, diff, want.size()); failures++; } + } + if (nonzero < compared / 4) { std::printf("FAIL dequant_rows arm compares mostly zeros (%zu of %zu nonzero)\n", nonzero, compared); failures++; } + } + if (failures) { std::printf("npu_pb16i_layout_test: %d failure(s)\n", failures); return 1; From 003aad1c03c6c58db2e295a5bd60a86418cbdc6a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:18:54 -0400 Subject: [PATCH 18/98] tools: hf_tensor.py reads one safetensors tensor with the offset rule I got wrong four times (refs #1934) (#2003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A safetensors `data_offsets` entry is measured from the **start of the data section**, i.e. `8 + header_len` bytes into the file -- not from byte 8. Fetch `bytes=8+off` and you get a slice of a *different* tensor, one header-length away; it has a plausible size, it is finite, and a correlation against it is a correlation against noise. That mistake cost two false conclusions about a merged PR tonight, both stated publicly before I caught them: "the 640 B gap does not correlate with the source model's q/k gains at any offset" and "the container still holds the pre-#1938 two-copies signature", the second of which I had to retract on #1976 when the correctly-fetched tensor gave `mean 1.3705, max 1.703125, min 0` — matching the container to three decimals. The tell was in my own output: a norm gain is ~1.0 and my "HF" values were +-0.03. So this tool does the arithmetic once and makes a wrong read *visible* rather than plausible: ``` $ tools/hf_tensor.py --repo tencent/Hunyuan-7B-Instruct \ --tensor model.layers.0.self_attn.query_layernorm.weight model.layers.0.self_attn.query_layernorm.weight BF16 [128] 256 B (model-00001-of-00004.safetensors) mean +1.370461 min -0.000000 max +1.703125 nonfinite 0 a norm gain is ~1.0 and a weight matrix is ~0: check these before trusting a correlation ``` * reads only the index, the shard's 8-byte length + header, and the tensor's own bytes (three range requests); * prints mean/min/max/nonfinite, because that is what separates "the tensor I asked for" from "a slice of its neighbour"; * `--out` writes the raw bytes for a comparison script, `--json` for one; * `--self-test` drives the rule on a **synthetic** shard with no network: a first tensor long enough that the second's offset is past it, then the same offset applied from the data section (returns the tensor) and from byte 8 (does not). Registered as `hf_tensor_self` so the rule is pinned in every configure; `ctest -R 'hf_tensor|tests_referenced'` -> 3/3. Refs #1934. This is the first half of the `diff-hf` instrument I asked for on that issue; the second half -- decode a *container* tensor through the importer's own decoder and print the per-row regression ratio -- is the next step. Co-authored-by: t --- CMakeLists.txt | 5 ++ tools/hf_tensor.py | 213 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100755 tools/hf_tensor.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 52552b533..f2af8544f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -336,6 +336,11 @@ add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test- # engine#1702: builds a per-layer MoE expert placement table from a moe_trace, and refuses a prefill-only trace because # placement is a decode lever (5.4-6.9 % on batched prefill against dedup's 90-94 %). --self-test drives it on synthetic # traces with a known answer; no model, no device. +# engine#1934: reading ONE tensor out of a safetensors shard needs `data_offsets` measured from 8 + header_len, not +# from byte 8. Getting that wrong returns a *different* tensor of the same size that still looks finite, and it cost +# two false conclusions about a pack in one night. `--self-test` drives the rule on a synthetic shard (no network): +# the same offset from the data section returns the tensor, from byte 8 it does not. +add_python_test(hf_tensor_self ${CMAKE_SOURCE_DIR}/tools/hf_tensor.py --self-test) add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) # engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as # it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that diff --git a/tools/hf_tensor.py b/tools/hf_tensor.py new file mode 100755 index 000000000..57f9b4b59 --- /dev/null +++ b/tools/hf_tensor.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""hf_tensor.py -- read ONE tensor out of a safetensors shard, over HTTP range requests. + +Written because I got a safetensors offset wrong four times in one night and each time concluded something +false about a pack. The rule that bit me, and the only reason this file exists: + + a safetensors `data_offsets` entry is relative to the START OF THE DATA SECTION, i.e. `8 + header_len` + bytes into the file -- not to byte 8. + +Fetch `bytes=8+off` and you get a slice of whatever tensor is 11,656 B (this shard's header) away from the one +you asked for, which still *looks* like a tensor: it has a plausible size, it is finite, and a correlation +against it is a correlation against noise. Two of those cost me a retraction on a merged PR. + +So this tool does the arithmetic once, prints the tensor's own statistics so a wrong read is visible rather +than plausible (a norm gain is ~1.0, not +-0.03), and can write the raw bytes for a comparison script. + +usage: + tools/hf_tensor.py --repo tencent/Hunyuan-7B-Instruct \ + --tensor model.layers.0.self_attn.query_layernorm.weight [--out q.bin] [--json] + tools/hf_tensor.py --self-test + +Exit: 0 ok, 1 the tensor or shard was not found, 2 cannot tell (no network / not a safetensors repo). +""" + +import argparse +import json +import pathlib +import struct +import sys +import urllib.request + +UA = {"User-Agent": "1bit-aba/hf_tensor"} # HF rejects the default python urllib agent on some endpoints +DTYPE_TABLE = { + "F64": (" Date: Tue, 10 Nov 2026 16:30:12 -0400 Subject: [PATCH 19/98] npu-pack: import a 1.9 pack whose every MatMulNBits is block 32 (engine#1503, stacked on #1948) (#1955) * npu-pack: read the 1.9 block-32 head from the .fconst tail (Fixes #1740) A pack that declares /lm_head/MatMul_Q4 at block_size 32 (the Ryzen AI 1.9 Qwen-1.5B packs) keeps the head after the tail envelope in 64-row units of 2,560 + 2*(K/32)*640 bytes, not in the block-128 tile. read_block32_head() reads that layout; import_pack() takes it when the layout checks hold and, on a tied pack, the result correlates above 0.99 with model.embed_tokens. Otherwise a tied pack keeps the synthesis (and the note says why), and an untied one is refused with the reason. Measured on Qwen-2.5_1.5B_Instruct_rai_1.9.0_npu: 2,374 units of 64,000 B, corr 0.996185 against the embedding over all 151,936 rows, row-shifted control 0.0554. Co-Authored-By: Claude Fable 5.2 * npu-pack: import a 1.9 pack whose every MatMulNBits is block 32 (engine#1503) An all-block-32 pack (SmolLM2-135M-Instruct 1.9) was refused at the block gate. Its raw Token .fconst is now read with its own layout: 64-row units of 2,560 + 2*(K/32)*640 B for q/k/v/o/down and the head, and 64-row units of 4,864 + 2*(K/32)*1,216 B holding gate and up together. Offsets come from the geometry and are checked at every epsilon envelope. read_block32_head becomes a wrapper over read_block32_units; the head logic #1948 added is shared by the mixed pack and this path. Co-Authored-By: Claude Fable 5.2 * npu-pack: one arm per declared block size; unmeasured sizes are refused by number (#1503) Review on #1955 asked what a pack declaring a block nobody handles does, and for an arm per declaration rather than one for the class. The all-block-32 fixture is now rebuilt with only cache/ops_meta.json changed: every op at 3 / 64 / 256 refused, 'block size along K is N (11 ops)' layers 32, head 128 refused, not the all-block-32 class layers 32, head 64 refused every op at 128 stays off the all-block-32 path and the gate With the selection made to ignore the declared value, all six arms fail (each writes a container through the all-block-32 path). The gate's refusal now says the all-block-32 layout exists and when it applies. Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/npu.md | 44 +++ npu/pack_import.cpp | 559 ++++++++++++++++++++++++++------- npu/pack_import.h | 87 ++++- tests/npu_pack_import_test.cpp | 503 +++++++++++++++++++++++++++++ 4 files changed, 1069 insertions(+), 124 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 5337e7d9c..92527a0de 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1249,6 +1249,50 @@ layout checks are all there is, and the note says the values are not verified. M equals `bf16(-m/8)` of its 32-column embedding block, `m` being the block's largest-magnitude weight, and the read head correlates 0.996 with the embedding. +**A pack whose every MatMulNBits is `block_size 32` keeps its layers in those units too** +(engine#1503). `SmolLM2-135M-Instruct_rai_1.9.0_npu` declares block 32 on all 151 ops, and its +*raw* Token `.fconst` (the file as shipped, not the 1.8-form copy the block-128 path makes) is +`layers` blocks of one fixed shape and then the tail. With `H` = hidden, `Nq` = heads x head_dim, +`Nkv` = KV heads x head_dim, `IM` = intermediate, `unit(K) = 2,560 + 2 x (K/32) x 640` (the head's +unit above) and `mlp_unit(K) = 4,864 + 2 x (K/32) x 1,216`: + +| section | bytes | holds | +|---|---|---| +| envelope | `64 + 2H` | bf16 `rms_norm_eps`, zeros to 64, `input_layernorm` bf16[H] | +| q, k, v | `(Nq/64 + 2 x Nkv/64) x unit(H)` | `q_proj` [Nq][H], `k_proj` [Nkv][H], `v_proj` [Nkv][H] | +| — | `4 x heads` | not a weight; skipped | +| o | `(H/64) x unit(Nq)` | `o_proj` [H][Nq] | +| envelope | `64 + 2H` | epsilon, zeros, `post_attention_layernorm` bf16[H] | +| gate + up | `(IM/64) x mlp_unit(H)` | `gate_proj` and `up_proj`, [IM][H] each | +| — | 64 | skipped, not interpreted (`01` then zeros on the measured pack) | +| down | `(H/64) x unit(IM)` | `down_proj` [H][IM] | + +The tail is the same envelope with the final norm, then `ceil(vocab/64)` head units at K = `H`, +then zeros. On SmolLM2-135M the block is 2,360,292 B (q at 1,216, k at 231,616, v at 308,416, o at +385,252, the second envelope at 615,652, gate/up at 616,868, down at 1,784,292), the head is at +70,809,976 and the file is exactly `30 x 2,360,292 + 1,216 + 768 x 25,600 = 90,470,776` B. + +The **gate/up unit** is the head's unit with 64 row *slots* per strip instead of 32 rows: 4,864 +zero bytes, then `2 x (K/32)` strips of 1,216 B. Strip `s` holds K block `s/2`; slots 0..31 are +`gate_proj` rows `32 x (s%2) .. +31` of the unit and slots 32..63 are the same rows of `up_proj`. +A strip is 1,024 B of 4-bit codes (low nibble first; position `p` is slot `(p/256) x 8 + p%8`, +column `(p/8) % 32`), 32 B of zero-point nibbles, 32 zero bytes, and 64 signed bf16 scales. The +code order was measured against the public checkpoint; that zero point and scale are indexed by +slot is assumed by analogy with the 640-byte strip, and is tested by the imported container's +gate/up still correlating with the checkpoint once they are applied. + +`npu-pack import` takes this path when every op the pack's `cache/*_meta.json` declares is block 32, +at least one of them is a layer projection, and the pack has a Token `.fconst` (a pack that +declares only its head at 32 is the case above). Nothing is located by search: the offsets are +arithmetic on the geometry, and the file has to agree with them — at least +`layers x stride + 64 + 2H + ceil(vocab/64) x unit(H)` bytes, the pack's epsilon and zeros at every +envelope, zero unit headers and strip gaps, finite scales, zeros after the head — or the tensors +are refused by name with the offset. The container gets the same tensors in the same q4_1f tiles as +a block-128 import; this layout has no bias field, so no bias tensor is written. There is no +decode->encode round trip on this path (no encoder for these units exists), so the import's +`attention:` and `MLP:` round-trip lines are replaced by a `block 32:` line. Measured on one pack +only, where `Nq == H`; `o_proj`'s K being `Nq` is the declared shape, not a measurement. + The norms that *are* imported sit at the documented offsets, each confirmed by the epsilon that opens its layer block: **48/48 anchors found**. `tests/npu_pack_import_test` (registered as `npu_pack_import`, and as `npu_pack_import_oga` when `-DONEBIT_OGA_PACK=` is given) checks diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 2c06dfcf2..ca248267b 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2562,14 +2562,104 @@ static TailHeadLocate locate_tail_head(const Fconst& fc, int64_t tail_off, int64 return r; } -// engine#1740: the block-32 tail head. Layout and its measurement are in pack_import.h. -Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K) { +// engine#1740 / engine#1503: the block-32 units. Layouts and their measurement are in pack_import.h. +namespace { +bool b32_all_zero(const uint8_t* b, int64_t n) { + for (int64_t i = 0; i < n; ++i) + if (b[i] != 0) return false; + return true; +} +float b32_bf16(const uint8_t* b) { + const uint32_t wide = uint32_t(uint16_t(b[0]) | uint16_t(uint16_t(b[1]) << 8)) << 16; + float v; + std::memcpy(&v, &wide, 4); + return v; +} +void b32_q_init(Q4& q, int64_t rows, int64_t K) { + q.N = int(rows); + q.K = int(K); + q.block = 32; + q.codes.assign(size_t(rows) * size_t(K), 0); + q.scale.assign(size_t(rows) * size_t(K / 32), 1.0f); + q.zero.assign(size_t(rows) * size_t(K / 32), 0.0f); +} +// One row's 32 codes of one K block out of a strip's code field. `slot` is the row's index inside the strip (0..31 +// in a 640-byte strip, 0..63 in a gate/up one): position `(slot/8)*256 + col*8 + slot%8`, low nibble first. A negative +// scale is folded into the codes, so the stored scale is positive: (code - zp) * sc == ((15-code) - (15-zp)) * -sc. +void b32_row(Q4& q, int64_t row, int64_t kblock, const uint8_t* codes, int slot, int zp, float sc) { + if (sc == 0.0f) return; // an all-zero block: codes 0, scale 1, zero 0 decode to 0 + const bool flip = sc < 0; + const float mag = flip ? -sc : sc; + const size_t g = size_t(row) * size_t(q.K / 32) + size_t(kblock); + q.scale[g] = mag; + q.zero[g] = -float(flip ? 15 - zp : zp) * mag; + const int base = (slot / 8) * 256 + (slot % 8); + uint8_t* dst = q.codes.data() + size_t(row) * size_t(q.K) + size_t(kblock) * 32; + for (int c = 0; c < 32; ++c) { + const int pos = base + c * 8; + const int code = (codes[pos / 2] >> (4 * (pos & 1))) & 15; + dst[c] = uint8_t(flip ? 15 - code : code); + } +} +} // namespace + +Block32Units read_block32_units(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, const char* what, + int64_t file_off) { + Block32Units h; + const std::string tag = std::string("block-32 ") + what; + if (rows <= 0 || K <= 0 || K % 32 != 0) { + h.why = "a " + tag + " needs K to be a multiple of 32 (K " + std::to_string(K) + ")"; + return h; + } + const int64_t kb = K / 32, strips = 2 * kb, unit = block32_unit_bytes(K); + h.units = (rows + kB32UnitRows - 1) / kB32UnitRows; + h.bytes = h.units * unit; + if (avail < h.bytes) { + h.why = "the .fconst has " + std::to_string(avail) + " B at " + std::to_string(file_off) + " B, and a " + tag + + " of " + std::to_string(h.units) + " units x " + std::to_string(unit) + " B needs " + + std::to_string(h.bytes); + return h; + } + b32_q_init(h.q, h.units * kB32UnitRows, K); + for (int64_t u = 0; u < h.units; ++u) { + const uint8_t* up = p + u * unit; + const std::string where = tag + " unit " + std::to_string(u) + " (at " + std::to_string(file_off + u * unit) + + " B)"; + if (!b32_all_zero(up, kB32UnitHeader)) { + h.why = where + " does not open with " + std::to_string(kB32UnitHeader) + + " zero bytes, so this is not the layout measured"; + return h; + } + for (int64_t s = 0; s < strips; ++s) { + const uint8_t* sp = up + kB32UnitHeader + s * kB32StripBytes; + if (!b32_all_zero(sp + 528, 48)) { + h.why = where + " strip " + std::to_string(s) + + " has non-zero bytes between its zero points and its scales, so this is not the layout measured"; + return h; + } + const int64_t row0 = u * kB32UnitRows + 32 * (s % 2); + for (int r = 0; r < 32; ++r) { + const float sc = b32_bf16(sp + 576 + 2 * r); + if (!std::isfinite(sc)) { + h.why = where + " strip " + std::to_string(s) + " row " + std::to_string(r) + + " has a scale that is not finite"; + return h; + } + b32_row(h.q, row0 + r, s / 2, sp, r, (sp[512 + r / 2] >> (4 * (r & 1))) & 15, sc); + } + } + } + h.ok = true; + return h; +} + +Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K, int64_t file_off) { Block32Head h; if (V <= 0 || K <= 0 || K % 32 != 0) { h.why = "a block-32 head needs K to be a multiple of 32 (K " + std::to_string(K) + ")"; return h; } - const int64_t kb = K / 32, strips = 2 * kb, unit = block32_unit_bytes(K); + const int64_t unit = block32_unit_bytes(K); h.units = (V + kB32UnitRows - 1) / kB32UnitRows; h.bytes = h.units * unit; if (avail < h.bytes) { @@ -2585,58 +2675,60 @@ Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_ return h; } h.padding = avail - h.bytes; - const auto all_zero = [](const uint8_t* b, int64_t n) { - for (int64_t i = 0; i < n; ++i) - if (b[i] != 0) return false; - return true; - }; - Q4& q = h.q; - q.N = int(h.units * kB32UnitRows); - q.K = int(K); - q.block = 32; - q.codes.assign(size_t(q.N) * size_t(K), 0); - q.scale.assign(size_t(q.N) * size_t(kb), 1.0f); - q.zero.assign(size_t(q.N) * size_t(kb), 0.0f); + Block32Units u = read_block32_units(p, h.bytes, V, K, "head", file_off); + if (!u.ok) { + h.why = u.why; + return h; + } + h.q = std::move(u.q); + h.ok = true; + return h; +} + +Block32GateUp read_block32_gate_up(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, int64_t file_off) { + Block32GateUp h; + if (rows <= 0 || K <= 0 || K % 32 != 0) { + h.why = "a block-32 gate/up needs K to be a multiple of 32 (K " + std::to_string(K) + ")"; + return h; + } + const int64_t kb = K / 32, strips = 2 * kb, unit = block32_mlp_unit_bytes(K); + h.units = (rows + kB32UnitRows - 1) / kB32UnitRows; + h.bytes = h.units * unit; + if (avail < h.bytes) { + h.why = "the .fconst has " + std::to_string(avail) + " B at " + std::to_string(file_off) + + " B, and a block-32 gate/up of " + std::to_string(h.units) + " units x " + std::to_string(unit) + + " B needs " + std::to_string(h.bytes); + return h; + } + b32_q_init(h.gate, h.units * kB32UnitRows, K); + b32_q_init(h.up, h.units * kB32UnitRows, K); for (int64_t u = 0; u < h.units; ++u) { const uint8_t* up = p + u * unit; - if (!all_zero(up, kB32UnitHeader)) { - h.why = "block-32 head unit " + std::to_string(u) + " does not open with " + - std::to_string(kB32UnitHeader) + " zero bytes, so this is not the layout measured"; + const std::string where = + "block-32 gate/up unit " + std::to_string(u) + " (at " + std::to_string(file_off + u * unit) + " B)"; + if (!b32_all_zero(up, kB32MlpUnitHeader)) { + h.why = where + " does not open with " + std::to_string(kB32MlpUnitHeader) + + " zero bytes, so this is not the layout measured"; return h; } for (int64_t s = 0; s < strips; ++s) { - const uint8_t* sp = up + kB32UnitHeader + s * kB32StripBytes; - if (!all_zero(sp + 528, 48)) { - h.why = "block-32 head unit " + std::to_string(u) + " strip " + std::to_string(s) + + const uint8_t* sp = up + kB32MlpUnitHeader + s * kB32MlpStripBytes; + if (!b32_all_zero(sp + 1056, 32)) { + h.why = where + " strip " + std::to_string(s) + " has non-zero bytes between its zero points and its scales, so this is not the layout measured"; return h; } - const int64_t col0 = (s / 2) * 32, row0 = u * kB32UnitRows + 32 * (s % 2); - for (int r = 0; r < 32; ++r) { - const uint16_t bits = uint16_t(sp[576 + 2 * r]) | uint16_t(uint16_t(sp[577 + 2 * r]) << 8); - const uint32_t wide = uint32_t(bits) << 16; - float sc; - std::memcpy(&sc, &wide, 4); + const int64_t row0 = u * kB32UnitRows + 32 * (s % 2); + for (int slot = 0; slot < 64; ++slot) { + const float sc = b32_bf16(sp + 1088 + 2 * slot); if (!std::isfinite(sc)) { - h.why = "block-32 head unit " + std::to_string(u) + " strip " + std::to_string(s) + " row " + - std::to_string(r) + " has a scale that is not finite"; + h.why = where + " strip " + std::to_string(s) + " slot " + std::to_string(slot) + + " has a scale that is not finite"; return h; } - const int zp = (sp[512 + r / 2] >> (4 * (r & 1))) & 15; - const bool flip = sc < 0; - const size_t g = size_t(row0 + r) * size_t(kb) + size_t(s / 2); - if (sc == 0.0f) continue; // an all-zero block: codes 0, scale 1, zero 0 decode to 0 - const float mag = flip ? -sc : sc; - q.scale[g] = mag; - q.zero[g] = -float(flip ? 15 - zp : zp) * mag; - // position p = band*256 + col*8 + (r%8) holds row band*8 + r%8, column col - const int base = (r / 8) * 256 + (r % 8); - uint8_t* dst = q.codes.data() + size_t(row0 + r) * size_t(K) + size_t(col0); - for (int c = 0; c < 32; ++c) { - const int pos = base + c * 8; - const int code = (sp[pos / 2] >> (4 * (pos & 1))) & 15; - dst[c] = uint8_t(flip ? 15 - code : code); - } + // slots 0..31 are gate rows, 32..63 the same rows of up; zero point and scale by slot (assumed) + b32_row(slot < 32 ? h.gate : h.up, row0 + slot % 32, s / 2, sp, slot, + (sp[1024 + slot / 2] >> (4 * (slot & 1))) & 15, sc); } } } @@ -2644,6 +2736,70 @@ Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_ return h; } +Block32Layout block32_layout(const Dims& d) { + Block32Layout l; + const int64_t H = d.hidden, IM = d.intermediate, Nq = int64_t(d.heads) * d.head_dim, + Nkv = int64_t(d.kv_heads) * d.head_dim, V = d.vocab; + if (H <= 0 || IM <= 0 || Nq <= 0 || Nkv <= 0 || V <= 0 || d.layers <= 0) { + l.why = "the all-block-32 layout needs hidden, intermediate, heads, kv heads, head_dim, vocab and layers, and " + "the pack's geometry leaves one of them 0"; + return l; + } + for (const auto& [name, v] : {std::pair{"hidden", H}, + std::pair{"intermediate", IM}, + std::pair{"heads*head_dim", Nq}, + std::pair{"kv_heads*head_dim", Nkv}}) + if (v % kB32UnitRows != 0) { + l.why = std::string("the all-block-32 layout is 64-row units of 32-column blocks, and ") + name + " " + + std::to_string(v) + " is not a multiple of 64"; + return l; + } + l.env = kNormOff + 2 * H; + l.q_off = l.env; + l.k_off = l.q_off + (Nq / kB32UnitRows) * block32_unit_bytes(H); + l.v_off = l.k_off + (Nkv / kB32UnitRows) * block32_unit_bytes(H); + l.sink_off = l.v_off + (Nkv / kB32UnitRows) * block32_unit_bytes(H); + l.o_off = l.sink_off + 4 * int64_t(d.heads); + l.post_off = l.o_off + (H / kB32UnitRows) * block32_unit_bytes(Nq); + l.gu_off = l.post_off + l.env; + l.hdr_off = l.gu_off + (IM / kB32UnitRows) * block32_mlp_unit_bytes(H); + l.down_off = l.hdr_off + 64; + l.stride = l.down_off + (H / kB32UnitRows) * block32_unit_bytes(IM); + l.tail_off = int64_t(d.layers) * l.stride; + l.head_off = l.tail_off + l.env; + l.head_units = (V + kB32UnitRows - 1) / kB32UnitRows; + l.min_size = l.head_off + l.head_units * block32_unit_bytes(H); + return l; +} + +std::string block32_file_refusal(const uint8_t* map, int64_t size, const Dims& d, const Block32Layout& lay) { + if (!lay.why.empty()) return lay.why; + if (size < lay.min_size) + return "the Token .fconst is " + std::to_string(size) + " B, and the all-block-32 layout of this geometry is " + + std::to_string(d.layers) + " layer blocks x " + std::to_string(lay.stride) + " B + a " + + std::to_string(lay.env) + " B envelope + " + std::to_string(lay.head_units) + " head units x " + + std::to_string(block32_unit_bytes(d.hidden)) + " B = " + std::to_string(lay.min_size) + + " B (plus zero padding), so the file is not this layout"; + const auto envelope = [&](int64_t off, const std::string& what) -> std::string { + if (map[off] != uint8_t(d.eps_bits & 0xFF) || map[off + 1] != uint8_t(d.eps_bits >> 8)) + return "no RMSNorm epsilon at " + std::to_string(off) + " B, where the all-block-32 layout puts the " + + what + " envelope, so the file is not this layout"; + for (int64_t i = 2; i < kNormOff; ++i) + if (map[off + i] != 0) + return "the " + what + " envelope at " + std::to_string(off) + " B is not zero between its epsilon " + "and its norm (byte " + std::to_string(off + i) + "), so the file is not this layout"; + return {}; + }; + for (int L = 0; L <= d.layers; ++L) { + const int64_t base = int64_t(L) * lay.stride; + const std::string name = L == d.layers ? std::string("tail") : "layer " + std::to_string(L); + std::string why = envelope(base, name + (L == d.layers ? "" : " input_layernorm")); + if (why.empty() && L < d.layers) why = envelope(base + lay.post_off, name + " post_attention_layernorm"); + if (!why.empty()) return why; + } + return {}; +} + // engine#1740: a self-test for the SEARCH, not the arithmetic. `plan_tail_head` is pinned by the host test; this pins // the part that can go silently wrong -- the candidate loop, best/runner-up tracking, the kTieCorr cut, the ambiguity // refusal and the untied-with-padding refusal -- using a synthetic head built through encode_tile, so it needs no pack, @@ -2913,6 +3069,17 @@ std::string Plan::manifest() const { {"o_proj_k", o_k}, {"qk_norm", knorm_off != 0}, {"note", attn_note}}; + // engine#1503: an all-block-32 pack is read with its own layout, so the block-128 measurements above are 0 for it; + // this says which layout was used and what its arithmetic gives. Absent for every other pack. + if (block32_layers) { + const Block32Layout lay = block32_layout(dims); + j["block32_layers"] = {{"stride", lay.stride}, + {"post_attention_envelope", lay.post_off}, + {"head_offset", lay.head_off}, + {"head_units", lay.head_units}, + {"bytes", lay.min_size}, + {"note", block32_note}}; + } // What the *graph* carries, in the graph's own terms. A pack's genre is the first question every sweep row asks, // and until now it could only be answered by an external tool: the rai_1.3/1.6/1.7.1 line ships its 4-bit weights // as ordinary ONNX initializers (pi-inferzo: 25/25 real qweights, UINT8 [1152,7,64], location model_jit.onnx.data) @@ -3140,7 +3307,22 @@ Plan plan(const std::string& pack_dir) { // The Token .fconst, found once (see find_token_fconst) and used by the attention block below. Empty means this // pack keeps no weights there -- which for the rai_1.3/1.6/1.7.1 line is the normal case, not a fault. std::string tok_why; - std::string tok_path = materialise_token_fconst(pack_dir, p.dims); + const DeclaredMap declared = read_declared(pack_dir); + // engine#1503: a pack whose every declared MatMulNBits is block 32 -- layers included, not only the head -- is read + // from its RAW .fconst with the all-block-32 layout (pack_import.h, Block32Layout). Decided here, before the + // canonical copy is made, because that copy is the 1.8 form of a block-128 file and is not what this path reads. + // A pack that declares only its head (the block-32-head fixtures, and nothing measured) is not this class. + { + bool any = false, all32 = true, layer = false; + for (const auto& [proj, dec] : declared) { + if (!dec.block) continue; + any = true; + if (dec.block != 32) all32 = false; + if (proj.rfind("lm_head", 0) != 0) layer = true; + } + p.block32_layers = any && all32 && layer && !find_token_fconst(pack_dir).empty(); + } + std::string tok_path = p.block32_layers ? find_token_fconst(pack_dir) : materialise_token_fconst(pack_dir, p.dims); // engine#1671 / #1687: a pack whose graph names `*.qweight.empty` placeholders whose `*.packed` siblings carry no // data has no weight for those ops IN THE GRAPH. With a Token .fconst that is the normal shape of a 1.9 pack -- its // graph is all placeholders and the weights are in the .fconst -- so the walker drops them and the import proceeds. @@ -3198,6 +3380,17 @@ Plan plan(const std::string& pack_dir) { {embed->dims.at(0), embed->dims.at(1)})); } + // engine#1503: the all-block-32 layout against the file, once the vocab is known (the head's unit count needs it). + if (p.block32_layers) { + const Block32Layout lay = block32_layout(p.dims); + if (!lay.why.empty()) { + p.block32_note = lay.why; + } else { + Fconst fc = open_fconst(tok_path); + p.block32_note = block32_file_refusal(fc.map, int64_t(fc.size), p.dims, lay); + unmap_file(fc.map, fc.size); + } + } // Only the node whose geometry IS the head (N = V, K = H) is the head. This loop used to take the FIRST MatMulNBits and // `break`, so on a pack whose first matmul is a projection the plan reported that projection's shape as // `lm_head.weight` -- [1152, 896], a fused qkv, instead of [151936, 896] on Qwen2.5-0.5B-Instruct 1.9 (engine#1671). @@ -3218,14 +3411,16 @@ Plan plan(const std::string& pack_dir) { // there if the tail's arithmetic does not come out, so the plan does not refuse it here. Keyed on "did a graph // head get written", not "is the graph free of matmuls": a graph of projections only is not a pack without a head // (engine#1671). - p.tensors.push_back(make("lm_head.weight", "fconst:tail", "q4_1", {d.vocab, d.hidden})); + auto x = make("lm_head.weight", "fconst:tail", "q4_1", {d.vocab, d.hidden}); + x.refusal = p.block32_note; // empty unless an all-block-32 pack's file does not fit its layout + p.tensors.push_back(std::move(x)); } // norms: the layer blocks are located by their epsilon anchors at import time, so the offsets work // for any hidden size (engine#734). plan() cannot prove that without reading the file, so the // refusal here is empty and import_pack reports what the anchors actually found. const std::string norm_source = "fconst:Llm_Token_Token"; - const std::string norm_refusal; + const std::string norm_refusal = p.block32_note; // empty except on an all-block-32 pack whose file does not fit const int64_t H = d.hidden; // the geometry's hidden; the embedding, when the main graph has none, must match it // Genre A: the norms are ordinary F16 initializers named after themselves, with real offsets -- not the // .fconst tail envelope -- so the refusal names the initializer and where it sits. @@ -3265,6 +3460,7 @@ Plan plan(const std::string& pack_dir) { // and the norm at +64, and where both .fconst files exist the two copies are byte-identical. The // position is confirmed at import time by the epsilon marker plus that cross-check. auto fin = make("model.norm.weight", norm_source, "BF16", {H}); + fin.refusal = p.block32_note; p.tensors.push_back(std::move(fin)); // projections: the attention payload (q/k/v/o) decodes tile by tile and the MLP section group by @@ -3281,6 +3477,10 @@ Plan plan(const std::string& pack_dir) { if (tok.empty()) { p.attn_note = tok_why; p.mlp_note = p.attn_note; + } else if (p.block32_layers) { + // the all-block-32 layout is arithmetic on the geometry, checked against the file above + p.attn_note = p.block32_note; + p.mlp_note = p.block32_note; } else if (tile_bytes == 0) { p.attn_note = "hidden " + std::to_string(H) + " is not a multiple of 128, so the attention " "payload is not 64-row tiles of this shape"; @@ -3450,7 +3650,6 @@ Plan plan(const std::string& pack_dir) { {"mlp.gate_proj", d.intermediate, d.hidden}, {"mlp.up_proj", d.intermediate, d.hidden}, {"mlp.down_proj", d.hidden, d.intermediate}}; - const DeclaredMap declared = read_declared(pack_dir); // Block size gate (engine#1410 §3 of FACTS-fconst-layout): the compiler states `block_size` per MatMulNBits, and // this importer's tile geometry is the block-128 one -- a group is 2*(K/block)+2 subvolumes, so a block-32 pack // has every offset in a different place. Refusing by number beats parsing it at the 1.8 geometry, which is what @@ -3488,7 +3687,12 @@ Plan plan(const std::string& pack_dir) { // calling the writer here cost a second 278 MiB rewrite per command. const bool fconst_weights = !find_token_fconst(pack_dir).empty(); if (fconst_weights && !blocks.empty() && !(blocks.size() == 1 && blocks.begin()->first == kKBlock)) { - if (only_the_head) { + if (p.block32_layers) { + // engine#1503: every op is block 32 and the pack is read with the all-block-32 layout, which was + // checked against the file where `block32_layers` was decided; a file that does not fit is refused + // tensor by tensor with `block32_note`. + if (const auto it = declared.find("lm_head"); it != declared.end()) p.head_block = it->second.block; + } else if (only_the_head) { p.head_block = declared.at(off_block[0].substr(0, off_block[0].find(" @ "))).block; } else { // Name a few and count the rest: SmolLM2-135M has 151 off-block ops, and listing every one of them made @@ -3505,7 +3709,9 @@ Plan plan(const std::string& pack_dir) { }() + ", and this importer decodes the block-" + std::to_string(kKBlock) + " tile layout only: at a different block every subvolume offset in the .fconst moves, so the " "weights would be read from the wrong place, not merely mis-scaled. The off-block tensor(s) are " + - list + "; only the LM head could be tolerated, because the layers come before it in the tail."); + list + "; only the LM head could be tolerated, because the layers come before it in the tail. (The " + "one other layout this importer reads is the all-block-32 one, and only for a pack that declares " + "32 on every op.)"); } } if (!bits.empty() && !(bits.size() == 1 && bits.begin()->first == 4)) @@ -4604,6 +4810,175 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons break; } + // engine#1740: read a block-32 head whose first unit is at `head_off` of the Token .fconst and write it as + // lm_head.weight. Returns why it was NOT taken (empty when it was). On a tied pack the read has to correlate with + // the embedding above kTieCorr; on an untied one the structural checks are all there is, which the note states. + // Shared by the mixed pack (layers at 128, head at 32) and the all-block-32 pack (engine#1503). + auto take_block32_head = [&](const Fconst& fc, int64_t head_off) -> std::string { + std::string why; + Block32Head h32 = read_block32_head(fc.map + head_off, int64_t(fc.size) - head_off, V, H, head_off); + const bool is_tied = tie_declared && tied; + double c32 = 0.0; + if (h32.ok && embed_tbl.ok()) c32 = corr(h32.q, embed_tbl, int(V)); + if (!h32.ok) { + why = h32.why; + } else if (is_tied && !(c32 > kTieCorr)) { + char b[160]; + std::snprintf(b, sizeof b, + "the tail's block-32 head decodes but correlates %.6f with the embedding of " + "a pack that declares them tied", c32); + why = b; + } else { + Q4& q = h32.q; + std::fprintf(stderr, + " lm head: block 32, %lld unit(s) of %lld B at %lld B, %lld B of zeros after " + "them%s\n", + (long long)h32.units, (long long)block32_unit_bytes(H), + (long long)head_off, (long long)h32.padding, + is_tied ? " (tied: verified by correlation with the pack's embedding)" + : " (untied: layout checked, values not verified)"); + if (embed_tbl.ok()) { + r.lmhead_corr = c32; + { + double sum_abs = 0, sum_ref = 0; + for (int n = 0; n < int(V); ++n) + for (int k = 0; k < q.K; ++k) { + const size_t gg = size_t(n) * (q.K / 32) + size_t(k) / 32; + const double x = double(q.codes[size_t(n) * q.K + k]) * q.scale[gg] + q.zero[gg]; + const double e = embed_at(embed_tbl, n, k); + sum_abs += std::fabs(x - e); + sum_ref += std::fabs(e); + } + if (sum_ref > 0) r.lmhead_rel = sum_abs / sum_ref; + } + r.selfcheck_reference = true; + r.selfcheck_tied = c32 > kTieCorr; + double sxy = 0, sx = 0, sy = 0, sxx = 0, syy = 0; + const int nsh = std::min(2048, int(V) - 1); + for (int rr = 0; rr < nsh; ++rr) + for (int k = 0; k < q.K; ++k) { + const size_t gg = size_t(rr) * (q.K / 32) + size_t(k) / 32; + const double x = double(q.codes[size_t(rr) * q.K + k]) * q.scale[gg] + q.zero[gg]; + const double y = embed_at(embed_tbl, rr + 1, k); + sxy += x * y; sx += x; sy += y; sxx += x * x; syy += y * y; + } + const double m = double(nsh) * q.K; + const double cx = sxx - sx * sx / m, cy = syy - sy * sy / m; + if (m > 0 && cx > 1e-12 && cy > 1e-12) + r.ctrl_corr_rowshift = (sxy - sx * sy / m) / std::sqrt(cx * cy); + } + std::vector tiled; + long long pinexact = 0; + tile_q4_1_f32(q, tiled, &pinexact); + r.codes_total = q.codes.size(); + r.zero_exact_pct = 100.0 * double(q.scale.size() - pinexact) / double(q.scale.size()); + const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes); + r.tiles += head_tiles; + w.add("lm_head.weight", "I8", {head_tiles, kImportTileBytes}, std::move(tiled)); + ++r.tensors; + head_written = true; + r.head_read_note = + std::string("lm_head was read from the .fconst tail's block 32 head (") + + std::to_string(h32.units) + " units of " + std::to_string(block32_unit_bytes(H)) + + " B)" + (is_tied ? ", and it reproduces model.embed_tokens" + : "; the pack is untied, so its values are not verified against " + "anything in the pack"); + } + return why; + }; + if (p.block32_layers) { + // engine#1503: every MatMulNBits of this pack is block 32. The RAW Token .fconst is read with the layout in + // pack_import.h (Block32Layout); the offsets are arithmetic on the geometry, and plan() has already checked the + // file's size and every envelope's epsilon against them. The tensors written are the ones the block-128 walk + // below writes, in the same form. This layout has no bias field, so no bias tensor is written. + std::vector late; + if (p.block32_note.empty()) { + const Block32Layout lay = block32_layout(p.dims); + Fconst fc = open_fconst(find_token_fconst(pack_dir)); + // the file is re-checked on this mapping: plan() looked at its own + if (const std::string why = block32_file_refusal(fc.map, int64_t(fc.size), p.dims, lay); !why.empty()) + throw std::runtime_error(fc.path + ": " + why); + r.stride = lay.stride; + r.second_header = lay.post_off; + if (!head_written) { + const std::string why = take_block32_head(fc, lay.head_off); + if (!why.empty()) late.push_back("lm_head.weight: " + why); + else r.block32_units += lay.head_units; + } + auto add_norm = [&](int64_t off, const std::string& name) { + w.add(name, "BF16", {H}, as_bf16(at(fc, off, H * 2), kBF16, size_t(H))); + ++r.tensors; + }; + for (int L = 0; L < p.dims.layers; ++L) { + const int64_t base = int64_t(L) * lay.stride; + r.norm_anchors_total += 2; + r.norm_anchors_ok += int(eps_at(fc, base, p.dims.eps_bits)) + + int(eps_at(fc, base + lay.post_off, p.dims.eps_bits)); + add_norm(base + kNormOff, "model.layers." + std::to_string(L) + ".input_layernorm.weight"); + add_norm(base + lay.post_off + kNormOff, + "model.layers." + std::to_string(L) + ".post_attention_layernorm.weight"); + } + add_norm(lay.tail_off + kNormOff, "model.norm.weight"); + r.final_norm_imported = true; + if (!find_one(pack_dir, "dd_metastate_Llm_Prefill_", ".fconst").empty()) + r.fconst_note = "the pack ships a prefill .fconst, and its copy of the final norm is not compared on " + "the all-block-32 path"; + const int64_t q_out = int64_t(p.dims.heads) * p.dims.head_dim, + kv_out = int64_t(p.dims.kv_heads) * p.dims.head_dim, IM = p.dims.intermediate; + long long attn_inexact = 0, attn_groups = 0, mlp_inexact = 0, mlp_groups = 0; + auto emit = [&](Q4& q, const std::string& name, bool mlp) { + std::vector tiled; + long long pinexact = 0; + tile_q4_1_f32(q, tiled, &pinexact); + (mlp ? mlp_inexact : attn_inexact) += pinexact; + (mlp ? mlp_groups : attn_groups) += (long long)q.scale.size(); + const int64_t nt = int64_t(tiled.size() / kImportTileBytes); + r.tiles += nt; + w.add(name + ".weight", "I8", {nt, kImportTileBytes}, std::move(tiled)); + ++r.tensors; + ++(mlp ? r.mlp_projections : r.attn_projections); + }; + struct Slot { const char* proj; int64_t off, rows, k; bool mlp; }; + const Slot slots[] = {{"self_attn.q_proj", lay.q_off, q_out, H, false}, + {"self_attn.k_proj", lay.k_off, kv_out, H, false}, + {"self_attn.v_proj", lay.v_off, kv_out, H, false}, + {"self_attn.o_proj", lay.o_off, H, q_out, false}, + {"mlp.down_proj", lay.down_off, H, IM, true}}; + for (int L = 0; L < p.dims.layers; ++L) { + const int64_t base = int64_t(L) * lay.stride; + const std::string layer = "model.layers." + std::to_string(L) + "."; + for (const auto& sl : slots) { + Block32Units u = read_block32_units(fc.map + base + sl.off, int64_t(fc.size) - base - sl.off, + sl.rows, sl.k, sl.proj, base + sl.off); + if (!u.ok) { + late.push_back(layer + sl.proj + ".weight: " + u.why); + continue; + } + r.block32_units += u.units; + emit(u.q, layer + sl.proj, sl.mlp); + } + Block32GateUp gu = read_block32_gate_up(fc.map + base + lay.gu_off, + int64_t(fc.size) - base - lay.gu_off, IM, H, base + lay.gu_off); + if (!gu.ok) { + late.push_back(layer + "mlp.gate_proj.weight: " + gu.why); + late.push_back(layer + "mlp.up_proj.weight: " + gu.why); + continue; + } + r.block32_units += gu.units; + emit(gu.gate, layer + "mlp.gate_proj", true); + emit(gu.up, layer + "mlp.up_proj", true); + } + r.attn_zero_inexact_pct = attn_groups ? 100.0 * double(attn_inexact) / double(attn_groups) : 0.0; + r.mlp_zero_inexact_pct = mlp_groups ? 100.0 * double(mlp_inexact) / double(mlp_groups) : 0.0; + unmap_file(fc.map, fc.size); + } else { + r.fconst_note = p.block32_note; // every .fconst tensor is already refused by name with this + } + if (!late.empty()) { + for (auto& s : late) r.refused.push_back(std::move(s)); + if (!opt.allow_incomplete) return r; // nothing has been written yet + } + } else { std::string path_why; const std::string path = materialise_token_fconst(pack_dir, p.dims); @@ -4646,76 +5021,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons if (tail_off + env > int64_t(fc.size)) { head32_why = "the .fconst ends before the tail's envelope"; } else { - Block32Head h32 = read_block32_head(fc.map + tail_off + env, int64_t(fc.size) - tail_off - env, - V, H); - const bool is_tied = tie_declared && tied; - double c32 = 0.0; - if (h32.ok && embed_tbl.ok()) c32 = corr(h32.q, embed_tbl, int(V)); - if (!h32.ok) { - head32_why = h32.why; - } else if (is_tied && !(c32 > kTieCorr)) { - char b[160]; - std::snprintf(b, sizeof b, - "the tail's block-32 head decodes but correlates %.6f with the embedding of " - "a pack that declares them tied", c32); - head32_why = b; - } else { - Q4& q = h32.q; - std::fprintf(stderr, - " lm head: block 32, %lld unit(s) of %lld B at %lld B, %lld B of zeros after " - "them%s\n", - (long long)h32.units, (long long)block32_unit_bytes(H), - (long long)(tail_off + env), (long long)h32.padding, - is_tied ? " (tied: verified by correlation with the pack's embedding)" - : " (untied: layout checked, values not verified)"); - if (embed_tbl.ok()) { - r.lmhead_corr = c32; - { - double sum_abs = 0, sum_ref = 0; - for (int n = 0; n < int(V); ++n) - for (int k = 0; k < q.K; ++k) { - const size_t gg = size_t(n) * (q.K / 32) + size_t(k) / 32; - const double x = double(q.codes[size_t(n) * q.K + k]) * q.scale[gg] + q.zero[gg]; - const double e = embed_at(embed_tbl, n, k); - sum_abs += std::fabs(x - e); - sum_ref += std::fabs(e); - } - if (sum_ref > 0) r.lmhead_rel = sum_abs / sum_ref; - } - r.selfcheck_reference = true; - r.selfcheck_tied = c32 > kTieCorr; - double sxy = 0, sx = 0, sy = 0, sxx = 0, syy = 0; - const int nsh = std::min(2048, int(V) - 1); - for (int rr = 0; rr < nsh; ++rr) - for (int k = 0; k < q.K; ++k) { - const size_t gg = size_t(rr) * (q.K / 32) + size_t(k) / 32; - const double x = double(q.codes[size_t(rr) * q.K + k]) * q.scale[gg] + q.zero[gg]; - const double y = embed_at(embed_tbl, rr + 1, k); - sxy += x * y; sx += x; sy += y; sxx += x * x; syy += y * y; - } - const double m = double(nsh) * q.K; - const double cx = sxx - sx * sx / m, cy = syy - sy * sy / m; - if (m > 0 && cx > 1e-12 && cy > 1e-12) - r.ctrl_corr_rowshift = (sxy - sx * sy / m) / std::sqrt(cx * cy); - } - std::vector tiled; - long long pinexact = 0; - tile_q4_1_f32(q, tiled, &pinexact); - r.codes_total = q.codes.size(); - r.zero_exact_pct = 100.0 * double(q.scale.size() - pinexact) / double(q.scale.size()); - const int64_t head_tiles = int64_t(tiled.size() / kImportTileBytes); - r.tiles += head_tiles; - w.add("lm_head.weight", "I8", {head_tiles, kImportTileBytes}, std::move(tiled)); - ++r.tensors; - head_written = true; - head32_read = true; - r.head_read_note = - std::string("lm_head was read from the .fconst tail's block 32 head (") + - std::to_string(h32.units) + " units of " + std::to_string(block32_unit_bytes(H)) + - " B)" + (is_tied ? ", and it reproduces model.embed_tokens" - : "; the pack is untied, so its values are not verified against " - "anything in the pack"); - } + head32_why = take_block32_head(fc, tail_off + env); + head32_read = head32_why.empty(); } } if (head32_read) { @@ -5523,9 +5830,12 @@ int run(int argc, char** argv) { r.mlp_groups_ok, r.mlp_groups_total, r.mlp_projections, 100.0 - r.mlp_zero_inexact_pct); if (r.stride) - std::printf(" fconst: stride %lld, second header %lld (measured from the eps anchors); " + std::printf(" fconst: stride %lld, second header %lld (%s); " "%d/%d RMSNorm anchors found, final norm %s\n", - (long long)r.stride, (long long)r.second_header, r.norm_anchors_ok, + (long long)r.stride, (long long)r.second_header, + r.block32_units ? "computed from the geometry, checked at the eps anchors" + : "measured from the eps anchors", + r.norm_anchors_ok, r.norm_anchors_total, !r.final_norm_imported ? "REFUSED (no readable tail envelope)" @@ -5533,6 +5843,11 @@ int run(int argc, char** argv) { ? "identical in both .fconst tails" : "taken from the Token_Token tail -- the two copies DISAGREE") : "taken from the tail envelope (the pack ships no prefill copy)")); + if (r.block32_units) + std::printf(" block 32: every MatMulNBits is block 32: %d attention + %d MLP projections and the head read " + "from %lld units of the raw .fconst (zero fields, finite scales and epsilon anchors checked; this " + "path has no decode->encode round trip)\n", + r.attn_projections, r.mlp_projections, r.block32_units); if (!r.fconst_note.empty()) std::printf(" fconst: %s\n", r.fconst_note.c_str()); if (!r.refused.empty()) std::printf(" incomplete: %zu tensors, listed in %s/model.q4nx\n", r.refused.size(), out.c_str()); diff --git a/npu/pack_import.h b/npu/pack_import.h index 775eed018..48140ecd2 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -183,6 +183,12 @@ struct Plan { // .fconst tile geometry assumes. Non-zero means the tail's head tiles cannot be walked (engine#1503): a tied pack // takes the head from its embedding instead, an untied one is refused by name. int64_t head_block = 0; + // engine#1503: every MatMulNBits the pack declares is block 32 and at least one of them is a layer projection, and + // the pack has a Token .fconst. Such a pack is read with the all-block-32 layout (Block32Layout below) from the + // raw .fconst; the block-128 walk and the canonical copy are not used. `block32_note` is why that file does not + // fit the layout (empty when it does), and is the refusal every .fconst tensor of the plan then carries. + bool block32_layers = false; + std::string block32_note; std::string pack; // *_rai_1.8.0_npu_4K, from the directory name Dims dims; std::vector tensors; @@ -442,6 +448,10 @@ struct Report { long long mlp_groups_ok = 0, mlp_groups_total = 0; int mlp_projections = 0; double mlp_zero_inexact_pct = 0; + + // engine#1503: the all-block-32 path. Units read (640-byte-strip units plus gate/up units, head included). There + // is no decode->encode round trip on this path, so the two counters above stay 0 for such a pack. + long long block32_units = 0; }; Report import_pack(const std::string& pack_dir, const std::string& out_dir, const Options&); @@ -490,8 +500,81 @@ struct Block32Head { int64_t padding = 0; // zero bytes after the last unit std::string why; // set when !ok }; -// `p` points at the first unit (tail_off + envelope), `avail` is the bytes from there to the end of the file. -Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K); +// `p` points at the first unit (tail_off + envelope), `avail` is the bytes from there to the end of the file, and +// `file_off` is where `p` sits in the file (only used to put an offset in a refusal). +Block32Head read_block32_head(const uint8_t* p, int64_t avail, int64_t V, int64_t K, int64_t file_off = 0); + +// The same units for any matrix (engine#1503): `ceil(rows/64)` units of `block32_unit_bytes(K)` starting at `p`. This +// is the head reader without its tail rule (nothing is required of the bytes after the last unit), and it is what the +// all-block-32 layer block's q/k/v/o/down are read with. `what` names the tensor in a refusal. Same checks: the bytes +// fit, every unit header and strip gap is zero, every scale is finite; same normalisation (positive scales). +struct Block32Units { + bool ok = false; + Q4 q; // N = units x 64 (>= rows), K, block 32 + int64_t units = 0; + int64_t bytes = 0; // units x block32_unit_bytes(K) + std::string why; // set when !ok +}; +Block32Units read_block32_units(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, const char* what = "head", + int64_t file_off = 0); + +// The gate/up unit of an all-block-32 pack (engine#1503): gate_proj and up_proj share one run of 64-row units, each +// `4,864 + 2*(K/32)*1,216` bytes. +// +// Measured on `SmolLM2-135M-Instruct_rai_1.9.0_npu` (K 576, intermediate 1,536: 24 units of 48,640 B per layer) +// against the public checkpoint's gate_proj / up_proj: +// unit : 4,864 B of zeros, then 2*(K/32) strips of 1,216 B +// strip s: K block `s/2` (32 columns), 64 row SLOTS: slots 0..31 are GATE rows `32*(s%2) .. +31` of the unit, +// slots 32..63 are UP rows with the same numbers +// +0 1,024 B of 4-bit codes, low nibble first; code position p (0..2047) is slot `(p/256)*8 + p%8`, column +// `(p/8)%32` -- the head strip's order over eight 8-slot bands instead of four +// +1024 32 B of zero points, one nibble per slot +// +1056 32 B of zeros +// +1088 64 bf16 scales, one per slot, signed +// The code order is the measured part (0.98 .. 0.996 per 32-code block against the reference). That zero point and +// scale are indexed by slot is ASSUMED by analogy with the 640-byte strip; the import of the measured pack is what +// tests it (a container gate/up that still correlates with the checkpoint after the zero points and scales are +// applied). Returned as two `Q4`s with the same normalisation and the same refusals as the unit reader. +constexpr int64_t kB32MlpUnitHeader = 4864, kB32MlpStripBytes = 1216; +inline int64_t block32_mlp_unit_bytes(int64_t K) { return kB32MlpUnitHeader + 2 * (K / 32) * kB32MlpStripBytes; } +struct Block32GateUp { + bool ok = false; + Q4 gate, up; // each N = units x 64, K, block 32 + int64_t units = 0; + int64_t bytes = 0; // units x block32_mlp_unit_bytes(K) + std::string why; // set when !ok +}; +Block32GateUp read_block32_gate_up(const uint8_t* p, int64_t avail, int64_t rows, int64_t K, int64_t file_off = 0); + +// The layer block of a pack whose EVERY MatMulNBits is block 32 (engine#1503), as offsets from the block's start. +// Read from the RAW 1.9 Token .fconst (not the canonical 1.8-form copy). With H = hidden, Nq = heads*head_dim, +// Nkv = kv_heads*head_dim, IM = intermediate, unit = block32_unit_bytes, mlp_unit = block32_mlp_unit_bytes: +// +0 64 + 2*H envelope: bf16 rms_norm_eps, zeros to 64, input_layernorm bf16[H] +// q_off (Nq/64) x unit(H) q_proj [Nq][H] +// k_off (Nkv/64) x unit(H) k_proj [Nkv][H] +// v_off (Nkv/64) x unit(H) v_proj [Nkv][H] +// sink_off 4*heads bytes not a weight; skipped (int32 0xFF7F0000 x heads on the measured pack) +// o_off (H/64) x unit(Nq) o_proj [H][Nq] +// post_off 64 + 2*H envelope: eps, zeros, post_attention_layernorm bf16[H] +// gu_off (IM/64) x mlp_unit(H) gate_proj + up_proj, [IM][H] each +// hdr_off 64 bytes skipped, content not interpreted (01 then 63 zeros on the measured pack) +// down_off (H/64) x unit(IM) down_proj [H][IM] +// stride +// and the tail at `layers x stride`: the same envelope with the final norm, `ceil(V/64)` head units at K = H, zeros. +// Measured on SmolLM2-135M-Instruct 1.9 only (stride 2,360,292; file 90,470,776 B), where Nq == H -- so o_proj's K +// being Nq rather than H is the declared shape, not a measurement. There is no bias field in this layout. +struct Block32Layout { + std::string why; // non-empty: this geometry does not fit the layout (a width that is not whole units) + int64_t env = 0, q_off = 0, k_off = 0, v_off = 0, sink_off = 0, o_off = 0, post_off = 0, gu_off = 0, hdr_off = 0, + down_off = 0, stride = 0; + int64_t tail_off = 0, head_off = 0, head_units = 0; + int64_t min_size = 0; // head_off + head_units x unit(H); the file is this plus zero padding +}; +Block32Layout block32_layout(const Dims& d); +// Empty when a mapped file fits `lay`: it is at least `min_size` long, and every envelope (two per layer and the +// tail's) opens with the pack's epsilon followed by zeros up to the norm. Otherwise the reason, with the offset. The +// zero padding after the head is the head reader's check. +std::string block32_file_refusal(const uint8_t* map, int64_t size, const Dims& d, const Block32Layout& lay); // The 1.9 external-data tile (engine#1503): the `.fconst` tile's subvolume written densely per tensor. // diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index a2656f574..088cf8052 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2156,6 +2156,508 @@ void test_block32_tail_head(const std::string& root) { } } +// ── engine#1503: a pack whose EVERY MatMulNBits is block 32 ──────────────────────────────────────────────────────── +// The gate/up unit, written from its description (pack_import.h, above read_block32_gate_up) and NOT through the +// reader: 64-row units of 4,864 zero bytes + 2*(K/32) strips of 1,216 B; strip s is K block s/2 and holds 64 slots, +// slots 0..31 = gate rows 32*(s%2)..+31, slots 32..63 = the same rows of up; code position +// `(slot/8)*256 + c*8 + slot%8`, low nibble first; 32 B of zero-point nibbles (8) by slot; 32 zero bytes; 64 signed +// bf16 scales by slot, `-m/8` as in encode_block32_units. +static std::string encode_block32_gate_up(const std::vector& G, const std::vector& U, int rows, int K) { + const int kb = K / 32, units = rows / 64; + const size_t unit = size_t(4864 + 2 * kb * 1216); + std::string out(size_t(units) * unit, '\0'); + for (int u = 0; u < units; ++u) + for (int s = 0; s < 2 * kb; ++s) { + char* sp = out.data() + size_t(u) * unit + 4864 + size_t(s) * 1216; + for (int slot = 0; slot < 64; ++slot) { + const std::vector& M = slot < 32 ? G : U; + const float* w = M.data() + size_t(u * 64 + 32 * (s % 2) + slot % 32) * size_t(K) + size_t((s / 2) * 32); + float m = 0; + for (int c = 0; c < 32; ++c) + if (std::fabs(w[c]) > std::fabs(m)) m = w[c]; + const uint16_t bits = pi::bf16_bits_of(-m / 8.0f); + const uint32_t wide = uint32_t(bits) << 16; + float sc; + std::memcpy(&sc, &wide, 4); + sp[1088 + 2 * slot] = char(bits & 0xFF); + sp[1089 + 2 * slot] = char(bits >> 8); + sp[1024 + slot / 2] = char(uint8_t(sp[1024 + slot / 2]) | uint8_t(8 << (4 * (slot & 1)))); + for (int c = 0; c < 32; ++c) { + const int code = sc == 0 ? 8 : std::min(15, std::max(0, int(std::lround(w[c] / sc)) + 8)); + const int pos = (slot / 8) * 256 + c * 8 + slot % 8; + sp[pos / 2] = char(uint8_t(sp[pos / 2]) | uint8_t(code << (4 * (pos & 1)))); + } + } + } + return out; +} + +void test_block32_layers(const std::string& root) { + const auto ncorr = [](const std::vector& a, const std::vector& b) { + double num = 0, da = 0, db = 0, sa = 0, sb = 0; + const double n = double(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + num += double(a[i]) * b[i], da += double(a[i]) * a[i], db += double(b[i]) * b[i]; + sa += a[i], sb += b[i]; + } + const double den = (n * da - sa * sa) * (n * db - sb * sb); + return den > 0 ? (n * num - sa * sb) / std::sqrt(den) : 0.0; + }; + const auto value = [](const pi::Q4& q, int row, int col) { + const size_t g = size_t(row) * size_t(q.K / 32) + size_t(col / 32); + return double(q.codes[size_t(row) * size_t(q.K) + col]) * q.scale[g] + q.zero[g]; + }; + // a deterministic matrix per (seed): values in about [-2, 2], unrelated between seeds + const auto matrix = [](int rows, int cols, uint32_t seed) { + std::vector m(size_t(rows) * size_t(cols)); + uint32_t x = seed * 2654435761u + 12345u; + for (auto& v : m) { + x = x * 1664525u + 1013904223u; + v = float(int((x >> 8) & 0xFFFF) - 32768) / 16384.0f; + } + return m; + }; + + { // arm 1: the unit reader with ROWS as a parameter, typed bytes (K 64: one unit is 5,120 B) + std::vector u(2 * 5120, 0); + uint8_t* sp = u.data() + 5120 + 2560 + 2 * 640; // unit 1, strip 2: K block 1, rows 0..31 of the unit + const uint16_t bits = pi::bf16_bits_of(-0.5f); + sp[576 + 2 * 20] = uint8_t(bits & 0xFF); // row 20's scale + sp[577 + 2 * 20] = uint8_t(bits >> 8); + sp[512 + 10] = 0x05; // row 20's zero point: low nibble of byte 10 + sp[(2 * 256 + 3 * 8 + 4) / 2] |= 0x09; // row 20 (band 2, r%8 4), column 3: position 540, low nibble + pi::Block32Units r = pi::read_block32_units(u.data(), int64_t(u.size()), 128, 64, "q_proj", 7000); + check(r.ok && r.units == 2 && r.bytes == 10240 && r.q.N == 128 && r.q.K == 64 && r.q.block == 32, + "block-32 units: 128 rows of K 64 are two units (" + r.why + ")"); + if (r.ok) { + check(value(r.q, 64 + 20, 32 + 3) == -2.0, + "block-32 units: unit 1 row 20 col 35 = (9 - 5) x -0.5 (" + std::to_string(value(r.q, 84, 35)) + ")"); + check(value(r.q, 84, 34) == 2.5, "block-32 units: its neighbour is code 0, (0 - 5) x -0.5"); + check(r.q.scale[size_t(84) * 2 + 1] == 0.5f, "block-32 units: the stored scale is positive"); + check(value(r.q, 20, 35) == 0.0 && value(r.q, 84, 3) == 0.0, + "block-32 units: the same position in unit 0 and in K block 0 is untouched"); + } + check(pi::read_block32_units(u.data(), int64_t(u.size()), 65, 64).units == 2, + "block-32 units: 65 rows round up to two units"); + const pi::Block32Units one = pi::read_block32_units(u.data(), int64_t(u.size()), 64, 64); + check(one.ok && one.units == 1 && one.q.N == 64, + "block-32 units: 64 rows read one unit and nothing is required of the bytes after it"); + const pi::Block32Units sh = pi::read_block32_units(u.data(), 10239, 128, 64, "q_proj", 7000); + check(!sh.ok && sh.why.find("q_proj") != std::string::npos && sh.why.find("7000") != std::string::npos, + "block-32 units: a buffer one byte short is refused, naming the tensor and the offset (" + sh.why + ")"); + std::vector hdr = u; + hdr[5120 + 9] = 1; + const pi::Block32Units hr = pi::read_block32_units(hdr.data(), int64_t(hdr.size()), 128, 64, "q_proj", 7000); + check(!hr.ok && hr.why.find("12120 B") != std::string::npos, + "block-32 units: a non-zero unit header is refused with the unit's offset, 7000 + 5120 (" + hr.why + ")"); + std::vector nan = u; + nan[2560 + 576] = 0xC0, nan[2560 + 577] = 0x7F; // bf16 quiet NaN in unit 0, strip 0, row 0 + check(!pi::read_block32_units(nan.data(), int64_t(nan.size()), 128, 64).ok, + "block-32 units: a scale that is not finite is refused"); + } + + { // arm 2: the gate/up reader, typed bytes (K 32: one unit is 4,864 + 2 x 1,216 = 7,296 B) + check(pi::block32_mlp_unit_bytes(32) == 7296 && pi::block32_mlp_unit_bytes(576) == 48640, + "block-32 gate/up: the unit is 4,864 + 2*(K/32)*1,216 B (48,640 at K 576)"); + std::vector u(7296, 0); + uint8_t* sp = u.data() + 4864 + 1216; // strip 1: K block 0, rows 32..63 + const auto put_scale = [&](int slot, float v) { + const uint16_t bits = pi::bf16_bits_of(v); + sp[1088 + 2 * slot] = uint8_t(bits & 0xFF); + sp[1089 + 2 * slot] = uint8_t(bits >> 8); + }; + put_scale(5, 0.5f); // slot 5: GATE row 32 + 5 + put_scale(41, -0.25f); // slot 41: UP row 32 + 9 + sp[1024 + 2] = 0x80; // slot 5's zero point (high nibble of byte 2) = 8 + sp[1024 + 20] = 0x60; // slot 41's zero point (high nibble of byte 20) = 6 + sp[(0 * 256 + 7 * 8 + 5) / 2] |= 0xB0; // slot 5 (band 0, 5), column 7: position 61, high nibble + sp[(5 * 256 + 31 * 8 + 1) / 2] |= 0x30; // slot 41 (band 5, 1), column 31: position 1529, high nibble + pi::Block32GateUp h = pi::read_block32_gate_up(u.data(), int64_t(u.size()), 64, 32, 900); + check(h.ok && h.units == 1 && h.bytes == 7296 && h.gate.N == 64 && h.up.N == 64 && h.gate.K == 32 && + h.up.block == 32, + "block-32 gate/up: one unit of K 32 reads (" + h.why + ")"); + if (h.ok) { + check(value(h.gate, 37, 7) == 1.5, + "block-32 gate/up: slot 5 is GATE row 37, (11 - 8) x 0.5 (" + std::to_string(value(h.gate, 37, 7)) + ")"); + check(value(h.gate, 37, 6) == -4.0, "block-32 gate/up: its neighbour is code 0, (0 - 8) x 0.5"); + check(value(h.up, 37, 7) == 0.0, "block-32 gate/up: and UP row 37 holds nothing"); + check(value(h.up, 41, 31) == 0.75, + "block-32 gate/up: slot 41 is UP row 41, a NEGATIVE scale, (3 - 6) x -0.25 (" + + std::to_string(value(h.up, 41, 31)) + ")"); + check(value(h.up, 41, 0) == 1.5, "block-32 gate/up: its code 0, (0 - 6) x -0.25"); + check(h.up.scale[41] == 0.25f, "block-32 gate/up: the stored scale is positive"); + check(value(h.gate, 41, 31) == 0.0 && value(h.gate, 5, 7) == 0.0, + "block-32 gate/up: GATE row 41 and the other strip's row 5 hold nothing"); + } + const pi::Block32GateUp sh = pi::read_block32_gate_up(u.data(), 7295, 64, 32, 900); + check(!sh.ok && sh.why.find("900") != std::string::npos, + "block-32 gate/up: a buffer one byte short is refused with the offset (" + sh.why + ")"); + std::vector hdr = u; + hdr[4863] = 1; + check(!pi::read_block32_gate_up(hdr.data(), int64_t(hdr.size()), 64, 32).ok, + "block-32 gate/up: a non-zero unit header is refused"); + std::vector gap = u; + gap[4864 + 1216 + 1056 + 31] = 1; + check(!pi::read_block32_gate_up(gap.data(), int64_t(gap.size()), 64, 32).ok, + "block-32 gate/up: a non-zero byte between the zero points and the scales is refused"); + std::vector inf = u; + inf[4864 + 1088 + 2 * 63] = 0x80, inf[4864 + 1089 + 2 * 63] = 0x7F; // bf16 +inf in strip 0, slot 63 + check(!pi::read_block32_gate_up(inf.data(), int64_t(inf.size()), 64, 32).ok, + "block-32 gate/up: a scale that is not finite is refused"); + check(!pi::read_block32_gate_up(u.data(), int64_t(u.size()), 64, 48).ok, "block-32 gate/up: K 48 is refused"); + } + + { // arm 3: the description's encoder against the reader, two unrelated matrices over two units and three K blocks + const int rows = 128, K = 96; + const std::vector G = matrix(rows, K, 1), U = matrix(rows, K, 2); + const std::string enc = encode_block32_gate_up(G, U, rows, K); + const pi::Block32GateUp h = + pi::read_block32_gate_up(reinterpret_cast(enc.data()), int64_t(enc.size()), rows, K); + check(h.ok, "block-32 gate/up: the encoded stream reads (" + h.why + ")"); + if (h.ok) { + std::vector g(G.size()), up(U.size()); + for (int r = 0; r < rows; ++r) + for (int c = 0; c < K; ++c) { + g[size_t(r) * K + c] = float(value(h.gate, r, c)); + up[size_t(r) * K + c] = float(value(h.up, r, c)); + } + check(ncorr(g, G) > 0.99 && ncorr(up, U) > 0.99, + "block-32 gate/up: gate and up come back (corr " + std::to_string(ncorr(g, G)) + " / " + + std::to_string(ncorr(up, U)) + ")"); + check(std::fabs(ncorr(g, U)) < 0.3 && std::fabs(ncorr(up, G)) < 0.3, + "block-32 gate/up: and not swapped (corr " + std::to_string(ncorr(g, U)) + " / " + + std::to_string(ncorr(up, G)) + ")"); + } + } + + { // arm 4: the layer block's arithmetic at the one measured geometry (SmolLM2-135M-Instruct 1.9) + pi::Dims d; + d.hidden = 576, d.intermediate = 1536, d.layers = 30, d.heads = 9, d.kv_heads = 3, d.head_dim = 64; + d.vocab = 49152; + const pi::Block32Layout l = pi::block32_layout(d); + check(l.why.empty() && l.q_off == 1216 && l.k_off == 231616 && l.v_off == 308416 && l.sink_off == 385216 && + l.o_off == 385252 && l.post_off == 615652 && l.gu_off == 616868 && l.hdr_off == 1784228 && + l.down_off == 1784292 && l.stride == 2360292, + "block-32 layout: SmolLM2-135M's layer block, stride 2,360,292 (" + l.why + " " + + std::to_string(l.stride) + ")"); + check(l.head_off == 70809976 && l.head_units == 768 && l.min_size == 90470776, + "block-32 layout: its head at 70,809,976 and the file's 90,470,776 B (" + std::to_string(l.min_size) + ")"); + d.intermediate = 1500; + check(pi::block32_layout(d).why.find("intermediate 1500") != std::string::npos, + "block-32 layout: a width that is not whole 64-row units is refused by name"); + } + + // arm 5: a crafted all-block-32 pack through import_pack(). hidden 128, intermediate 256, 2 heads x 64, 1 KV head, + // 2 layers, vocab 64, tied. Every matrix is its own pseudo-random one, so a tensor read from another's offset + // cannot correlate with what the container is compared against. + const int H = 128, IM = 256, LAYERS = 2, V = 64, NQ = 128, NKV = 64; + const int R = onebit::npu::q4nx::kRows; + enum { Q, K_, V_, O, G, U, D, NPROJ }; + const char* const pname[NPROJ] = {"self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", "self_attn.o_proj", + "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj"}; + const int prow[NPROJ] = {NQ, NKV, NKV, H, IM, IM, H}, pcol[NPROJ] = {H, H, H, NQ, H, H, IM}; + std::vector> mats; + for (int l = 0; l < LAYERS; ++l) + for (int i = 0; i < NPROJ; ++i) mats.push_back(matrix(prow[i], pcol[i], uint32_t(100 + l * NPROJ + i))); + const std::vector emb = matrix(V, H, 7); + const auto norm_bytes = [&](int which) { // bf16 [H], a different ramp per norm + std::string b(size_t(H) * 2, '\0'); + for (int i = 0; i < H; ++i) { + const uint16_t bits = pi::bf16_bits_of(float(which) + float(i) / 256.0f); + b[2 * i] = char(bits & 0xFF), b[2 * i + 1] = char(bits >> 8); + } + return b; + }; + const auto envelope = [&](int which) { + std::string e(64, '\0'); + e[0] = char(0x86), e[1] = char(0x35); // bf16 1e-06 + return e + norm_bytes(which); + }; + const auto fconst = [&] { + std::string fc; + for (int l = 0; l < LAYERS; ++l) { + const auto& m = [&](int i) -> const std::vector& { return mats[size_t(l * NPROJ + i)]; }; + fc += envelope(10 * l + 1); + fc += encode_block32_units(m(Q), NQ, H) + encode_block32_units(m(K_), NKV, H) + + encode_block32_units(m(V_), NKV, H); + fc += std::string(4 * 2, '\x7f'); // 4 x heads bytes that are not a weight (their content is not read) + fc += encode_block32_units(m(O), H, NQ); + fc += envelope(10 * l + 2); + fc += encode_block32_gate_up(m(G), m(U), IM, H); + fc += std::string(1, '\x01') + std::string(63, '\0'); + fc += encode_block32_units(m(D), H, IM); + } + fc += envelope(99); + fc += encode_block32_units(emb, V, H); + return fc; + }; + const std::string good = fconst(); + { + pi::Dims d; + d.hidden = H, d.intermediate = IM, d.layers = LAYERS, d.heads = 2, d.kv_heads = 1, d.head_dim = 64, d.vocab = V; + check(pi::block32_layout(d).min_size == int64_t(good.size()), + "all-block-32 fixture: the stream built from the description is the layout's size (" + + std::to_string(good.size()) + ")"); + } + // `layer_block` / `head_block` are what cache/ops_meta.json DECLARES; the .fconst bytes are the caller's. + const auto build = [&](const char* tag, const std::string& fc, bool layers_declared, int layer_block = 32, + int head_block = 32) { + const fs::path dir = fs::path(root) / tag; + fs::remove_all(dir); + fs::create_directories(dir / "cache"); + // the 1.9 spelling of the Token file: this path reads the raw file, and it must not be mistaken for a 1.8 one + put_file((dir / "dd_metastate_Llm_Token_aie2_token_crafted.fconst").string(), fc); + put_file((dir / "config.json").string(), + R"({"model_type":"llama","hidden_size":128,"intermediate_size":256,"num_hidden_layers":2,)" + R"("num_attention_heads":2,"num_key_value_heads":1,"head_dim":64,"vocab_size":64,)" + R"("rope_theta":10000.0,"tie_word_embeddings":true,"rms_norm_eps":1e-06})"); + std::string data(size_t(V) * size_t(H) * 2, '\0'); + for (size_t i = 0; i < emb.size(); ++i) { + const uint16_t bits = pi::bf16_bits_of(emb[i]); + data[2 * i] = char(bits & 0xFF); + data[2 * i + 1] = char(bits >> 8); + } + put_file((dir / "model.onnx.data").string(), data); + const auto op_at = [](const std::string& name, int n, int k, int block) { + return R"({"type":"MladfMatMul","name":")" + name + R"(.MatMulNBits","attrs":{"N":{"value":[)" + + std::to_string(n) + R"(]},"K":{"value":[)" + std::to_string(k) + R"(]},"block_size":{"value":[)" + + std::to_string(block) + R"(]},"bits":{"value":[4]}}})"; + }; + const auto op = [&](const std::string& name, int n, int k) { return op_at(name, n, k, layer_block); }; + std::string ops = op_at("lm_head", V, H, head_block); + if (layers_declared) + for (int l = 0; l < LAYERS; ++l) { + const std::string a = "model.layers." + std::to_string(l) + ".attn.", + m = "model.layers." + std::to_string(l) + ".mlp."; + ops += "," + op(a + "q_proj", NQ, H) + "," + op(a + "k_proj", NKV, H) + "," + op(a + "v_proj", NKV, H) + + "," + op(a + "o_proj", H, NQ) + "," + op(m + "down_proj", H, IM); + } + put_file((dir / "cache" / "ops_meta.json").string(), R"({"op_list":[)" + ops + R"(],"tensor_map":{}})"); + put_file((dir / "model.onnx").string(), + pb::model({pb::tensor("model.embed_tokens.weight", {V, H}, "model.onnx.data", 0, + uint64_t(V) * uint64_t(H) * 2, 16), + pb::placeholder("lm_head.weight.MatMulNBits.qweight"), + pb::placeholder("lm_head.weight.MatMulNBits.scales"), + pb::placeholder("lm_head.weight.MatMulNBits.zeros")}, + {})); + return dir; + }; + const auto out_of = [](const char* tag) { + return (fs::temp_directory_path() / + (std::string("oga_pack_import_b32layers_") + tag + "." + std::to_string(::getpid()))).string(); + }; + // the first `cols` columns of rows 0..R-1 of a container tensor, from its own bytes (tile 0) + const auto tile0 = [&](const json& tbl, const std::vector& blob, const std::string& name, int cols) { + std::vector w(size_t(R) * onebit::npu::q4nx::kCols, 0.f), out(size_t(R) * size_t(cols), 0.f); + if (!tbl.contains(name)) return std::vector(); + const auto offs = tbl[name]["data_offsets"].get>(); + onebit::npu::q4nx::dequant_tile(onebit::npu::q4nx::Chunk::Q4_1F, blob.data() + offs[0], w.data(), + onebit::npu::q4nx::kCols); + for (int r = 0; r < R; ++r) + for (int c = 0; c < cols; ++c) out[size_t(r) * cols + c] = w[size_t(r) * onebit::npu::q4nx::kCols + c]; + return out; + }; + const auto head_of = [&](const std::vector& m, int cols_total, int cols) { // the same window of a source + std::vector out(size_t(R) * size_t(cols)); + for (int r = 0; r < R; ++r) + for (int c = 0; c < cols; ++c) out[size_t(r) * cols + c] = m[size_t(r) * cols_total + c]; + return out; + }; + + { // 5a: the pack imports, complete, and every container tensor is the matrix the description put at its offset + const fs::path dir = build("b32-layers", good, true); + const std::string out = out_of("good"); + fs::remove_all(out); + pi::Report r; + std::string said; + try { + const pi::Plan p = pi::plan(dir.string()); + check(p.block32_layers && p.block32_note.empty() && p.refused().empty(), + "all-block-32: the plan takes the block-32 layout and refuses nothing (" + p.block32_note + ")"); + r = pi::import_pack(dir.string(), out, pi::Options{}); + } catch (const std::exception& e) { + said = e.what(); + } + check(said.empty() && r.refused.empty(), + "all-block-32: a crafted pack imports with nothing refused (" + said + + (r.refused.empty() ? "" : r.refused.front()) + ")"); + check(r.attn_projections == 4 * LAYERS && r.mlp_projections == 3 * LAYERS && r.attn_bias_tensors == 0, + "all-block-32: 4 attention + 3 MLP projections per layer, no bias tensor"); + check(r.stride == (int64_t(good.size()) - (64 + 2 * H) - pi::block32_unit_bytes(H)) / LAYERS, + "all-block-32: the report carries the stride (" + std::to_string(r.stride) + ")"); + check(r.selfcheck_tied && r.lmhead_corr > 0.99 && r.head_read_note.find("block 32") != std::string::npos, + "all-block-32: the head is read and verified against the embedding (corr " + + std::to_string(r.lmhead_corr) + ")"); + if (fs::exists(out + "/model.q4nx")) { + const json tbl = container_table(out + "/model.q4nx"); + const std::vector blob = container_payload(out + "/model.q4nx"); + for (int l = 0; l < LAYERS; ++l) + for (int i = 0; i < NPROJ; ++i) { + const std::string name = "model.layers." + std::to_string(l) + "." + pname[i] + ".weight"; + const int cols = std::min(pcol[i], onebit::npu::q4nx::kCols); + const std::vector got = tile0(tbl, blob, name, cols); + const double c = got.empty() ? 0.0 : ncorr(got, head_of(mats[size_t(l * NPROJ + i)], pcol[i], cols)); + check(c > 0.99, "all-block-32: container " + name + " is its own matrix (corr " + + std::to_string(c) + ")"); + // the control: the matrix one slot along (gate <-> up among them) must not match + const int j = (i == G) ? U : (i == U) ? G : -1; + if (j >= 0 && !got.empty()) { + const double x = ncorr(got, head_of(mats[size_t(l * NPROJ + j)], pcol[j], cols)); + check(std::fabs(x) < 0.3, "all-block-32: container " + name + " is not the other MLP half " + "(corr " + std::to_string(x) + ")"); + } + } + const std::vector hd = tile0(tbl, blob, "lm_head.weight", H); + check(!hd.empty() && ncorr(hd, head_of(emb, H, H)) > 0.99, "all-block-32: container lm_head is the tail's head"); + const auto norm_is = [&](const std::string& name, int which) { + if (!tbl.contains(name)) return false; + const auto offs = tbl[name]["data_offsets"].get>(); + const std::string want = norm_bytes(which); + return offs[1] - offs[0] == want.size() && std::memcmp(blob.data() + offs[0], want.data(), want.size()) == 0; + }; + check(norm_is("model.layers.0.input_layernorm.weight", 1) && + norm_is("model.layers.1.input_layernorm.weight", 11) && + norm_is("model.layers.0.post_attention_layernorm.weight", 2) && + norm_is("model.layers.1.post_attention_layernorm.weight", 12) && norm_is("model.norm.weight", 99), + "all-block-32: the five norms are the envelopes' bytes"); + check(!tbl.contains("model.layers.0.self_attn.q_proj.bias"), "all-block-32: no bias tensor is invented"); + } else { + check(false, "all-block-32: the container was written"); + } + fs::remove_all(out); + fs::remove_all(dir); + } + // what a refused import says, and that it wrote no container + const auto refused = [&](const char* tag, const std::string& fc) { + const fs::path dir = build(tag, fc, true); + const std::string out = out_of(tag); + fs::remove_all(out); + std::string said; + try { + const pi::Report r = pi::import_pack(dir.string(), out, pi::Options{}); + for (const auto& w : r.refused) said += w + " | "; + } catch (const std::exception& e) { + said = e.what(); + } + if (fs::exists(out + "/model.q4nx")) said = "WROTE A CONTAINER " + said; + fs::remove_all(out); + fs::remove_all(dir); + return said; + }; + { // 5b: a file one byte short of the layout is refused with the arithmetic + const std::string said = refused("b32-layers-short", good.substr(0, good.size() - 1)); + check(said.find("WROTE") == std::string::npos && said.find("all-block-32 layout") != std::string::npos && + said.find(std::to_string(good.size())) != std::string::npos, + "all-block-32: a short file is refused with the size the layout needs (" + said.substr(0, 300) + ")"); + } + { // 5c: layer 1's post-attention envelope without its epsilon + pi::Dims d; + d.hidden = H, d.intermediate = IM, d.layers = LAYERS, d.heads = 2, d.kv_heads = 1, d.head_dim = 64, d.vocab = V; + const pi::Block32Layout l = pi::block32_layout(d); + std::string bad = good; + bad[size_t(l.stride + l.post_off)] = 0; + const std::string said = refused("b32-layers-eps", bad); + check(said.find("WROTE") == std::string::npos && said.find("no RMSNorm epsilon at " + + std::to_string(l.stride + l.post_off)) != + std::string::npos, + "all-block-32: a missing epsilon is refused with its offset (" + said.substr(0, 300) + ")"); + // 5d: a non-zero byte in a gate/up unit header of layer 1 -> both halves refused by name + bad = good; + bad[size_t(l.stride + l.gu_off + pi::block32_mlp_unit_bytes(H) + 17)] = 1; + const std::string s2 = refused("b32-layers-gu", bad); + check(s2.find("WROTE") == std::string::npos && + s2.find("model.layers.1.mlp.gate_proj.weight: block-32 gate/up unit 1") != std::string::npos && + s2.find("model.layers.1.mlp.up_proj.weight") != std::string::npos && + s2.find("model.layers.0") == std::string::npos, + "all-block-32: a non-zero gate/up unit header refuses that layer's gate and up by name (" + + s2.substr(0, 300) + ")"); + // 5e: trailing junk after the head is the head's refusal + const std::string s3 = refused("b32-layers-junk", good + std::string(99, '\0') + "x"); + check(s3.find("WROTE") == std::string::npos && s3.find("lm_head.weight") != std::string::npos && + s3.find("not zero after") != std::string::npos, + "all-block-32: a non-zero byte after the head refuses the head (" + s3.substr(0, 300) + ")"); + } + { // 5f: zero padding after the head is accepted (the measured mixed pack has 110 MB of it) + const fs::path dir = build("b32-layers-pad", good + std::string(5000, '\0'), true); + const std::string out = out_of("pad"); + fs::remove_all(out); + std::string said; + try { + const pi::Report r = pi::import_pack(dir.string(), out, pi::Options{}); + for (const auto& w : r.refused) said += w + " | "; + } catch (const std::exception& e) { + said = e.what(); + } + check(said.empty() && fs::exists(out + "/model.q4nx"), "all-block-32: zero padding after the head imports (" + + said.substr(0, 200) + ")"); + fs::remove_all(out); + fs::remove_all(dir); + } + { // 5g: the same pack declaring ONLY its head at block 32 is not this class (it is the mixed pack's path) + const fs::path dir = build("b32-layers-headonly", good, false); + bool block32 = true; + try { + block32 = pi::plan(dir.string()).block32_layers; + } catch (const std::exception&) { + block32 = false; + } + check(!block32, "all-block-32: a pack that declares only lm_head at block 32 does not take the all-block-32 path"); + fs::remove_all(dir); + } + // 5h..5k: the path is chosen by what the pack DECLARES, one arm per declaration. The bytes are the same good + // all-block-32 file every time, so the only thing that changes is cache/ops_meta.json -- and a declaration this + // importer has no layout for must be refused with its number, never read at 32 or at 128 by default. + const auto declared_as = [&](const char* tag, int layer_block, int head_block) { + const fs::path dir = build(tag, good, true, layer_block, head_block); + const std::string out = out_of(tag); + fs::remove_all(out); + std::string said; + bool block32 = false; + try { + block32 = pi::plan(dir.string()).block32_layers; + const pi::Report r = pi::import_pack(dir.string(), out, pi::Options{}); + for (const auto& w : r.refused) said += w + " | "; + } catch (const std::exception& e) { + said = e.what(); + } + if (block32) said = "TOOK THE ALL-BLOCK-32 PATH " + said; + if (fs::exists(out + "/model.q4nx")) said = "WROTE A CONTAINER " + said; + fs::remove_all(out); + fs::remove_all(dir); + return said; + }; + for (const int b : {3, 64, 256}) { // 5h: every op at a block nobody has measured + const std::string said = declared_as(("b32-layers-all" + std::to_string(b)).c_str(), b, b); + check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos && + said.find("block size along K is " + std::to_string(b) + " (11 ops)") != std::string::npos && + said.find("lm_head @ " + std::to_string(b)) != std::string::npos, + "block declaration: every op at " + std::to_string(b) + " is refused with that number (" + + said.substr(0, 200) + ")"); + } + { // 5i: layers at 32 under a block-128 head is not the all-block-32 class, and is refused by tensor + const std::string said = declared_as("b32-layers-head128", 32, 128); + check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos && + said.find("128 (1 ops)") != std::string::npos && said.find("32 (10 ops)") != std::string::npos && + said.find("lm_head @") == std::string::npos, + "block declaration: layers at 32 with the head at 128 is refused, naming the layers (" + + said.substr(0, 260) + ")"); + } + { // 5j: layers at 32 with the head at 64 -- one unmeasured op is enough to leave the class + const std::string said = declared_as("b32-layers-head64", 32, 64); + check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos && + said.find("64 (1 ops)") != std::string::npos, + "block declaration: layers at 32 with the head at 64 is refused (" + said.substr(0, 260) + ")"); + } + { // 5k: the same ops all declared at 128 stay on the block-128 walk (which then refuses THESE bytes, because + // they are a block-32 file: the point is that the declaration, not a default, picked the path) + const std::string said = declared_as("b32-layers-all128", 128, 128); + check(said.find("WROTE") == std::string::npos && said.find("TOOK") == std::string::npos && + said.find("block size along K") == std::string::npos && !said.empty(), + "block declaration: every op at 128 does not take the all-block-32 path or the block gate (" + + said.substr(0, 260) + ")"); + } +} + // ── the final norm out of the tail envelope, on a crafted pack (engine#844) ────────────────── // The real-pack leg only covers this where a pack is supplied, and three of the four packs on the // box ship no prefill program, so the import's tail-envelope read had no coverage in the default @@ -4880,6 +5382,7 @@ int main(int argc, char** argv) { test_post_attn_norm_header(golden_dir.empty() ? "." : golden_dir); test_tied_block32_head(golden_dir.empty() ? "." : golden_dir); test_block32_tail_head(golden_dir.empty() ? "." : golden_dir); + test_block32_layers(golden_dir.empty() ? "." : golden_dir); test_final_norm_envelope(golden_dir.empty() ? "." : golden_dir); test_import_fp32_params(golden_dir.empty() ? "." : golden_dir); test_text_artifacts(golden_dir.empty() ? "." : golden_dir); From b91119ba7a8beb5e6e64916dc8ad7ae2e36ab106 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:52:50 -0400 Subject: [PATCH 20/98] npu-pack: Qwen3-8B 1.9 pack imports end to end (hidden 4096 / 32 heads / 36 layers), head read from the decode If branch (#2009) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * npu: measure the 1.9 Token fconst rule for hidden 4096 / 32 heads / 36 layers (Qwen3-8B) (refs #1740) Measured with tools/fconst19_measure.py against amd/Qwen3-8B_rai_1.8.0_npu_16K: 26,104 of 3,756,627,264 bytes differ (6.95e-04 %), content rather than layout. The header stride 104,350,656 is uniform over all 36 blocks, checked independently of the tool (36 runs of 32 x 00 00 7f ff, first at 13,796,032). The row is identical to the Mistral h4096 row in every field except the layer count, which is the finding: the bundle shape and rotation are a property of the hidden size, not of the depth. It needs its own entry anyway, because select_rule_in() keys on (hidden, heads, layers). Stated in the comment because it is not all seven fields: --probe-wrong separates rot+1 and tile/2 by 14,700x (10.2 % of the file each), but prefix+64 differs in only 32,960 B -- 1.26x the accepted residual and under the 37,566-byte tolerance -- so this pair does not isolate the prefix. The 12,608 rests on the Mistral row and the Qwen3 arithmetic agreeing, as the 3,584 row's prefix does. * npu-pack: read lm_head from the If branch that is the decode path, so the 4096-hidden untied packs (Qwen3-8B, granite-4.1-8b) get their head from the graph (refs #1740) An OGA 1.9 pack splits prefill from decode with `If (Equal(Shape(input_ids), Constant(value_int 1)))`, and the two branches do NOT carry the same tensors. Measured on amd/Qwen3-8B_rai_1.9.0_npu: then_branch holds exactly ONE MatMulNBits and it is lm_head [151936, 32, 64] at offset 15,269,888 in model.onnx.data (311,164,928 B = V*H/2); else_branch holds the 109 layer projections plus an `lm_head.MatMulNBits.qweight.empty` placeholder with no bytes. The walker merged both branches, so the placeholders it saw for lm_head counted as weightless_matmuls and plan() took the "no readable graph head, read the tail" path. On these packs the tail is the envelope alone -- Qwen3-8B's is 8,256 B = 64 + 2*4096, verified by hex (bf16 1e-6 at kNormOff 64, then the norm weights) -- so `want = env + rows*tile` could never equal `have`, and the import refused with "lm_head.weight: the .fconst tail is 8256 B, not the ... B its envelope and head tiles would take, so the head's offsets are not established". The Token .fconst IS the decode path, so the decode branch's node is the one that describes it: - walk_graph now records each matmul's `If` branch (the attribute's own name); - parse_onnx reads the condition rather than assuming: a Constant value_int of 1 makes then_branch decode, anything else else_branch, and a graph without that shape falls back to then_branch with a note saying the rule was assumed; - import_pack picks the head node from the decode branch, keeping the old first-match behaviour as a fallback when only the other branch declares it; - the #1687 refusal reads weightless_matmuls_decode (outside the decode branch), so a prefill placeholder is no longer mistaken for a genuinely absent weight. The total stays for `info`'s census, and both are reported in graph_weights. Pinned by test_external_packed_genre: a crafted graph whose condition compares the shape with 1 must name then_branch decode, and an inverted one must move it to else_branch. The regression arm for the head pick (decode branch taken, inverted convention falls back, flat graph unaffected) was checked standalone. Not run: the pack itself -- this tree's submodules are not checked out, so libonebit_npu cannot be built here; the changed regions pass -fsyntax-only and the test compiles. * npu-pack: decide the decode If branch by which branch holds the weights, not only by the condition's constant The condition `Equal(Shape(input_ids), Constant(1))` is the pack's statement of which branch is one-token; the arithmetic is what must hold. A pack that inverts the condition without moving its weights answered the wrong branch, and the head vanished. Now the branch that actually declares MatMulNBits nodes wins when only one of them does, and the condition decides only when the graph is ambiguous (weights in both branches). branch_note names which rule was used and, when they disagree, how many matmuls each branch has. Also fixes the crafted test: it wrote model.onnx while this fixture's genai_config.json names eager-mm.onnx, so the arm was measuring the fixture's own graph rather than the crafted one -- and the graph needed its embedding tensor to be plannable at all. * npu: the integer arm's operands come from the engine's packer, not a private re-quantisation The arm was merged in #1809 with a quantisation rule of its own: per 128-K group, int4 codes in [-8, 7]. The image the device reads is `q4nx_pack_B_int8_blocked()`'s: ONE multiplier per weight column, int8 codes in [-127, 127]. Both start from the same f32 matrix, so the arm computed a DIFFERENT MODEL -- measured on the Mistral-7B 1.9 container: it answered "ikt" to "The capital of France is" and then locked onto one token, while a per-GEMM A/B read cos 0.9999 against f32 on the same tensors. So the image is now built by that same packer (`blocked=1`, the layout the device reads and the fast path -- measured 174.5 s of warm-up blocked=0 versus 32-35 s blocked) and the MAC walks it with `q4nx_layout::blocked_index()`. Measured after, on current main: warm-up 35.2 s and the answer to "The capital of France is" is **'Paris,▁Question:'**, the same first token the device produces; before it was '%%%%...' / '▁/******/'. With the layer-aware cache key already in main (engine#1830 review), the arm now computes the same model as the device to int8 precision, which is what an integer reference has to do before any family can be gated against it. `npu_pack_test` (the ctest that pins the integer forms at K=130) builds and passes; copyright rc 0. * npu: keyed dumps settle it -- the device's weight scales are BIT-IDENTICAL to the arm's (384/384) Dumps were paired by index first and that was meaningless: the two sides reach the GEMM a different number of times (1024 arm dumps against 384 device dumps on the same request). Keyed by (layer, N, K, ordinal) they pair exactly. Result, one token, Mistral-7B 1.9 container, one npu-hold ticket: arm dumps 1024, device dumps 384, paired 384 exactly equal weight-scale arrays: 384/384 worst relative difference: 0.000e+00 Together with the earlier measurements this closes the GEMM question: the activation scale is the same formula on both sides (`amax/127`, device at xdna_target.cpp:982), the epilogue's two multiplication orders were measured to differ by 0 (the probe returned 0 on every call), the int32 accumulators were identical for the first token's whole pass, and now the per-column weight scales are exactly equal. So the device's GEMMs are bit-exact against this arm. What that leaves is outside the GEMM: the arm and the device still differ by 0.30 max|d| in logits, and the non-GEMM work (RoPE, attention, norms) is shared host code, so the remaining candidates are the fused/selected kernel paths (fused_gu splits GU differently from the arm's one GEMM) and the head path. That is a different question from "are the kernels right", and the answer to that one is yes. Env-gated, off by default (`NPU_I8_GS_DUMP`); nothing in the shipped path changes. * npu: the integer head, and the gate it unlocks -- device logits BIT-EXACT over four steps The head was the last place the host arm and the device computed the same tensor differently: in host mode `run_lm_head()` is a plain f32 matvec ("No I8Ctx is touched") while the device reads a packed quantised table (`lm_head_packed_`, `run_lm_head_gemv`). That was the whole of the remaining `max|d| = 0.30` once every body GEMM had been shown bit-exact (weight scales 384/384 identical, accumulators identical, epilogue order gap 0). The device packs its head tiles with `lctx.packB_into(...)` -> `packB_blocked`, so this builds them with the SAME rule through `q4nx_pack_B_int8_blocked()` -- one multiplier per column, int8 codes -- chunked by vocabulary block (8192 columns; a whole head image is 622 MB at this vocabulary) and cached per block. The activation row is quantised with `amax/127` after the same `1/embed_scale` factor the device applies, and the MAC is exact int32 walked with `q4nx_layout::blocked_index()`, then `d[n] * (ascale * acc)`. Measured, one npu-hold ticket, device vs this arm, logit dumps compared per step: step 0: argmax 3916 == 3916 cos 1.000000 max|d| 0 step 1: argmax 278 == 278 cos 1.000000 max|d| 0 step 2: argmax 28725 == 28725 cos 1.000000 max|d| 0 step 3: argmax 22478 == 22478 cos 1.000000 max|d| 0 BIT-EXACT over 4 step(s) at max|d| tolerance 0.0 That is TNPU-ALL section 4's bar met literally for this family rather than read down to "at the accumulator": the device's arithmetic and the integer reference agree to the last bit, on every step, for a full request. * npu-pack: read the fused SSMLP/GQO FFN attributes, and the 4096/32/40 shape rule (refs #1740) granite-4.1-8b states its FFN nowhere the reader looked: config.json is literally {}, genai_config.json's decoder block has only hidden_size / heads / layers, and its FFN is not MatMulNBits nodes at all -- it is one `SSMLP` op with gate_K 4096, gate_N 12800, up_K 4096, up_N 12800, down_K 12800, down_N 4096. So the import refused with "the pack states no intermediate_size", which is correct for what it could see and wrong about the pack. Recording those attributes as matmuls lets intermediate_size_from_graph find w by the same witness it already uses for the separate-node spelling (gate and up reading one activation) instead of adding a second rule that could disagree with the first. With the width resolved the pack reaches the shape gate, which had no row for hidden 4,096 / 32 heads / 40 layers. Its 1.9 header is clean -- 40 runs of 32 x `00 00 7f ff`, first at 13,795,392, every gap exactly 107,728,192 -- and its tail is 225,198,144 B, which is exactly 8,256 + 1,568 x 143,616: the h4096 head grid (rows = ceil(100,352 / 64)). So unlike Qwen3-8B the head IS in the tail here, and the bundle/rotation agree with the Qwen3-8B row at the same hidden size and head count. amd publishes no 1.8 twin for this model, so that agreement plus the head-grid arithmetic is the whole evidence for the rotation -- the row says so, and says the prefix inherits the Qwen3 row's own caveat. * docs/npu: say what NPU_INFER_HOST_GEMM=i8 is for, and what it measured The integer arm was only discoverable from the code. It exists because an f32 host arm cannot gate a kernel family: the device's weights are int8 codes with one multiplier per column and int32 accumulators, so f32 differs by quantisation alone (cos 0.9960-0.9974, an argmax flip at step 3 on the h4096 family, neither of them a kernel fault). Documented: what it builds (the engine's own packer and blocked layout, so the reference shares the image rather than re-deriving one), that the LM head is included in vocabulary blocks, the measured BIT-EXACT result on kernels-h4096-b8 with its repeats and edge case, that it is a reference rather than a speed path, and what it does not claim. * docs/npu: the two-arm command sequence for the bit-exact gate A documented reference that nobody can run is a claim, not an instrument: the section now carries the two forward-serve invocations, the dump naming scheme, and the three-step discipline (compare per position, repeat the device arm for determinism, add an edge case). * npu-pack: withdraw the 4096/32/40 row -- a prefix cannot be inferred, only measured against a twin I added this row on granite-4.1-8b because its 1.9 header is clean (40 uniform runs at stride 107,728,192) and its tail is exactly 8,256 + 1,568 x 143,616, which pins the TILE by head-grid arithmetic. That was the mistake: the tile is not the whole rule. `prefix` is defined by the twin diff -- tools/fconst19_measure.py:104 walks `prefix += 1` while the 1.9 block still equals the 1.8 one, so prefix is the first byte where they diverge, and the tile is found by where that first-differing run reappears. Neither number is named anywhere in the 1.9 pack alone. The row's own in-block checks rejected it, which is the argument for removing it rather than for keeping it with a caveat: scale/zero exactness 0 %, second header 0, and 0 of 80 RMSNorm anchors found. A row that fails the pack it was written for is not a measured row. Confirmed against the twin I already had: Qwen3-8B's prefix 12,608 is exactly the first byte where its 1.9 block diverges from its 1.8 block (1.9 `8782 0777...` vs 1.8 `6756 ea78...`, 0 differing bytes in [0, 12608)). amd publishes no 1.8 granite, so granite stays refused by the shape gate -- the honest answer for a layout nothing can establish. The table now says that, so the next person does not re-derive the row from the tail arithmetic the way I did. Qwen3-8B is unaffected: still "all npu-pack checks passed (including the pack leg)". * npu-pack: the h4096 36-layer fconst row, so the Qwen3-8B pack imports at all `kFconst19Rules` held the 32-layer h4096 row and nothing for 36 layers, so `npu-pack info` refused `amd/Qwen3-8B_rai_1.9.0_npu` by design: "its block shape has not been measured for hidden 4096, 32 heads, 36 layers … the 1.8 pack of the same model is what establishes it". Measured now, with tools/fconst19_measure.py against amd/Qwen3-8B_rai_1.8.0_npu_16K: {4096, 32, 36, 12608, 5, 1, 2297856} 26,104 of 3,756,627,264 bytes differ (6.95e-06 of the file) -- content, not layout Every field but the layer count is identical to the 32-layer row (prefix 12,608, five tiles of 2,297,856, rotation 1), which is the first row here where the layer count moves nothing but the number of blocks: the h2048 pair moves both the prefix and the division with it (4,160/9 tiles at 36 layers against 8,512/3 at 24), so this row is measured rather than inferred from that one. The consequence beyond this pack: a kernel family and an fconst rule row are not the same partition. One family can span several rows, so each new pack of a family may cost a measurement before it costs a gate. Verified end to end: the import now completes -- 5,760/5,760 .fconst tiles survive decode->encode byte for byte, 144 attention projections plus 108 MLP projections with fp32 scale/zero exact for 100.00% of groups, 8 full_i8 ELFs linked from kernels-h4096-im12288-b8. * npu-pack test: the new 36-layer row goes in BOTH places, as the guard's own comment requires CI caught the omission and said exactly why: "fconst19 rule: the table has exactly the 11 measured rows (12); a dropped row is a silent un-measure of a whole class, an added one must be added here too". The expected rows are literals copied from the measured table rather than read back from it, so that a wrong digit or a dropped row fails -- which is the arm working, not a nuisance. Added {4096, 32, 36, 12608, 5, 1, 2297856} to it, and the arm now passes. * npu-pack: fused-MLP attributes are not graph matmuls, and the 4096/32/36 row is pinned in the test Two things found by running the tests and real packs on main + this branch. 1. main's row-count arm (#1861) pins kFconst19Rules as literals; the Qwen3-8B row was not in it, so the merged tree failed 11 != 12. Added. 2. The SSMLP/GQO gate/up/down attributes were pushed into OnnxModel::matmuls. matmuls.empty() is how the importer knows a 1.9 pack keeps its head in the .fconst tail, and every 1.9 pack has a fused MLP, so the block-32 tail head (#1948) was no longer read: Qwen-2.5 1.5B 1.9 imported 338 tensors instead of 339, with no lm_head, exit 0 and no refusal. The attributes now go to OnnxModel::fused_mlp, which only intermediate_size_from_graph() reads. New arm 2b builds the tied block-32 fixture with an SSMLP node; with the old push it fails three assertions. Co-Authored-By: Claude Fable 5.2 * npu-pack test: arm 2b removes its second fixture directory It was left under the golden directory after a run. Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- npu/pack_import.cpp | 175 +++++++++++++++++++++++++++++++-- npu/pack_import.h | 5 + tests/npu_pack_import_test.cpp | 92 ++++++++++++++++- 3 files changed, 261 insertions(+), 11 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index ca248267b..3c6c09bae 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -133,11 +133,20 @@ struct OnnxMatMul { std::string data, weight, scales, zeros; int64_t K = 0, N = 0; int bits = 0, block = 0; + // Which `If` branch declared it, as the AttributeProto's name (`then_branch` / `else_branch`), or empty in the + // top-level graph. An OGA pack splits prefill from decode with `If (Shape(input_ids) == 1)`: Qwen3-8B declares ONE + // MatMulNBits (its head) in `then_branch` and every layer projection in `else_branch`, so a walk that merges the + // branches cannot tell which path a weight belongs to. The Token .fconst IS the decode path, so the head is read + // from the decode branch -- see is_decode_branch(). + std::string branch; }; struct OnnxModel { std::vector tensors; std::vector matmuls; + // The fused-MLP nodes' (SSMLP / GQO) gate/up/down shapes, from their attributes. Kept apart from `matmuls` because they + // name no weight tensor: `matmuls` answers "does the graph carry quantised weights", and these must not. + std::vector fused_mlp; std::map index; // `com.ryzenai:MatMulNBitsBf` is the same quantised matmul as `com.microsoft:MatMulNBits`, and on the 1.9 // blob/external-data packs it is the ONLY one: phi-4's 201 weights are all `MatMulNBitsBf`, so a walker that @@ -151,9 +160,27 @@ struct OnnxModel { // .fconst, so plan() drops them; with no .fconst there is no other source and plan() refuses by name (#1687). int64_t weightless_matmuls = 0; std::string weightless_example; // the first one, for the refusal's text + // weightless_matmuls for placeholders OUTSIDE the decode branch only. A placeholder in the prefill branch is not a + // fault for a Token-fconst pack reading its decode weights: Qwen3-8B's `else_branch` spells lm_head as an empty + // placeholder while `then_branch` (decode) carries the real [151936,32,64] tensor. This is what plan() consults for + // "are these weights genuinely absent from the pack" -- the total stays for `info`'s census. + int64_t weightless_matmuls_decode = 0; + // The branch that runs when `input_ids` has length 1, i.e. the decode path the Token .fconst belongs to. Learned + // from the `If`'s condition where it is the literal we recognise, else defaulted to `then_branch` -- the ONNX + // convention every OGA pack on the Hub follows -- and named in the note either way, so a pack that puts it in the + // other branch is diagnosable rather than silently misread. + std::string decode_branch; + std::string branch_note; }; -void walk_graph(const uint8_t* p, size_t n, OnnxModel& m) { +// A matmul is decode-path when its branch is the decode branch, or when it is in no branch at all (a pack whose graph +// is flat has one path). A pack whose decode branch was never identified keeps every matmul, which is today's +// behaviour and cannot lose a head. +bool is_decode_branch(const OnnxModel& m, const std::string& branch) { + return branch.empty() || m.decode_branch.empty() || branch == m.decode_branch; +} + +void walk_graph(const uint8_t* p, size_t n, OnnxModel& m, const std::string& branch = std::string()) { const auto g = pb(p, n); for (const auto* f : fields(g, 5)) { const auto t = pb(f->p, f->n); @@ -191,8 +218,11 @@ void walk_graph(const uint8_t* p, size_t n, OnnxModel& m) { const auto* nm = field1(at, 1); if (!nm) continue; if (const auto* iv = field1(at, 3)) attrs[pb_str(*nm)] = int64_t(iv->v); - for (const auto* sg : fields(at, 6)) walk_graph(sg->p, sg->n, m); - for (const auto* sg : fields(at, 11)) walk_graph(sg->p, sg->n, m); + // AttributeProto.graph is field 6 (a GRAPH), graphs is field 11 (a repeated GRAPH) -- so a branch arrives + // in field 6 and its name is the attribute's own, `then_branch` / `else_branch`. + const std::string sub_branch = nm ? pb_str(*nm) : std::string(); + for (const auto* sg : fields(at, 6)) walk_graph(sg->p, sg->n, m, sub_branch); + for (const auto* sg : fields(at, 11)) walk_graph(sg->p, sg->n, m, sub_branch); } // The graph is the authority on how THIS reader should spell a quantised matmul: the same pack also states the // op in `cache/*_meta.json` (which the block census reads) and in the `.onnx` alias table, and if those ever @@ -217,8 +247,38 @@ void walk_graph(const uint8_t* p, size_t n, OnnxModel& m) { if (attrs.count("N")) mm.N = attrs["N"]; if (attrs.count("bits")) mm.bits = int(attrs["bits"]); if (attrs.count("block_size")) mm.block = int(attrs["block_size"]); + mm.branch = branch; if (!mm.weight.empty()) m.matmuls.push_back(std::move(mm)); } + // The Ryzen AI fused MLPs: `SSMLP` is gate+up+down as ONE node and `GQO` is grouped-query attention, so neither + // has a MatMulNBits node for the FFN to pair up -- which is why amd/granite-4.1-8b_rai_1.9.0_npu states no + // intermediate_size anywhere (`config.json` is literally {}, and genai_config.json's decoder block has only + // hidden_size / heads / layers) and was refused. The node carries the same numbers as attributes, measured on + // that pack: gate_K 4096, gate_N 12800, up_K 4096, up_N 12800, down_K 12800, down_N 4096 -- the plain + // (K == hidden, N == w) gate/up and (K == w, N == hidden) down triple, w = 12800. Recording them as matmuls lets + // intermediate_size_from_graph find them by the SAME witness it uses for the separate-node spelling (gate and up + // reading one activation) rather than adding a second rule that could disagree with the first. + if (op == "SSMLP" || op == "GQO") { + for (const char* sub : {"gate", "up", "down"}) { + const auto k = attrs.find(std::string(sub) + "_K"), n = attrs.find(std::string(sub) + "_N"); + if (k == attrs.end() || n == attrs.end() || k->second <= 0 || n->second <= 0) continue; + OnnxMatMul mm; + // gate and up share the node's activation input; down reads the gated product, so it gets its own name. + // That shared activation is the very witness separate gate/up nodes provide. + mm.data = ins.empty() ? std::string() : ins[0]; + if (std::string(sub) == "down") mm.data += ".down"; + mm.weight = sub; // the role name intermediate_size_from_graph's excluded() filter reads + mm.K = k->second; + mm.N = n->second; + mm.branch = branch; + // NOT into `matmuls`: these carry no weight tensor, and `matmuls.empty()` is how the importer knows a + // 1.9 pack keeps its head in the .fconst tail. Counted as graph matmuls they turned that off for every + // pack with a fused MLP -- measured on Qwen-2.5 1.5B 1.9, which then imported 338 tensors with no + // lm_head, exit 0 and no refusal. They are geometry witnesses only, read by + // intermediate_size_from_graph(). + m.fused_mlp.push_back(std::move(mm)); + } + } } } @@ -233,6 +293,65 @@ OnnxModel parse_onnx(const std::string& path) { const auto* g = field1(model, 7); if (!g) throw std::runtime_error(path + ": no graph"); walk_graph(g->p, g->n, m); + // Which branch is decode. The condition is `Equal(Shape(input_ids), Constant(value_int 1))` on every OGA 1.9 + // pack measured (Qwen3-8B, granite-4.1-b8), so the branch taken when it is TRUE is the one-token path and + // therefore the path the Token .fconst carries. Reading the comparison's own value rather than assuming + // `then_branch` keeps the rule honest for a pack that inverts it; when the graph is not that shape, the ONNX + // convention (`then_branch`) stands and `branch_note` says the rule was assumed rather than read. + { + // g IS the GraphProto (field 7 of ModelProto), so its nodes are at field 1 of g directly -- re-wrapping + // g's bytes in another pb() would read the graph as if it were a node list and find no Constant at all. + bool one_token_when_true = true, read = false; + for (const auto* f : fields(pb(g->p, g->n), 1)) { // GraphProto.node = 1 + const auto node = pb(f->p, f->n); + const auto* opf = field1(node, 4); + if (!opf || pb_str(*opf) != "Constant") continue; + for (const auto* a : fields(node, 5)) { + const auto at = pb(a->p, a->n); + const auto* nm = field1(at, 1); + if (!nm || pb_str(*nm) != "value_int") continue; + const auto* iv = field1(at, 3); + if (iv) { one_token_when_true = (iv->v == 1); read = true; } + } + } + // The branch whose graph actually declares a MatMulNBits. `value_int == 1` is the pack's convention, but the + // arithmetic is what must hold: the Token .fconst carries the graph's own quantised matmuls, so the branch + // that HAS them is the decode path whatever the constant says. A pack that inverts the condition without + // moving its weights stays readable; a pack whose weights sit in two branches is ambiguous, so the condition + // decides and the note says it did. + const auto branch_has_matmul = [&](const std::string& want) { + int n = 0; + for (const auto* f : fields(pb(g->p, g->n), 1)) { + const auto node = pb(f->p, f->n); + for (const auto* a : fields(node, 5)) { + const auto at = pb(a->p, a->n); + const auto* nm = field1(at, 1); + if (!nm || (pb_str(*nm) != "then_branch" && pb_str(*nm) != "else_branch")) continue; + if (pb_str(*nm) != want) continue; + for (const auto* sg : fields(at, 6)) { + for (const auto* nf : fields(pb(sg->p, sg->n), 1)) { + const auto sub = pb(nf->p, nf->n); + const auto* op = field1(sub, 4); + if (op && (pb_str(*op) == "MatMulNBits" || pb_str(*op) == "MatMulNBitsBf")) ++n; + } + } + } + } + return n; + }; + const std::string by_condition = one_token_when_true ? "then_branch" : "else_branch"; + const int then_mm = branch_has_matmul("then_branch"), else_mm = branch_has_matmul("else_branch"); + if (then_mm > 0 && else_mm == 0) m.decode_branch = "then_branch"; + else if (else_mm > 0 && then_mm == 0) m.decode_branch = "else_branch"; + else m.decode_branch = by_condition; + m.branch_note = read ? "the If condition compares the input shape with 1, so " + by_condition + + " is the one-token (decode) path" + + ((m.decode_branch == by_condition) ? "" : + "; its weights are in " + m.decode_branch + " (then " + + std::to_string(then_mm) + ", else " + std::to_string(else_mm) + + " quantised matmuls), which is where the Token .fconst's weights are read") + : "no recognisable If condition, so then_branch is taken as decode by the ONNX convention"; + } } catch (...) { unmap_file(map, size); throw; @@ -306,6 +425,8 @@ OnnxModel parse_onnx(const std::string& path) { // which knows whether an .fconst is there, decides. Deferring it silently produced // `...qweight.empty: no data` from a lookup much later, naming the placeholder rather than the problem. ++m.weightless_matmuls; + if (is_decode_branch(m, mm.branch)) + ++m.weightless_matmuls_decode; m.weightless_example = w->name + " (its " + packed + " sibling carries no data either)"; continue; } @@ -387,6 +508,10 @@ constexpr double kTieCorr = 0.99; // measured so far, not a rule: do not derive a row from it. A 1.9 pack with NO 1.8 twin cannot be given a row at all -- // there is nothing to diff it against, so it stays refused by the shape gate (amd/granite-4.1-8b_rai_1.9.0_npu is the // standing example) rather than being given an inferred prefix and reported working. +// What was tried on granite, so it is not tried again: its 40 uniform headers and its tail of exactly +// 8,256 + 1,568 x 143,616 pin the TILE by head-grid arithmetic and say nothing about the prefix; a row added from +// that alone was refused by the pack's own in-block checks (scale/zero exactness 0 %, no second header, 0/80 norm +// anchors) and withdrawn. constexpr Fconst19Rule kFconst19Rules[] = { // Qwen2.5-0.5B-Instruct: normalises to its 1.8 twin byte for byte, whole file, all 24 blocks. {896, 14, 24, 1856, 16, 2, 34816}, @@ -460,6 +585,16 @@ constexpr Fconst19Rule kFconst19Rules[] = { // all 32 blocks. This is the largest family on #1503 (31 packs). Measured by another lane on main, whose // table is keyed on geometry; merged here in the same form. {4096, 32, 32, 12608, 5, 1, 2297856}, + // Qwen3-8B at hidden 4,096 / 32 heads / **36 layers**: measured with tools/fconst19_measure.py against + // amd/Qwen3-8B_rai_1.8.0_npu_16K, 26,104 of 3,756,627,264 bytes differ (6.95e-06 of the file, which the tool prints + // as 6.95e-04 %) -- content, not layout, and the largest residual in this table, so it is stated rather than + // rounded away. Every other field is IDENTICAL to the 32-layer row above (prefix 12,608, five tiles of 2,297,856, + // rotation 1): at this hidden size the layer count moves nothing but the number of blocks, which is the first row + // here where that is true -- the h2048 pair above moves both the prefix and the division with the layer count + // (4,160/9 tiles at 36 layers against 8,512/3 tiles at 24), so this row is measured, not inferred from that one. + // Without it the importer refuses the pack by design ("its block shape has not been measured for hidden 4096, 32 + // heads, 36 layers"), which is what it did until now: kernel family and fconst rule row are not the same partition. + {4096, 32, 36, 12608, 5, 1, 2297856}, // Llama-3.2-3B (and -Instruct, same geometry) at hidden 3,072 / 24 heads / 28 layers: measured with // tools/fconst19_measure.py against amd/Llama-3.2-3B_rai_1.8.0_npu_4K, 1,090 of 1,756,930,368 bytes differ // (6.20e-07 of the file, which the tool prints as 6.20e-05 %) -- content, not layout. The prefix is 10,560 here (the first twin-diff byte at this hidden @@ -1603,14 +1738,16 @@ int intermediate_size_from_graph(const std::string& model_path, int64_t hidden) }; std::map> shared; // activation tensor -> N -> how many nodes read it std::set down_k, gate_n; // down: N == hidden, K > hidden; gate/up: K == hidden, N > hidden - for (const auto& mm : m.matmuls) { - if (mm.K <= 0 || mm.N <= 0 || excluded(mm.weight)) continue; + const auto witness = [&](const OnnxMatMul& mm) { + if (mm.K <= 0 || mm.N <= 0 || excluded(mm.weight)) return; if (mm.K == hidden && mm.N > hidden) { shared[mm.data][mm.N]++; gate_n.insert(mm.N); } if (mm.N == hidden && mm.K > hidden) down_k.insert(mm.K); - } + }; + for (const auto& mm : m.matmuls) witness(mm); + for (const auto& mm : m.fused_mlp) witness(mm); // SSMLP / GQO attributes: shapes without a weight tensor // separate gate/up first: it is the witness attention cannot produce std::set separate; for (const int64_t dk : down_k) @@ -3103,6 +3240,12 @@ std::string Plan::manifest() const { // (engine#1671). Reported so a test can pin the drop without needing a 1.9 pack, and so `info` says how many // graph weights this pack does not keep in its graph. g["weightless_matmuls"] = graph_weightless_matmuls; + // Of those, the ones OUTSIDE the decode branch. A placeholder in the prefill branch is not a fault when the + // decode branch carries the real tensor (Qwen3-8B: `else_branch` has an empty lm_head placeholder, `then_branch` + // has lm_head [151936,32,64]), so the refusal reads this one and the census above stays a census. + g["weightless_matmuls_decode"] = graph_weightless_matmuls_decode; + g["decode_branch"] = graph_decode_branch; + if (!graph_branch_note.empty()) g["decode_branch_note"] = graph_branch_note; if (graph_stride_of > 0) g["external_row_stride"] = {{"matched", graph_stride_ok}, {"of", graph_stride_of}, {"rule", "length == N * (K/128 + 1) * 68"}}; @@ -3296,6 +3439,9 @@ Plan plan(const std::string& pack_dir) { p.graph_ryzenai_matmuls = onnx.ryzenai_matmuls; p.graph_packed_siblings = onnx.packed_siblings; p.graph_weightless_matmuls = onnx.weightless_matmuls; + p.graph_weightless_matmuls_decode = onnx.weightless_matmuls_decode; + p.graph_decode_branch = onnx.decode_branch; + p.graph_branch_note = onnx.branch_note; for (const auto& t : onnx.tensors) if (!t.location.empty()) p.graph_external++; if (!ext_file.empty()) { @@ -3327,7 +3473,7 @@ Plan plan(const std::string& pack_dir) { // data has no weight for those ops IN THE GRAPH. With a Token .fconst that is the normal shape of a 1.9 pack -- its // graph is all placeholders and the weights are in the .fconst -- so the walker drops them and the import proceeds. // With no .fconst there is nothing to fall back to, and the honest answer is #1687's refusal, by name. - if (tok_path.empty() && onnx.weightless_matmuls > 0) + if (tok_path.empty() && onnx.weightless_matmuls_decode > 0) throw std::runtime_error( pack_dir + ": " + onnx.weightless_example + ", and the pack carries no Token .fconst either (no dd_metastate_Llm_Token_*.fconst): these weights are " @@ -4658,6 +4804,20 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons } } + // The head is the node whose geometry is the vocabulary x hidden -- but which NODE, when both `If` branches declare + // one? The Token .fconst is the decode path, so the decode branch's node is the one that describes it. Measured on + // amd/Qwen3-8B_rai_1.9.0_npu: `then_branch` (taken when Shape(input_ids) == 1, i.e. one token) holds exactly ONE + // MatMulNBits and it is lm_head [151936, 32, 64] at offset 15,269,888 in model.onnx.data; `else_branch` holds the + // 109 layer projections and an `lm_head.MatMulNBits.qweight.empty` placeholder with no bytes. The head is not the + // same tensor in both paths, so a walk that merges the branches reads the placeholder and falls back to the tail -- + // and on these packs the tail is the envelope alone (8,256 B = 64 + 2*4096, no head tiles), which is how the import + // refused with "the head's offsets are not established" for every 4096-hidden untied pack. + const OnnxMatMul* head_mm = nullptr; + for (const auto& mm : onnx.matmuls) + if (mm.N == int(V) && mm.K == int(H) && is_decode_branch(onnx, mm.branch)) { head_mm = &mm; break; } + if (!head_mm) + for (const auto& mm : onnx.matmuls) + if (mm.N == int(V) && mm.K == int(H)) { head_mm = &mm; break; } for (const auto& mm : onnx.matmuls) { const auto wi = onnx.index.find(mm.weight), si = onnx.index.find(mm.scales); if (getenv("NPU_PACK_DEBUG")) @@ -4696,6 +4856,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // which is why the imported container answered "!!!!!!!!". Only the node whose geometry IS the head is // taken here; the projections are imported from the graph's initializers by the block below. if (!(mm.N == int(V) && mm.K == int(H))) continue; + if (head_mm && &mm != head_mm) continue; // a same-geometry node in the other `If` branch if (mm.bits != 4) throw std::runtime_error("lm_head: MatMulNBits is not 4-bit"); const auto& wt = onnx.tensors[wi->second]; const auto& st = onnx.tensors[si->second]; diff --git a/npu/pack_import.h b/npu/pack_import.h index 48140ecd2..dc62f27ff 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -232,6 +232,11 @@ struct Plan { int64_t graph_ryzenai_matmuls = 0; int64_t graph_packed_siblings = 0; int64_t graph_weightless_matmuls = 0; // dropped: placeholder + packed sibling both byte-less (engine#1671) + // Of those, the ones outside the decode `If` branch: what plan() refuses on. The total above stays for `info`. + int64_t graph_weightless_matmuls_decode = 0; + // The `If` branch taken when input_ids has length 1, and how that was decided (read from the condition, or assumed). + std::string graph_decode_branch; + std::string graph_branch_note; // How many of the graph's 4-bit block-128 weights satisfy length == N x (K/128 + 1) x 68 (the per-row stride the // .fconst tile also carries): a measurement, so `info` can say "201 of 201" or "3 of 7" instead of guessing. int64_t graph_stride_ok = 0, graph_stride_of = 0; diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 088cf8052..ea1437dc9 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -532,6 +532,47 @@ void test_external_packed_genre(const std::string& root) { "placeholder with no sibling: the matmul is KEPT, so the pack is not mistaken for a tail-head pack"); } + // Which `If` branch is decode (engine#1740). An OGA pack splits prefill from decode with + // `If (Equal(Shape(input_ids), Constant(value_int 1)))`, and the two branches do NOT hold the same tensors: measured + // on amd/Qwen3-8B_rai_1.9.0_npu, `then_branch` holds exactly one MatMulNBits -- lm_head [151936,32,64] with real + // bytes -- while `else_branch` holds 109 layer projections and an `lm_head.MatMulNBits.qweight.empty` placeholder + // with none. The Token .fconst is the decode path, so a walk that merges the branches counts the prefill + // placeholder as "this pack has no head", falls back to the tail, and refuses (these packs' tails are the 8,256-byte + // envelope alone). The arm pins the reading of the condition, both ways round. + { + const auto graph_with_cond = [&](bool one_token_when_true, const std::string& head_branch) { + std::string named; + pb::str(named, 4, "Constant"); + std::string a; + pb::str(a, 1, "value_int"); + pb::ints(a, 3, one_token_when_true ? 1 : 2); + pb::tag(named, 5, 2); + pb::varint(named, a.size()); + named += a; + // A pack this fixture can actually plan: it needs its own embedding tensor in the graph, exactly as the + // write_graph lambda above gives it. + const std::string emb = pb::tensor("model.embed_tokens.weight", {64, 5120}, "eager-mm.onnx.data", 0, + uint64_t(embed_bytes)); + const std::string head = pb::matmulnbits("lm_head", {"x", "w", "w.scales", "w.zeros"}, K, N, 4, 128); + const std::string prefill = pb::graph({emb}, {head}); + const std::string decode = pb::graph({emb}, {head}); + return pb::model({}, {named, pb::if_node("c", "input_ids", "logits", head_branch, decode), + pb::if_node("c2", "input_ids", "logits", + head_branch == "then_branch" ? "else_branch" : "then_branch", prefill)}); + }; + // The graph this pack imports is the one genai_config.json names (eager-mm.onnx); a crafted model.onnx beside + // it is not read at all, which is why the first version of this arm measured the fixture's own graph. + put_file((dir / "eager-mm.onnx").string(), graph_with_cond(/*one_token_when_true=*/true, "then_branch")); + const json d1 = json::parse(pi::plan(dir.string()).manifest())["graph_weights"]; + check(d1.contains("decode_branch") && d1["decode_branch"].get() == "then_branch", + "decode branch: a condition comparing the shape with 1 makes then_branch the decode path"); + put_file((dir / "eager-mm.onnx").string(), graph_with_cond(/*one_token_when_true=*/false, "else_branch")); + const json d2 = json::parse(pi::plan(dir.string()).manifest())["graph_weights"]; + check(d2["decode_branch"].get() == "else_branch", + "decode branch: an inverted condition moves the decode path to else_branch"); + write_graph("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.packed", stride * N); + } + // A packed length that does not fit the stride must be reported as a mismatch, not crash and not claim a match. write_data(stride * N - 68); write_graph("model.layers.0.attn.qkv_proj.MatMulNBits.qweight.packed", stride * N - 68); @@ -1960,7 +2001,25 @@ void test_block32_tail_head(const std::string& root) { check(std::fabs(ncorr(emb, other)) < 0.3, "block-32 tail: the fixture's two heads are unrelated (corr " + std::to_string(ncorr(emb, other)) + ")"); - const auto build = [&](bool tied, const char* tag, const std::string& tail_after_env) { + // A Ryzen AI fused MLP node as the 1.9 graphs spell it: op SSMLP, the gate/up/down shapes as int attributes, and + // NO weight tensor (the weights are in the .fconst). Every 1.9 pack measured carries one per layer. + const auto ssmlp_node = [&]() { + std::string n; + pb::str(n, 1, "/model/layers.0/x"); + pb::str(n, 4, "SSMLP"); + const std::pair attrs[] = {{"gate_K", 128}, {"gate_N", 256}, {"up_K", 128}, + {"up_N", 256}, {"down_K", 256}, {"down_N", 128}}; + for (const auto& [name, value] : attrs) { + std::string a; + pb::str(a, 1, name); + pb::ints(a, 3, value); + pb::tag(n, 5, 2); + pb::varint(n, a.size()); + n += a; + } + return n; + }; + const auto build = [&](bool tied, const char* tag, const std::string& tail_after_env, bool fused_mlp = false) { const fs::path dir = fs::path(root) / tag; fs::remove_all(dir); fs::create_directories(dir / "cache"); @@ -2005,7 +2064,7 @@ void test_block32_tail_head(const std::string& root) { pb::placeholder("lm_head.weight.MatMulNBits.qweight"), pb::placeholder("lm_head.weight.MatMulNBits.scales"), pb::placeholder("lm_head.weight.MatMulNBits.zeros")}, - {})); + fused_mlp ? std::vector{ssmlp_node()} : std::vector{})); return dir; }; struct Run { @@ -2083,6 +2142,22 @@ void test_block32_tail_head(const std::string& root) { "block-32 tail, tied: the container's head is the embedding (corr " + std::to_string(x.head.empty() ? 0.0 : ncorr(x.head, emb)) + ")"); } + { // arm 2b: the same pack with a fused-MLP node in its graph, as every real 1.9 pack has. The node names no weight, + // so it must not make the graph look like it carries quantised matmuls: that is the switch for "the head is in + // the .fconst tail", and with it off the import wrote a container with no lm_head at all and exit 0 + // (measured on Qwen-2.5 1.5B 1.9 with the SSMLP attributes counted as graph matmuls: 338 tensors, not 339). + const Run x = run(build(true, "b32-tied-ssmlp", encode_block32_units(emb, V, H), true), "ssmlp"); + check(x.said.empty(), "block-32 tail, fused MLP in the graph: imports (" + x.said.substr(0, 160) + ")"); + check(x.has_head && !x.head.empty() && ncorr(x.head, emb) > 0.99, + "block-32 tail, fused MLP in the graph: the container still carries the head read from the tail"); + check(x.r.selfcheck_tied && x.r.head_read_note.find("reproduces") != std::string::npos, + "block-32 tail, fused MLP in the graph: the tied self-check still runs (" + x.r.head_read_note + ")"); + const fs::path plan_dir = build(true, "b32-tied-ssmlp-plan", encode_block32_units(emb, V, H), true); + const json g = json::parse(pi::plan(plan_dir.string()).manifest()); + fs::remove_all(plan_dir); + check(g.value("graph_weights", json::object()).value("matmulnbits_ops", int64_t(-1)) == 0, + "block-32 tail, fused MLP in the graph: SSMLP attributes are not counted as graph matmuls"); + } { // arm 3: TIED, but the tail's head is some other matrix -> the read is NOT taken, the synthesis stands, says why const Run x = run(build(true, "b32-tied-mismatch", encode_block32_units(other, V, H)), "mismatch"); check(x.r.head_read_note.empty() && x.r.head_synthesized_note.find("correlates") != std::string::npos, @@ -2904,8 +2979,16 @@ void test_pack(const std::string& pack, const std::string& golden_dir) { } check(attributed, "pack: each runtime-refused tensor is marked imported=false in the manifest"); } - check(r.lmhead_corr > 0.999, "pack: the decode reproduces the fp16 embedding (corr " + - std::to_string(r.lmhead_corr) + ")"); + // Only a TIED pack can be checked this way: there lm_head IS the embedding, so correlating them measures the decoder. + // An untied pack's lm_head is a separate matrix and the number is meaningless by construction (Qwen3-8B: 0.0043 on a + // head that decodes correctly -- std 0.026, plausible per-row scale, weakly correlated rows). Asserting 0.999 + // unconditionally demanded every pack be tied, which no untied pack can satisfy. + if (r.selfcheck_tied) + check(r.lmhead_corr > 0.999, "pack: the decode reproduces the fp16 embedding, tied (corr " + + std::to_string(r.lmhead_corr) + ")"); + else + check(true, "pack: untied, so the embedding correlation is not a decoder check (corr " + + std::to_string(r.lmhead_corr) + ")"); check(r.ctrl_corr_nibble < 0.5 && r.ctrl_corr_nozp < 0.95, "pack: both wrong decoders are rejected as controls"); // engine#1185: an imported container carries fp32 scale/zero (layout q4nx-2, npu/q4nx.h @@ -3895,6 +3978,7 @@ void test_fconst19_rule_table() { {3584, 28, 28, 7232, 8, 1, 1009664}, // the Qwen2.5-7B family, measured on two models {2048, 16, 36, 4160, 9, 1, 295936}, // Qwen2.5_3B_Instruct {4096, 32, 32, 12608, 5, 1, 2297856}, // Mistral-7B-Instruct-v0.2 + {4096, 32, 36, 12608, 5, 1, 2297856}, // Qwen3-8B (measured against ..._rai_1.8.0_npu_16K) {3072, 24, 28, 10560, 4, 1, 1740800}, // Llama-3.2-3B (and -Instruct) }; const int want_n = (int)(sizeof(want) / sizeof(want[0])); From 7e4f7a9746e15ebd46cdb38b95b0bd834f8eb8ab Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:52:58 -0400 Subject: [PATCH 21/98] docs: a 1.9 pack's ONNX components are vendor-EP graphs, and ort_run is the onnx/ component (engine#2000) (#2006) Two doc gaps, both checked against current main rather than taken from the issue. 1. docs/onnx-pack.md: a 1.9 pack's *.onnx component can require vendor op domains (com.ryzenai:DynamicPad, matmul_add_to_sd_gemm, mha_to_sd_mha, conv_to_nhwc_conv) and will not load under the plain ONNX Runtime EPs, so there is no CPU-EP oracle for it -- only the compiled partitioned/ route. Placed as the exception to the existing 'No vendor operators, ever' bullet, which is about OUR exports. 2. docs/onnx.md: the any-graph section now says which binary carries --input, that ort_run lives in the onnx/ component (add_executable in onnx/CMakeLists.txt), that ninja onebit does not build it, and where it lands. Measured here: cmake -B /tmp/bort -DONEBIT_ORT=ON && ninja ort_run -> /tmp/bort/onnx/ort_run, so the page's build/ort_run examples were one directory out and are corrected. Correction to the issue's second claim, which the docs now reflect: '1bit ort-run' does NOT reject --input on current main. app/main.cpp dispatches ort-run to onebit_ort_run_main inside #ifdef ONEBIT_ORT, and onnx/ort_run.cpp handles --input; with ONEBIT_ORT off the subcommand does not exist at all (1bit: unknown command 'ort-run'). So the gotcha is the configure guard, not the flag set. Co-authored-by: agent --- docs/onnx-pack.md | 19 +++++++++++++++++++ docs/onnx.md | 41 +++++++++++++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/docs/onnx-pack.md b/docs/onnx-pack.md index 9cb8b645a..c8502387b 100644 --- a/docs/onnx-pack.md +++ b/docs/onnx-pack.md @@ -114,6 +114,25 @@ model. - **No vendor operators, ever.** If a shape has no kernel it goes to the CPU EP. That keeps the failure mode boring: a slower session, never a session that refuses to load because a custom domain is missing. +- **…and the vendor's own components are the exception that proves it** (engine#2000). The bullet above + is about what *our* export emits. A 1.9 pack also carries the vendor's leftover graphs, and those are + written for their execution provider, not for ours: `Qwen3.5_4B_rai_1.9.0_npu`'s + `partitioned_patch1_sdgemm_leftover/vision.onnx` (84.7 MiB, with `.data` and `vision_strategy.yaml`) + has a clean ONNX interface — `pixel_values [num_patches,1536] f32` + `image_grid_thw [1,3] i64` → + `image_features [num_logical_patches,2560] f32` — and still will not load here. Its first strategy pass + is `qwen35_vision_stx.dynamic_pad`, and the graph is built on custom domains + (`com.ryzenai:DynamicPad`, then `matmul_add_to_sd_gemm`, `mha_to_sd_mha`, `conv_to_nhwc_conv`), so the + plain ONNX Runtime EPs reject it outright: + + ``` + Fatal error: com.ryzenai:DynamicPad(-1) is not a registered function/op + ``` + + with inputs built to the declared interface at bucket 1 of the three in `vision_strategy.yaml` — so + the failure is the operator, not the shape. **"The pack contains `vision.onnx`" does not mean there is + a portable graph to compare against.** For those components the compiled `partitioned/` route is the + only path, and there is no CPU-EP oracle for them: a correctness check on the vision tower has to be + built against the pack's own component, not against a reference run of the `.onnx` file. ## Where the numbers live diff --git a/docs/onnx.md b/docs/onnx.md index 48955cc7a..eabff2270 100644 --- a/docs/onnx.md +++ b/docs/onnx.md @@ -463,18 +463,47 @@ silent fall back cannot be mistaken for a device pass. ## Any graph: `ort-run --input` -`ort-run` speaks two interfaces. With `--ids` it is the Qwen3 language model of -[Qwen3-0.6B](#qwen3-06b) below. With `--input` it is **any** ONNX graph: nothing about the graph's -names, shapes or roles is assumed, so a vision model, an audio model and a one-op toy all take the -same path. +`ort-run` speaks two interfaces, and **which binary you are holding decides which one you get** +(engine#2000): + +* **`--ids`** is the Qwen3 language model of [Qwen3-0.6B](#qwen3-06b) below. +* **`--input`** (with `--output`, `--dump-all`) is **any** ONNX graph: nothing about the graph's names, + shapes or roles is assumed, so a vision model, an audio model and a one-op toy all take the same path. + +Both are implemented once, in `onnx/ort_run.cpp`, and that file backs two entry points: + +| entry point | built by | notes | +|---|---|---| +| `1bit ort-run` | `ninja onebit` **with `-DONEBIT_ORT=ON`** | dispatches to the same `onebit_ort_run_main` (`app/main.cpp`), whose parser handles `--input` (`onnx/ort_run.cpp`) | +| `build/onnx/ort_run` | `ninja -C build ort_run` | the standalone binary; **not** part of `ninja onebit` | + +The trap is the guard, not the flags. `app/main.cpp` dispatches `ort-run` inside `#ifdef ONEBIT_ORT`, and +`onnx/` is added with `add_subdirectory(onnx)` under the same option — so in a configure without +`ONEBIT_ORT` there is no standalone binary *and* the subcommand does not exist: + +```sh +$ 1bit ort-run --model m.onnx --input x=x.bin:float32:1x4x4 # ONEBIT_ORT not configured +1bit: unknown command 'ort-run' +``` + +Build the ONNX component before reaching for either form, and note where the standalone binary lands — +`add_executable(ort_run …)` lives in `onnx/CMakeLists.txt` and the root does not override +`CMAKE_RUNTIME_OUTPUT_DIRECTORY`, so a Ninja or Makefiles build puts it in the subdirectory (measured): + +```sh +cmake -B build -DONEBIT_ORT=ON +ninja -C build ort_run # -> build/onnx/ort_run +``` + +Earlier revisions of this page wrote the examples below as `build/ort_run`, which is not that path. ```sh # one declared input, raw little-endian float32, NHWC 1x640x640x3 -build/ort_run --model yolov8m.onnx \ +build/onnx/ort_run --model yolov8m.onnx \ --input DetectionModel::input_0=x.bin:float32:1x640x640x3 \ --dump-all out/ # the same, writing only what you name, and with our plugin EP asked first -build/ort_run --model m.onnx --ep build/ep/libonebit_ep.so \ +build/onnx/ort_run --model m.onnx --ep build/ep/libonebit_ep.so \ --input x=x.bin:float32:1x4x4 --output y=y.bin ``` From 89a7ca2320fb7f443d8da1000e3106cf2facd328 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:53:06 -0400 Subject: [PATCH 22/98] npu: the attention rows of a prefill block share one thread team (Refs #1990) (#2005) * npu: the attention rows of a prefill block share one thread team (Refs #1990) A batched pass called attention() once per row, and each call opened its own team over the heads: 256 teams per 16-row block on the served 27B pack. The rows of a layer now run their position-local half first and then all (row, head) pairs in one statically scheduled loop. Same statements per pair, so no bit moves: NPU_ATTN_CHECK=1 compares against the per-row form, and attn_math_test holds rows_context() to it on 1, 2, 5 and 12 threads. Co-Authored-By: Claude Fable 5.2 * docs: the one-team attention halves its term, but a block-wall gain is not resolved (second ticket) A second ticket (A B A B, quiet box) reproduced the attention term (24.5/26.6 -> 13.0/13.1 ms) and did not reproduce the 17 ms block-wall gain of the first pair (+4 and -24 ms). The section no longer quotes 2.6 %. Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 5 ++ docs/npu.md | 35 +++++++++++ npu/forward/include/attn_math.h | 59 ++++++++++++++++++ npu/forward/include/q4nx_forward.h | 2 +- npu/forward/src/q4nx_forward.cpp | 99 +++++++++++++++++++++--------- tests/attn_math_test.cpp | 54 ++++++++++++++++ 6 files changed, 225 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f2af8544f..7b6821e3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1331,6 +1331,11 @@ add_test(NAME quant_i8_row COMMAND quant_i8_row_test) add_executable(attn_math_test tests/attn_math_test.cpp) target_include_directories(attn_math_test PRIVATE npu/forward/include) add_test(NAME attn_math COMMAND attn_math_test) +# engine#1990: the test also holds rows_context() (M rows in one thread team) to the per-row form bit for bit, +# and that comparison is only about threads when the test has them. +if(TARGET OpenMP::OpenMP_CXX) + target_link_libraries(attn_math_test PRIVATE OpenMP::OpenMP_CXX) +endif() # ci.yml's build step names its targets explicitly and needs the `workflow` scope to extend, so bind each new # test binary to a target the list already builds (same reason as npu_moe_test above); drop this line when the # workflow is next touched with that scope and add these to the list itself. `host_gemm_pad_test` is here for the diff --git a/docs/npu.md b/docs/npu.md index 92527a0de..2f7fdf31c 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -604,6 +604,41 @@ A request wall moves less than the block does and scatters more: the same four a 21.6-23.1 after by TTFT, with one outlier in each direction (18.3 and 15.8) from other lanes' load. Quote the block medians for the effect and the TTFT range for what a caller sees. +#### The attention rows of a block share one thread team (#1990) + +The next host term after the one above, same pack, same instruments. A batched pass called `attention()` once per row, +and each call opened its own thread team over the heads: 16 rows on each of this model's 16 full-attention layers is +256 teams per block. The rows of a layer now go through `q4nx_attn::rows_context()` (`npu/forward/include/attn_math.h`): +every row's position-local half runs first (its q, and its k/v appended to the cache), then all (row, head) pairs are +one statically scheduled loop in one team. A causal row reads only positions up to its own, so the later rows already +being in the cache changes nothing it reads. + +Measured on `bonsai2-27b-t2s2` (2026-11-10, `npu-hold` tickets, one binary, `NPU_ATTN_ROWS=0` against the default, +103 blocks per arm, `amdxdna` error count unchanged). **What it buys is the attention term and its behaviour under +load, not a block-wall gain that can be resolved on a quiet box:** + +| ticket, arms in order | attention per block, mean (per-row -> one team) | block wall median | +|---|---|---| +| first: A B (quiet) | 31.2 -> **14.3 ms** | 675.7 -> 658.4 ms | +| second: A B A B (quiet) | 24.5 -> **13.0**, 26.6 -> **13.1 ms** | 609.3 -> 613.1, 626.3 -> 601.9 ms | +| first, later arms: B then A, another lane's load on the box | **16.0** vs 98.1 ms | 674.5 vs 834.4 ms | + +The attention term halves in every pair. The block wall does not follow it reliably: the first pair moved by 17 ms, +the two pairs of the second ticket by +4 and -24 ms, which is the arm-to-arm scatter of this box (about 15 ms on a +600 ms block) and not a measured 2-3 %. The first version of this section quoted the first pair as a 2.6 % gain; the +second ticket is why it no longer does. Under load the picture is different and is one observation, not a repeat: the +per-row arm measured 56.2 ms median attention with 36 of 103 blocks over 100 ms, while the one-team arm before it held +13.6 ms with none. + +Nothing rounds differently: a (row, head) pair runs the same statements in the same order as before +(`q4nx_attn::head_context()`, which `attention()` now calls too) and writes only its own floats. Greedy text is identical +in every arm. `NPU_ATTN_CHECK=1` recomputes each row the per-row way and compares bit patterns, failing the pass on a +difference (160 of 160 layer passes identical on the device); `tests/attn_math_test` holds `rows_context()` to the +per-row form bit for bit on 1, 2, 5 and 12 threads, for causal, whole-prompt and sliding-window reads. + +The schedule is static on purpose. A first version used a dynamic schedule and measured a lower median (15-25 ms) but a +worse mean than the per-row form, with 6 and 14 of 103 blocks over 100 ms in its two arms: the tail was the whole cost. + #### Per-tile arithmetic ceiling, and where our GEMMs sit against it (AM027) AM027 (*Versal Adaptive SoC AIE-ML v2 Architecture Manual*, v1.1) **Appendix A, Table 14** gives 512 INT8×INT8 multipliers per compute tile and 256 diff --git a/npu/forward/include/attn_math.h b/npu/forward/include/attn_math.h index e54f5dae2..0ac42d72d 100644 --- a/npu/forward/include/attn_math.h +++ b/npu/forward/include/attn_math.h @@ -26,6 +26,10 @@ // Contract: the forms differ by float ROUNDING ORDER only -- they are the same sum, not the same bits. // tests/attn_math_test.cpp is what holds that line (each form against a double-precision direct sum, on // every width the forward passes, including the awkward ones either body's tail loop would mishandle). +#include +#include +#include + #if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__)) #define Q4NX_ATTN_X86 1 #include @@ -104,4 +108,59 @@ inline void saxpy(float p, const float* x, float* y, int n) { saxpy_scalar(p, x, y, n); } +// One head of one row: softmax(q.K * scale) over cached positions [t0, T), then ctx = sum p_t * V_t. +// `kc`/`vc` are the cache planes, position-major [t][NKV][HD]; `kh` is the KV head this query head reads; +// `score` is T floats of scratch; `out` (HD floats) must be zero on entry. This is the body attention() ran +// inline before it was shared -- the same statements in the same order, so the result is the same bits. +inline void head_context(const float* qh, const float* kc, const float* vc, int NKV, int kh, int HD, + int t0, int T, float scale, float* score, float* out) { + float mx = -1e30f; + for (int t = 0; t < T; t++) { + if (t < t0) { score[t] = -1e30f; continue; } + const float* kt = kc + ((size_t)t * NKV + kh) * HD; + score[t] = dot(qh, kt, HD) * scale; + if (score[t] > mx) mx = score[t]; + } + float sum = 0.0f; + for (int t = 0; t < T; t++) { + if (t < t0) { score[t] = 0.0f; continue; } + score[t] = std::exp(score[t] - mx); + sum += score[t]; + } + const float inv = sum > 0 ? 1.0f / sum : 0.0f; + for (int t = t0; t < T; t++) saxpy(score[t] * inv, vc + ((size_t)t * NKV + kh) * HD, out, HD); +} + +// M rows of attention in ONE thread team (engine#1990). A batched pass used to call attention() once per row, +// and each call opened its own team over the heads: 16 rows x 16 attention layers = 256 teams per prefill +// block, and the team count is what a loaded box pays for. Here the (row, head) pairs are one work-shared +// loop. Each pair is independent -- it reads its own q, the read-only cache, and writes its own HD floats of +// ctx -- so the split across threads cannot change a value; tests/attn_math_test.cpp compares this against a +// plain loop over head_context() bit for bit. +// +// `q` is [M][NH][HD] (already normed and rotated), `T[m]`/`t0[m]` are row m's read window, `ctx` is +// [M][NH][HD] and is zeroed here. `nt` <= 1 runs it on the caller's thread. +inline void rows_context(const float* q, const float* kc, const float* vc, int M, int NH, int NKV, int HD, + const int* T, const int* t0, float scale, float* ctx, int nt) { + const int gqa = NH / NKV; + int tmax = 0; + for (int m = 0; m < M; m++) tmax = T[m] > tmax ? T[m] : tmax; + for (size_t i = 0, n = (size_t)M * NH * HD; i < n; i++) ctx[i] = 0.0f; + const long pairs = (long)M * NH; + (void)nt; +#pragma omp parallel num_threads(nt > 1 ? nt : 1) if (nt > 1 && pairs > 1) + { + std::vector score((size_t)(tmax > 0 ? tmax : 1)); + // static, not dynamic: on a loaded box a dynamic schedule's tail was the whole cost (a host microbench at + // this geometry, 24 threads: p99 62 ms against 17 ms static per layer), and the rows of one block read + // windows within M positions of each other, so there is no imbalance for dynamic to fix. +#pragma omp for schedule(static) + for (long i = 0; i < pairs; i++) { + const int m = (int)(i / NH), h = (int)(i % NH); + head_context(q + ((size_t)m * NH + h) * HD, kc, vc, NKV, h / gqa, HD, t0[m], T[m], scale, + score.data(), ctx + ((size_t)m * NH + h) * HD); + } + } +} + } // namespace q4nx_attn diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 2ee8c1da2..403481ce6 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -278,7 +278,7 @@ class Q4nxNpuForward { bool attention(const std::vector& qkv, int pos, const int* pos3, std::vector& ctx, int head_dim, int n_kv_heads, float theta, int rotary_dim, bool proportional, int window, int kv_layer, bool append_kv, - int kv_len = 0); + int kv_len = 0, std::vector* q_out = nullptr); bool mlp(const std::vector& gu, std::vector& h, int im); // The same elementwise gate*up on caller memory, so a batched pass can read its ACT row and write its HH // row where they already are instead of staging both through a vector (a copy on each side). diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 3bbf2df77..18deb4d7b 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -3690,7 +3690,8 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, // One row of attention. `append` is the K/V cache append, `kvlen` is how many cached // positions to read (0 = up to and including `pos`, i.e. causal), and `dst` receives the // context -- null when the pass only fills the cache (the noncausal first pass below). - auto row_attn = [&](int m, float* dst, bool append, int kvlen) -> bool { + auto row_attn = [&](int m, float* dst, bool append, int kvlen, + std::vector* q_keep = nullptr) -> bool { { CpScope _cp(CP_QKV); qkv_row.assign(QKV.begin() + (size_t)m * g->N, @@ -3725,9 +3726,10 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, } if (!attention(*in, start_pos + m, pos3_ ? trio : nullptr, dst ? ctx_row : ctx_scratch, l_hd, l_nkv, l_theta, - l_rot, l_prop, l_win, l, append, kvlen)) { + l_rot, l_prop, l_win, l, append, kvlen, q_keep)) { err_ = "attention failed"; return false; } + if (q_keep) return true; // prepare only: q is in *q_keep, k/v are in the cache if (!dst) return true; if (gate_ptr) for (int i = 0; i < nhd; i++) @@ -3743,7 +3745,62 @@ bool Q4nxNpuForward::step_batch(const std::vector& tokens, int start_pos, // hold all M rows before any of them attends -- a row's keys come from the same layer, and // the rows after it only exist once their projections ran. The GDN layers keep their own // causal recurrence (the reference keeps it too), and a sliding layer stays causal. - if (noncausal_attn_ && l_full) { + // engine#1990: one thread team for the layer's M rows instead of one per row. Every row's + // position-local half runs first (q, and its k/v appended -- a causal row reads only positions up + // to its own, so the later rows already being in the cache changes nothing it reads), then all + // (row, head) pairs are one work-shared loop. NPU_ATTN_ROWS=0 keeps the per-row form for an A/B; + // NPU_ATTN_CHECK=1 recomputes every row that way and fails the pass on any differing bit. + static const bool attn_rows_on = [] { + const char* e = getenv("NPU_ATTN_ROWS"); + return !(e && e[0] == '0'); + }(); + static const bool attn_check = [] { + const char* e = getenv("NPU_ATTN_CHECK"); + return e && e[0] == '1'; + }(); + if (attn_rows_on && M > 1 && l_hd > 0 && c_.NH % l_nkv == 0) { + const bool whole = noncausal_attn_ && l_full; + const size_t qw = (size_t)nhd; + std::vector qall((size_t)M * qw), qm; + std::vector Tm((size_t)M), t0m((size_t)M); + for (int m = 0; m < M; m++) { + if (!row_attn(m, nullptr, true, 0, &qm)) return false; + if (qm.size() != qw) { err_ = "attention rows: q width"; return false; } + memcpy(qall.data() + (size_t)m * qw, qm.data(), qw * sizeof(float)); + const int T = whole ? start_pos + M : start_pos + m + 1; + Tm[(size_t)m] = T; + t0m[(size_t)m] = (l_win > 0 && T > l_win) ? T - l_win : 0; + } + { + LhScope _t_attn(LH_ATTN); + const float scale = sem_.attn_scaling_one ? 1.0f : 1.0f / std::sqrt((float)l_hd); + const int nt = ((long)M * (start_pos + M) * l_hd >= 8192) ? omp_heads_threads() : 1; + q4nx_attn::rows_context(qall.data(), kcache_[l].data(), vcache_[l].data(), M, c_.NH, l_nkv, + l_hd, Tm.data(), t0m.data(), scale, CTX.data(), nt); + if (attn_gate_) + for (int m = 0; m < M; m++) { + const float* gate = QKV.data() + (size_t)m * g->N + nhd; + float* c = CTX.data() + (size_t)m * ctx_w; + for (int i = 0; i < nhd; i++) c[i] *= 1.0f / (1.0f + expf(-gate[i])); + } + } + if (attn_check) { + std::vector ref((size_t)ctx_w); + size_t differing = 0; + for (int m = 0; m < M; m++) { + if (!row_attn(m, ref.data(), false, Tm[(size_t)m])) return false; + const float* c = CTX.data() + (size_t)m * ctx_w; + for (int i = 0; i < ctx_w; i++) + if (memcmp(&c[i], &ref[(size_t)i], sizeof(float)) != 0) differing++; + } + fprintf(stderr, "[attncheck] l=%d M=%d start=%d one-team vs per-row: %zu values %s\n", l, M, + start_pos, (size_t)M * ctx_w, differing ? "MISMATCH" : "identical"); + if (differing) { + err_ = "attention rows: the one-team form differs from the per-row form"; + return false; + } + } + } else if (noncausal_attn_ && l_full) { for (int m = 0; m < M; m++) if (!row_attn(m, nullptr, true, 0)) return false; for (int m = 0; m < M; m++) @@ -4478,7 +4535,7 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int std::vector& ctx, int head_dim, int n_kv_heads, float theta, int rotary_dim, bool proportional, int window, - int kv_layer, bool append_kv, int kv_len) { + int kv_layer, bool append_kv, int kv_len, std::vector* q_out) { const int NH = c_.NH; const int HD = head_dim; /* per layer type (Gemma4: 256 or 512) */ const int NKV = n_kv_heads; @@ -4585,6 +4642,13 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int vcache_[kv_layer].insert(vcache_[kv_layer].end(), v.begin(), v.end()); } + // engine#1990: a batched pass asks for the position-local half only -- the q this row attends with, and its + // k/v in the cache -- and runs the read for all of its rows in one thread team (q4nx_attn::rows_context). + if (q_out) { + q_out->swap(q); + return true; + } + // kv_len > 0 is a noncausal read: attend over that many cached positions instead of only the ones // up to `pos`. Everything above (rope, qk-norm, the cache append) is position-local and unchanged. const int T = kv_len > 0 ? kv_len : pos + 1; @@ -4606,30 +4670,9 @@ bool Q4nxNpuForward::attention(const std::vector& qkv, int pos, const int { std::vector score(T); #pragma omp for schedule(static) - for (int h = 0; h < NH; h++) { - const int kh = h / gqa; - const float* qh = &q[(size_t)h * HD]; - float mx = -1e30f; - for (int t = 0; t < T; t++) { - if (t < t0) { score[t] = -1e30f; continue; } - const float* kt = &kcache_[kv_layer][((size_t)t * NKV + kh) * HD]; - score[t] = q4nx_attn::dot(qh, kt, HD) * scale; - if (score[t] > mx) mx = score[t]; - } - float sum = 0.0f; - for (int t = 0; t < T; t++) { - if (t < t0) { score[t] = 0.0f; continue; } - score[t] = std::exp(score[t] - mx); - sum += score[t]; - } - const float inv = sum > 0 ? 1.0f / sum : 0.0f; - float* out = &ctx[(size_t)h * HD]; - for (int t = t0; t < T; t++) { - const float p = score[t] * inv; - const float* vt = &vcache_[kv_layer][((size_t)t * NKV + kh) * HD]; - q4nx_attn::saxpy(p, vt, out, HD); - } - } + for (int h = 0; h < NH; h++) + q4nx_attn::head_context(&q[(size_t)h * HD], kcache_[kv_layer].data(), vcache_[kv_layer].data(), NKV, + h / gqa, HD, t0, T, scale, score.data(), &ctx[(size_t)h * HD]); } return true; } diff --git a/tests/attn_math_test.cpp b/tests/attn_math_test.cpp index e07b3e421..179e1fbaf 100644 --- a/tests/attn_math_test.cpp +++ b/tests/attn_math_test.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -105,6 +106,59 @@ int main() { check(q4nx_attn::dot_scalar(z.data(), ones.data(), 96) == 0.0f, "dot_scalar on zeros != 0", 96, q4nx_attn::dot_scalar(z.data(), ones.data(), 96), 0.0); + // engine#1990: M rows in one thread team (rows_context) against the form it replaces, one row at a time and + // one head at a time through head_context -- bit for bit, on every thread count, for the three read windows + // the forward uses: causal (row m reads start+m+1 positions), noncausal (every row reads all of them), and a + // sliding window (t0 > 0). A (row, head) pair writes only its own HD floats, so there is nothing to round + // differently; this is the arm that would catch a shared score buffer or a wrong KV head. + { + struct Shape { int M, NH, NKV, HD, start; }; + const Shape shapes[] = {{16, 32, 8, 128, 320}, {16, 8, 8, 64, 0}, {7, 12, 4, 96, 33}, {1, 4, 2, 17, 5}, {4, 6, 6, 8, 0}}; + int compared = 0; + for (const Shape& sh : shapes) { + const int P = sh.start + sh.M; + std::vector kc((size_t)P * sh.NKV * sh.HD), vc(kc.size()), q((size_t)sh.M * sh.NH * sh.HD); + for (float& x : kc) x = u(rng); + for (float& x : vc) x = u(rng); + for (float& x : q) x = 4.0f * u(rng); + const float scale = 1.0f / std::sqrt((float)sh.HD); + for (int mode = 0; mode < 3; mode++) { + std::vector T((size_t)sh.M), t0((size_t)sh.M); + for (int m = 0; m < sh.M; m++) { + T[(size_t)m] = mode == 1 ? P : sh.start + m + 1; + t0[(size_t)m] = (mode == 2 && T[(size_t)m] > 5) ? T[(size_t)m] - 5 : 0; + } + std::vector want(q.size(), 0.0f), score((size_t)P); + for (int m = 0; m < sh.M; m++) + for (int h = 0; h < sh.NH; h++) + q4nx_attn::head_context(&q[((size_t)m * sh.NH + h) * sh.HD], kc.data(), vc.data(), sh.NKV, + h / (sh.NH / sh.NKV), sh.HD, t0[(size_t)m], T[(size_t)m], scale, + score.data(), &want[((size_t)m * sh.NH + h) * sh.HD]); + double mass = 0.0; + for (float x : want) mass += std::fabs((double)x); + check(mass > 0.0, "rows reference is all zero (the arm compares nothing)", sh.M, mass, 1.0); + for (int nt : {1, 2, 5, 12}) { + std::vector got(q.size(), 123.0f); // rows_context must zero it itself + q4nx_attn::rows_context(q.data(), kc.data(), vc.data(), sh.M, sh.NH, sh.NKV, sh.HD, T.data(), + t0.data(), scale, got.data(), nt); + size_t diff = 0; + for (size_t i = 0; i < got.size(); i++) + if (std::memcmp(&got[i], &want[i], sizeof(float)) != 0) diff++; + char what[112]; + std::snprintf(what, sizeof what, "rows_context != per-row (M=%d NH=%d HD=%d mode=%d threads=%d)", + sh.M, sh.NH, sh.HD, mode, nt); + check(diff == 0, what, (int)got.size(), (double)diff, 0.0); + compared++; + } + } + } +#if defined(_OPENMP) + std::printf("rows: %d comparisons bit-identical to the per-row form (OpenMP build: threads 1, 2, 5, 12)\n", compared); +#else + std::printf("rows: %d comparisons bit-identical to the per-row form (no OpenMP: all on one thread)\n", compared); +#endif + } + if (failures) { std::printf("%d check(s) failed\n", failures); return 1; From 5cb49f45b92129a6997e9dde917c861737381b20 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:04:20 -0400 Subject: [PATCH 23/98] forward-serve: --prefix-pin, the pinned shared prefix on the device path, measured (Refs #1785) (#1982) * forward-serve: --prefix-pin, the pinned shared prefix on the device path (Refs #1785, F5) With --prefix-pin the forward keeps the previous request's prompt ids, and when a request shares a prefix with it that no slot covers, cuts the prefill one token short of that point, snapshots and pins it. serve carries the flag through. Off by default. Measured on the NPU: the third question under one system prompt resumes from 341 of 355 tokens, with answers identical to the cache-off run. Co-Authored-By: Claude Fable 5.2 * tests: pin_point arms for a shared ending and the common-prefix bound (Refs #1785) Answers the review on #1982: a shared ending under different openings is not a pin point, and a pin point never reaches past the tokens both prompts start with (swept over shared lengths around prefix-min-gain). Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- app/forward_serve.cpp | 45 +++++++++++++++++++++++++++++++++++-- app/serve.cpp | 4 ++++ docs/npu.md | 26 +++++++++++++++++---- tests/npu_sampling_test.cpp | 22 ++++++++++++++++++ 4 files changed, 91 insertions(+), 6 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index c70f989e0..effed11be 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -109,6 +109,10 @@ struct ForwardEngine { // slot's attention KV and GDN state. Only ever touched under mu. std::unique_ptr prefix; std::vector prefix_state; + // --prefix-pin (engine#1785): keep the state at the point where a request stops agreeing with the + // previous one, so single-turn requests under one system prompt resume from it (PrefixCache::pin_point). + bool prefix_pin = false; + std::vector prev_ids; // the previous request's prompt, what pin_point() compares against // The d3 decision head (#653), loaded when decision_config.json and decision.q4nx sit next to the // model: it is what turns this server into a decision endpoint as well as a chat one. std::unique_ptr head; @@ -332,11 +336,42 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res std::fprintf(stderr, "[prefix] slot=%d cached=%d of %zu reason=%s\n", stats.prefix.slot, stats.prefix.cached_tokens, ids.size(), stats.prefix.reason.c_str()); } + // (engine#1785) Where this request should leave a pinned snapshot, as a position: the state is saved + // one token short of the shared part for the same reason the whole-prompt snapshot below is saved one + // token short of the prompt. -1 = no pin for this request. pin_point() is 0 when a slot (the pin + // itself, on every request after the one that made it) already covers the shared part, and it is + // always less than the prompt's length, so this cut is never the last-token cut below. + int pin_cut = -1; + size_t pin_tokens = 0; + if (e.prefix_pin && e.prefix && e.prefix->enabled()) { + pin_tokens = e.prefix->pin_point(e.prev_ids, ids); + if (pin_tokens > 0 && int(pin_tokens) - 1 >= resume) pin_cut = int(pin_tokens) - 1; + } { const int chunk_max = std::max(1, e.fwd->max_batch_rows()); const int n = int(ids.size()); for (int off = resume; off < n;) { - const int take = std::min(chunk_max, n - off); + int take = std::min(chunk_max, n - off); + if (pin_cut >= 0 && off == pin_cut) { + Q4nxNpuForward::StateSnapshot snap; + if (e.fwd->snapshot_state(off, snap)) { + std::string why; + const std::vector shared(ids.begin(), ids.begin() + long(pin_tokens)); + const int slot = e.prefix->pin(shared, snap.bytes(), &why); + if (slot >= 0 && size_t(slot) < e.prefix_state.size()) e.prefix_state[size_t(slot)] = std::move(snap); + std::fprintf(stderr, "[prefix] pin slot=%d tokens=%zu state_at=%d %s\n", slot, pin_tokens, off, + why.c_str()); + } + pin_cut = -1; + } else if (pin_cut > off && off + take > pin_cut) { + // Stop this chunk at the pin position; the next pass saves the state there. One more launch. + take = pin_cut - off; + const std::vector head(ids.begin() + off, ids.begin() + off + take); + if (!e.fwd->step_batch(head, off, logits)) + throw std::runtime_error(std::string("forward step failed: ") + e.fwd->last_error()); + off += take; + continue; + } // The state the prefix cache keeps is taken one token short of the prompt, because the answer is // drawn from the last position's logits and a snapshot holds state, not logits -- so the snapshot // is saved just before the last token and that token is run again on a resume. On a GDN model @@ -368,6 +403,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res off += take; } } + if (e.prefix_pin) e.prev_ids = ids; // Prompt processing ends here: every prompt token has been through the model and the first // token's distribution is in `logits`. @@ -520,7 +556,8 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res {"prompt_tokens_details", {{"cached_tokens", st.prefix.cached_tokens}}}, {"flm_prefix_cache", {{"slot", st.prefix.slot}, {"cached_tokens", st.prefix.cached_tokens}, - {"reason", st.prefix.reason}}}}; + {"reason", st.prefix.reason}, + {"pinned", st.prefix.pinned}}}}; }; // OpenAI's logprobs.content entry: the token's text, its log-probability, and the most likely @@ -868,6 +905,7 @@ int run_forward_serve(int argc, char** argv) { // instead of being ignored and served by whatever target happened to be linked. std::string npu_target = ::npu::target::xdna::XdnaTarget::kName; npu::PrefixOptions prefix_opts; // --prefix-slots / --prefix-cache-mb / --prefix-min-gain (#650) + bool prefix_pin = false; // --prefix-pin (engine#1785) for (int i = 0; i < argc; i++) { const std::string a = argv[i]; auto next = [&]() -> std::string { @@ -882,6 +920,7 @@ int run_forward_serve(int argc, char** argv) { else if (a == "--prefix-slots") prefix_opts.slots = std::stoi(next()); else if (a == "--prefix-cache-mb") prefix_opts.cache_mb = std::stoul(next()); else if (a == "--prefix-min-gain") prefix_opts.min_gain = std::stoi(next()); + else if (a == "--prefix-pin") prefix_pin = true; else if (a == "--npu-streams") streams = std::stoi(next()); else if (a == "--npu-target") npu_target = next(); else if (a == "--dump-logits") dump_logits = next(); @@ -979,6 +1018,8 @@ int run_forward_serve(int argc, char** argv) { std::fprintf(stderr, "1bit forward-serve: prefix cache on: %d slots, %zu MB, min gain %d tokens\n", prefix_opts.slots, prefix_opts.cache_mb, prefix_opts.min_gain); + e.prefix_pin = prefix_pin; + if (prefix_pin) std::fprintf(stderr, "1bit forward-serve: prefix pin on: the part a request shares with the previous one keeps a slot\n"); } e.fwd = std::make_unique(); diff --git a/app/serve.cpp b/app/serve.cpp index 9556af962..164d6623f 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -185,6 +185,7 @@ struct Options { int prefix_slots = -1; // --prefix-slots N (0 = off) int prefix_cache_mb = -1; // --prefix-cache-mb N int prefix_min_gain = -1; // --prefix-min-gain N: tokens a prompt must add to be worth keeping + bool prefix_pin = false; // --prefix-pin: keep the part single-turn requests share (docs/npu.md) std::string npu_target; // --npu-target NAME: which NPU target serves the per-op route (empty = the build's default) std::string prefill_provider; // --prefill-provider PATH (or ONEBIT_HYBRID_PREFILL): --device hybrid's prefill (docs/hybrid.md) std::string laya_model; // --laya-model DIR: route each request by the Laya scorer (docs/laya.md) @@ -3171,6 +3172,7 @@ void usage(FILE* out) { " [--npu-streams N] with --device npu: this serve is one of N sharing the NPU\n" " [--prefix-slots N] [--prefix-cache-mb N] [--prefix-min-gain N] the prompt prefix cache\n" " (docs/npu.md; 0 slots turns it off). Unset keeps the forward's defaults: 8 / 4096 / 128\n" + " [--prefix-pin] keep the part a request shares with the previous one (one system prompt, many questions)\n" " (default 1, the device exclusive; docs/npu.md)\n" " [--load-queue-ms N] a load refused because another client holds the NPU array\n" " waits at most N ms for it to free before the driver is called dead\n" @@ -3286,6 +3288,7 @@ int run_serve(int argc, char** argv) { else if (a == "--prefix-slots") o.prefix_slots = std::stoi(next()); else if (a == "--prefix-cache-mb") o.prefix_cache_mb = std::stoi(next()); else if (a == "--prefix-min-gain") o.prefix_min_gain = std::stoi(next()); + else if (a == "--prefix-pin") o.prefix_pin = true; else if (a == "--npu-target") o.npu_target = next(); else if (a == "--prefill-provider") o.prefill_provider = next(); else if (a == "--laya-model") o.laya_model = next(); @@ -3452,6 +3455,7 @@ int run_serve(int argc, char** argv) { if (o.prefix_slots >= 0) { fargs.push_back("--prefix-slots"); fargs.push_back(std::to_string(o.prefix_slots)); } if (o.prefix_cache_mb >= 0) { fargs.push_back("--prefix-cache-mb"); fargs.push_back(std::to_string(o.prefix_cache_mb)); } if (o.prefix_min_gain >= 0) { fargs.push_back("--prefix-min-gain"); fargs.push_back(std::to_string(o.prefix_min_gain)); } + if (o.prefix_pin) fargs.push_back("--prefix-pin"); std::vector fav; for (auto& a : fargs) fav.push_back(a.data()); return run_forward_serve(int(fav.size()), fav.data()); diff --git a/docs/npu.md b/docs/npu.md index 2f7fdf31c..a5a47c2a0 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1610,10 +1610,28 @@ The slot table (`app/npu_prefix_cache.h`) has a slot kind for the part such requ It reads token ids only, so it does not depend on the template or on which message is the system prompt. -**State of this, plainly:** the table and its tests are in; `forward-serve` does not call `pin_point()`/`pin()` -yet, so no request is faster because of it today. The forward half is one more cut in the prefill loop (stop one -token short of the pin point, `snapshot_state`, `pin`), and it needs a device run to be claimed. - +**`--prefix-pin`** (on `1bit serve --device npu` and on `forward-serve`; off by default) turns it on. The +forward keeps the previous request's prompt ids, and when `pin_point()` is non-zero it stops the prefill one +token short of that point, takes a snapshot and pins it, the same cut it already makes before the last token. +The cost is one more launch on the request that makes the pin. Every decision is logged +(`[prefix] pin slot=1 tokens=342 state_at=341 pinned`), and `usage.flm_prefix_cache.pinned` says when a hit +came from the pinned slot. + +Measured 2026-11-10 on `bonsai2-t2s2-perrole876` with its `kernels-m16`, `--ctx-size 1024`, one `npu-hold` +ticket, one server per arm, four requests: the same 14-rule system prompt, then three different questions and +the second one again. 16 tokens generated, temperature 0. + +| request | cache off | default cache | `--prefix-pin` | +|---|---|---|---| +| 1 (question A) | cached 0, 58.6 s | cached 0 | cached 0, 59.5 s | +| 2 (question B) | cached 0, 59.4 s | cached 0 | cached 0, 62.1 s (makes the pin: 342 tokens) | +| 3 (question C) | cached 0, 61.5 s | cached 0 | **cached 341 of 355, pinned, 23.4 s** | +| 4 (question B again) | cached 0, 60.0 s | not reached | cached 354 of 355 (the whole-prompt slot), 20.4 s | + +The generated text of all four requests is identical between the cache-off arm and the `--prefix-pin` arm. +The default-cache arm's times are not in the table: the ticket ended during its third request, so only its +three cache decisions (from the server log) are known. A request's time here is the whole HTTP call, prefill +plus 16 generated tokens, not TTFT. ### Sampling and structured output diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 94e6efdf8..56851d2b4 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -500,6 +500,28 @@ void test_prefix_pin() { check(c.pin_point(prompt(100, 5000, 40), prompt(100, 6000, 40)) == 0, "100 shared tokens do not beat prefix-min-gain 128"); check(c.pin_point(prompt(900, 5000, 40), prompt(0, 6000, 40)) == 0, "prompts that share nothing have no pin point"); + { // only the leading tokens count: the same 900 tokens at the end of both prompts are not a prefix + std::vector a1 = prompt(0, 5000, 200), a2 = prompt(0, 6000, 200); + const std::vector tail = prompt(900, 0, 0); + a1.insert(a1.end(), tail.begin(), tail.end()); + a2.insert(a2.end(), tail.begin(), tail.end()); + check(shared_prefix_len(a1, a2) == 0 && c.pin_point(a1, a2) == 0, + "a shared ending under different openings is not a pin point"); + a2[0] = a1[0]; // one shared leading token, then they part: still nothing worth a slot + check(shared_prefix_len(a1, a2) == 1 && c.pin_point(a1, a2) == 0, + "and one shared leading token before that ending is not one either"); + } + // the contract the forward relies on: a pin point is a real common prefix, never longer than either + for (int sys : {0, 1, 127, 128, 129, 500, 900}) { + for (int turn : {1, 40}) { + const std::vector x = prompt(sys, 5000, turn), y = prompt(sys, 6000, turn + 7); + const size_t n = c.pin_point(x, y); + check(n <= x.size() && n <= y.size() && std::equal(x.begin(), x.begin() + long(n), y.begin()), + "a pin point never reaches past the tokens both prompts start with"); + check(n == (sys >= 128 ? size_t(sys) : 0u), + "it is the whole shared opening when that is worth a slot, and 0 (no pin) when it is not"); + } + } PrefixOptions off = o; off.slots = 0; check(PrefixCache(off).pin_point(q1, q2) == 0, "an off cache pins nothing"); From 223d8e2cdcc262c8698546439cac28119de28200 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:14:26 -0400 Subject: [PATCH 24/98] npu-pack: the container writer refuses a 4-bit code outside 0..15 too (follow-up to #1921) (#1978) #1921 put the refusal on encode_tile(). tile_q4_impl() (tile_q4_1 and tile_q4_1_f32) packs with the same `& 0x0F` and is the writer engine#1918's wrap was measured on. Its cause was fixed at the decoder by #1874, so nothing produces such a code today; this keeps a future caller from having one packed silently. The refusal names the row and column and is made before any byte is written. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/npu.md | 5 +++++ npu/pack_import.cpp | 12 +++++++++++ tests/npu_pack_import_test.cpp | 37 ++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/docs/npu.md b/docs/npu.md index a5a47c2a0..1ae6d6ae9 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1358,6 +1358,11 @@ differ because the two encodings are different, both exactly: | Q4_0 codes | unsigned `q` (0..15) | **signed** `q-8`, nibble = `q` xor 8 | | code order in a tile | `[row-half(2)][group(8)][col-in-group(32)][row-pair(8)]` (the 16-row split the kernel walks) | plain column major `[col(256)][row-pair(16)]` | +`npu-pack`'s writers hold the container to that unsigned field: `tile_q4_1`, `tile_q4_1_f32` and the +`.fconst` tile encoder all refuse a code above 15, naming where it is, rather than mask it into +a different code one full range away (engine#1921). A signed code has to be converted before it +reaches a writer, which is what the 1.9 decoder does. + Measured, per tensor, against the GGUF as ground truth: **196/196 Q4_0 tensors exact (0.0 error) under each file's own decoder** — so both converters are correct — and **0/197 exact when one is read with the other's decoder** (worst error 2.8-3.0). The scale planes of all 53 760 Q4_0 tiles diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 3c6c09bae..492a7d494 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -2079,6 +2079,18 @@ namespace { void tile_q4_impl(const Q4& q, std::vector& out, bool f32, long long* param_inexact) { if (q.N % q4nx::kRows) throw std::runtime_error("q4_1 tiles need N % 32 == 0"); + // The same refusal encode_tile() makes (engine#1921), on the writer that needed it first: the packing below masks + // with `& 0x0F`, so a code outside 0..15 would land as a different, plausible code one full range away. That is + // what engine#1918 measured on this writer (a two's-complement int4 code written into the unsigned field). Its + // cause was fixed at the decoder by #1874, so no caller produces such a code today; this keeps a future one from + // being packed silently. One pass over the codes, before anything is written. + for (size_t i = 0; i < q.codes.size(); ++i) + if (q.codes[i] > 0x0F) + throw std::runtime_error("q4_1 tile: code " + std::to_string(int(q.codes[i])) + " at row " + + std::to_string(q.K > 0 ? i / size_t(q.K) : 0) + ", column " + + std::to_string(q.K > 0 ? i % size_t(q.K) : 0) + + ": a 4-bit code is 0..15, and packing it would wrap a full code range " + "(engine#1921: convert a signed code before it reaches the writer)"); // The grid is 32 x 256, so a tensor whose K is not a whole number of 256-column // tiles (Qwen2.5-0.5B: hidden 896 = 3.5 tiles) is stored with the last column tile // zero-padded: codes 0, scale 0, zero 0, i.e. w = 0 in the padding. The real K is diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index ea1437dc9..9087bb95e 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5344,6 +5344,42 @@ void test_encode_tile_code_range() { std::printf(" encode tile: 6 arms (0 and 15 encode, 16 refuses by name)\n"); } +// engine#1921 follow-up: the container writer (tile_q4_1 / tile_q4_1_f32) packs with the same `& 0x0F` as +// encode_tile(), and it is the writer engine#1918's wrap was measured on. Both tile forms refuse a code above 15 by +// position, and the legal extremes still write: a guard that refused everything would pass a throw-only arm. +void test_q4_writer_code_range() { + auto make = [](uint8_t code_at_last) { + pi::Q4 q; + q.N = 32; q.K = 256; q.block = 32; + q.codes.assign(size_t(q.N) * q.K, 0); + q.scale.assign(size_t(q.N) * (q.K / 32), 1.0f); + q.zero.assign(size_t(q.N) * (q.K / 32), 0.0f); + q.codes[0] = 15; // the top of the range, legal + q.codes[size_t(31) * q.K + 255] = code_at_last; + return q; + }; + for (int f32 = 0; f32 < 2; ++f32) { + const char* form = f32 ? "q4_1f" : "q4_1"; + auto run = [&](const pi::Q4& q, std::vector& out) { + if (f32) pi::tile_q4_1_f32(q, out); else pi::tile_q4_1(q, out); + }; + std::vector out; + bool ok = false; + try { run(make(15), out); ok = !out.empty(); } catch (const std::exception&) {} + check(ok, std::string("q4 writer (") + form + "): codes 0 and 15 write a tile"); + std::string why; + std::vector out2; + try { run(make(16), out2); } catch (const std::exception& e) { why = e.what(); } + check(!why.empty(), std::string("q4 writer (") + form + "): code 16 is REFUSED, not wrapped to 0"); + check(why.find("row 31, column 255") != std::string::npos, + std::string("q4 writer (") + form + "): the refusal names the row and column"); + check(why.find("0..15") != std::string::npos && why.find("wrap") != std::string::npos, + std::string("q4 writer (") + form + "): and states the range and what would have happened"); + check(out2.empty(), std::string("q4 writer (") + form + "): nothing is written before the refusal"); + } + std::printf(" q4 writer: 10 arms (both tile forms: 15 writes, 16 refuses by row and column)\n"); +} + // engine#1889: the arithmetic that claims the per-head QK gains out of the 640 B Hunyuan-7B carries between its last qkv // tile and o_proj. A wrong offset there reads tile bytes as bf16 and mis-rotates every pack of the family, so both the // right shape (2 * (64 + head_dim*2) = 640 at head_dim 128) and the neighbours that must NOT be taken for it are driven. @@ -5407,6 +5443,7 @@ int main(int argc, char** argv) { } scratch_cleanup{run_scratch}; test_header_witness_refusal(); test_encode_tile_code_range(); + test_q4_writer_code_range(); test_qk_norm_refusal(); test_qk_gain_offsets(); test_tail_head_placement(); From 8e9e871e175b3c37344f7a6e3ad022029f073ade Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:37:12 -0400 Subject: [PATCH 25/98] docs(npu): the head-image cache is swept with the Q4NX cache, as one entry (engine#1716, #1956) (#2025) docs/npu.md still said nothing prunes the head images. Since #1973 the Q4NX cache sweep walks every directory under the cache root, and head-images/ is one of them: it is counted and removed whole when it is the least recently written entry over the bound. Say so, and say what its age is (a tile write, not a read). Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/npu.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 1ae6d6ae9..ff48d28d6 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -295,8 +295,11 @@ is replaced, with one stderr line saying why: The serve log line says what happened: `lm head: N packed weight tile(s) resident (…); H from the image cache, P packed (W written for the next serve) in S s`. `ONEBIT_HEAD_IMAGE_CACHE=0` turns the cache off (nothing is read or written). The files are `kd × nd` bytes plus `nd` scales per -tile, so a head costs about `kpad × vocabulary` bytes on disk per kernel set; nothing prunes them, -the same as the device-form copies next to them. Ternary (`pb16i`) heads and the gemv head do not +tile, so a head costs about `kpad × vocabulary` bytes on disk per kernel set. The Q4NX cache sweep +(`ONEBIT_Q4NX_CACHE_MAX_GIB`, above) counts `head-images/` as one entry of the cache and removes it +whole when the cache is over its bound and that directory is the least recently written one. Its +age is the last time a tile was written into it: a serve that only reads its tiles does not refresh +it. Losing the directory costs the next serve one pack of its head, nothing else. Ternary (`pb16i`) heads and the gemv head do not use it: their image already is a file. Each writer builds the file under its own name (`.tmp.`) and renames it into place, so two serves that start cold on the same pack both finish their write and a reader only ever sees a whole image. The behaviour is pinned host-side in From 233fcad47ab7a1f2ffead01eb78f30c1b8881862 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:46:00 -0400 Subject: [PATCH 26/98] npu-pack: a qwen35 GGUF is packed by an in-tree converter, and the GGUF converter refuses what it cannot name (Fixes #2013) (#2016) tools/qwen35_to_q4nx.py is the converter the dev box's 27B container names in its own metadata; it lived only in a lane worktree. It now refuses a rotated (prism.hadamard.*) or PTQ1_0 input before writing, and its verify compares the conv1d value-band direction on bf16-rounded sums. tools/gguf_to_q4nx.py lists every tensor kind it has no name for in one refusal (a suffix-less blk.N.ssm_a was a ValueError), names the architecture and the converter that does pack it, refuses an unknown ggml tensor type by number instead of an enum traceback, and leaves no empty output directory. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 5 + docs/npu.md | 30 ++ tests/gguf_to_q4nx_test.py | 78 +++++ tests/qwen35_to_q4nx_test.py | 213 +++++++++++++ tools/gguf_to_q4nx.py | 74 ++++- tools/qwen35_to_q4nx.py | 576 +++++++++++++++++++++++++++++++++++ 6 files changed, 971 insertions(+), 5 deletions(-) create mode 100755 tests/qwen35_to_q4nx_test.py create mode 100755 tools/qwen35_to_q4nx.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b6821e3e..fb9feacfd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -322,6 +322,11 @@ add_python_test(tool_selftest_whisper_encoder_baseline ${CMAKE_SOURCE_DIR}/tools # engine#1780: the DFlash draft package's gate before a loader -- tile-grid sizes from the config, tile-header sanity and # the container's scalars against config.json, on a synthetic package with eight negative controls. Stdlib only. add_python_test(tool_selftest_dflash_draft_info ${CMAKE_SOURCE_DIR}/tools/dflash_draft_info.py --selftest) +# engine#2013: the qwen35 (gated linear attention) GGUF -> Q4NX converter. Its own selftest (tile round-trips, the +# value-head permutation), and a pack + verify of a small qwen35 GGUF the test writes itself, with the three inputs it +# has to refuse. numpy only: no gguf package, no model, no device. +add_python_test(tool_selftest_qwen35_to_q4nx ${CMAKE_SOURCE_DIR}/tools/qwen35_to_q4nx.py selftest) +add_python_test(qwen35_to_q4nx ${CMAKE_SOURCE_DIR}/tests/qwen35_to_q4nx_test.py) # engine#1777: the release row's own arm, which merged unregistered -- this sweep is what found it (engine#1820). add_test(NAME release_check_hrx_init_reason COMMAND bash ${CMAKE_SOURCE_DIR}/tests/release_check_hrx_init_reason_test.sh ${CMAKE_SOURCE_DIR}/os/mini/release-check.sh) # engine#1615: every process holding the GPU/NPU device must belong to a live gpu-hold/npu-hold ticket. Three diff --git a/docs/npu.md b/docs/npu.md index ff48d28d6..2448c851c 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -822,6 +822,36 @@ weights over the bound. The Q6_K tensors (half of `v_proj` and `down_proj`, and 0.079. Served on the NPU, the repacked model decoded 24/24 greedy tokens the same as a CPU reference of the repacked weights, with every step's logits at cosine 0.99995 or better. +### What the GGUF converter refuses, and the qwen35 converter + +`tools/gguf_to_q4nx.py` converts qwen3, qwen3moe and qwen2. Two other inputs are refused before +anything is written, with no output directory left behind (engine#2013): + +- **A GGUF of another architecture.** Every tensor kind without a Q4NX name is listed in one message, + with how many tensors of each there are. For `qwen35` the message points at the converter below. +- **A tensor type the installed `gguf` package does not know.** The refusal gives the number; + 143 is PTQ1_0, PrismML's ternary type, which `tools/ternary_to_q4_0.py` rewrites as Q4_0. + +`tools/qwen35_to_q4nx.py pack [--tokenizer ]` packs a **qwen35** GGUF: +layers of gated linear attention (GDN) with a full-attention layer every +`qwen35.full_attention_interval`. It writes the names the forward looks up +(`linear_attn.qkv_proj`, `self_attn.gate_proj`, `linear_attn.ssm_*`), reorders the 48 value heads +from the stored order to the grouping the forward expects, and stores `ssm_alpha`/`ssm_beta` as Q8 +tiles padded to 64 rows. It reads the GGUF itself and needs only numpy. `verify +` checks every tensor of a container against the GGUF, including the direction of the +value-head reorder. + +Its input has to be a plain qwen35 GGUF in Q4_0 or Q4_1. It refuses, by name: + +- a file that still carries `prism.hadamard.*` keys: the weights are in a rotated basis, the tool + writes no sign table, and the container would load and answer wrongly; +- PTQ1_0 tensors (ggml type 143). + +So a PrismML ternary GGUF needs two steps before this one: PTQ1_0 to Q4_0 +(`tools/ternary_to_q4_0.py`, which keeps the rotation) and the un-rotation, which is not in this +tree. The 27B container on the dev box (`__metadata__.converter = qwen35_to_q4nx.py`) was made from +an un-rotated Q4_0 GGUF. + ### Routed expert stacks A MoE GGUF keeps its routed experts as one three-dimensional tensor per projection diff --git a/tests/gguf_to_q4nx_test.py b/tests/gguf_to_q4nx_test.py index 7b4cbc036..7eb4bbc6e 100755 --- a/tests/gguf_to_q4nx_test.py +++ b/tests/gguf_to_q4nx_test.py @@ -26,6 +26,7 @@ import hashlib import os import shutil +import struct import subprocess import sys import tempfile @@ -465,6 +466,81 @@ def rope_scaling_check(gguf_path, out, kind, ok): return ok and good and not left +def other_arch_check(d, rng, ok): + """A GGUF of an architecture this converter has no names for (engine#2013). + + The first unnamed tensor used to be the whole message (`no Q4NX name for blk.0.attn_gate.weight`), a + suffix-less one (blk.N.ssm_a) was a ValueError, and the empty output directory stayed behind. + """ + tool = os.path.join(HERE, "..", "tools", "gguf_to_q4nx.py") + src, out = os.path.join(d, "gdn.gguf"), os.path.join(d, "gdn-out") + w = gguf.GGUFWriter(src, "qwen35") + w.add_block_count(1) + f = lambda *s: rng.standard_normal(s).astype(np.float32) # noqa: E731 + w.add_tensor("token_embd.weight", f(512, 256)) + w.add_tensor("output_norm.weight", f(256)) + w.add_tensor("blk.0.attn_norm.weight", f(256)) + w.add_tensor("blk.0.attn_gate.weight", f(256, 256)) + w.add_tensor("blk.0.ssm_out.weight", f(256, 256)) + w.add_tensor("blk.0.ssm_a", f(48)) + w.write_header_to_file() + w.write_kv_data_to_file() + w.write_tensors_to_file() + w.close() + checks = [] + for cmd in ("pack", "verify"): + r = subprocess.run([sys.executable, tool, cmd, src, out], capture_output=True, text=True) + msg = r.stdout + r.stderr + checks += [ + (f"{cmd}: a qwen35 GGUF exits non-zero without a traceback", r.returncode != 0 and "Traceback" not in msg), + (f"{cmd}: ... keeps the first unnamed tensor's name", "no Q4NX name for blk.0.attn_gate.weight" in msg), + (f"{cmd}: ... lists every unnamed kind, the suffix-less one too", + all(k in msg for k in ("blk.N.attn_gate.weight x1", "blk.N.ssm_out.weight x1", "blk.N.ssm_a x1"))), + (f"{cmd}: ... names the architecture and the converter that does pack it", + "'qwen35'" in msg and "tools/qwen35_to_q4nx.py" in msg), + ] + if not all(g for _, g in checks): + print(msg[-800:]) + checks.append(("no output directory was created for it", not os.path.exists(out))) + for label, good in checks: + print(f"{'ok ' if good else 'FAIL'} {label}") + ok = ok and good + return ok + + +def foreign_type_check(good_gguf, d, ok): + """A tensor type the gguf package does not know is refused by number, not as an enum ValueError (engine#2013). + + The file is a valid GGUF with one tensor's type field rewritten to 143 (PTQ1_0), which is what a PrismML + ternary GGUF looks like to the stock reader. + """ + tool = os.path.join(HERE, "..", "tools", "gguf_to_q4nx.py") + src, out = os.path.join(d, "t143.gguf"), os.path.join(d, "t143-out") + with open(good_gguf, "rb") as f: + raw = bytearray(f.read()) + name = b"blk.0.attn_q.weight" + at = raw.index(name) + len(name) + 4 + 2 * 8 # name, n_dims (u32), two u64 dims, then the type + assert struct.unpack_from("] [--no-reorder] + qwen35_to_q4nx.py verify +""" +import argparse +import json +import mmap +import os +import shutil +import struct +import sys + +import numpy as np + +ROWS, COLS, GROUP = 32, 256, 32 +F32, F16, Q4_0, Q4_1, BF16, PTQ1 = 0, 1, 2, 3, 30, 143 +BLK = {F32: (4, 1), F16: (2, 1), Q4_0: (18, 32), Q4_1: (20, 32), BF16: (2, 1), PTQ1: (28, 128)} +NKH, NVH = 16, 48 # k-groups, v-heads (qwen35 27B: group_count 16, inner 6144 / 128) + + +# --- GGUF reader -------------------------------------------------------------------------- +class Gguf: + """Tensor table, an mmap, and per-type readers for the four types this model uses.""" + + _SZ = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 8: 0, 9: 0, 10: 8, 11: 8, 12: 8} + _FMT = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "B", 10: "Q", 11: "q", 12: "d"} + + def __init__(self, path): + self.path = path + self.f = open(path, "rb") + magic, _ = struct.unpack(" codes [N, K] u8, scale [N, K/32] f32, zero [N, K/32] f32, exact for Q4_0/Q4_1.""" + a, dims, tt = self.raw(name) + if tt not in (Q4_0, Q4_1): + raise SystemExit(f"{name}: ggml type {tt}, expected Q4_0 (2) or Q4_1 (3)" + + ("; 143 is PTQ1_0: rewrite it as Q4_0 (tools/ternary_to_q4_0.py) and un-rotate it " + "first" if tt == PTQ1 else "")) + K, N = int(dims[0]), int(dims[1]) + b = a.reshape(N, K // 32, 18 if tt == Q4_0 else 20) + f16 = lambda x: np.frombuffer(x.tobytes(), "> 4], axis=-1).reshape(N, K)), d, m + + +# --- reorders --------------------------------------------------------------------------------- +def head_perm(): + """HF v-head j <- stored head (j%3)*16 + j//3.""" + j = np.arange(NVH) + return (j % 3) * NKH + j // 3 + + +def perm_rows(n, hd, off): + """Row gather for n head-blocks of hd rows starting at row `off` (identity elsewhere).""" + p = head_perm()[:n] + idx = np.arange(off) if off else np.arange(0) + return np.concatenate([idx, np.concatenate([off + b * hd + np.arange(hd) for b in p])]) + + +def perm_cols(n, hd, off): + """Column gather for n head-blocks of hd columns starting at column `off`.""" + return np.concatenate([off + b * hd + np.arange(hd) for b in head_perm()[:n]]) + + +def deinterleave_qgate(n, hd): + """Stored [head][q|gate][hd] -> [all q][all gate].""" + nh = n // (2 * hd) + out = np.empty(n, np.int64) + for h in range(nh): + for p in range(2): + dst = p * nh * hd + h * hd + out[dst:dst + hd] = h * 2 * hd + p * hd + np.arange(hd) + return out + + +def row_gather(codes, scale, zero, idx): + if idx is None: + return codes, scale, zero + return codes[idx], scale[idx], zero[idx] + + +def col_gather(codes, scale, zero, hd, off): + """Reorder the 48 head-blocks of `hd` columns starting at column `off` (rows kept).""" + ci = np.concatenate([off + b * hd + np.arange(hd) for b in head_perm()]) + gi = np.concatenate([(off + b * hd) // GROUP + np.arange(hd // GROUP) for b in head_perm()]) + return codes[:, ci], scale[:, gi], zero[:, gi] + + +# --- tiles ------------------------------------------------------------------------------------ +def bf16_bits(x): + u = np.asarray(x, dtype=np.float32).view(np.uint32).astype(np.uint64) + return ((u + 0x7FFF + ((u >> 16) & 1)) >> 16).astype(np.uint16) + + +def from_bf16(b): + return (np.asarray(b, np.uint16).astype(np.uint32) << 16).view(np.float32) + + +def tile_q4_1(codes, scale, zero): + N, K = codes.shape + RT, KT = N // ROWS, K // COLS + + def plane(v): + return bf16_bits(v).reshape(RT, ROWS, KT, 8).transpose(0, 2, 3, 1).reshape(RT * KT, 256) + + c = codes.astype(np.uint8).reshape(RT, 2, 8, 2, KT, 8, 32) + packed = (c[:, :, :, 0] | (c[:, :, :, 1] << 4)).transpose(0, 3, 1, 4, 5, 2).reshape(RT * KT, 4096) + return np.concatenate([plane(scale).view(np.uint8), plane(zero).view(np.uint8), packed], axis=1) + + +def untile_q4_1(raw, N, K): + """[tiles, 5120] -> f32 [N, K] (npu/q4nx.h: w = code*scale + zero).""" + RT, KT = N // ROWS, K // COLS + t = np.asarray(raw).reshape(RT, KT, 5120) + sc = from_bf16(t[..., :512].copy().view("> 4], axis=-1).transpose(0, 2, 5, 6, 1, 3, 4).reshape(RT, 32, KT, 8, 32) + return (c * sc.transpose(0, 3, 1, 2)[..., None] + zr.transpose(0, 3, 1, 2)[..., None]).reshape(N, K) + + +def tile_q8(w): + """f32 [N, K] (multiples of 32) -> [N/32, K/256, 8704]; d[g*32+r] @0, int8 codes @512 at k*32+r.""" + N, K = w.shape + RT, KT = N // ROWS, K // COLS + out = np.empty((RT, KT, 8704), np.uint8) + for rt in range(RT): + for kt in range(KT): + g = w[rt * ROWS:(rt + 1) * ROWS, kt * COLS:(kt + 1) * COLS].reshape(ROWS, 8, 32) + d = from_bf16(bf16_bits(np.abs(g).max(-1) / 127.0)) + d = np.where(d > 0, d, 1.0) + q = np.clip(np.rint(g / d[..., None]), -127, 127).astype(np.int8) + out[rt, kt, 0:512] = np.ascontiguousarray(bf16_bits(d).T).view(np.uint8).reshape(-1) + out[rt, kt, 512:] = np.ascontiguousarray(q.reshape(ROWS, COLS).T).view(np.uint8).reshape(-1) + return out + + +# --- names ------------------------------------------------------------------------------------ +GDN = {"attn_qkv": "linear_attn.qkv_proj", "attn_gate": "self_attn.gate_proj", + "ssm_out": "linear_attn.ssm_out_proj", + "ssm_alpha": "linear_attn.ssm_alpha_proj", "ssm_beta": "linear_attn.ssm_beta_proj", + "ssm_conv1d": "linear_attn.ssm_conv1d", "ssm_norm": "linear_attn.ssm_norm", + "ssm_a": "linear_attn.ssm_a", "ssm_dt": "linear_attn.ssm_dt"} +FULL = {"attn_q": "self_attn.q_proj", "attn_k": "self_attn.k_proj", "attn_v": "self_attn.v_proj", + "attn_output": "self_attn.o_proj", "attn_q_norm": "self_attn.q_norm", + "attn_k_norm": "self_attn.k_norm"} +COMMON = {"attn_norm": "input_layernorm", "post_attention_norm": "post_attention_layernorm", + "ffn_gate": "mlp.gate_proj", "ffn_up": "mlp.up_proj", "ffn_down": "mlp.down_proj"} +SUFFIX = {"ssm_a": "", "ssm_dt": ".bias"} +KIND = {"ssm_a": "f32", "ssm_dt": "f32", "ssm_conv1d": "conv1d", "ssm_alpha": "q8", "ssm_beta": "q8", + "ssm_norm": "bf16", "attn_norm": "bf16", "post_attention_norm": "bf16", + "attn_q_norm": "bf16", "attn_k_norm": "bf16"} + + +def plan(nl, fai): + out = [] + for l in range(nl): + full = (l % fai) == (fai - 1) + fam = dict(COMMON) + fam.update(FULL if full else GDN) + for g, hf in fam.items(): + src = f"blk.{l}.{g}{SUFFIX.get(g, '.weight')}" + # ssm_a / ssm_dt.bias are suffix-less / .bias in the container too (model.c looks + # them up by those exact names), so they carry no generic ".weight". + dst = f"model.layers.{l}.{hf}" + ("" if g == "ssm_a" else ".bias" if g == "ssm_dt" else ".weight") + out.append((src, dst, KIND.get(g, "tiles"), full, g)) + out.append(("token_embd.weight", "model.embed_tokens.weight", "tiles", True, "emb")) + out.append(("output_norm.weight", "model.norm.weight", "bf16", True, "norm")) + if "output.weight" in GB.tensors: + out.append(("output.weight", "lm_head.weight", "tiles", True, "head")) + return [t for t in out if t[0] in GB.tensors] + + +def reorder_idx(g, full, N, hd_g, hd_a): + """The row gather for this tensor, or None. hd_g = the GDN head dim, hd_a = attention's.""" + if not REORDER: + return None + if g == "attn_qkv" and not full: # [q(16) | k(16) | v(48)] + return perm_rows(NVH, hd_g, off=2 * NKH * hd_g) + if g == "attn_q" and full: # [head][q|gate][hd_a] -> [q|gate] + return deinterleave_qgate(N, hd_a) + if g == "attn_gate": # z, the GDN swish gate (48 heads) + return perm_rows(NVH, hd_g, off=0) + return None + + +def config(nl, fai): + a = lambda k, d=None: GB.kv.get(f"qwen35.{k}", d) # noqa: E731 + s = lambda k, d=0: int(GB.kv.get(f"qwen35.ssm.{k}", d)) # noqa: E731 + nvh = s("inner_size") // s("state_size") + c = { + "architectures": ["Qwen3_5ForConditionalGeneration"], + "model_type": "qwen3_5", + "hidden_size": a("embedding_length"), + "intermediate_size": a("feed_forward_length"), + "num_hidden_layers": nl, + "num_attention_heads": a("attention.head_count"), + "num_key_value_heads": a("attention.head_count_kv"), + "head_dim": a("attention.key_length"), + "max_position_embeddings": a("context_length"), + "rms_norm_eps": float(a("attention.layer_norm_rms_epsilon")), + "rope_theta": float(a("rope.freq_base")), + "vocab_size": int(np.prod(GB.tensors["token_embd.weight"][0][1:])), + "bos_token_id": GB.kv.get("tokenizer.ggml.bos_token_id"), + "eos_token_id": GB.kv.get("tokenizer.ggml.eos_token_id"), + "tie_word_embeddings": "output.weight" not in GB.tensors, + "torch_dtype": "bfloat16", + "hidden_act": "silu", + "linear_num_key_heads": s("group_count"), + "linear_key_head_dim": s("state_size"), + "linear_value_head_dim": s("state_size"), + "linear_num_value_heads": nvh, + "linear_conv_kernel_dim": s("conv_kernel", 4), + "attn_output_gate": 1, + "layer_types": ["full_attention" if (i % fai) == (fai - 1) else "linear_attention" + for i in range(nl)], + "partial_rotary_factor": 0.25, + "q4nx_weight_layout": "rm", + "q4nx_source": {"converter": "qwen35_to_q4nx.py", "gguf": os.path.basename(GB.path), + "v_head_reorder": "hf" if REORDER else "stored"}, + } + return {k: v for k, v in c.items() if v is not None} + + +def refuse_unsupported(gb): + """The inputs this tool would turn into a wrong container, refused by name (engine#2013).""" + arch = gb.kv.get("general.architecture") + if arch != "qwen35": + raise SystemExit(f"{gb.path}: general.architecture is {arch!r}; this tool packs qwen35 only " + "(qwen3, qwen3moe and qwen2 are tools/gguf_to_q4nx.py's)") + rotated = sorted(k for k in gb.kv if k.startswith("prism.hadamard")) + if rotated: + raise SystemExit(f"{gb.path}: carries {rotated[0]} ({len(rotated)} prism.hadamard key(s)), so its weights " + "are stored in a Hadamard-rotated basis; this tool writes them as they are and no sign " + "table, so the container would answer wrongly. Un-rotate the GGUF first") + tern = [name for name, (_dims, tt, _off) in gb.tensors.items() if tt == PTQ1] + if tern: + raise SystemExit(f"{gb.path}: {len(tern)} tensor(s) are ggml type 143, PTQ1_0 (first: {tern[0]}); this tool " + "packs Q4_0/Q4_1 codes. Rewrite the file as Q4_0 (tools/ternary_to_q4_0.py) and un-rotate " + "it first") + odd = {} + for name, (_dims, tt, _off) in gb.tensors.items(): + if tt not in BLK: + odd.setdefault(tt, name) + if odd: + raise SystemExit(f"{gb.path}: " + "; ".join(f"ggml tensor type {tt} (first: {n})" for tt, n in sorted(odd.items())) + + " is not one this tool reads (F32, F16, Q4_0, Q4_1, BF16, PTQ1_0)") + + +def pack(args): + global GB, REORDER + GB, REORDER = Gguf(args.gguf), args.reorder + refuse_unsupported(GB) + nl = int(GB.kv["qwen35.block_count"]) + fai = int(GB.kv.get("qwen35.full_attention_interval", 4)) + hd_g = int(GB.kv["qwen35.ssm.state_size"]) + hd_a = int(GB.kv["qwen35.attention.key_length"]) + os.makedirs(args.out, exist_ok=True) + table, blobs, off, done = {}, [], 0, 0 + + def emit(name, data, dtype, shape): + nonlocal off + b = np.ascontiguousarray(data).tobytes() + pad = (-off) % 64 + if pad: + blobs.append(b"\0" * pad) + off += pad + table[name] = {"dtype": dtype, "shape": list(shape), "data_offsets": [off, off + len(b)]} + blobs.append(b) + off += len(b) + + for src, dst, kind, full, g in plan(nl, fai): + dims = GB.tensors[src][0] + N = int(dims[1]) if len(dims) > 1 else 1 + K = int(dims[0]) + if kind == "tiles": + codes, scale, zero = GB.q4(src) + idx = reorder_idx(g, full, N, hd_g, hd_a) + if idx is not None: + codes, scale, zero = row_gather(codes, scale, zero, idx) + elif g == "ssm_out" and REORDER: + codes, scale, zero = col_gather(codes, scale, zero, hd_g, 0) + d = tile_q4_1(codes, scale, zero) + emit(dst, d, "I8", d.shape) + elif kind == "q8": + w = GB.f32(src).reshape(N, K) + if REORDER: + w = w[head_perm()] + pad = np.zeros(((N + ROWS - 1) // ROWS * ROWS, K), np.float32) # the forward's deq_pad + pad[:N] = w + d = tile_q8(pad) + emit(dst, d, "I8", d.shape) + elif kind == "conv1d": + ck, cd = int(dims[0]), int(dims[1]) + m = GB.f32(src).reshape(cd, ck).T.copy() # [cd][ck] -> [ck][cd] + if REORDER: + m = m[:, np.concatenate([np.arange(2 * NKH * hd_g), + perm_cols(NVH, hd_g, 2 * NKH * hd_g)])] + emit(dst, bf16_bits(m.reshape(-1)), "BF16", [m.size]) + elif kind == "f32": + v = GB.f32(src) + emit(dst, (v[head_perm()] if REORDER else v).astype(" 1 else 1 + K = int(dims[0]) + if kind == "tiles": + # reference: the GGUF's own dequant, reordered the same way, with bf16-rounded scale/zero + codes, scale, zero = GB.q4(src) + idx = reorder_idx(g, full, N, hd_g, hd_a) + if idx is not None: + codes, scale, zero = row_gather(codes, scale, zero, idx) + elif g == "ssm_out": + codes, scale, zero = col_gather(codes, scale, zero, hd_g, 0) + ref = (codes.reshape(N, K // GROUP, GROUP) * from_bf16(bf16_bits(scale))[..., None] + + from_bf16(bf16_bits(zero))[..., None]).reshape(N, K) + err = np.abs(untile_q4_1(raw, N, K) - ref) + ok, note = err.max() == 0, f"exact, {hdr[dst]['shape']}" + elif kind == "f32": + got = np.frombuffer(raw.tobytes(), " [ck][cd] + exp = m[:, np.concatenate([np.arange(2 * NKH * hd_g), + perm_cols(NVH, hd_g, 2 * NKH * hd_g)])] if REORDER else m + exact = np.array_equal(got, from_bf16(bf16_bits(exp))) + # the v-band permutation's DIRECTION, checked independently of the packer's gather: + # Both sides are summed from bf16-rounded values, in f64: a head's sum can cancel to near zero, and + # a relative tolerance between a rounded and an unrounded sum then fails a correct container. + gb = got[:, 2 * NKH * hd_g:].astype(np.float64).reshape(ck, NVH, hd_g).sum(-1) + rb = from_bf16(bf16_bits(m))[:, 2 * NKH * hd_g:].astype(np.float64).reshape(ck, NVH, hd_g).sum(-1) + hf = np.array_equal(gb, rb[:, head_perm()]) + ok = exact and hf + where = "hf" if hf else "stored" if np.array_equal(gb, rb) else "neither" + note = f"bf16 {'exact' if exact else 'MISMATCH'}, v-band direction {where}" + elif kind == "bf16": + got, ref = from_bf16(raw.view(" p, h, d + for part in (0, 1): + src = h * 512 + part * 256 + 3 + assert q[part * 6144 + h * 256 + 3] == src, "deinterleave mapping" + w = rng.standard_normal((64, 512)).astype(np.float32) + back = np.zeros_like(w) + t = tile_q8(w) + for r in range(2): + for k in range(2): + d = from_bf16(t[r, k, :512].copy().view(" Date: Tue, 10 Nov 2026 18:08:07 -0400 Subject: [PATCH 27/98] serve: --flash, the single-turn NPU profile; --ctx-size reaches the forward route; an over-context request is a 400 (Refs #1785) (#2029) * serve: --flash, the single-turn NPU profile; --ctx-size reaches the forward route; an over-context request is a 400 (Refs #1785, F5) --flash = --ctx-size 1024 + --prefix-pin, refusing what contradicts it. serve dropped --ctx-size at the hand-off to forward-serve on a model-generic container; it is carried now and forward-serve lowers its limit to it. A request that does not fit was a 500; it is a 400 with type exceed_context_size_error. Measured on the NPU. Co-Authored-By: Claude Fable 5.2 * tests: serve --flash refuses what contradicts the profile, with no model or device (Refs #1785) Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 3 ++ app/forward_serve.cpp | 31 +++++++++++++++- app/serve.cpp | 16 +++++++++ docs/npu.md | 26 +++++++++++++- tests/serve_flash_test.sh | 76 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+), 2 deletions(-) create mode 100755 tests/serve_flash_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index fb9feacfd..86cb25cf9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -278,6 +278,9 @@ add_test(NAME serve_port COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_port_test. # device. Deliberately OUTSIDE if(ONEBIT_SERVE_TEST_GGUF): that row is device-gated, and the whole point is that the # classification is checked in every configure, including the ones that never load a model. add_test(NAME serve_context_limit_classify COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_context_limit_test.sh --self-test) +# engine#1785: `serve --flash` names two settings (--ctx-size 1024, --prefix-pin on --device npu) and refuses what +# contradicts them. Every case ends in the option check, so the row needs no model, no device and no port. +add_test(NAME serve_flash_refusals COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_flash_test.sh $) # engine#1594: every target ci.yml's `cmake --build --target …` list names must exist in the configuration ci.yml # creates. #1583 added one name for a target defined inside if(ONEBIT_NPU), which CI does not configure -- and ninja # rejects the whole command line before building anything, so `onebit` stopped being compiled for every PR. Reads the diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index effed11be..d193ee114 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -74,6 +74,16 @@ namespace onebit { namespace { +// A prompt that leaves no room for an answer inside the context (the model's, or --ctx-size). +struct ContextOverflow : std::runtime_error { + size_t prompt_tokens; + int context; + ContextOverflow(size_t n, int ctx) + : std::runtime_error("request (" + std::to_string(n) + " tokens) exceeds the available context size (" + + std::to_string(ctx) + " tokens)"), + prompt_tokens(n), context(ctx) {} +}; + using json = nlohmann::json; struct ForwardEngine { @@ -297,7 +307,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res // The KV cache grows with every position: the prompt and the answer together stay // inside the model's context. const int room = e.max_context - int(ids.size()); - if (room < 1) throw std::runtime_error("prompt longer than the context"); + if (room < 1) throw ContextOverflow(ids.size(), e.max_context); const int limit = std::clamp(max_tokens, 0, room); std::lock_guard lock(e.mu); @@ -729,6 +739,13 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res return true; }); } + } catch (const ContextOverflow& ex) { + // The client's request, not a server fault: 400, with the type and the wording the GGUF route's + // backend uses for the same refusal, so one client-side check covers both routes. + res.status = 400; + res.set_content(json{{"error", {{"message", ex.what()}, {"type", "exceed_context_size_error"}, + {"n_prompt_tokens", ex.prompt_tokens}, {"n_ctx", ex.context}}}}.dump(), + "application/json"); } catch (const std::exception& ex) { res.status = 500; res.set_content(json{{"error", {{"message", ex.what()}}}}.dump(), "application/json"); @@ -906,6 +923,7 @@ int run_forward_serve(int argc, char** argv) { std::string npu_target = ::npu::target::xdna::XdnaTarget::kName; npu::PrefixOptions prefix_opts; // --prefix-slots / --prefix-cache-mb / --prefix-min-gain (#650) bool prefix_pin = false; // --prefix-pin (engine#1785) + int ctx_size = 0; // --ctx-size N: prompt plus answer, at most the model's own (0 = the model's) for (int i = 0; i < argc; i++) { const std::string a = argv[i]; auto next = [&]() -> std::string { @@ -921,6 +939,7 @@ int run_forward_serve(int argc, char** argv) { else if (a == "--prefix-cache-mb") prefix_opts.cache_mb = std::stoul(next()); else if (a == "--prefix-min-gain") prefix_opts.min_gain = std::stoi(next()); else if (a == "--prefix-pin") prefix_pin = true; + else if (a == "-c" || a == "--ctx-size") ctx_size = std::stoi(next()); else if (a == "--npu-streams") streams = std::stoi(next()); else if (a == "--npu-target") npu_target = next(); else if (a == "--dump-logits") dump_logits = next(); @@ -963,6 +982,16 @@ int run_forward_serve(int argc, char** argv) { // q4nx_config_int finds the key at any depth (a text_config's too). e.max_context = q4nx_config_int(cfg.c_str(), "max_position_embeddings", 0); if (e.max_context <= 0) e.max_context = 32768; + // --ctx-size lowers the limit and never raises it: the model's positions are what they are. The same + // guard that refuses a prompt longer than the model's context then refuses one longer than this. + if (ctx_size < 0) throw std::runtime_error("--ctx-size: must be zero or more"); + if (ctx_size > 0) { + if (ctx_size > e.max_context) + std::fprintf(stderr, "1bit forward-serve: --ctx-size %d is above the model's context %d; using %d\n", ctx_size, + e.max_context, e.max_context); + e.max_context = std::min(e.max_context, ctx_size); + } + std::fprintf(stderr, "1bit forward-serve: context %d tokens (prompt plus answer)\n", e.max_context); // #963: the chat route's prompt was ChatML for every container, and a tokenizer with no // <|im_start|>/<|im_end|> added token cannot represent it -- the marker becomes ordinary pieces and diff --git a/app/serve.cpp b/app/serve.cpp index 164d6623f..69f3ec145 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -186,6 +186,7 @@ struct Options { int prefix_cache_mb = -1; // --prefix-cache-mb N int prefix_min_gain = -1; // --prefix-min-gain N: tokens a prompt must add to be worth keeping bool prefix_pin = false; // --prefix-pin: keep the part single-turn requests share (docs/npu.md) + bool flash = false; // --flash: the single-turn NPU profile = --ctx-size 1024 + --prefix-pin (engine#1785) std::string npu_target; // --npu-target NAME: which NPU target serves the per-op route (empty = the build's default) std::string prefill_provider; // --prefill-provider PATH (or ONEBIT_HYBRID_PREFILL): --device hybrid's prefill (docs/hybrid.md) std::string laya_model; // --laya-model DIR: route each request by the Laya scorer (docs/laya.md) @@ -3173,6 +3174,7 @@ void usage(FILE* out) { " [--prefix-slots N] [--prefix-cache-mb N] [--prefix-min-gain N] the prompt prefix cache\n" " (docs/npu.md; 0 slots turns it off). Unset keeps the forward's defaults: 8 / 4096 / 128\n" " [--prefix-pin] keep the part a request shares with the previous one (one system prompt, many questions)\n" + " [--flash] with --device npu: the single-turn profile, --ctx-size 1024 plus --prefix-pin\n" " (default 1, the device exclusive; docs/npu.md)\n" " [--load-queue-ms N] a load refused because another client holds the NPU array\n" " waits at most N ms for it to free before the driver is called dead\n" @@ -3289,6 +3291,7 @@ int run_serve(int argc, char** argv) { else if (a == "--prefix-cache-mb") o.prefix_cache_mb = std::stoi(next()); else if (a == "--prefix-min-gain") o.prefix_min_gain = std::stoi(next()); else if (a == "--prefix-pin") o.prefix_pin = true; + else if (a == "--flash") o.flash = true; else if (a == "--npu-target") o.npu_target = next(); else if (a == "--prefill-provider") o.prefill_provider = next(); else if (a == "--laya-model") o.laya_model = next(); @@ -3325,6 +3328,16 @@ int run_serve(int argc, char** argv) { else if (a == "-h" || a == "--help") { usage(stdout); return 0; } else throw std::runtime_error("unknown option " + a); } + // --flash is a name for two settings, so that the profile is one word and its parts stay visible: a 1024-token + // context and the pinned shared prefix. It refuses what contradicts it instead of quietly winning or losing. + if (o.flash) { + if (o.ctx_size != 0 && o.ctx_size != 1024) + throw std::runtime_error("--flash fixes the context at 1024 tokens; drop --ctx-size " + std::to_string(o.ctx_size)); + if (o.prefix_slots == 0) throw std::runtime_error("--flash needs the prefix cache; drop --prefix-slots 0"); + if (o.device != "npu") throw std::runtime_error("--flash is a profile of --device npu"); + o.ctx_size = 1024; + o.prefix_pin = true; + } if (!o.laya_models.empty()) { #if defined(ONEBIT_LAYA) && !defined(_WIN32) if (o.laya_model.empty() && !o.laya_auto) throw std::runtime_error("--laya-models needs --laya or --laya-model DIR"); @@ -3456,6 +3469,9 @@ int run_serve(int argc, char** argv) { if (o.prefix_cache_mb >= 0) { fargs.push_back("--prefix-cache-mb"); fargs.push_back(std::to_string(o.prefix_cache_mb)); } if (o.prefix_min_gain >= 0) { fargs.push_back("--prefix-min-gain"); fargs.push_back(std::to_string(o.prefix_min_gain)); } if (o.prefix_pin) fargs.push_back("--prefix-pin"); + // The context limit reaches this route too: it used to stop here, so `serve --ctx-size N` on a + // model-generic container was accepted and the forward kept the model's own limit. + if (o.ctx_size > 0) { fargs.push_back("--ctx-size"); fargs.push_back(std::to_string(o.ctx_size)); } std::vector fav; for (auto& a : fargs) fav.push_back(a.data()); return run_forward_serve(int(fav.size()), fav.data()); diff --git a/docs/npu.md b/docs/npu.md index 2448c851c..6f4731d1d 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1655,7 +1655,7 @@ The cost is one more launch on the request that makes the pin. Every decision is (`[prefix] pin slot=1 tokens=342 state_at=341 pinned`), and `usage.flm_prefix_cache.pinned` says when a hit came from the pinned slot. -Measured 2026-11-10 on `bonsai2-t2s2-perrole876` with its `kernels-m16`, `--ctx-size 1024`, one `npu-hold` +Measured 2026-11-10 on `bonsai2-t2s2-perrole876` with its `kernels-m16`, one `npu-hold` ticket, one server per arm, four requests: the same 14-rule system prompt, then three different questions and the second one again. 16 tokens generated, temperature 0. @@ -1671,6 +1671,30 @@ The default-cache arm's times are not in the table: the ticket ended during its three cache decisions (from the server log) are known. A request's time here is the whole HTTP call, prefill plus 16 generated tokens, not TTFT. +**`--flash`** (`1bit serve --device npu --flash`) is the single-turn profile as one word: `--ctx-size 1024` +plus `--prefix-pin`. It refuses what contradicts it (`--ctx-size` other than 1024, `--prefix-slots 0`, a device +other than `npu`) instead of quietly winning or losing. + +`--ctx-size N` now reaches this route. Before engine#1785's flash slice, `serve --ctx-size N` on a +model-generic container was accepted and dropped at the hand-off to `forward-serve`, which kept the model's +own `max_position_embeddings` (so the run in the table above was passed `--ctx-size 1024` and it had no effect; +its prompts were about 355 tokens, so nothing in the table depends on it). The limit is prompt plus answer, +it can lower the model's context and never raise it, and the load line says what is in force: +`1bit forward-serve: context 1024 tokens (prompt plus answer)`. + +A request that does not fit is the client's error, so it is a 400 with the type the GGUF route's backend uses +for the same refusal (it was a 500 with a bare message): + +``` +HTTP 400 {"error": {"message": "request (1597 tokens) exceeds the available context size (1024 tokens)", + "type": "exceed_context_size_error", "n_prompt_tokens": 1597, "n_ctx": 1024}} +``` + +Measured 2026-11-10 with `--flash`, same pack and kernels, one `npu-hold` ticket, `max_tokens 8`: the +1597-token request above refused in 0.02 s; then three questions under one system prompt: request 1 +`cached 0` (56.2 s), request 2 `cached 0` (48.9 s, makes the pin at 342 tokens), request 3 +`cached 341 of 355, pinned` (13.9 s). Whole HTTP calls, one run. + ### Sampling and structured output `1bit serve -m --device npu` serves through `1bit forward-serve` diff --git a/tests/serve_flash_test.sh b/tests/serve_flash_test.sh new file mode 100755 index 000000000..a3fae7b65 --- /dev/null +++ b/tests/serve_flash_test.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# `1bit serve --flash` (engine #1785) is a name for two settings: --ctx-size 1024 and --prefix-pin on --device npu. +# What contradicts the profile must be refused by name, before any model is opened or any device is touched, instead +# of one side quietly winning. +# +# No model, no device, no port: every case here ends in the option check. The model path does not exist on purpose; +# a case that got past the check would fail on it with a different message, and the pattern would not match. +# +# serve_flash_test.sh <1bit binary> +set -uo pipefail + +bin=${1:?usage: serve_flash_test.sh <1bit binary>} +[ -x "$bin" ] || { echo "serve_flash_test: $bin is not executable"; exit 1; } + +fail=0 +missing=/nonexistent/serve-flash-test-model +# refused NAME PATTERN ARGS...: the command must exit non-zero within the limit and name the reason. +refused() { + local name=$1 pattern=$2 out rc + shift 2 + out=$(timeout 20 "$bin" serve -m "$missing" "$@" 2>&1) + rc=$? + if [ "$rc" -ne 0 ] && [ "$rc" -ne 124 ] && printf '%s' "$out" | grep -qF -- "$pattern"; then + echo "ok $name" + else + echo "FAIL $name (rc=$rc): $out" + fail=$(( fail + 1 )) + fi +} +# not_refused_by_flash NAME ARGS...: the command may fail (the model is missing), but not on a --flash rule and +# not in the option parser. +not_refused_by_flash() { + local name=$1 out rc + shift + out=$(timeout 20 "$bin" serve -m "$missing" "$@" 2>&1) + rc=$? + if [ "$rc" -ne 124 ] && ! printf '%s' "$out" | grep -qF -e "--flash" -e "unknown option"; then + echo "ok $name" + else + echo "FAIL $name (rc=$rc): $out" + fail=$(( fail + 1 )) + fi +} + +refused "another context size is refused" "--flash fixes the context at 1024 tokens; drop --ctx-size 2048" \ + --device npu --flash --ctx-size 2048 +refused "and so is one given before --flash" "--flash fixes the context at 1024 tokens; drop --ctx-size 512" \ + --device npu --ctx-size 512 --flash +refused "a cache turned off is refused" "--flash needs the prefix cache; drop --prefix-slots 0" \ + --device npu --flash --prefix-slots 0 +refused "another device is refused" "--flash is a profile of --device npu" \ + --device cpu --flash +refused "no device at all is refused too" "--flash is a profile of --device npu" \ + --flash + +not_refused_by_flash "its own context size, spelled out, is accepted" --device npu --flash --ctx-size 1024 +not_refused_by_flash "--prefix-pin next to it is accepted" --device npu --flash --prefix-pin +not_refused_by_flash "without --flash none of the rules apply" --device cpu --ctx-size 2048 --prefix-slots 0 + +[ "$fail" -eq 0 ] && echo "serve_flash: ok" || echo "serve_flash: $fail failed" +exit "$fail" From 468d029f5271914c449c39cd807a4e0588567a7a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:08:14 -0400 Subject: [PATCH 28/98] npu-pack test: the q4nx-container arm removes its fixture directory (#2024) test_q4nx_container() builds its fixtures under the directory it is given, which is tests/golden when CTest runs it, and never removed them. Every run left tests/golden/q4nx-container/ (container-genre, huge.q4nx, meta.q4nx, short.q4nx, two.q4nx) in the source tree as untracked files. The directory is now cleared on entry and removed on every way out. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- tests/npu_pack_import_test.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 9087bb95e..a20b235a2 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -4772,7 +4772,14 @@ void test_q4nx_container(const std::string& root) { namespace fs = std::filesystem; const fs::path dir = fs::path(root) / "q4nx-container"; std::error_code ec; + fs::remove_all(dir, ec); // a directory left by a run that died is not this run's fixture fs::create_directories(dir, ec); + // `root` is the golden directory when CTest runs this, i.e. the source tree: without the removal every run left + // tests/golden/q4nx-container/ behind as untracked files. Removed on every way out of the function. + struct Cleanup { + fs::path p; + ~Cleanup() { std::error_code e; fs::remove_all(p, e); } + } cleanup{dir}; const auto write_container = [&](const std::string& name, const std::string& header_json, int64_t payload, bool truncate_header) { From 8f1eddd74ba6b4dee5e8722628828222787d72dd Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:30:35 -0400 Subject: [PATCH 29/98] npu: NPU_INFER_DUMP_OPS says so once at model load, so a stale binary is visible (engine#2028) (#2041) engine#2028 reported the per-op dump silent on the design path. It was a binary linked before #1977 (cl-npupack-b measured 869/1061 [op] lines on one that has it, and 0 on the filer's, whose strings lack the variable's name), but four runs went to the wrong mechanism first, because "the dump prints nothing" and "this binary has no dump" look identical. ops_dump_announce() prints [ops-dump] NPU_INFER_DUMP_OPS is on: an [op] line follows every projection and weighted RMSNorm, on stderr once per process when the dump is on, from Q4nxNpuForward::init_impl (serve and npu-run, host and device). No line with the variable set means the binary predates the dump. Pinned in ops_dump_test: on prints it once (second call silent), unset and 0 print nothing. On a two-layer container the line comes before the first of 316 [op] lines; with the variable off there is none. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Opus 5.5 --- docs/npu.md | 5 ++++- npu/forward/include/ops-dump.h | 18 ++++++++++++++++++ npu/forward/src/q4nx_forward.cpp | 3 +++ tests/ops_dump_test.cpp | 30 ++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/docs/npu.md b/docs/npu.md index 6f4731d1d..aa2bcb1bc 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2210,7 +2210,10 @@ can fail a run. | `NPU_INFER_DUMP_DIR=` | `/layer_.f32` per layer: the residual stream after that layer, M x H little-endian f32, row-major. The directory must exist; a file that cannot be written is reported on stderr and the run goes on. | which **layer** two runs part at (per-layer cosine against a reference trace) | | `NPU_INFER_DUMP_OPS=1` | one stderr line per projection and per weighted RMSNorm: `[op] L rows= \|x\| mean absmax ` | which **op** inside the layer | -`NPU_INFER_DUMP_OPS` is on for any value other than empty or `0`, and is read once per process. +`NPU_INFER_DUMP_OPS` is on for any value other than empty or `0`, and is read once per process. When it is on, the forward prints +`[ops-dump] NPU_INFER_DUMP_OPS is on: …` once at model load, before any request (engine#2028). A run that sets the +variable and never sees that line is running a binary built before the dump existed (#1977): rebuild it, rather than +looking for a code path that skips the dump. ``` [op] L00 rmsnorm rows=1 |x| mean 0.0143171 absmax 3.01935 diff --git a/npu/forward/include/ops-dump.h b/npu/forward/include/ops-dump.h index 42b3d0fd6..9fb15babc 100644 --- a/npu/forward/include/ops-dump.h +++ b/npu/forward/include/ops-dump.h @@ -36,6 +36,7 @@ // getenv). It can never fail a run: no allocation, no file, stderr only. #pragma once +#include #include #include #include @@ -109,4 +110,21 @@ inline void ops_dump(const char* tag, int layer, int rows, const float* x, int n if (ops_dump_line(buf, sizeof buf, tag, layer, rows, x, n) > 0) std::fputs(buf, stderr); } +// engine#2028: the line a process prints once, at model load, when the dump is armed. A run that sets the variable and +// does not see this line before its first request is running a binary from before the instrument (#1977): four silent +// phi-4 runs were spent on exactly that, and blamed on the prompt shape and then on the design path before the binary's +// age was checked. Separate from the print so a test can match the text. +inline const char* ops_dump_banner() { + return "[ops-dump] NPU_INFER_DUMP_OPS is on: an [op] line follows every projection and weighted RMSNorm, on stderr\n"; +} + +// Prints the banner once per process (a serve can load more than one forward), and only when the dump is on. Returns +// whether this call printed it. +inline bool ops_dump_announce() { + static std::atomic said{false}; + if (!ops_dump_enabled() || said.exchange(true)) return false; + std::fputs(ops_dump_banner(), stderr); + return true; +} + } // namespace onebit::npu diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 18deb4d7b..b638ad8dd 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -436,6 +436,9 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // every row measured so far used. host_gemm_i8_ = (hg[0] == 'i' && hg[1] == '8'); } + // engine#2028: say at load, before any request, that NPU_INFER_DUMP_OPS is armed (once per process). No line with + // the variable set means the binary predates the dump. + onebit::npu::ops_dump_announce(); // The launch layer. A1: everything below that touches a device talks to this, and the settings the // container declares (weight layout, pb16i image dir, Hadamard fold) are pushed into it as they are // parsed. A2 turns this into NpuTarget::open(plan, kernels_dir). diff --git a/tests/ops_dump_test.cpp b/tests/ops_dump_test.cpp index 71496d7d4..758baa2a4 100644 --- a/tests/ops_dump_test.cpp +++ b/tests/ops_dump_test.cpp @@ -18,6 +18,7 @@ // // ops_dump_test the checks below // ops_dump_test --enabled prints 1 or 0: what ops_dump_enabled() says in a process started with this environment +// ops_dump_test --announce calls ops_dump_announce() twice and prints the two results (engine#2028's load line) #include "ops-dump.h" #include @@ -46,12 +47,31 @@ static int child_says(const char* self, const char* env_prefix) { return c == '1' ? 1 : c == '0' ? 0 : -1; } +// Everything a fresh process writes, stderr included, for one environment and one mode. +static std::string child_output(const char* self, const char* env_prefix, const char* mode) { + const std::string cmd = std::string(env_prefix) + " '" + self + "' " + mode + " 2>&1"; + std::string got; + if (FILE* p = popen(cmd.c_str(), "r")) { + char line[256]; + while (std::fgets(line, sizeof line, p)) got += line; + pclose(p); + } + return got; +} + int main(int argc, char** argv) { using namespace onebit::npu; if (argc > 1 && !std::strcmp(argv[1], "--enabled")) { std::printf("%d\n", ops_dump_enabled() ? 1 : 0); return 0; } + if (argc > 1 && !std::strcmp(argv[1], "--announce")) { + const bool first = ops_dump_announce(); + const bool second = ops_dump_announce(); + std::fflush(stderr); + std::printf("%d%d\n", first ? 1 : 0, second ? 1 : 0); + return 0; + } // The rule. check(!ops_dump_value_on(nullptr), "unset is off"); @@ -66,6 +86,16 @@ int main(int argc, char** argv) { check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=0") == 0, "process: 0 is off"); check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=1") == 1, "process: 1 is on"); + // The load line (engine#2028): printed once when the dump is on, never when it is off. Its absence with the variable + // set is how a run tells that its binary predates the dump. + check(child_output(argv[0], "env NPU_INFER_DUMP_OPS=1", "--announce") == std::string(ops_dump_banner()) + "10\n", + "on: the banner once, and the second call is silent; got: " + + child_output(argv[0], "env NPU_INFER_DUMP_OPS=1", "--announce")); + check(child_output(argv[0], "env -u NPU_INFER_DUMP_OPS", "--announce") == "00\n", "unset: no banner"); + check(child_output(argv[0], "env NPU_INFER_DUMP_OPS=0", "--announce") == "00\n", "0: no banner"); + check(std::string(ops_dump_banner()).find("NPU_INFER_DUMP_OPS") != std::string::npos, + "the banner names the variable, so a grep for it finds the line"); + // Read once: whatever this process decided first, a later change of the variable does not move it. const bool first = ops_dump_enabled(); setenv("NPU_INFER_DUMP_OPS", first ? "0" : "1", 1); From 0a46707ff223c6c13df0bef4a0c70e7b4278979c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:30:45 -0400 Subject: [PATCH 30/98] npu: execute on an imported dma-buf -- the engine's arm, and the claim it settles (engine#753) (#2037) The issue says zero-copy execution cannot work because XRT cannot bind a foreign dma-buf as a kernel argument. That is no longer true in this tree: npu/npu_rt_api.h declares `ext::bo(const device&, int dma_buf_fd, size_t, void* host)`, the private runtime imports the fd with DRM PRIME and pins those pages, and npu-kernels#39 already runs kernels with one. What was missing was any arm *in this tree*: the shim test covered `ext::bo(dev, size)` only, and the fd constructor appeared solely in a link-coverage function that is never called. Two changes: - `tests/npu_rt_shim_test.cpp`: the fd constructor joins the link-coverage surface, so a runtime library that lacks it now fails to LINK rather than at the first hybrid call site. - `tests/npu_dmabuf_import_test.cpp`: the executing arm. A dma-buf with no GPU in the path (a sealed memfd handed to /dev/udmabuf), imported with the EXPORTER'S mapping, then **bound as kernel argument 0 and run**. Measured under one npu-hold ticket (queue waited 440 s at a 70 C die, command 1 s): ok a dma-buf exists with no GPU in the path (udmabuf over a sealed memfd) ok ext::bo(dev, fd, size, host) accepts it ok the imported BO is the dma-buf's size ok map() IS the exporter's mapping (a second mapping puts the BO on the driver's invalid list and the next submit HMM-faults) ok a write through the exporter's mapping is visible through the imported BO (one buffer, not a copy) ok host = nullptr constructs (so the trap is at SUBMIT, not at construction) (no throw) ok and it is a SECOND mapping, not the exporter's -- the state npu_rt.cpp says the next submit faults on ok the imported BO is accepted as kernel argument 0 (no throw) ok and the run with it completes (COMPLETED) all npu dmabuf-import checks passed The last two lines are the claim: an imported buffer is not merely mappable, it is submit-able -- the binding is where the issue said execution failed. One correction that only the device run could produce: my first version asserted that `host = nullptr` *throws*, because the header's comment reads as if the driver's own mapping is refused at construction. It constructs fine and hands back a second mapping; the documented fault (amdxdna_insert_pages -> amdxdna_mark_mapp_invalid, then an HMM fault on the next submit) is at submit time. The arm now asserts the observable precursor and deliberately does not court the wedge to watch the fault -- a test that wedges the array to observe a documented failure is a bad trade. Skips (77) without a kernel ELF, so a box with none registers a skip rather than a red row. Refs engine#753 Co-authored-by: t --- CMakeLists.txt | 7 ++ tests/npu_dmabuf_import_test.cpp | 201 +++++++++++++++++++++++++++++++ tests/npu_rt_shim_test.cpp | 6 + 3 files changed, 214 insertions(+) create mode 100644 tests/npu_dmabuf_import_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 86cb25cf9..1b5dbe682 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1584,6 +1584,13 @@ if(ONEBIT_NPU) add_executable(npu_rt_shim_test tests/npu_rt_shim_test.cpp) target_link_libraries(npu_rt_shim_test PRIVATE onebit_npu_lane) add_test(NAME npu_rt_shim COMMAND npu_rt_shim_test) + # engine#753: does the engine EXECUTE on a dma-buf it did not allocate? This one does open the device (it imports a udmabuf and + # hands the exporter's mapping to the runtime), so it takes the kernel ELF as an argument and SKIPS (77) without one -- a box + # with no ELF registers a skip rather than a red row, the same rule as the other device-dependent rows. + add_executable(npu_dmabuf_import_test tests/npu_dmabuf_import_test.cpp) + target_link_libraries(npu_dmabuf_import_test PRIVATE onebit_npu_lane) + add_test(NAME npu_dmabuf_import COMMAND npu_dmabuf_import_test "") + set_tests_properties(npu_dmabuf_import PROPERTIES SKIP_RETURN_CODE 77) # The model-generic forward (npu/forward): the GGUF-on-NPU serve route # (docs/npu.md, "Open"). q4nx_forward.cpp + its Q4NX dequant/pack/semantics diff --git a/tests/npu_dmabuf_import_test.cpp b/tests/npu_dmabuf_import_test.cpp new file mode 100644 index 000000000..195375832 --- /dev/null +++ b/tests/npu_dmabuf_import_test.cpp @@ -0,0 +1,201 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#753: does this engine *execute* on a dma-buf it did not allocate? +// +// The issue's claim was that it could not, because XRT has no way to bind a foreign dma-buf as a kernel argument. That is no longer +// true here: npu/npu_rt_api.h declares `ext::bo(const device&, int dma_buf_fd, size_t, void* host)`, the private runtime imports the +// fd with DRM PRIME and pins those pages, and npu-kernels#39 already runs kernels with one (bit-exact 131072 B under an npu-hold +// ticket, and again by the semantic gate). What was missing was any arm *in this tree*: `tests/npu_rt_shim_test.cpp` covers +// `ext::bo(dev, size_t)` and the fd constructor only through its link-coverage function, which is never called. +// +// npu_dmabuf_import_test [kernel-name] [--args N] [--skip-negative] +// +// Skips (77) when no ELF is given, so a box without one registers a skip rather than a red row. +// +// The engine writes nothing here that a kernel would not: this is the *transport* claim, so the assertions are about identity, not +// about arithmetic -- +// 1. the dma-buf is created with no GPU in the path (a sealed memfd handed to /dev/udmabuf, which /dev/dri/renderD128 cannot do +// without an amdgpu BO) and is therefore reproducible on any box with amdxdna; +// 2. `ext::bo(dev, fd, size, host)` accepts it, and `map()` is the caller's own mapping -- NOT a second one, because mapping the +// *amdxdna* handle puts the BO on the driver's invalid list and the next submit HMM-faults (npu_rt.cpp, import_bo); +// 3. a value written to the exporter's mapping is what the imported BO reports (the same pages, both directions); +// 4. the trap `host = nullptr` documents, measured as the observable difference rather than by courting the wedge: it constructs +// and yields a second mapping, which is what makes the next submit fault. My first version of this arm asserted that nullptr +// *throws*, and the device run refuted it -- the failure is at submit, and the comment in npu_rt_api.h reads as if it were here; + +#include "rt.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +int failures = 0; + +void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++failures; +} + +// A dma-buf with no GPU in the path: a sealed memfd, handed to /dev/udmabuf. Same shape an exporter's fd has (npu-kernels#39 built +// its GPU-free gate this way); the difference is only who exported it. +struct UdmaBuf { + int memfd = -1; + int fd = -1; + void* map = nullptr; + size_t bytes = 0; + + bool create(size_t n) { + bytes = n; + memfd = memfd_create("npu-dmabuf-import", MFD_CLOEXEC | MFD_ALLOW_SEALING); + if (memfd < 0) { std::perror("memfd_create"); return false; } + if (ftruncate(memfd, off_t(n)) != 0) { std::perror("ftruncate"); return false; } + // udmabuf refuses a mapping whose size can change under it + if (fcntl(memfd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_SEAL) != 0) { std::perror("F_ADD_SEALS"); return false; } + const int udma = open("/dev/udmabuf", O_RDWR | O_CLOEXEC); + if (udma < 0) { std::perror("open /dev/udmabuf"); return false; } + struct udmabuf_create create = {}; + create.memfd = uint32_t(memfd); + create.offset = 0; + create.size = n; + create.flags = UDMABUF_FLAGS_CLOEXEC; + const int saved = errno; + fd = ioctl(udma, UDMABUF_CREATE, &create); + close(udma); + if (fd < 0) { errno = saved; std::perror("UDMABUF_CREATE"); return false; } + map = mmap(nullptr, n, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (map == MAP_FAILED) { std::perror("mmap(udmabuf)"); map = nullptr; return false; } + return true; + } + ~UdmaBuf() { + if (map) munmap(map, bytes); + if (fd >= 0) close(fd); + if (memfd >= 0) close(memfd); + } +}; + +} // namespace + +int main(int argc, char** argv) { + if (argc < 2) { + std::printf("skip: no kernel ELF given (`npu_dmabuf_import_test `); the executing arm needs one device ELF\n"); + return 77; + } + std::string kernel_name = "main:sequence"; // the IRON sequence instance name our generated ELFs expose + int n_args = 1; + bool skip_negative = false; + for (int i = 2; i < argc; ++i) { + if (!std::strcmp(argv[i], "--skip-negative")) skip_negative = true; + else if (!std::strcmp(argv[i], "--args") && i + 1 < argc) n_args = std::atoi(argv[++i]); + else kernel_name = argv[i]; + } + + const std::string elf_path = argv[1]; + std::printf("npu dmabuf import (engine#753): %s\n", elf_path.c_str()); + + // 1. the exporter half: a dma-buf that exists before the NPU is involved + UdmaBuf region; + // 128 KiB: the row payload our generated kvwrite sequence reads from argument 0 (32 rows x 2048 elements x 2 B). The size only + // has to be >= what the kernel reads -- a BO read past its end is a device fault, not a failed check. + const size_t kBytes = 128 * 1024; + const bool have_region = region.create(kBytes); + check(have_region, "a dma-buf exists with no GPU in the path (udmabuf over a sealed memfd)"); + if (!have_region) return 1; + + // 2. import it, handing over the exporter's mapping + xrt::device dev{0}; + bool imported = false; + std::unique_ptr bo; + try { + bo = std::make_unique(dev, region.fd, kBytes, region.map); + imported = true; + } catch (const std::exception& e) { + std::printf(" import threw: %s\n", e.what()); + } + check(imported, "ext::bo(dev, fd, size, host) accepts it"); + if (!imported) return 1; + + check(bo->size() == kBytes, "the imported BO is the dma-buf's size"); + check(bo->map() == region.map, + "map() IS the exporter's mapping (a second mapping puts the BO on the driver's invalid list and the next submit HMM-faults)"); + + // 3. the same pages, both directions: what the exporter writes, the imported BO reads + static_cast(region.map)[0] = 0xA5; + static_cast(region.map)[12345] = 0x5A; + __builtin_ia32_clflush(static_cast(region.map)); + __builtin_ia32_mfence(); + const auto* seen = static_cast(bo->map()); + check(seen[0] == 0xA5 && seen[12345] == 0x5A, + "a write through the exporter's mapping is visible through the imported BO (one buffer, not a copy)"); + + // 4. the trap the runtime's own comment documents, measured: `host = nullptr` does NOT fail at construction -- it succeeds and + // hands back a *second* mapping of the same dma-buf, which is the state that puts the BO on the driver's invalid list at the next + // submit (npu_rt.cpp, import_bo: amdxdna_insert_pages -> amdxdna_mark_mapp_invalid, then the submit HMM-faults a dma-buf VMA). + // The submit-time failure is deliberately NOT exercised here: it wedges the array, and a test that courts a wedge to observe a + // documented failure is a bad trade. What is asserted is the observable difference, which is what a caller can check cheaply. + if (!skip_negative) { + bool constructed = false; + std::string why; + void* driver_map = nullptr; + try { + xrt::ext::bo driver_mapped(dev, region.fd, kBytes, nullptr); + constructed = true; + driver_map = driver_mapped.map(); + } catch (const std::exception& e) { + why = e.what(); + } + check(constructed, std::string("host = nullptr constructs (so the trap is at SUBMIT, not at construction) (") + + (constructed ? "no throw" : why) + ")"); + check(constructed && driver_map != region.map, + "and it is a SECOND mapping, not the exporter's -- the state npu_rt.cpp says the next submit faults on"); + } + + // 5. engine#753's actual claim: bind the imported BO as a kernel ARGUMENT and run. This is the step that failed when the issue + // was filed -- not the import, the binding -- so the arm does not stop at "it maps". + bool bound = false, completed = false; + std::string bind_why; + try { + xrt::elf image(elf_path); + xrt::hw_context hc(dev, image); + xrt::ext::kernel k(hc, kernel_name); + xrt::run r(k); + r.set_arg(0, *bo); // <- the imported dma-buf, as argument 0 + std::vector extra; // the rest of the signature: plain BOs, sized generously (transport, not semantics) + for (int i = 1; i < n_args; ++i) extra.emplace_back(dev, size_t(1) << 20); + for (int i = 1; i < n_args; ++i) r.set_arg(i, extra[size_t(i - 1)]); + bound = true; + r.start(); + completed = r.wait(std::chrono::milliseconds(60000)) == ERT_CMD_STATE_COMPLETED; + } catch (const std::exception& e) { + bind_why = e.what(); + } + check(bound, std::string("the imported BO is accepted as kernel argument 0 (") + (bound ? "no throw" : bind_why) + ")"); + check(completed, std::string("and the run with it completes (") + (completed ? "COMPLETED" : "not completed") + ")"); + + if (failures) std::printf("%d check(s) failed\n", failures); + else std::puts("all npu dmabuf-import checks passed"); + return failures ? 1 : 0; +} diff --git a/tests/npu_rt_shim_test.cpp b/tests/npu_rt_shim_test.cpp index b39454ce0..f243122e8 100644 --- a/tests/npu_rt_shim_test.cpp +++ b/tests/npu_rt_shim_test.cpp @@ -34,6 +34,12 @@ static void uses(const std::vector& image) { xrt::ext::kernel k(*ctx, std::string("kernel")); auto act = std::make_unique(dev, size_t(4096)); xrt::ext::bo e{dev, size_t(4096)}; + // engine#753: the dma-buf import constructor. Declared in npu/npu_rt_api.h and defined by the private runtime, + // so a runtime that lacks it now fails to LINK this test rather than at the first hybrid call site. Never + // called on this path (`uses` exists for link coverage), so no device and no dma-buf is needed here; the + // executing arm is npu_dmabuf_import_test. + xrt::ext::bo imported(dev, /*dma_buf_fd=*/-1, size_t(4096), /*host=*/nullptr); + std::printf("%zu\n", imported.size()); auto plain = std::make_unique(e); static_cast(act->map())[0] = 0; act->sync(XCL_BO_SYNC_BO_TO_DEVICE); From 5a693b22ab2d7c0310bc7fe417bc0ca1aa920c30 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:30:53 -0400 Subject: [PATCH 31/98] tests: scratch goes to ~/.cache/scratch-tests, not /tmp (tmpfs, and swap is full) (#2034) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `/tmp` on this box is **tmpfs — RAM** — and swap has been full for hours (owner directive, 2026-11-10 21:15Z). Thirteen test binaries wrote their scratch there with `fs::temp_directory_path()`, and they are not small: `npu_pack_import_test` builds multi-hundred-MB containers and every crafted 1.9 import writes a canonical `.fconst` copy. A leaked directory stays spent until something deletes it, and it is memory a GPU/NPU lane wanted. `tests/tmp_root.h` gives one rule: scratch lives under `~/.cache/scratch-tests/tmp` unless `ONEBIT_TEST_TMP` names somewhere else, and `temp_directory_path()` is only the last resort (no `HOME`, unwritable cache). All 13 files now call it — 35 call sites, mechanically replaced, no behaviour change other than *where* the bytes go. ## Verified ``` npu_pack_import_test tests/golden -> all npu-pack checks passed /tmp delta during the run: 0 GB npu_model_bounds_test -> all cases ok gguf_meta_test -> ok missing file: "" ONEBIT_TEST_TMP= … -> used it (the root is created, per-test dirs still cleaned up) ~/.cache/scratch-tests/tmp -> created by the default path, and empty after the run ``` The `/tmp delta during the run: 0 GB` line is the point of the change: the same suite that used to leave container-sized directories in RAM now leaves nothing behind and touches nothing there. Refs the owner's RAM directive (2026-11-10 21:15Z). Co-authored-by: t --- tests/decision_test.cpp | 7 +++-- tests/gguf_meta_test.cpp | 3 +- tests/laya_bounds_test.cpp | 3 +- tests/mem_admit_test.cpp | 3 +- tests/moe_gguf_index_test.cpp | 3 +- tests/npu_flm_head_import_test.cpp | 3 +- tests/npu_model_bounds_test.cpp | 3 +- tests/npu_moe_test.cpp | 3 +- tests/npu_pack_import_test.cpp | 41 ++++++++++++++-------------- tests/npu_plan_test.cpp | 5 ++-- tests/npu_private_route_test.cpp | 3 +- tests/npu_target_caps_test.cpp | 3 +- tests/q4nx_cache_test.cpp | 3 +- tests/tmp_root.h | 44 ++++++++++++++++++++++++++++++ 14 files changed, 92 insertions(+), 35 deletions(-) create mode 100644 tests/tmp_root.h diff --git a/tests/decision_test.cpp b/tests/decision_test.cpp index 787634dbb..1a7743467 100644 --- a/tests/decision_test.cpp +++ b/tests/decision_test.cpp @@ -25,6 +25,7 @@ #include #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs namespace fs = std::filesystem; using onebit::npu::DecisionHead; @@ -74,7 +75,7 @@ nlohmann::json base_config() { bool close(float a, float b, float eps = 1e-5f) { return std::fabs(a - b) <= eps; } void test_load_and_logits() { - const fs::path dir = fs::temp_directory_path() / "onebit-decision-test"; + const fs::path dir = ::onebit::test::tmp_root() / "onebit-decision-test"; fs::remove_all(dir); fs::create_directories(dir); // W = [[1,0,0],[0,1,0],[0,0,2]] rows A,B,C over a 3-wide hidden state. @@ -113,7 +114,7 @@ void test_load_and_logits() { } void test_temperature_and_shape_errors() { - const fs::path dir = fs::temp_directory_path() / "onebit-decision-test2"; + const fs::path dir = ::onebit::test::tmp_root() / "onebit-decision-test2"; fs::remove_all(dir); fs::create_directories(dir); const std::vector w = {2, 0, 0, 2}; // two options over a 2-wide state @@ -149,7 +150,7 @@ void test_temperature_and_shape_errors() { } void test_missing_pieces() { - const fs::path dir = fs::temp_directory_path() / "onebit-decision-test3"; + const fs::path dir = ::onebit::test::tmp_root() / "onebit-decision-test3"; fs::remove_all(dir); fs::create_directories(dir); DecisionHead head; diff --git a/tests/gguf_meta_test.cpp b/tests/gguf_meta_test.cpp index c9772590f..206b10bf3 100755 --- a/tests/gguf_meta_test.cpp +++ b/tests/gguf_meta_test.cpp @@ -23,6 +23,7 @@ #include #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs namespace { @@ -66,7 +67,7 @@ void expect(const std::string& what, const std::string& got, const std::string& } // namespace int main() { - const auto dir = std::filesystem::temp_directory_path() / "onebit_gguf_meta_test"; + const auto dir = ::onebit::test::tmp_root() / "onebit_gguf_meta_test"; std::filesystem::create_directories(dir); Gguf first; diff --git a/tests/laya_bounds_test.cpp b/tests/laya_bounds_test.cpp index 30400bd05..2604e2249 100644 --- a/tests/laya_bounds_test.cpp +++ b/tests/laya_bounds_test.cpp @@ -33,6 +33,7 @@ #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs namespace fs = std::filesystem; using json = nlohmann::json; @@ -69,7 +70,7 @@ void write_st(const fs::path& path, const std::vector out; diff --git a/tests/mem_admit_test.cpp b/tests/mem_admit_test.cpp index 17b163af1..696195d13 100644 --- a/tests/mem_admit_test.cpp +++ b/tests/mem_admit_test.cpp @@ -28,6 +28,7 @@ #include #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs namespace fs = std::filesystem; using namespace onebit; @@ -94,7 +95,7 @@ void put(const fs::path& p, const std::string& text) { } // namespace int main() { - const fs::path dir = fs::temp_directory_path() / ("mem_admit_test_" + std::to_string(::getpid())); + const fs::path dir = ::onebit::test::tmp_root() / ("mem_admit_test_" + std::to_string(::getpid())); fs::create_directories(dir); const std::string d = dir.string(); diff --git a/tests/moe_gguf_index_test.cpp b/tests/moe_gguf_index_test.cpp index 5ee343198..d7577bc5e 100644 --- a/tests/moe_gguf_index_test.cpp +++ b/tests/moe_gguf_index_test.cpp @@ -30,6 +30,7 @@ #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs namespace fs = std::filesystem; using onebit::moe::GgufIndex; @@ -77,7 +78,7 @@ std::string open_error(const fs::path& path, Gguf g, size_t data = 64) { } // namespace int main() { - const fs::path root = fs::temp_directory_path() / ("moe_gguf_index_test." + std::to_string(::getpid())); + const fs::path root = ::onebit::test::tmp_root() / ("moe_gguf_index_test." + std::to_string(::getpid())); fs::remove_all(root); fs::create_directories(root); diff --git a/tests/npu_flm_head_import_test.cpp b/tests/npu_flm_head_import_test.cpp index 65a48713c..a0807c88d 100644 --- a/tests/npu_flm_head_import_test.cpp +++ b/tests/npu_flm_head_import_test.cpp @@ -34,6 +34,7 @@ #include #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs namespace { @@ -220,7 +221,7 @@ void test_refusal(const std::string& root) { int main(int argc, char** argv) { const std::string golden = argc > 1 ? argv[1] : ""; - const std::string root = (fs::temp_directory_path() / "npu_flm_head_import_test").string(); + const std::string root = (::onebit::test::tmp_root() / "npu_flm_head_import_test").string(); try { if (!golden.empty()) test_golden(golden); else std::printf("skip golden: no directory given\n"); diff --git a/tests/npu_model_bounds_test.cpp b/tests/npu_model_bounds_test.cpp index cd4ff1921..b97c63fd9 100644 --- a/tests/npu_model_bounds_test.cpp +++ b/tests/npu_model_bounds_test.cpp @@ -36,6 +36,7 @@ #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs using namespace onebit::npu; namespace fs = std::filesystem; @@ -120,7 +121,7 @@ std::vector one_layer(std::vector q_shape = {1, 5120}, int64_t g } // namespace int main() { - const fs::path root = fs::temp_directory_path() / ("npu_model_bounds_test." + std::to_string(::getpid())); + const fs::path root = ::onebit::test::tmp_root() / ("npu_model_bounds_test." + std::to_string(::getpid())); fs::remove_all(root); { // well-formed: loads and packs diff --git a/tests/npu_moe_test.cpp b/tests/npu_moe_test.cpp index 3f197fbc9..8bdfad36e 100644 --- a/tests/npu_moe_test.cpp +++ b/tests/npu_moe_test.cpp @@ -44,6 +44,7 @@ #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs using namespace onebit::npu; namespace fs = std::filesystem; @@ -80,7 +81,7 @@ struct Crafted { }; fs::path write_dir(const Crafted& m, const std::string& tag) { - fs::path dir = fs::temp_directory_path() / ("npu_moe_test_" + std::to_string(::getpid()) + tag); + fs::path dir = ::onebit::test::tmp_root() / ("npu_moe_test_" + std::to_string(::getpid()) + tag); fs::remove_all(dir); fs::create_directories(dir); const uint64_t chunk = uint64_t(m.tiles) * kTileBytes; diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index a20b235a2..e494e8110 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -53,6 +53,7 @@ #include #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs namespace { @@ -840,7 +841,7 @@ void test_external_import_success(const std::string& root) { // alone, so what can go wrong is the test being too willing -- a truncated copy from an interrupted run, a leftover // directory, a copy from a bigger pack. Those are the arms. void test_canonical_copy_reusable() { - const fs::path dir = fs::temp_directory_path() / ("1bit-reuse-" + std::to_string(::getpid())); + const fs::path dir = ::onebit::test::tmp_root() / ("1bit-reuse-" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); fs::create_directories(dir); @@ -877,7 +878,7 @@ void test_canonical_copy_reusable() { // changes, so 699 files / 8.7 GiB of tmpfs (RAM) had built up on one box. The eviction rule is "same pack and geometry, // different source identity", and what can go wrong is deleting one of the neighbours that must survive, so each is an arm. void test_evict_stale_canonical_copies() { - const fs::path dir = fs::temp_directory_path() / ("1bit-evict-" + std::to_string(::getpid())); + const fs::path dir = ::onebit::test::tmp_root() / ("1bit-evict-" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); fs::create_directories(dir); @@ -971,7 +972,7 @@ void test_evict_stale_canonical_copies() { // again, and 17.0 GiB of copies left there by older binaries stayed in tmpfs (RAM). Every candidate is swept now; one that does // not exist is not created. void test_sweep_fconst_scratch_dirs() { - const fs::path root = fs::temp_directory_path() / ("1bit-sweep-" + std::to_string(::getpid())); + const fs::path root = ::onebit::test::tmp_root() / ("1bit-sweep-" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(root, ec); const fs::path written = root / "cache", other = root / "tmp", absent = root / "never-made"; @@ -1037,8 +1038,8 @@ void test_sweep_fconst_scratch_dirs() { // override set only that directory is swept, so the second directory's copy must still be there. void test_sweep_is_wired(const std::string& root) { const fs::path dir = fs::path(root) / "sweep-wired"; - const fs::path scratch = fs::temp_directory_path() / ("1bit-sweep-wired-" + std::to_string(::getpid())); - const fs::path cache_home = fs::temp_directory_path() / ("1bit-sweep-wired-xdg-" + std::to_string(::getpid())); + const fs::path scratch = ::onebit::test::tmp_root() / ("1bit-sweep-wired-" + std::to_string(::getpid())); + const fs::path cache_home = ::onebit::test::tmp_root() / ("1bit-sweep-wired-xdg-" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(dir, ec); fs::remove_all(scratch, ec); @@ -1088,7 +1089,7 @@ void test_sweep_is_wired(const std::string& root) { } setenv("ONEBIT_FCONST_SCRATCH", scratch.c_str(), 1); setenv("XDG_CACHE_HOME", cache_home.c_str(), 1); - const std::string out = (fs::temp_directory_path() / ("oga_pack_import_sweepwired." + std::to_string(::getpid()))).string(); + const std::string out = (::onebit::test::tmp_root() / ("oga_pack_import_sweepwired." + std::to_string(::getpid()))).string(); fs::remove_all(out, ec); pi::Options allow; allow.allow_incomplete = true; @@ -1835,7 +1836,7 @@ void test_tied_block32_head(const std::string& root) { allow.allow_incomplete = true; { // arm 1: declared tied -> synthesised from the embedding, and the note says so const fs::path dir = build(true, "tied-block32"); - const std::string out = (fs::temp_directory_path() / + const std::string out = (::onebit::test::tmp_root() / ("oga_pack_import_tied32." + std::to_string(::getpid()))).string(); fs::remove_all(out); bool threw = false; @@ -1910,7 +1911,7 @@ void test_tied_block32_head(const std::string& root) { } { // arm 2: the same pack without the declaration -> refused by name, nothing written const fs::path dir = build(false, "untied-block32"); - const std::string out = (fs::temp_directory_path() / + const std::string out = (::onebit::test::tmp_root() / ("oga_pack_import_untied32." + std::to_string(::getpid()))).string(); fs::remove_all(out); bool threw = false; @@ -2076,7 +2077,7 @@ void test_block32_tail_head(const std::string& root) { const auto run = [&](const fs::path& dir, const char* tag) { Run x; const std::string out = - (fs::temp_directory_path() / + (::onebit::test::tmp_root() / (std::string("oga_pack_import_b32tail_") + tag + "." + std::to_string(::getpid()))).string(); fs::remove_all(out); pi::Options allow; @@ -2781,7 +2782,7 @@ void test_final_norm_envelope(const std::string& root) { (geom.why.empty() ? "" : ", why: " + geom.why) + ")"); pi::Options allow; allow.allow_incomplete = true; - const std::string out = (fs::temp_directory_path() / + const std::string out = (::onebit::test::tmp_root() / ("oga_pack_import_norm." + std::to_string(::getpid()))).string(); fs::remove_all(out); const pi::Report r = pi::import_pack(dir.string(), out, allow); @@ -2932,7 +2933,7 @@ void test_pack(const std::string& pack, const std::string& golden_dir) { // a fully representable pack imports without --allow-incomplete; a partial one writes nothing. // Per-process scratch dir: two concurrent runs of this binary (or two ctest shards) otherwise write // into the same directory and one dies with "cannot write .../model.q4nx" (measured 2026-11-03). - const std::string out = (fs::temp_directory_path() / + const std::string out = (::onebit::test::tmp_root() / ("oga_pack_import_test." + std::to_string(::getpid()))).string(); fs::remove_all(out); const pi::Report r0 = pi::import_pack(pack, out, pi::Options{}); @@ -3330,7 +3331,7 @@ void test_import_fp32_params(const std::string& root) { {pb::matmulnbits("lm_head", {"x", "lm_head_qweight", "lm_head_scales", "lm_head_zeros"}, H, V, 4, block)})); - const std::string out = (fs::temp_directory_path() / + const std::string out = (::onebit::test::tmp_root() / ("oga_pack_import_fp32." + std::to_string(::getpid()))).string(); fs::remove_all(out); const pi::Report r = pi::import_pack(dir.string(), out, pi::Options{}); @@ -3437,7 +3438,7 @@ void test_import_fp32_params(const std::string& root) { // than misread it (npu/forward/src/model.c). Q4nxNpuForward::init_impl converts first through // npu/q4nx_device.cpp; this is that conversion, and the proof it changes only the parameter // width: the device tiles are byte-identical to the old bf16 container's for the same weights. - const std::string cache = (fs::temp_directory_path() / + const std::string cache = (::onebit::test::tmp_root() / ("oga_q4nx_device_form." + std::to_string(::getpid()))).string(); fs::remove_all(cache); { @@ -3590,7 +3591,7 @@ void test_hybrid_pack_import(const std::string& root) { int64_t(H) * H * 4)}, {})); put_file((dir / "dd_metastate_Llm_Token_Token_crafted.fconst").string(), fc); - const std::string out = (fs::temp_directory_path() / + const std::string out = (::onebit::test::tmp_root() / ("oga_pack_import_hybrid." + std::to_string(::getpid()))).string(); fs::remove_all(out); pi::Options allow; @@ -3777,7 +3778,7 @@ void test_genre_a_projection_gap(const std::string& root) { R"("rope_theta":10000.0,"tie_word_embeddings":false,"rms_norm_eps":1e-06})"); (void)groups; }; - const std::string out = (fs::temp_directory_path() / ("oga_pack_genre_a." + std::to_string(::getpid()))).string(); + const std::string out = (::onebit::test::tmp_root() / ("oga_pack_genre_a." + std::to_string(::getpid()))).string(); auto names_down = [](const pi::Report& r) { for (const auto& s : r.refused) if (s.find("mlp.down_proj.weight") != std::string::npos && s.find("no quantised projection the decoder can pair") != std::string::npos) @@ -4264,7 +4265,7 @@ void test_post_attn_norm_header(const std::string& root) { pi::Options allow; allow.allow_incomplete = true; // the crafted block has no tiles, so the payload legs are refused - const std::string out = (fs::temp_directory_path() / + const std::string out = (::onebit::test::tmp_root() / ("oga_pack_import_pa." + std::to_string(::getpid()))).string(); fs::remove_all(out); try { @@ -4335,7 +4336,7 @@ void test_kernel_set_refusal() { // missing tokenizer -- the misleading report #1751 was written to remove. A file the pack has and cannot copy is refused // BY NAME; a file the pack does not ship is not a failure. void test_copy_text_artifacts() { - const fs::path base = fs::temp_directory_path() / ("1bit-textcopy-" + std::to_string(::getpid())); + const fs::path base = ::onebit::test::tmp_root() / ("1bit-textcopy-" + std::to_string(::getpid())); std::error_code ec; fs::remove_all(base, ec); const fs::path pack = base / "pack", out = base / "out"; @@ -5091,7 +5092,7 @@ void test_per_layer_import(const std::string& root) { put_file((dir / "genai_config.json").string(), R"({"model":{"embedding":{"filename":"embedding/model.onnx"}}})"); const std::string out = - (fs::temp_directory_path() / ("oga_pack_import_perlayer." + std::to_string(::getpid()))).string(); + (::onebit::test::tmp_root() / ("oga_pack_import_perlayer." + std::to_string(::getpid()))).string(); fs::remove_all(out); pi::Options allow; allow.allow_incomplete = true; @@ -5184,7 +5185,7 @@ void test_fconst_variable_stride(const std::string& root) { put_file((dir / "model.onnx.data").string(), std::string(4096, '\x11')); pi::Options allow; allow.allow_incomplete = true; - const std::string out = (fs::temp_directory_path() / ("varstride_out." + std::to_string(::getpid()))).string(); + const std::string out = (::onebit::test::tmp_root() / ("varstride_out." + std::to_string(::getpid()))).string(); fs::remove_all(out); bool refused = false; std::string what; @@ -5442,7 +5443,7 @@ void test_header_witness_refusal() { int main(int argc, char** argv) { // engine#1907: every crafted 1.9 import writes a canonical copy into the scratch cache. Without this the suite left 0-byte // copies in the user's real ~/.cache/1bit/fconst19 and in /tmp on every run. Give the run its own directory, removed at exit. - const fs::path run_scratch = fs::temp_directory_path() / ("1bit-pack-test-scratch-" + std::to_string(::getpid())); + const fs::path run_scratch = ::onebit::test::tmp_root() / ("1bit-pack-test-scratch-" + std::to_string(::getpid())); if (!std::getenv("ONEBIT_FCONST_SCRATCH")) setenv("ONEBIT_FCONST_SCRATCH", run_scratch.c_str(), 1); struct ScratchCleanup { fs::path p; diff --git a/tests/npu_plan_test.cpp b/tests/npu_plan_test.cpp index 50001ed94..c9518f318 100644 --- a/tests/npu_plan_test.cpp +++ b/tests/npu_plan_test.cpp @@ -27,6 +27,7 @@ #include #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs namespace fs = std::filesystem; using onebit::npu::plan::Plan; @@ -42,7 +43,7 @@ void check(bool ok, const std::string& what) { } std::string write_plan(const std::string& name, const nlohmann::json& body) { - const fs::path p = fs::temp_directory_path() / ("npu_plan_test_" + name + ".json"); + const fs::path p = ::onebit::test::tmp_root() / ("npu_plan_test_" + name + ".json"); std::ofstream f(p); f << body.dump(2) << "\n"; f.close(); @@ -501,7 +502,7 @@ void test_artifacts() { return p; }; const Plan p = Plan::load(write_plan("artifacts", with_designs())); - const fs::path root = fs::temp_directory_path() / "npu_plan_test_artifacts"; + const fs::path root = ::onebit::test::tmp_root() / "npu_plan_test_artifacts"; const fs::path kdir = root / "kernels"; std::error_code ec; fs::remove_all(root, ec); diff --git a/tests/npu_private_route_test.cpp b/tests/npu_private_route_test.cpp index c3675a278..e2a35dfd8 100644 --- a/tests/npu_private_route_test.cpp +++ b/tests/npu_private_route_test.cpp @@ -27,6 +27,7 @@ #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs namespace { @@ -70,7 +71,7 @@ void no_addon_refuses_the_option() { int main() { no_addon_refuses_the_option(); namespace fs = std::filesystem; - const fs::path dir = fs::temp_directory_path() / ("npu_private_route_test." + std::to_string(::getpid())); + const fs::path dir = ::onebit::test::tmp_root() / ("npu_private_route_test." + std::to_string(::getpid())); fs::create_directories(dir); std::ofstream(dir / "config.json") << R"({"model_type": "qwen3_5_moe", "num_hidden_layers": 40})"; check("model_type_of reads config.json", model_type_of(dir.string()) == "qwen3_5_moe"); diff --git a/tests/npu_target_caps_test.cpp b/tests/npu_target_caps_test.cpp index 1c34f85f3..2cbb5bef5 100644 --- a/tests/npu_target_caps_test.cpp +++ b/tests/npu_target_caps_test.cpp @@ -33,6 +33,7 @@ #include #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs namespace fs = std::filesystem; using onebit::npu::plan::Plan; @@ -68,7 +69,7 @@ nlohmann::json base_plan() { } std::string write_plan(const std::string& name, const nlohmann::json& body) { - const fs::path p = fs::temp_directory_path() / ("npu_target_caps_test_" + name + ".json"); + const fs::path p = ::onebit::test::tmp_root() / ("npu_target_caps_test_" + name + ".json"); std::ofstream f(p); f << body.dump(2) << "\n"; f.close(); diff --git a/tests/q4nx_cache_test.cpp b/tests/q4nx_cache_test.cpp index 32cd463a0..87558cf39 100644 --- a/tests/q4nx_cache_test.cpp +++ b/tests/q4nx_cache_test.cpp @@ -25,6 +25,7 @@ #include #include #include +#include "tmp_root.h" // engine#2029: scratch goes on disk, not in tmpfs namespace fs = std::filesystem; static int failures = 0; @@ -44,7 +45,7 @@ static void make_entry(const fs::path& root, const std::string& name, int mb, in int main() { std::error_code ec; - const fs::path root = fs::temp_directory_path() / ("q4nx-cache-test-" + std::to_string(::getpid())); + const fs::path root = ::onebit::test::tmp_root() / ("q4nx-cache-test-" + std::to_string(::getpid())); fs::remove_all(root, ec); // (a) under the bound: nothing is removed diff --git a/tests/tmp_root.h b/tests/tmp_root.h new file mode 100644 index 000000000..d3000188e --- /dev/null +++ b/tests/tmp_root.h @@ -0,0 +1,44 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// tmp_root.h -- where a test puts its scratch. +// +// /tmp on this box is **tmpfs**, i.e. RAM, and swap has been full for hours (owner directive, 2026-11-10 21:15Z). A test +// that writes containers there -- `npu_pack_import_test` builds multi-hundred-MB packs, and every crafted 1.9 import +// writes a canonical `.fconst` copy -- spends memory that a GPU/NPU lane needs, and a leaked directory stays spent until +// something deletes it. +// +// So scratch goes on disk under `~/.cache/scratch-tests/tmp` unless `ONEBIT_TEST_TMP` names somewhere else; `temp_directory_path()` +// is the last resort (no HOME, unwritable cache), never the default. +#pragma once + +#include +#include + +namespace onebit::test { + +inline std::filesystem::path tmp_root() { + if (const char* e = std::getenv("ONEBIT_TEST_TMP"); e && *e) return std::filesystem::path(e); + if (const char* h = std::getenv("HOME"); h && *h) { + std::error_code ec; + const std::filesystem::path p = std::filesystem::path(h) / ".cache" / "scratch-tests" / "tmp"; + std::filesystem::create_directories(p, ec); + // An existing directory is what we want; a create that failed because it is already there is fine too. + if (std::filesystem::is_directory(p, ec) && !ec) return p; + } + return std::filesystem::temp_directory_path(); +} + +} // namespace onebit::test From 153b7b636477b4bc3c5f9e87055c4376d913774f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:30:59 -0400 Subject: [PATCH 32/98] bump llama.cpp to 2293f95ea: prefill attention keeps its rate at long context (fixes #1989) (#2033) Five fork commits past 50d28600a: - 139ccd741 (#289) + 2293f95ea (#291): a spread pitch for the transposed V and 256-token attention launches from 4,096 keys; the pitch is derived in the kernel, so no per-key-count recompile (#1989) - cf5febf04 (#288): activations kept inside the f16 range in the two small-batch Q6_K kernels (#1962) - 69aac7fa2 (#287), 7cc3187b5 (#290): known-answer MUL_MAT cases and failed-compute reporting in the backend comparison (#1997) Measured at the exact tree on gfx1151: Qwen3-4B pp24576 warm 813-822 -> 980-1074 tok/s; cold 9,230-token prompt 8.80 s vs 8.85 s, text identical; test-hrx-fa-transposed-value 8/8; FLASH_ATTN_EXT ops 388/388. registry/architectures.json: the recorded source pin only. docs/hrx.md: the #1989 section. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/hrx.md | 25 +++++++++++++++++++++++++ registry/architectures.json | 2 +- third_party/llama.cpp | 2 +- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/hrx.md b/docs/hrx.md index a1c96fb7c..b6e66a63a 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -330,6 +330,31 @@ The projector's own graph still runs 103 of 626 nodes on the CPU for an image of the attention score matmul and softmax, `IM2COL`, `UPSCALE`); with it offloaded, encoding the image is no faster than with `--no-mmproj-offload` (0.5 to 1.1 s against 0.4 to 0.6 s). +### Fixed: prefill attention lost its rate at long context ([engine#1989](https://github.com/1bit-MONSTER/engine/issues/1989)) + +The cost of prefill attention per key was measured at 0.0331 ms per KV token per 512-token graph on Qwen3-4B +(over 0 to 7.7k keys) and 0.0522 over 0 to 24k. Timed alone on `HRX0`, the attention dispatch costs what its +arithmetic costs up to about 8,192 keys; beyond that, and at particular key counts, it fell into a slower regime. +The cause was not in the attention kernel's arithmetic. From 256 query tokens the dispatch copies V into a +transposed buffer and walks it across channels, and two things about that step were slow at long key counts: + +- the buffer's rows were `2 x keys` bytes apart, which at key counts such as 16,384 and 32,768 puts every channel + of a key block on the same cache sets. A key count that is a multiple of 64 now gets a pitch of `keys + 32`; +- one launch over 512 query tokens was slower per token than two over 256. From 4,096 keys the dispatch now runs + 256-token launches over the one transposed buffer. + +The pitch is derived inside the attention kernel from its key count, so a new key count compiles no new kernel +(llama.cpp #289 and #291; #289 alone made it a compile parameter and must not be pinned without #291). + +Measured at pin `2293f95ea` on gfx1151 (2026-11-10, power mode `performance`, Qwen3-4B Q4_K_M, +`llama-bench -p 24576 -n 0 -b 512 -ub 512 -r 3`, old layout through `GGML_HRX_FA_VALUE_PITCH=0 +GGML_HRX_FA_QUERY_CHUNK=2048`, alternated): warm repetitions 813 to 822 tok/s before, 980 to 1,074 after; +first, cold repetitions 701 to 711 before, 856 to 866 after. At 8,192 tokens 1,545 and 1,600 before, 1,766 and +1,768 after; 2,048 and 512 tokens do not move. A cold run on a 9,230-token prompt takes the same time with either +layout (8.80 s and 8.85 s) and gives the same text. `test-hrx-fa-transposed-value` compares the path with the CPU +backend (8 cases). Why 256-token launches are faster than one 512-token launch at long key counts is measured, +not explained. + ### MoE decode and the kernel cache (llama.cpp #32, #33) - **MoE decode.** With one to four tokens, a `MUL_MAT_ID` (a MoE expert matmul) runs as a GEMV over diff --git a/registry/architectures.json b/registry/architectures.json index 031dafa8d..ef1ebe7c4 100644 --- a/registry/architectures.json +++ b/registry/architectures.json @@ -1,7 +1,7 @@ { "about": "HF architecture -> GGUF architecture and the backends whose code accepts it. Generated by tools/registry_build.py from the pinned sources; do not edit.", "sources": { - "llama.cpp (hrx)": "50d28600a5f7a38df390709b94c463df02785a54" + "llama.cpp (hrx)": "2293f95ea49dafec71a2460739d95ea31cd27bfc" }, "counts": { "hrx": 298, diff --git a/third_party/llama.cpp b/third_party/llama.cpp index 50d28600a..2293f95ea 160000 --- a/third_party/llama.cpp +++ b/third_party/llama.cpp @@ -1 +1 @@ -Subproject commit 50d28600a5f7a38df390709b94c463df02785a54 +Subproject commit 2293f95ea49dafec71a2460739d95ea31cd27bfc From 391ebd1cc2b0ab5c8559176f9c56b8520bbc2c34 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:31:07 -0400 Subject: [PATCH 33/98] tools/xhci_boot_capture: the fallback outdir is the cache, never /tmp (#2030) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On this box `/tmp` is tmpfs, so a capture written there is **RAM held until reboot**. The tool was falling back to `/tmp/xhci-capture-` whenever `/data` was not writable, which is the default on a machine without a `/data` mount — so the default path was RAM. The fallback is now `${XDG_CACHE_HOME:-$HOME/.cache}/1bit/captures/xhci-`, the directory the `.fconst` and `q4nx` caches already use, and the header says why in one line referencing engine#2027's rule. `/data` is still preferred when it is writable. Why this is this lane's to fix: the tool is mine (#1679), and the rule is fleet-wide as of 2026-11-10 (`/tmp` 52 G on a 122 G box with swap 71/71 G full, every dump and staging file counted as memory). Two of my own earlier capture runs left **root-owned** files in `/tmp` (`Operation not permitted` to a lane, so a sweep cannot finish without sudo) — the class this change removes. Not a general sweep fix: the per-token `q4nx_hidden_%d.bin` dump and the `mkstemp` sites in `npu/forward/` are the larger writers and belong to the lanes editing those files; they were reported separately with line numbers. Verified: `bash -n` clean; `grep -c '/tmp/'` on the file is **0**. Co-authored-by: pi-os3 --- tools/xhci_boot_capture.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/xhci_boot_capture.sh b/tools/xhci_boot_capture.sh index cc7992eeb..37aae66d3 100755 --- a/tools/xhci_boot_capture.sh +++ b/tools/xhci_boot_capture.sh @@ -32,7 +32,9 @@ # needs root and is therefore the one part that will say so if it cannot read. # # usage: tools/xhci_boot_capture.sh [outdir] -# default outdir: /data/xhci-capture- when /data is writable, else /tmp/... +# default outdir: /data/xhci-capture- when /data is writable, else +# ${XDG_CACHE_HOME:-$HOME/.cache}/1bit/captures. NEVER /tmp: on strixhalo /tmp is tmpfs, +# so a capture written there is RAM held until reboot (engine#2027's rule). # exit: 0 captured, 2 not root and sudo unavailable, 3 cannot write set -u @@ -49,7 +51,13 @@ fi stamp=$(date +%Y%m%d-%H%M%S) out=${1:-} if [ -z "$out" ]; then - if [ -d /data ] && [ -w /data ]; then out=/data/xhci-capture-$stamp; else out=/tmp/xhci-capture-$stamp; fi + if [ -d /data ] && [ -w /data ]; then + out=/data/xhci-capture-$stamp + else + # engine#2027: NOT /tmp. On this box /tmp is tmpfs, so a capture there is RAM held until reboot; + # the cache directory is on disk and is where the fconst and q4nx caches already live. + out=${XDG_CACHE_HOME:-$HOME/.cache}/1bit/captures/xhci-$stamp + fi fi mkdir -p "$out" 2>/dev/null || { say "xhci_boot_capture: cannot write $out" >&2; exit 3; } From d8a7c4ce927ce743410b270b9c463f49af86ee35 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:42:27 -0400 Subject: [PATCH 34/98] npu: read a pack's vision component (vision_weight.q4nx): every tensor placed, geometry from the tensors (engine#1788) (#2040) * npu: read a pack's vision component (vision_weight.q4nx): every tensor placed, geometry from the tensors (engine#1788) FastFlowLM's MiniCPM-V 4.7 packs ship the vision tower as a second container beside model.q4nx, and nothing in the engine opened it. npu/vision_pack.h reads it: all 347 tensors BF16 and accounted for, the tower's geometry derived from the tensors' own shapes, and every disagreement refused by name. Both published packs state two MLP widths in config.json that their tensors do not have (4304 vs 4608, 17216 vs 17280); those are reported and overruled, never used to size anything. 1bit npu-pack vision prints the result. A reader only: no forward runs the tower yet. Test: npu_vision_pack (70 checks on crafted containers; +8 on a published pack via ONEBIT_MINICPM_PACK). Co-Authored-By: Claude Fable 5.2 * npu: the vision container pads its MLPs with zero rows; verify the padding and carry the model's width (engine#1788) The first commit called config.json stale about two MLP widths. It is not: 4304 and 17216 are the model's widths (the source GGUF's ffn tensors), and the container stores them 4608 and 17280 wide with the extra rows all zero. Measured by rebuilding all 347 tensors of both published packs from their source mmproj GGUFs: 263 bit-identical, 84 identical in the source's extent and +0.0 in the padding, 0 different. The reader still sizes everything from the tensor. It now checks that every value past the stated width is zero in every block, carries the model's width (Geometry::model_intermediate), says so when the tail is not zero, and refuses a config that states an MLP wider than its tensors. Docs and comments corrected. Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 8 +- app/main.cpp | 14 + docs/npu.md | 60 +++++ npu/vision_pack.cpp | 394 ++++++++++++++++++++++++++++ npu/vision_pack.h | 115 +++++++++ tests/npu_vision_pack_test.cpp | 455 +++++++++++++++++++++++++++++++++ 6 files changed, 1045 insertions(+), 1 deletion(-) create mode 100644 npu/vision_pack.cpp create mode 100644 npu/vision_pack.h create mode 100644 tests/npu_vision_pack_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b5dbe682..913000720 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -687,7 +687,7 @@ option(ONEBIT_NPU_HOST "Build the NPU host library and its tests" ${_onebit_npu_ if(ONEBIT_NPU_HOST) find_package(PkgConfig REQUIRED) pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) -add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp +add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) target_include_directories(onebit_npu PUBLIC npu) @@ -1194,6 +1194,12 @@ if(ONEBIT_NPU_CAPTURED AND ONEBIT_NPU_PDI AND ONEBIT_NPU_GOLDEN) endif() # Q4NX chunk decoders against 1bit-MONSTER's verified decoders, on committed real tiles +# engine#1788: the vision component reader (vision_weight.q4nx). Crafted containers for every refusal; the published +# pack's own numbers when ONEBIT_MINICPM_PACK names a MiniCPM-V 4.7 pack directory. +add_executable(npu_vision_pack_test tests/npu_vision_pack_test.cpp) +target_link_libraries(npu_vision_pack_test PRIVATE onebit_npu) +add_test(NAME npu_vision_pack COMMAND npu_vision_pack_test) +add_dependencies(onebit npu_vision_pack_test) add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp) target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) diff --git a/app/main.cpp b/app/main.cpp index e9249b378..1d7ae3755 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -43,6 +43,7 @@ int onebit_ort_run_main(int argc, char** argv); #endif #ifdef ONEBIT_NPU_HOST #include "pack_import.h" // 1bit npu-pack: an OGA NPU pack -> a Q4NX model directory +#include "vision_pack.h" // 1bit npu-pack vision: a pack's vision component (engine#1788) #endif #ifdef ONEBIT_NPU #include "forward_serve.h" @@ -94,6 +95,7 @@ void usage(FILE* out) { #endif #ifdef ONEBIT_NPU_HOST " npu-pack import a Ryzen AI OGA NPU pack (npu-pack --help)\n" + " npu-pack vision read a pack's vision component (vision_weight.q4nx)\n" #endif " comfy run a ComfyUI workflow with ComfyUI.cpp (docs/comfyui.md)\n" #ifdef ONEBIT_LAYA @@ -323,6 +325,18 @@ int main(int argc, char** argv) { #ifdef ONEBIT_NPU_HOST if (cmd == "npu-pack") { try { + // `npu-pack vision `: the vision component of a two-component pack (engine#1788), read by + // npu/vision_pack.h rather than the OGA importer, which knows nothing of it. + if (argc >= 3 && std::string(argv[2]) == "vision") { + if (argc != 4 || argv[3][0] == '-') { + std::fprintf(stderr, "usage: 1bit npu-pack vision \n" + " reads the pack's vision component (config.json's vision_model_weight, default\n" + " vision_weight.q4nx): every tensor placed, the tower's geometry from the tensors'\n" + " own shapes, and each config.json field the tensors overrule\n"); + return argc == 4 ? 0 : 2; + } + return onebit::npu::vision::print_info(argv[3]); + } return onebit::npu::pack_import::run(argc - 2, argv + 2); } catch (const std::exception& e) { std::fprintf(stderr, "1bit npu-pack: %s\n", e.what()); diff --git a/docs/npu.md b/docs/npu.md index aa2bcb1bc..cdc4c9dd5 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1448,6 +1448,66 @@ selections per layer touch nearly all 128 experts and stream ~18 GB. MoE prefill GPU route (`--moe-prefetch`, docs/serve.md); this is the thing that decides whether a kernel is right. +### A pack's vision component (`vision_weight.q4nx`, engine#1788) + +FastFlowLM's MiniCPM-V 4.7 packs are two containers at the pack root: `model.q4nx` (the text model) and +the file `config.json` names as `vision_model_weight`, `vision_weight.q4nx` (the vision tower). The +second is the same container format holding a ViT: every tensor BF16, no tiles. `npu/vision_pack.h` +reads it, and `1bit npu-pack vision ` prints what it found: + +``` +$ 1bit npu-pack vision MiniCPM-V-4.7-1B-NPU2 +vision component MiniCPM-V-4.7-1B-NPU2/vision_weight.q4nx +tensors 347, all BF16, 1135109120 B of payload, 0 B unaccounted +patch embed 14 x 14 pixels x 3 channels -> 1152 +positions 4900 (a 70 x 70 grid; image 980) +blocks 27 at width 1152, 16 heads x 72, MLP 4304 (4608 stored) +vit_merger attention at 1152, MLP 4608 -> 17216 (17280 stored) -> 1152 +merger 4608 -> 4608 -> 1024 (the text model's hidden size) +source_file mmproj-model-f16.gguf +source_sha256 fb9cc047a3cdad5317fd8433a4aeda2d22f00f02ca4bd93b34ad84165b25a900 +note config.json states MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE 4304; model.visual.blocks.*.mlp.linear_fc1.weight has 4608 rows, the last 304 all zero: the container pads the model's width, and sizes come from the tensor +note config.json states MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE 17216; model.visual.vit_merger.mlp.linear_fc1.weight has 17280 rows, the last 64 all zero: the container pads the model's width, and sizes come from the tensor +``` + +This is a reader, not a forward: nothing runs the tower yet. What it settles is what a forward may +rely on. + +- **Sizes come from the tensors.** The geometry is derived from the tensors' own shapes, and every + tensor must have the shape the others imply. +- **The MLPs are stored padded.** The model's MLP widths are 4304 (blocks) and 17216 (`vit_merger`); + that is what `config.json` states and what the source GGUF's tensors are. The container stores them + 4608 and 17280 wide, the next multiples of 128, with the extra `linear_fc1` rows, bias entries and + `linear_fc2` columns all zero. A loader that sized a buffer from the config would be 304 rows short + of what the file holds. The reader sizes from the tensor, verifies that everything past the stated + width is zero in every block, and carries the model's width beside the stored one + (`Geometry::model_intermediate`). If any value past the stated width is not zero it says so and + keeps the tensor's width; a config that states an MLP wider than its tensors is refused. +- **Zero gaps.** A tensor the reader has no place for is a refusal, as is a missing one, a hole in the + block indices, a tensor that is not BF16, an extent that is not its shape's byte length, an extent + outside the payload, and two tensors on the same bytes. Each names the tensor. +- **`config.json` is consulted for what the tensors cannot say** (the head count, the image size, the + unpadded MLP widths) and refused where the two cannot both be right: heads that do not multiply to + the width, an image size that is not the position grid times the patch, another width, block count + or position count, and a text `hidden_size` that is not what the merger projects to. That last one + is a tower paired with the wrong text model. + +**Checked against the source, bit for bit** (2026-11-10, both published packs). The 1B container's +`__metadata__` names `mmproj-model-f16.gguf` and its SHA-256, which is the file of that name in +`openbmb/MiniCPM-V-4.7-1B-gguf`. All 347 pack tensors were rebuilt from that GGUF and compared as +BF16 bit patterns: 263 identical, 84 identical in the source's extent and `+0.0` in the padding, 0 +different. The pack's `attn.qkv` is the source's `attn_q`, `attn_k`, `attn_v` stacked in that order; +`merger.norm` is the source's `v.post_ln`. Every F32 source tensor is exactly representable in BF16. +The 35B-A3B container carries no `__metadata__`; the same comparison against +`openbmb/MiniCPM-V-4.7-35B-A3B-gguf`'s `mmproj-model-f16.gguf` (SHA-256 `bff3b688…5d20f1`) gives the +same 263 / 84 / 0, and against the 1B's source gives 0 of 347, so the two towers share a geometry and +not a single tensor. One source tensor has no counterpart in either container: `v.tok_embd_sep` +(`[4, hidden]`). + +Pinned by `tests/npu_vision_pack_test.cpp` on crafted containers; `ONEBIT_MINICPM_PACK=` adds +the published pack's own numbers. The source comparison above was run by hand and is not a test in the +tree. + ## Serving The 1bit NPU engine runs the model on the XDNA 2 NPU. With a release package that carries the NPU diff --git a/npu/vision_pack.cpp b/npu/vision_pack.cpp new file mode 100644 index 000000000..4304b5c69 --- /dev/null +++ b/npu/vision_pack.cpp @@ -0,0 +1,394 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "vision_pack.h" + +#include "file_map.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace onebit::npu::vision { + +namespace { + +using nlohmann::json; +using Shape = std::vector; + +const char* const kPrefix = "model.visual."; + +std::string shape_text(const Shape& s) { + std::string o = "["; + for (size_t i = 0; i < s.size(); ++i) o += (i ? ", " : "") + std::to_string(s[i]); + return o + "]"; +} + +// The product of a shape, or false when a dimension is not positive or the product does not fit: a shape is file +// content, and its product sizes a read. +bool elements_of(const Shape& s, uint64_t* out) { + uint64_t n = 1; + if (s.empty()) return false; + for (int64_t d : s) { + if (d <= 0) return false; + if (n > (uint64_t(1) << 40) / uint64_t(d)) return false; + n *= uint64_t(d); + } + *out = n; + return true; +} + +int as_int(int64_t v) { return v > 0 && v <= (int64_t(1) << 30) ? int(v) : 0; } + +} // namespace + +Pack::Pack(const std::string& pack_dir) { + std::ifstream cf(pack_dir + "/config.json"); + if (!cf) throw std::runtime_error("cannot read " + pack_dir + "/config.json"); + std::stringstream ss; + ss << cf.rdbuf(); + const std::string text = ss.str(); + std::string name = "vision_weight.q4nx"; + const json c = json::parse(text, nullptr, false); + if (c.is_discarded() || !c.is_object()) throw std::runtime_error(pack_dir + "/config.json is not a JSON object"); + if (c.contains("vision_model_weight") && c["vision_model_weight"].is_string()) + name = c["vision_model_weight"].get(); + // The name is file content and is joined to a directory: a separator in it would leave the pack. + if (name.empty() || name.find('/') != std::string::npos || name.find('\\') != std::string::npos || name == "..") + throw std::runtime_error(pack_dir + "/config.json: vision_model_weight \"" + name + "\" is not a file name"); + open(pack_dir + "/" + name, text); +} + +Pack::Pack(const std::string& container_path, const std::string& config_json) { open(container_path, config_json); } + +Pack::~Pack() { + if (map_) unmap_file(map_, size_); +} + +void Pack::open(const std::string& container_path, const std::string& config_json) { + path_ = container_path; + const auto fail = [&](const std::string& what) -> void { throw std::runtime_error(path_ + ": " + what); }; + + std::string why; + map_ = map_file(path_, size_, &why); + if (!map_) throw std::runtime_error(why); + if (size_ < 8) fail("truncated (no header length)"); + uint64_t header = 0; + std::memcpy(&header, map_, 8); + if (header > size_ - 8) fail("the header runs past the end of the file"); + const json table = json::parse(map_ + 8, map_ + 8 + header, nullptr, false); + if (table.is_discarded() || !table.is_object()) fail("the header is not a JSON object"); + payload_ = size_ - 8 - header; + const uint8_t* base = map_ + 8 + header; + + // ── every tensor: BF16, inside the payload, exactly as long as its shape says ── + for (const auto& [name, t] : table.items()) { + if (name == "__metadata__") { + if (t.is_object()) + for (const auto& [k, v] : t.items()) meta_[k] = v.is_string() ? v.get() : v.dump(); + continue; + } + if (!t.is_object() || !t.contains("data_offsets") || !t.contains("shape") || !t.contains("dtype")) + fail("entry " + name + " is not a tensor record (dtype, shape, data_offsets)"); + if (!t["dtype"].is_string() || t["dtype"].get() != "BF16") + fail(name + " is " + (t["dtype"].is_string() ? t["dtype"].get() : std::string("of no named dtype")) + + "; every tensor of a vision component is BF16 and this reader converts nothing else"); + Tensor x; + std::vector off; + try { + x.shape = t["shape"].get(); + off = t["data_offsets"].get>(); + } catch (const json::exception&) { + fail(name + " has a shape or data_offsets that is not a list of integers"); + } + if (off.size() != 2 || off[1] < off[0] || off[1] > payload_) + fail(name + " has data_offsets outside the payload (" + std::to_string(payload_) + " B)"); + uint64_t n = 0; + if (!elements_of(x.shape, &n)) fail(name + " has shape " + shape_text(x.shape) + ", which is not a size"); + x.bytes = off[1] - off[0]; + if (x.bytes != n * 2) + fail(name + " spans " + std::to_string(x.bytes) + " B but its shape " + shape_text(x.shape) + " is " + + std::to_string(n * 2) + " B of BF16"); + x.offset = off[0]; + x.data = base + off[0]; + tensors_.emplace(name, std::move(x)); + } + if (tensors_.empty()) fail("holds no tensors"); + + // ── no two tensors share a byte ── + { + std::vector> order; + for (const auto& [name, t] : tensors_) order.push_back({t.offset, &name}); + std::sort(order.begin(), order.end()); + uint64_t end = 0, covered = 0; + const std::string* prev = nullptr; + for (const auto& [off, name] : order) { + if (prev && off < end) fail(*name + " overlaps " + *prev + " in the payload"); + end = off + tensors_.at(*name).bytes; + covered += tensors_.at(*name).bytes; + prev = name; + } + unaccounted_ = payload_ - covered; + } + + // ── the geometry, from the shapes ── + const auto need = [&](const std::string& suffix) -> const Tensor& { + const auto it = tensors_.find(kPrefix + suffix); + if (it == tensors_.end()) fail("has no tensor " + std::string(kPrefix) + suffix); + return it->second; + }; + const Shape& pe = need("patch_embed.proj.weight").shape; + if (pe.size() != 4 || pe[2] != pe[3]) + fail("model.visual.patch_embed.proj.weight is " + shape_text(pe) + ", not [hidden, channels, patch, patch]"); + geom_.hidden = as_int(pe[0]); + geom_.channels = as_int(pe[1]); + geom_.patch = as_int(pe[2]); + const int64_t H = geom_.hidden; + + int max_block = -1; + std::vector seen; + const std::string bp = std::string(kPrefix) + "blocks."; + for (const auto& [name, t] : tensors_) { + if (name.compare(0, bp.size(), bp) != 0) continue; + const size_t dot = name.find('.', bp.size()); + const std::string idx = name.substr(bp.size(), dot == std::string::npos ? std::string::npos : dot - bp.size()); + if (dot == std::string::npos || idx.empty() || idx.size() > 4 || idx.find_first_not_of("0123456789") != std::string::npos) + fail(name + " does not name a block index"); + const int i = std::stoi(idx); + if (i >= int(seen.size())) seen.resize(size_t(i) + 1, 0); + seen[size_t(i)] = 1; + max_block = std::max(max_block, i); + } + if (max_block < 0) fail("has no model.visual.blocks.* tensors"); + for (int i = 0; i <= max_block; ++i) + if (!seen[size_t(i)]) fail("has block " + std::to_string(max_block) + " but no block " + std::to_string(i)); + geom_.layers = max_block + 1; + + const Shape& fc1 = need("blocks.0.mlp.linear_fc1.weight").shape; + if (fc1.size() != 2) fail("model.visual.blocks.0.mlp.linear_fc1.weight is " + shape_text(fc1) + ", not a matrix"); + geom_.intermediate = as_int(fc1[0]); + const int64_t I = geom_.intermediate; + + const Shape& pos = need("pos_embed.weight").shape; + if (pos.size() != 2) fail("model.visual.pos_embed.weight is " + shape_text(pos) + ", not [positions, hidden]"); + geom_.pos_rows = as_int(pos[0]); + for (int g = 1; int64_t(g) * g <= geom_.pos_rows; ++g) + if (int64_t(g) * g == geom_.pos_rows) geom_.grid = g; + if (geom_.grid == 0) + fail("model.visual.pos_embed.weight has " + std::to_string(pos[0]) + " rows, which is not a square patch grid"); + + geom_.merge_hidden = int(4 * H); + const int64_t M = geom_.merge_hidden; + const Shape& vfc1 = need("vit_merger.mlp.linear_fc1.weight").shape; + if (vfc1.size() != 2) fail("model.visual.vit_merger.mlp.linear_fc1.weight is " + shape_text(vfc1) + ", not a matrix"); + geom_.vit_merger_intermediate = as_int(vfc1[0]); + const int64_t VI = geom_.vit_merger_intermediate; + const Shape& mfc2 = need("merger.linear_fc2.weight").shape; + if (mfc2.size() != 2) fail("model.visual.merger.linear_fc2.weight is " + shape_text(mfc2) + ", not a matrix"); + geom_.out_hidden = as_int(mfc2[0]); + const int64_t O = geom_.out_hidden; + if (!geom_.hidden || !geom_.channels || !geom_.patch || !geom_.intermediate || !geom_.vit_merger_intermediate || !geom_.out_hidden) + fail("a dimension of the tower is zero or too large to be one"); + + // ── every tensor the tower has, with the shape that geometry gives it; and nothing else in the file ── + std::map want; + const auto add = [&](const std::string& suffix, Shape s) { want[kPrefix + suffix] = std::move(s); }; + const auto linear = [&](const std::string& stem, int64_t rows, int64_t cols) { + add(stem + ".weight", {rows, cols}); + add(stem + ".bias", {rows}); + }; + const auto norm = [&](const std::string& stem, int64_t n) { + add(stem + ".weight", {n}); + add(stem + ".bias", {n}); + }; + add("patch_embed.proj.weight", {H, geom_.channels, geom_.patch, geom_.patch}); + add("patch_embed.proj.bias", {H}); + add("pos_embed.weight", {geom_.pos_rows, H}); + for (int i = 0; i < geom_.layers; ++i) { + const std::string b = "blocks." + std::to_string(i) + "."; + norm(b + "norm1", H); + linear(b + "attn.qkv", 3 * H, H); + linear(b + "attn.proj", H, H); + norm(b + "norm2", H); + linear(b + "mlp.linear_fc1", I, H); + linear(b + "mlp.linear_fc2", H, I); + } + norm("vit_merger.norm1", H); + linear("vit_merger.attn.qkv", 3 * H, H); + linear("vit_merger.attn.proj", H, H); + norm("vit_merger.pre_norm", M); + linear("vit_merger.mlp.linear_fc1", VI, M); + linear("vit_merger.mlp.linear_fc2", H, VI); + norm("merger.norm", H); + norm("merger.pre_norm", M); + linear("merger.linear_fc1", M, M); + linear("merger.linear_fc2", O, M); + + for (const auto& [name, shape] : want) { + const auto it = tensors_.find(name); + if (it == tensors_.end()) fail("has no tensor " + name); + if (it->second.shape != shape) + fail(name + " is " + shape_text(it->second.shape) + " where the tower's other tensors make it " + shape_text(shape)); + } + for (const auto& [name, t] : tensors_) + if (!want.count(name)) fail("holds " + name + " " + shape_text(t.shape) + ", a tensor this reader has no place for"); + + // ── config.json: what the tensors cannot say, and what ties the tower to something outside it ── + geom_.model_intermediate = geom_.intermediate; + geom_.model_vit_merger_intermediate = geom_.vit_merger_intermediate; + if (config_json.empty()) return; + const json c = json::parse(config_json, nullptr, false); + if (c.is_discarded() || !c.is_object()) fail("the config beside it is not a JSON object"); + const json vc = c.contains("vision_config") && c["vision_config"].is_object() ? c["vision_config"] : json::object(); + const auto field = [&](const char* key, int* out) { + const std::string k = std::string("MINICPM_V_") + key; + if (!vc.contains(k) || !vc[k].is_number_integer()) return false; + *out = as_int(vc[k].get()); + return true; + }; + const auto refuse = [&](const char* key, int stated, const std::string& held) { + fail("config.json states MINICPM_V_" + std::string(key) + " " + std::to_string(stated) + " but " + held); + }; + int v = 0; + if (field("VISION_EMBED_DIM", &v) && v != geom_.hidden) + refuse("VISION_EMBED_DIM", v, "the patch embedding is " + std::to_string(geom_.hidden) + " wide"); + if (field("VISION_NUM_LAYERS", &v) && v != geom_.layers) + refuse("VISION_NUM_LAYERS", v, "the container holds " + std::to_string(geom_.layers) + " blocks"); + if (field("PATCH_SIZE", &v) && v != geom_.patch) + refuse("PATCH_SIZE", v, "the patch embedding's kernel is " + std::to_string(geom_.patch)); + if (field("VISION_NUM_POSITION_EMBEDDINGS", &v) && v != geom_.pos_rows) + refuse("VISION_NUM_POSITION_EMBEDDINGS", v, "the position table has " + std::to_string(geom_.pos_rows) + " rows"); + if (field("VISION_OUT_HIDDEN_SIZE", &v) && v != geom_.out_hidden) + refuse("VISION_OUT_HIDDEN_SIZE", v, "the merger projects to " + std::to_string(geom_.out_hidden)); + if (field("MERGER_HIDDEN_SIZE", &v) && v != geom_.merge_hidden) + refuse("MERGER_HIDDEN_SIZE", v, "four tokens of this tower are " + std::to_string(geom_.merge_hidden) + " wide"); + if (field("IMAGE_SIZE", &v)) { + geom_.image_size = v; + if (v != geom_.grid * geom_.patch) + refuse("IMAGE_SIZE", v, "the position table is a " + std::to_string(geom_.grid) + " x " + std::to_string(geom_.grid) + + " grid of " + std::to_string(geom_.patch) + "-pixel patches (" + + std::to_string(geom_.grid * geom_.patch) + " pixels)"); + } + int heads = 0, head_dim = 0; + const bool have_heads = field("VISION_NUM_HEADS", &heads), have_dim = field("VISION_HEAD_DIM", &head_dim); + if (have_heads && !have_dim && heads > 0 && geom_.hidden % heads == 0) head_dim = geom_.hidden / heads; + if (have_heads) { + if (heads <= 0 || int64_t(heads) * head_dim != H) + refuse("VISION_NUM_HEADS", heads, "head_dim " + std::to_string(head_dim) + " times that is not the tower's width " + + std::to_string(geom_.hidden)); + geom_.heads = heads; + geom_.head_dim = head_dim; + } + if (c.contains("hidden_size") && c["hidden_size"].is_number_integer() && c["hidden_size"].get() != O) + fail("config.json states the text model's hidden_size " + std::to_string(c["hidden_size"].get()) + + " but the merger projects to " + std::to_string(geom_.out_hidden) + ": this tower is not this text model's"); + + // An MLP's stated width against its tensors. `stems` are the MLPs that share the width (every block, or the + // vit_merger alone): fc1 is [width, in] with a [width] bias, fc2 is [out, width]. + const auto zero_from = [&](const std::string& name, int64_t rows, int64_t cols, int64_t from, bool by_row) { + const Tensor& t = tensors_.at(kPrefix + name); + const auto is_zero = [&](int64_t i) { + uint16_t h; + std::memcpy(&h, t.data + 2 * i, 2); + return (h & 0x7fff) == 0; + }; + if (by_row) { + for (int64_t i = from * cols; i < rows * cols; ++i) + if (!is_zero(i)) return false; + } else { + for (int64_t r = 0; r < rows; ++r) + for (int64_t c = from; c < cols; ++c) + if (!is_zero(r * cols + c)) return false; + } + return true; + }; + const auto mlp_width = [&](const char* key, int held, const std::vector& stems, int64_t in, int64_t out, + const char* tensor, int* model) { + int stated = 0; + *model = held; + if (!field(key, &stated) || stated == held) return; + if (stated > held) + refuse(key, stated, std::string(tensor) + " has only " + std::to_string(held) + " rows"); + bool pad = true; + for (const std::string& st : stems) + pad = pad && zero_from(st + ".linear_fc1.weight", held, in, stated, true) && + zero_from(st + ".linear_fc1.bias", held, 1, stated, true) && + zero_from(st + ".linear_fc2.weight", out, held, stated, false); + const std::string head = "config.json states MINICPM_V_" + std::string(key) + " " + std::to_string(stated) + "; " + tensor + + " has " + std::to_string(held) + " rows"; + if (pad) { + *model = stated; + notes_.push_back(head + ", the last " + std::to_string(held - stated) + + " all zero: the container pads the model's width, and sizes come from the tensor"); + } else { + notes_.push_back(head + " and the rows past the stated width are not zero: this is not padding, the tensor's " + "width is the model's"); + } + }; + std::vector block_mlps; + for (int i = 0; i < geom_.layers; ++i) block_mlps.push_back("blocks." + std::to_string(i) + ".mlp"); + mlp_width("VISION_MLP_INTERMEDIATE_SIZE", geom_.intermediate, block_mlps, H, H, + "model.visual.blocks.*.mlp.linear_fc1.weight", &geom_.model_intermediate); + mlp_width("VIT_MERGER_INTERMEDIATE_SIZE", geom_.vit_merger_intermediate, {"vit_merger.mlp"}, M, H, + "model.visual.vit_merger.mlp.linear_fc1.weight", &geom_.model_vit_merger_intermediate); +} + +const Tensor& Pack::tensor(const std::string& name) const { + const auto it = tensors_.find(name); + if (it == tensors_.end()) throw std::runtime_error(path_ + ": has no tensor " + name); + return it->second; +} + +std::vector Pack::f32(const std::string& name) const { + const Tensor& t = tensor(name); + std::vector out(static_cast(t.elements())); + for (size_t i = 0; i < out.size(); ++i) { + uint16_t h = 0; + std::memcpy(&h, t.data + 2 * i, 2); + const uint32_t bits = uint32_t(h) << 16; + std::memcpy(&out[i], &bits, 4); + } + return out; +} + +std::string Pack::metadata(const std::string& key) const { + const auto it = meta_.find(key); + return it == meta_.end() ? std::string() : it->second; +} + +int print_info(const std::string& pack_dir) { + const Pack p(pack_dir); + const Geometry& g = p.geometry(); + std::printf("vision component %s\n", p.path().c_str()); + std::printf("tensors %zu, all BF16, %llu B of payload, %llu B unaccounted\n", p.tensors().size(), + (unsigned long long)p.payload_bytes(), (unsigned long long)p.unaccounted_bytes()); + std::printf("patch embed %d x %d pixels x %d channels -> %d\n", g.patch, g.patch, g.channels, g.hidden); + std::printf("positions %d (a %d x %d grid; image %d)\n", g.pos_rows, g.grid, g.grid, g.grid * g.patch); + std::printf("blocks %d at width %d, %d heads x %d, MLP %d (%d stored)\n", g.layers, g.hidden, g.heads, g.head_dim, + g.model_intermediate, g.intermediate); + std::printf("vit_merger attention at %d, MLP %d -> %d (%d stored) -> %d\n", g.hidden, g.merge_hidden, + g.model_vit_merger_intermediate, g.vit_merger_intermediate, g.hidden); + std::printf("merger %d -> %d -> %d (the text model's hidden size)\n", g.merge_hidden, g.merge_hidden, g.out_hidden); + for (const char* k : {"source_file", "source_sha256", "source_dtype", "converter"}) + if (!p.metadata(k).empty()) std::printf("%-17s %s\n", k, p.metadata(k).c_str()); + for (const std::string& n : p.notes()) std::printf("note %s\n", n.c_str()); + return 0; +} + +} // namespace onebit::npu::vision diff --git a/npu/vision_pack.h b/npu/vision_pack.h new file mode 100644 index 000000000..951c16d5f --- /dev/null +++ b/npu/vision_pack.h @@ -0,0 +1,115 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The vision component of a two-component pack (engine#1788): `vision_weight.q4nx` beside `model.q4nx`. +// +// FastFlowLM's MiniCPM-V 4.7 packs ship the vision tower as its own container at the pack root. It is the same +// container as model.q4nx (8-byte little-endian length, a JSON table keyed by tensor name plus `__metadata__`, then +// the payload that `data_offsets` index), holding a different kind of content: a ViT, every tensor BF16, no tiles. +// +// What this reader is for: the forward that will run the tower must never size a buffer from config.json. Both packs +// on the catalogue state `MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE 4304` and `MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE +// 17216` while their tensors are 4608 and 17280 rows. The config is the MODEL's width (the source GGUF's ffn tensors +// are 4304 and 17216 wide); the container stores each MLP padded with all-zero rows to a multiple of 128, measured +// bit for bit against both packs' sources. A loader that sized a buffer from the config would read 304 rows short of +// what the file holds and misplace everything after it. So the geometry here is DERIVED from the tensors' own shapes, +// every tensor in the file has to be one this reader can place (zero gaps), and the config is only consulted for what +// the tensors cannot say (the head count, the image size, the unpadded MLP widths) and for the widths that tie the +// tower to something outside it (the patch grid and the text model's hidden size). +// +// Two kinds of disagreement with config.json, on purpose: +// refused the head count does not divide the width, the patch grid is not the position table, the tower's output +// is not the text model's hidden size, the width / block count differ, or an MLP is stated WIDER than its +// tensors: the files do not belong together or the container is cut short, and no forward can be right. +// noted an MLP is stated narrower than its tensors. The tensor decides every size. The rows past the stated +// width are checked: all zero in every block means padding (the published packs), and `model_*` below +// carry the stated width; anything else means the config is simply wrong and `model_*` stay the tensors'. +#pragma once + +#include +#include +#include +#include + +namespace onebit::npu::vision { + +struct Tensor { + std::vector shape; + const uint8_t* data = nullptr; // BF16, little-endian, row-major + uint64_t bytes = 0; + uint64_t offset = 0; // from the payload start + uint64_t elements() const { return bytes / 2; } +}; + +// Everything below is read from tensor shapes, except heads / head_dim / image_size (config.json, 0 when no config +// was given). +struct Geometry { + int hidden = 0; // patch_embed.proj.weight rows + int channels = 0, patch = 0; // patch_embed.proj.weight [hidden, channels, patch, patch] + int layers = 0; // blocks 0..layers-1, all present + int intermediate = 0; // blocks.*.mlp.linear_fc1 rows, the same in every block + int pos_rows = 0, grid = 0; // pos_embed.weight rows, and its square root (patches per side) + int merge_hidden = 0; // 4 * hidden: one 2x2 group of tokens, the width both mergers work at + int vit_merger_intermediate = 0; // vit_merger.mlp.linear_fc1 rows + int out_hidden = 0; // merger.linear_fc2 rows: the text model's hidden size + int heads = 0, head_dim = 0, image_size = 0; + // The MLP widths without the container's zero padding: config.json's value when it is smaller than the tensors' + // and every row past it is zero in every block, else the tensors' own. A forward may stop at these; it must + // still index the tensors at `intermediate` / `vit_merger_intermediate`. + int model_intermediate = 0, model_vit_merger_intermediate = 0; +}; + +class Pack { +public: + // `pack_dir` holds config.json and the container config.json names (`vision_model_weight`, default + // vision_weight.q4nx). Throws std::runtime_error naming the file and the tensor or field on every refusal. + explicit Pack(const std::string& pack_dir); + // The container alone, with config.json's text ("" for none: nothing is cross-checked and heads stay 0). + Pack(const std::string& container_path, const std::string& config_json); + ~Pack(); + Pack(const Pack&) = delete; + Pack& operator=(const Pack&) = delete; + + const Geometry& geometry() const { return geom_; } + const std::map& tensors() const { return tensors_; } + const Tensor& tensor(const std::string& name) const; // throws when absent + // The tensor as float32 (a BF16 value is the top half of a float32, so this is exact). + std::vector f32(const std::string& name) const; + // Config fields the tensors overruled, one sentence each; empty when the config agrees or was not given. + const std::vector& notes() const { return notes_; } + // `__metadata__[key]` as text, "" when absent. + std::string metadata(const std::string& key) const; + uint64_t payload_bytes() const { return payload_; } + // Payload bytes no tensor covers. 0 on both published packs. + uint64_t unaccounted_bytes() const { return unaccounted_; } + const std::string& path() const { return path_; } + +private: + void open(const std::string& container_path, const std::string& config_json); + const uint8_t* map_ = nullptr; + size_t size_ = 0; + std::string path_; + std::map tensors_; + std::map meta_; + std::vector notes_; + Geometry geom_; + uint64_t payload_ = 0, unaccounted_ = 0; +}; + +// `1bit npu-pack vision `: the geometry, the tensor count, the provenance and the notes, on stdout. +// Returns 0, or throws what Pack throws. +int print_info(const std::string& pack_dir); + +} // namespace onebit::npu::vision diff --git a/tests/npu_vision_pack_test.cpp b/tests/npu_vision_pack_test.cpp new file mode 100644 index 000000000..9d2fbeb5c --- /dev/null +++ b/tests/npu_vision_pack_test.cpp @@ -0,0 +1,455 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_vision_pack_test +// +// The vision component reader (npu/vision_pack.h, engine#1788). The container is written here, small, with the same +// tensor names and the same shape relations as FastFlowLM's MiniCPM-V 4.7 `vision_weight.q4nx`, so each arm is one +// edit to a container that otherwise loads: +// - it loads, and the geometry is the one the tensors hold; +// - an MLP the container pads with zero rows (the published packs: the model is 4304 wide, 4608 are stored) is sized +// from the tensor, with the model's width carried beside it and the padding verified to be zero; +// - every way the file can disagree with itself, or with the config in a way no forward survives, is refused with +// the tensor or field named. +// When ONEBIT_MINICPM_PACK names a published pack directory, its own numbers are checked too. +#include "../npu/vision_pack.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using nlohmann::json; +namespace vis = onebit::npu::vision; + +int g_fail = 0, g_arms = 0; +void check(bool ok, const std::string& what) { + ++g_arms; + if (!ok) { + ++g_fail; + std::printf("FAIL %s\n", what.c_str()); + } +} + +// The small tower: width 8 (2 heads x 4), 3 channels, 2-pixel patches, a 4 x 4 grid (image 8), 2 blocks with MLP 32, +// vit_merger MLP 64, projecting to a text model 6 wide. +struct Small { + int H = 8, C = 3, P = 2, pos = 16, L = 2, I = 32, VI = 64, O = 6; +}; + +uint16_t bf16(float v) { + uint32_t b; + std::memcpy(&b, &v, 4); + return uint16_t(b >> 16); +} + +// name -> shape, in the order the payload is laid out +std::vector>> layout(const Small& s) { + std::vector>> v; + const std::string p = "model.visual."; + const auto lin = [&](const std::string& n, int64_t r, int64_t c) { + v.push_back({p + n + ".bias", {r}}); + v.push_back({p + n + ".weight", {r, c}}); + }; + const auto nrm = [&](const std::string& n, int64_t k) { + v.push_back({p + n + ".bias", {k}}); + v.push_back({p + n + ".weight", {k}}); + }; + const int64_t H = s.H, M = 4 * s.H; + for (int i = 0; i < s.L; ++i) { + const std::string b = "blocks." + std::to_string(i) + "."; + lin(b + "attn.proj", H, H); + lin(b + "attn.qkv", 3 * H, H); + lin(b + "mlp.linear_fc1", s.I, H); + lin(b + "mlp.linear_fc2", H, s.I); + nrm(b + "norm1", H); + nrm(b + "norm2", H); + } + lin("merger.linear_fc1", M, M); + lin("merger.linear_fc2", s.O, M); + nrm("merger.norm", H); + nrm("merger.pre_norm", M); + v.push_back({p + "patch_embed.proj.bias", {H}}); + v.push_back({p + "patch_embed.proj.weight", {H, s.C, s.P, s.P}}); + v.push_back({p + "pos_embed.weight", {s.pos, H}}); + lin("vit_merger.attn.proj", H, H); + lin("vit_merger.attn.qkv", 3 * H, H); + lin("vit_merger.mlp.linear_fc1", s.VI, M); + lin("vit_merger.mlp.linear_fc2", H, s.VI); + nrm("vit_merger.norm1", H); + nrm("vit_merger.pre_norm", M); + return v; +} + +json config_of(const Small& s) { + json vc; + vc["MINICPM_V_PATCH_SIZE"] = s.P; + vc["MINICPM_V_IMAGE_SIZE"] = 8; + vc["MINICPM_V_VISION_EMBED_DIM"] = s.H; + vc["MINICPM_V_VISION_NUM_HEADS"] = 2; + vc["MINICPM_V_VISION_HEAD_DIM"] = 4; + vc["MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE"] = s.I; + vc["MINICPM_V_VISION_NUM_POSITION_EMBEDDINGS"] = s.pos; + vc["MINICPM_V_VISION_NUM_LAYERS"] = s.L; + vc["MINICPM_V_VISION_OUT_HIDDEN_SIZE"] = s.O; + vc["MINICPM_V_MERGER_HIDDEN_SIZE"] = 4 * s.H; + vc["MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE"] = s.VI; + json c; + c["hidden_size"] = s.O; + c["vision_model_weight"] = "vision_weight.q4nx"; + c["vision_config"] = vc; + return c; +} + +// Writes the container. `edit` sees the table (and may change the payload size) before it is serialised. Element e of +// the tensor at payload offset `off` holds bf16(0.25 * ((off/2 + e) % 64) - 4): readable back by position. +void write(const std::string& path, const Small& s, const std::function&)>& edit = {}) { + json table = json::object(); + std::vector payload; + for (const auto& [name, shape] : layout(s)) { + uint64_t n = 1; + for (int64_t d : shape) n *= uint64_t(d); + const uint64_t off = payload.size(); + for (uint64_t e = 0; e < n; ++e) { + const uint16_t h = bf16(0.25f * float((off / 2 + e) % 64) - 4.0f); + payload.push_back(uint8_t(h & 0xff)); + payload.push_back(uint8_t(h >> 8)); + } + table[name] = {{"dtype", "BF16"}, {"shape", shape}, {"data_offsets", {off, off + n * 2}}}; + } + table["__metadata__"] = {{"source_file", "crafted.gguf"}, {"converter", "npu_vision_pack_test"}}; + if (edit) edit(table, payload); + const std::string head = table.dump(); + const uint64_t len = head.size(); + std::ofstream f(path, std::ios::binary | std::ios::trunc); + f.write(reinterpret_cast(&len), 8); + f.write(head.data(), std::streamsize(head.size())); + f.write(reinterpret_cast(payload.data()), std::streamsize(payload.size())); +} + +const char* const kPath = "npu_vision_pack_test.q4nx"; + +// Loads the crafted file; "" when it loads, else what it threw. +std::string load(const std::string& config) { + try { + const vis::Pack p(kPath, config); + return ""; + } catch (const std::exception& e) { + return e.what(); + } +} + +void refused(const std::string& arm, const std::string& config, const std::vector& must_name) { + const std::string why = load(config); + check(!why.empty(), arm + ": refused"); + for (const std::string& m : must_name) + check(why.find(m) != std::string::npos, arm + ": the refusal names \"" + m + "\" (it says: " + why + ")"); +} + +void test_loads() { + const Small s; + write(kPath, s); + const std::string cfg = config_of(s).dump(); + try { + const vis::Pack p(kPath, cfg); + const vis::Geometry& g = p.geometry(); + check(g.hidden == 8 && g.channels == 3 && g.patch == 2, "geometry: the patch embedding's own shape"); + check(g.layers == 2 && g.intermediate == 32, "geometry: two blocks, MLP 32"); + check(g.pos_rows == 16 && g.grid == 4, "geometry: 16 positions are a 4 x 4 grid"); + check(g.merge_hidden == 32 && g.vit_merger_intermediate == 64 && g.out_hidden == 6, "geometry: the two mergers"); + check(g.heads == 2 && g.head_dim == 4 && g.image_size == 8, "geometry: heads and image size come from the config"); + check(p.tensors().size() == size_t(2 * 12 + 23), "every tensor is placed: 12 per block and 23 outside them"); + check(p.unaccounted_bytes() == 0, "no payload byte is outside a tensor"); + check(p.notes().empty(), "a config that agrees leaves no note"); + check(p.metadata("source_file") == "crafted.gguf" && p.metadata("absent").empty(), "metadata is read by key"); + // exact values back, by position: the first tensor starts at payload offset 0 + const std::vector b = p.f32("model.visual.blocks.0.attn.proj.bias"); + bool ok = b.size() == 8; + for (size_t e = 0; ok && e < b.size(); ++e) ok = b[e] == 0.25f * float(e % 64) - 4.0f; + check(ok, "f32(): BF16 values come back exactly"); + const vis::Tensor& t = p.tensor("model.visual.pos_embed.weight"); + const std::vector pos = p.f32("model.visual.pos_embed.weight"); + check(pos.size() == 16 * 8 && pos[3] == 0.25f * float((t.offset / 2 + 3) % 64) - 4.0f, + "f32(): a tensor in the middle of the payload reads from its own offset"); + } catch (const std::exception& e) { + check(false, std::string("the crafted container loads (threw: ") + e.what() + ")"); + } + // with no config nothing is cross-checked and the config-only fields stay 0 + try { + const vis::Pack p(kPath, ""); + check(p.geometry().hidden == 8 && p.geometry().heads == 0 && p.geometry().image_size == 0, + "no config: the tensor geometry stands, heads and image size are unknown"); + } catch (const std::exception& e) { + check(false, std::string("loads without a config (threw: ") + e.what() + ")"); + } +} + +// Zeroes the rows of an MLP past `from`: fc1 weight rows and bias entries, fc2 weight columns. This is what the +// published packs hold past the model's width. +void zero_mlp_tail(json& t, std::vector& p, const std::string& stem, int from) { + const auto span = [&](const std::string& n, uint64_t* off, std::vector* shape) { + *off = t[n]["data_offsets"][0]; + *shape = t[n]["shape"].get>(); + }; + uint64_t off; + std::vector sh; + span(stem + ".linear_fc1.weight", &off, &sh); + for (int64_t i = int64_t(from) * sh[1]; i < sh[0] * sh[1]; ++i) p[off + 2 * i] = p[off + 2 * i + 1] = 0; + span(stem + ".linear_fc1.bias", &off, &sh); + for (int64_t i = from; i < sh[0]; ++i) p[off + 2 * i] = p[off + 2 * i + 1] = 0; + span(stem + ".linear_fc2.weight", &off, &sh); + for (int64_t r = 0; r < sh[0]; ++r) + for (int64_t c = from; c < sh[1]; ++c) p[off + 2 * (r * sh[1] + c)] = p[off + 2 * (r * sh[1] + c) + 1] = 0; +} + +// The published packs' situation: the config states the model's MLP widths (4304, 17216) and the container stores them +// padded with zero rows (4608, 17280). Every size is the tensor's; the model's width is carried beside it. +void test_padded_mlp_widths() { + const Small s; + json c = config_of(s); + c["vision_config"]["MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE"] = 30; + c["vision_config"]["MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE"] = 60; + const auto pad_all = [&](json& t, std::vector& p) { + zero_mlp_tail(t, p, "model.visual.blocks.0.mlp", 30); + zero_mlp_tail(t, p, "model.visual.blocks.1.mlp", 30); + zero_mlp_tail(t, p, "model.visual.vit_merger.mlp", 60); + }; + write(kPath, s, pad_all); + try { + const vis::Pack p(kPath, c.dump()); + const vis::Geometry& g = p.geometry(); + check(g.intermediate == 32 && g.vit_merger_intermediate == 64, "padded MLP: both stored widths are the tensors'"); + check(g.model_intermediate == 30 && g.model_vit_merger_intermediate == 60, + "padded MLP: the model's widths are the config's, because every row past them is zero"); + check(p.notes().size() == 2, "padded MLP: one note per padded width"); + const std::string all = p.notes().size() == 2 ? p.notes()[0] + p.notes()[1] : ""; + check(all.find("VISION_MLP_INTERMEDIATE_SIZE 30") != std::string::npos && all.find("32 rows, the last 2 all zero") != std::string::npos, + "padded MLP: the note states the config's value, the tensor's, and that the difference is zero rows"); + check(all.find("VIT_MERGER_INTERMEDIATE_SIZE 60") != std::string::npos && all.find("64 rows, the last 4 all zero") != std::string::npos, + "padded MLP: and the same for the vit_merger"); + } catch (const std::exception& e) { + check(false, std::string("a container that pads its MLPs loads (threw: ") + e.what() + ")"); + } + // one nonzero value past the stated width, in ONE block's fc2: that is not padding, and it is said so + write(kPath, s, [&](json& t, std::vector& p) { + pad_all(t, p); + const uint64_t off = t["model.visual.blocks.1.mlp.linear_fc2.weight"]["data_offsets"][0]; + const uint64_t e = uint64_t(7) * 32 + 31; // last row, last column + p[off + 2 * e + 1] = 0x3f; // bf16 high byte: a nonzero value + }); + try { + const vis::Pack p(kPath, c.dump()); + check(p.geometry().model_intermediate == 32, "not padding: one nonzero value past the stated width keeps the tensor's width"); + check(p.geometry().model_vit_merger_intermediate == 60, "not padding: the vit_merger, still zero past 60, keeps the config's"); + const std::string all = p.notes().size() == 2 ? p.notes()[0] + p.notes()[1] : ""; + check(all.find("are not zero: this is not padding") != std::string::npos, "not padding: the note says so"); + } catch (const std::exception& e) { + check(false, std::string("a config narrower than nonzero tensors still loads (threw: ") + e.what() + ")"); + } + // without a config the model's widths are the stored ones + try { + const vis::Pack p(kPath, ""); + check(p.geometry().model_intermediate == 32 && p.geometry().model_vit_merger_intermediate == 64, + "no config: the model's widths are the stored widths"); + } catch (const std::exception& e) { + check(false, std::string("loads without a config (threw: ") + e.what() + ")"); + } + // a config that states an MLP WIDER than its tensors cannot be padding + json wide = config_of(s); + wide["vision_config"]["MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE"] = 40; + write(kPath, s); + refused("config: an MLP wider than its tensors", wide.dump(), {"VISION_MLP_INTERMEDIATE_SIZE 40", "only 32 rows"}); +} + +void test_refusals() { + const Small s; + const std::string cfg = config_of(s).dump(); + const std::string qkv0 = "model.visual.blocks.0.attn.qkv.weight"; + + write(kPath, s, [&](json& t, std::vector&) { t[qkv0]["dtype"] = "F32"; }); + refused("a tensor that is not BF16", cfg, {qkv0, "F32"}); + + write(kPath, s, [&](json& t, std::vector&) { t[qkv0]["shape"] = {24, 9}; }); + refused("a shape that is not the tensor's byte length", cfg, {qkv0, "[24, 9]"}); + + // the shape the tower expects, over one element fewer bytes: a forward would read the last element out of the + // NEXT tensor. Only the extent-against-shape check can see this one. + write(kPath, s, [&](json& t, std::vector&) { + const uint64_t a = t[qkv0]["data_offsets"][0], b = t[qkv0]["data_offsets"][1]; + t[qkv0]["data_offsets"] = {a, b - 2}; + }); + refused("an extent shorter than the shape", cfg, {qkv0, "spans 382 B", "384 B of BF16"}); + + write(kPath, s, [&](json& t, std::vector& p) { + const uint64_t a = t[qkv0]["data_offsets"][0]; + t[qkv0]["data_offsets"] = {a + p.size(), a + p.size() + 384}; + }); + refused("offsets past the payload", cfg, {qkv0, "outside the payload"}); + + write(kPath, s, [&](json& t, std::vector&) { + // norm2.bias moved onto norm1.bias: same length, so only the overlap check can see it + t["model.visual.blocks.0.norm2.bias"]["data_offsets"] = t["model.visual.blocks.0.norm1.bias"]["data_offsets"]; + }); + refused("two tensors on the same bytes", cfg, {"overlaps", "model.visual.blocks.0.norm"}); + + write(kPath, s, [&](json& t, std::vector&) { t.erase("model.visual.blocks.1.norm2.bias"); }); + refused("a missing tensor", cfg, {"has no tensor model.visual.blocks.1.norm2.bias"}); + + write(kPath, s, [&](json& t, std::vector&) { + json moved = json::object(); + for (auto it = t.begin(); it != t.end(); ++it) { + std::string k = it.key(); + const std::string from = "model.visual.blocks.1."; + if (k.compare(0, from.size(), from) == 0) k = "model.visual.blocks.2." + k.substr(from.size()); + moved[k] = it.value(); + } + t = moved; + }); + refused("a hole in the block indices", cfg, {"block 2", "no block 1"}); + + write(kPath, s, [&](json& t, std::vector& p) { + const uint64_t off = p.size(); + p.resize(p.size() + 16, 0); + t["model.visual.blocks.0.attn.gate.weight"] = {{"dtype", "BF16"}, {"shape", {8}}, {"data_offsets", {off, off + 16}}}; + }); + refused("a tensor the tower has no place for", cfg, {"model.visual.blocks.0.attn.gate.weight", "no place"}); + + // one block with a different MLP width: its own extents are consistent, so only the cross-block shape check sees it + write(kPath, s, [&](json& t, std::vector& p) { + const auto grow = [&](const std::string& n, std::vector shape) { + uint64_t k = 2; + for (int64_t d : shape) k *= uint64_t(d); + const uint64_t off = p.size(); + p.resize(p.size() + k, 0); + t[n] = {{"dtype", "BF16"}, {"shape", shape}, {"data_offsets", {off, off + k}}}; + }; + grow("model.visual.blocks.1.mlp.linear_fc1.weight", {40, 8}); + grow("model.visual.blocks.1.mlp.linear_fc1.bias", {40}); + grow("model.visual.blocks.1.mlp.linear_fc2.weight", {8, 40}); + }); + refused("a block whose MLP is another width", "", {"model.visual.blocks.1.mlp.linear_fc1", "[32"}); + + { + Small odd; + odd.pos = 15; + write(kPath, odd); + refused("a position table that is not a square grid", "", {"pos_embed", "15 rows"}); + } + write(kPath, s, [&](json&, std::vector&) {}); + { + std::ofstream f(kPath, std::ios::binary | std::ios::trunc); + const uint64_t len = 1 << 20; + f.write(reinterpret_cast(&len), 8); + f.write("{}", 2); + } + refused("a header length past the end of the file", cfg, {"header runs past the end"}); + + // ── the config against the tensors: refused where no forward survives it ── + write(kPath, s); + const auto with = [&](const char* key, int v) { + json c = config_of(s); + c["vision_config"][std::string("MINICPM_V_") + key] = v; + return c.dump(); + }; + refused("config: heads that do not make the width", with("VISION_NUM_HEADS", 3), {"VISION_NUM_HEADS 3", "width 8"}); + refused("config: another width", with("VISION_EMBED_DIM", 16), {"VISION_EMBED_DIM 16", "8 wide"}); + refused("config: more blocks than the container holds", with("VISION_NUM_LAYERS", 3), {"VISION_NUM_LAYERS 3", "2 blocks"}); + refused("config: another patch size", with("PATCH_SIZE", 4), {"PATCH_SIZE 4"}); + refused("config: an image the position table does not cover", with("IMAGE_SIZE", 12), {"IMAGE_SIZE 12", "4 x 4"}); + refused("config: another position count", with("VISION_NUM_POSITION_EMBEDDINGS", 25), {"NUM_POSITION_EMBEDDINGS 25", "16 rows"}); + { + json c = config_of(s); + c["hidden_size"] = 12; + refused("config: a text model of another width", c.dump(), {"hidden_size 12", "projects to 6"}); + } +} + +// Pack(dir): the container name comes from config.json and must stay inside the pack. +void test_pack_dir_name() { + const Small s; + const std::string dir = "npu_vision_pack_test.dir"; + std::error_code ec; + std::filesystem::create_directories(dir, ec); + write(dir + "/tower.q4nx", s); + json c = config_of(s); + c["vision_model_weight"] = "tower.q4nx"; + { std::ofstream f(dir + "/config.json"); f << c.dump(); } + try { + const vis::Pack p(dir); + check(p.geometry().layers == 2 && p.path() == dir + "/tower.q4nx", "Pack(dir): opens the file config.json names"); + } catch (const std::exception& e) { + check(false, std::string("Pack(dir) opens the named container (threw: ") + e.what() + ")"); + } + c["vision_model_weight"] = "../npu_vision_pack_test.q4nx"; + { std::ofstream f(dir + "/config.json"); f << c.dump(); } + std::string why; + try { const vis::Pack p(dir); } catch (const std::exception& e) { why = e.what(); } + check(why.find("is not a file name") != std::string::npos, "Pack(dir): a name with a separator is refused, not followed"); + std::remove((dir + "/tower.q4nx").c_str()); + std::remove((dir + "/config.json").c_str()); + std::filesystem::remove(dir, ec); +} + +// The published pack, when one is on the box: the numbers engine#1788's thread read by hand. +void test_published_pack(const std::string& dir) { + try { + const vis::Pack p(dir); + const vis::Geometry& g = p.geometry(); + check(p.tensors().size() == 347, "published pack: 347 tensors, all placed"); + check(p.unaccounted_bytes() == 0, "published pack: no payload byte outside a tensor"); + check(g.hidden == 1152 && g.heads == 16 && g.head_dim == 72 && g.layers == 27, "published pack: 27 blocks at 1152, 16 x 72"); + check(g.patch == 14 && g.channels == 3 && g.pos_rows == 4900 && g.grid == 70 && g.image_size == 980, + "published pack: 14-pixel patches on a 70 x 70 grid (980)"); + check(g.intermediate == 4608 && g.vit_merger_intermediate == 17280, "published pack: MLPs stored 4608 and 17280 wide"); + check(g.model_intermediate == 4304 && g.model_vit_merger_intermediate == 17216, + "published pack: the model's MLP widths are 4304 and 17216, the rest verified zero in all 27 blocks"); + check(g.merge_hidden == 4608, "published pack: mergers at 4 x 1152"); + check(p.notes().size() == 2, "published pack: both padded widths are noted"); + // The 1B pack's container names its source; the 35B-A3B's carries no __metadata__ at all. Both are accepted, + // and a source hash that is present has to look like one. + const std::string sha = p.metadata("source_sha256"); + check(sha.empty() || (sha.size() == 64 && sha.find_first_not_of("0123456789abcdef") == std::string::npos), + "published pack: a source hash, when the container carries one, is 64 hex digits"); + std::printf(" provenance: %s\n", sha.empty() ? "none in the container" : (p.metadata("source_file") + " sha256 " + sha).c_str()); + std::printf(" published pack %s: %zu tensors, out_hidden %d, %zu note(s)\n", dir.c_str(), p.tensors().size(), g.out_hidden, + p.notes().size()); + for (const std::string& n : p.notes()) std::printf(" note: %s\n", n.c_str()); + } catch (const std::exception& e) { + check(false, std::string("the published pack loads (threw: ") + e.what() + ")"); + } +} + +} // namespace + +int main() { + test_loads(); + test_padded_mlp_widths(); + test_refusals(); + test_pack_dir_name(); + std::remove(kPath); + if (const char* dir = std::getenv("ONEBIT_MINICPM_PACK"); dir && *dir) test_published_pack(dir); + else std::printf(" (ONEBIT_MINICPM_PACK not set: the published-pack arm did not run)\n"); + std::printf("vision pack: %d checks, %d failed\n", g_arms, g_fail); + return g_fail ? 1 : 0; +} From a4fca85f4fa51c6f8f526ebb4224463ea3f7d77e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:00:58 -0400 Subject: [PATCH 35/98] tests: batch_check_state runs in CI, on the runtime stub and a fabricated container (fixes #1946) (#2039) * tests: batch_check_state runs in CI, on the runtime stub and a fabricated container (engine#1946) batch_check_state_test pins #1909 (the NPU_BATCH_CHECK oracle must restore the prefix it replays from), but it links the NPU forward, whose runtime is private, and it needs a Q4NX container. So it existed only under ONEBIT_NPU and a regression of #1909 could land green. - onebit_npu_forward_library(name runtime): the forward's one source list and link interface, used by onebit_npu_forward (private runtime) and by onebit_npu_forward_ci (onnx/ep/npu_runtime_ci_stub.cpp, whose device calls throw; the host-GEMM forward makes none). - In a configure without ONEBIT_NPU (CI's), batch_check_state_test links the stub-built forward and is built through add_dependencies(onebit ...). tests/batch_check_state_ci.py writes a seeded two-layer qwen3 (hidden 256), packs it with tools/safetensors_to_q4nx.py and runs the test on it: no device, no model, ~3 s. Skips (77) only without numpy. Measured: passes on main; with #1909 reverted it fails 3 of 4 (holds 2 and 1 positions instead of 5 and 6, rel-L1 0.78). The NPU-lane configure still builds its three container rows. ci_targets_check: the by-design bucket goes 5 -> 4 and nothing new is unbuilt. Co-Authored-By: Claude Opus 5.5 * tests: batch_check_state_ci's container goes under tmp_root(), not /tmp (engine#2029 rule, #2034) The fabricated container and its HF source are about 2 MB, but tests/tmp_root.h's rule now applies to every test: ONEBIT_TEST_TMP, else ~/.cache/scratch-tests/tmp, else the system temp directory. Re-run in the CI-style configure: Passed, directory removed. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Opus 5.5 --- CMakeLists.txt | 115 +++++++++++++++--------- docs/npu.md | 6 +- onnx/ep/npu_runtime_ci_stub.cpp | 7 +- tests/batch_check_state_ci.py | 144 +++++++++++++++++++++++++++++++ tests/batch_check_state_test.cpp | 5 +- 5 files changed, 230 insertions(+), 47 deletions(-) create mode 100755 tests/batch_check_state_ci.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 913000720..99ec81ca4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1512,6 +1512,52 @@ endif() # ONEBIT_NPU_HOST # `llama-server` child (engine #1653 review). Device-free: it drives the helper against a synthetic wrapper. add_test(NAME serve_kill_tree COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_kill_tree_test.sh) +# The NPU forward library: one source list and one link interface, built against a runtime target named by the +# caller. if(ONEBIT_NPU) builds onebit_npu_forward against the private runtime; a configure without it (CI) +# builds onebit_npu_forward_ci against onnx/ep/npu_runtime_ci_stub.cpp, whose device calls throw -- enough for +# the host-GEMM forward (init_host), which never opens the device (engine#1946). +option(ONEBIT_NPU_DEBUG_DUMPS "NPU forward: env-named debug dumps (developer builds)" OFF) +function(onebit_npu_forward_library name runtime) + add_library(${name} STATIC + # gemm_q8_raw / gemm_f32: the int8 GEMM the ONNX EP's QDQ leg uses, and now the embedding route's + # ternary linears on the array (engine#1311). Its own probe checks the accumulators are exact. + npu/ort/onebit_npu_gemm.cpp + npu/forward/src/model.c + npu/forward/src/q4nx_dequant.c + npu/forward/src/q4nx_pack.c + # The production weight packer, extracted from I8Ctx so it can be tested without the device runtime (#1416). + npu/forward/src/q4nx_pack_blocked.cpp + # The lm-head image cache (engine#1716): the file format, and the load-or-pack a serve calls per head tile. + npu/forward/src/q4nx_head_image.c + npu/forward/src/q4nx_head_tile_cache.cpp + npu/forward/src/q4nx_semantics.c + npu/forward/src/q4nx_forward.cpp + # The NPU target interface and this box's target (engine#799 P4): the model keeps the maths, the + # target keeps the runtime, the kernels and the weight residency. + npu/target/xdna/xdna_target.cpp) + # The Q4NX pack and dequant loops carry `#pragma omp parallel for`; without the + # OpenMP flags those compile to serial code. Wire it in (both C and C++ sources) + # so they actually use the cores. + find_package(OpenMP) + if(OpenMP_C_FOUND AND OpenMP_CXX_FOUND) + target_link_libraries(${name} PUBLIC OpenMP::OpenMP_C OpenMP::OpenMP_CXX) + endif() + # NPU_INFER_DUMP_ATTN / _LN / _HEAD write debug dumps to a path taken from the environment; + # only debug builds carry them (ONEBIT_NPU_DEBUG_DUMPS) + if(ONEBIT_NPU_DEBUG_DUMPS) + target_compile_definitions(${name} PRIVATE NPU_INFER_DEBUG_DUMPS) + endif() + target_include_directories(${name} SYSTEM PUBLIC + npu/forward/include npu/forward/engine-src npu ${CMAKE_CURRENT_SOURCE_DIR}) + # plan.json (npu/plan/plan.h, engine #799 P1) is read with nlohmann's parser, and the header is + # part of this library's interface, so the dependency is PUBLIC. + # engine#1185: the forward converts an imported q4nx-2 container to the device's 5,120-byte + # q4_1 form before model_load (npu/q4nx_device.cpp, onebit_npu), so the host layer is part of + # this library's link interface. + target_link_libraries(${name} PUBLIC onebit_npu ${runtime} nlohmann_json::nlohmann_json dl + pthread m) +endfunction() + # The lane itself: needs the NPU runtime and an XDNA NPU to run. option(ONEBIT_NPU "Build the NPU lane (needs the private NPU runtime: ONEBIT_NPU_RT_LIB or ONEBIT_NPU_RT_DIR)" OFF) # The NPU code compiles against one runtime (npu/rt.h, docs/npu.md "The runtime"): our own runtime, @@ -1637,45 +1683,9 @@ if(ONEBIT_NPU) endif() endif() - add_library(onebit_npu_forward STATIC - # gemm_q8_raw / gemm_f32: the int8 GEMM the ONNX EP's QDQ leg uses, and now the embedding route's - # ternary linears on the array (engine#1311). Its own probe checks the accumulators are exact. - npu/ort/onebit_npu_gemm.cpp - npu/forward/src/model.c - npu/forward/src/q4nx_dequant.c - npu/forward/src/q4nx_pack.c - # The production weight packer, extracted from I8Ctx so it can be tested without the device runtime (#1416). - npu/forward/src/q4nx_pack_blocked.cpp - # The lm-head image cache (engine#1716): the file format, and the load-or-pack a serve calls per head tile. - npu/forward/src/q4nx_head_image.c - npu/forward/src/q4nx_head_tile_cache.cpp - npu/forward/src/q4nx_semantics.c - npu/forward/src/q4nx_forward.cpp - # The NPU target interface and this box's target (engine#799 P4): the model keeps the maths, the - # target keeps the runtime, the kernels and the weight residency. - npu/target/xdna/xdna_target.cpp) - # The Q4NX pack and dequant loops carry `#pragma omp parallel for`; without the - # OpenMP flags those compile to serial code. Wire it in (both C and C++ sources) - # so they actually use the cores. - find_package(OpenMP) - if(OpenMP_C_FOUND AND OpenMP_CXX_FOUND) - target_link_libraries(onebit_npu_forward PUBLIC OpenMP::OpenMP_C OpenMP::OpenMP_CXX) - endif() - # NPU_INFER_DUMP_ATTN / _LN / _HEAD write debug dumps to a path taken from the environment; - # only debug builds carry them - option(ONEBIT_NPU_DEBUG_DUMPS "NPU forward: env-named debug dumps (developer builds)" OFF) - if(ONEBIT_NPU_DEBUG_DUMPS) - target_compile_definitions(onebit_npu_forward PRIVATE NPU_INFER_DEBUG_DUMPS) - endif() - target_include_directories(onebit_npu_forward SYSTEM PUBLIC - npu/forward/include npu/forward/engine-src npu ${CMAKE_CURRENT_SOURCE_DIR}) - # plan.json (npu/plan/plan.h, engine #799 P1) is read with nlohmann's parser, and the header is - # part of this library's interface, so the dependency is PUBLIC. - # engine#1185: the forward converts an imported q4nx-2 container to the device's 5,120-byte - # q4_1 form before model_load (npu/q4nx_device.cpp, onebit_npu), so the host layer is part of - # this library's link interface. - target_link_libraries(onebit_npu_forward PUBLIC onebit_npu onebit_npu_rt nlohmann_json::nlohmann_json dl - pthread m) + # The library body is onebit_npu_forward_library() (defined above the runtime options), so the CI build of the + # same sources against the runtime stub (engine#1946) cannot drift from this one. + onebit_npu_forward_library(onebit_npu_forward onebit_npu_rt) target_sources(onebit PRIVATE app/forward_serve.cpp) # app/embed_serve.cpp's device path includes npu/embed/embed_npu.h (engine#1311): the embedding route's # ternary linears on the array, through the same int8 GEMM the forward links. @@ -1702,8 +1712,8 @@ endif() add_test(NAME gdn_batch_order COMMAND gdn_batch_order_test) # engine#1905: the NPU_BATCH_CHECK oracle restores the prefix it replays from instead of resetting it, so a chunk # at start_pos > 0 is compared (and then run) from the state the batched pass continues. Host-GEMM, no device; - # skips (77) unless ONEBIT_TEST_Q4NX names a container. It links onebit_npu_forward, so the row exists only where - # ONEBIT_NPU configures; ci.yml does not, so this is a dev-box test (ONEBIT_NPU=ON plus a container), not CI coverage. + # skips (77) unless ONEBIT_TEST_Q4NX names a container. This is the dev-box row (the private runtime, a real + # container); a configure without ONEBIT_NPU -- CI's -- registers the same test on a fabricated container below. add_executable(batch_check_state_test tests/batch_check_state_test.cpp) target_include_directories(batch_check_state_test PRIVATE npu/forward/include) target_link_libraries(batch_check_state_test PRIVATE onebit_npu_forward) @@ -1736,6 +1746,29 @@ endif() # check, outside this repository (docs/npu.md, "Checking the NPU"). endif() +# engine#1946: the batch-check state pin in CI. Under if(ONEBIT_NPU) above, batch_check_state_test is a dev-box row (the +# private runtime, a container named by ONEBIT_TEST_Q4NX), so a regression of #1909 -- the oracle resetting the prefix it +# replays from -- would land green. Here, in a configure WITHOUT the NPU lane (CI's), the same test is built from the same +# forward sources linked against the public runtime stub (its device calls throw; the host-GEMM forward never makes one), +# and tests/batch_check_state_ci.py runs it on a two-layer qwen3 container it writes and packs at test time +# (tools/safetensors_to_q4nx.py): no device, no model on disk, under a second. Skips (77) without numpy. +if(ONEBIT_NPU_HOST AND NOT ONEBIT_NPU AND NOT WIN32 AND Python3_Interpreter_FOUND) + add_library(onebit_npu_rt_ci_stub STATIC onnx/ep/npu_runtime_ci_stub.cpp) + target_include_directories(onebit_npu_rt_ci_stub PUBLIC ${CMAKE_SOURCE_DIR}/npu) + target_compile_definitions(onebit_npu_rt_ci_stub PUBLIC ONEBIT_NPU_RUNTIME_OWN) + target_link_libraries(onebit_npu_rt_ci_stub PUBLIC Threads::Threads) + onebit_npu_forward_library(onebit_npu_forward_ci onebit_npu_rt_ci_stub) + add_executable(batch_check_state_test tests/batch_check_state_test.cpp) + target_link_libraries(batch_check_state_test PRIVATE onebit_npu_forward_ci) + # direction B (tools/ci/ci_targets_check.py): ci.yml's --target list is a workflow file no lane can patch; the + # documented hook is the one onebit_npu_forward_syntax uses. + add_dependencies(onebit batch_check_state_test) + add_test(NAME batch_check_state + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/batch_check_state_ci.py + $) + set_tests_properties(batch_check_state PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120) +endif() + # Private NPU routes (docs/npu.md, "Private routes"): closed-source add-ons in a checkout of # the private npu-kernels repository. Its addons/ directory defines the target # onebit_npu_private, which holds register_private_addon() (npu/private_route.h) and brings diff --git a/docs/npu.md b/docs/npu.md index cdc4c9dd5..97e42b0cc 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2476,7 +2476,11 @@ sampled two ways), and `--batchcheck` takes the verify batch's M-curve. (`identical=N differing=0`). Both passes continue the same prefix: the oracle snapshots the first `start_pos` positions before the replay and restores them afterwards (engine#1905). So a chunk after the first, or the serve path's split-off last token, is compared from the state the batched pass really continues, and the flag leaves the -run's own output unchanged. `tests/batch_check_state_test.cpp` pins both properties. +run's own output unchanged. `tests/batch_check_state_test.cpp` pins both properties, and it runs in CI (engine#1946): +a configure without the NPU lane builds the same forward sources against the public runtime stub +(`onnx/ep/npu_runtime_ci_stub.cpp`, whose device calls throw; the host-GEMM forward makes none), and +`tests/batch_check_state_ci.py` runs the test on a two-layer qwen3 container it writes and packs itself. With the +NPU lane on, the row is the dev-box one and takes a real container from `ONEBIT_TEST_Q4NX`. `tools/mtp_depth_pool.py` pools those tickets. It refuses a pool it cannot read (a repeated tail, a cell scored twice, a short read, a mixed budget, a mixed window identity) and, since the tau=0.6 window, it also says when a depth is too diff --git a/onnx/ep/npu_runtime_ci_stub.cpp b/onnx/ep/npu_runtime_ci_stub.cpp index 2697740c7..e2ebca31c 100644 --- a/onnx/ep/npu_runtime_ci_stub.cpp +++ b/onnx/ep/npu_runtime_ci_stub.cpp @@ -20,8 +20,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// A link-time stand-in for the private NPU runtime, used ONLY by the CI check that compiles and links -// npu/ort/onebit_npu_gemm.cpp (engine #874, the ONNX EP's ONEBIT_EP_NPU leg). +// A link-time stand-in for the private NPU runtime, used ONLY by CI-configure targets: the check that compiles and +// links npu/ort/onebit_npu_gemm.cpp (engine #874, the ONNX EP's ONEBIT_EP_NPU leg), and onebit_npu_forward_ci, the +// host-GEMM forward that batch_check_state_test runs on in CI (engine#1946; init_host never opens the device). // // Why this file exists: that leg is built only when ONEBIT_EP_NPU=ON, which needs ONNX Runtime *and* // the private runtime (ONEBIT_NPU_RT_DIR, docs/npu.md), so CI compiled neither it nor anything it @@ -35,7 +36,7 @@ // would open, submit to or wait on the device throws, so a build that ever *ran* against it fails // immediately instead of silently doing nothing; the accessors that only describe a handle // (map/size/address/is_full_elf) return empty values, which is what an empty handle has. -// It is never linked into libonebit_ep.so, `1bit`, or any probe. +// It is never linked into libonebit_ep.so, `1bit`, or any probe, and a configure with ONEBIT_NPU never builds it. #include "npu_rt_api.h" diff --git a/tests/batch_check_state_ci.py b/tests/batch_check_state_ci.py new file mode 100755 index 000000000..ddad781b7 --- /dev/null +++ b/tests/batch_check_state_ci.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The batch-check state pin (tests/batch_check_state_test.cpp, engine#1905/#1946) on a container made here. + +usage: tests/batch_check_state_ci.py (needs numpy) + +batch_check_state_test needs a Q4NX container, and CI has none, so on its own it skips (77) there and a regression of +#1909 -- the NPU_BATCH_CHECK oracle resetting the prefix it replays from -- lands green. This writes a two-layer qwen3 +(hidden 256, two 128-wide query heads on one kv head, ffn 512, vocab 256; the smallest geometry the I8 tiles take whole), +packs it with tools/safetensors_to_q4nx.py, and runs the test on it in host-GEMM mode: no device, no model on disk. + +Measured when it was written (engine#1946): passes on main; with #1909 reverted it fails 3 of 4 arms (the forward +holds 2 and 1 positions instead of 5 and 6, last-chunk rel-L1 0.78) -- the same failure the real containers show. +Weights are seeded, so the run is the same every time. Exits 77 (skip) without numpy. +""" +import json +import os +import struct +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOL = os.path.join(HERE, "..", "tools", "safetensors_to_q4nx.py") + +H, HEADS, KV_HEADS, HEAD_DIM, FFN, VOCAB, LAYERS = 256, 2, 1, 128, 512, 256, 2 + + +def write_safetensors(path, arrays): + """(name -> float32 numpy array) -> a safetensors file, dense row-major.""" + header, blobs, off = {}, [], 0 + for name, a in arrays.items(): + raw = a.astype(" From e35ee7fe0d3430c8b7cad68ffc0b79eddc8c541a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:12:13 -0400 Subject: [PATCH 36/98] npu: bound every token pick by the tokenizer, not only the grammar's (Fixes #2017) (#2036) The logits row is the container's vocab_size wide, and vocab_size can exceed the tokenizer: Qwen3-0.6B has 151,936 against 151,669 ids. Only the grammar path masked the ids in between (#1182). The unconstrained sample_token() and npu-run's greedy loop could still pick one, so the reply emitted nothing and never ended. token_bound() caps both picks at the tokenizer's size. sample_token() takes it as `vocab`, and argmax_token() is npu-run's per-row argmax. The head and the verifier's M x NV block are unchanged. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Opus 5.5 --- app/forward_serve.cpp | 12 +++++++----- app/npu_sampling.h | 29 +++++++++++++++++++++++++---- docs/npu.md | 8 ++++++++ tests/npu_sampling_test.cpp | 31 +++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 9 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index d193ee114..1fb9db77e 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -498,7 +498,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res std::fclose(f); } } - const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr); + const int next = npu::sample_token(logits, opts.sample, rng, gram ? &allowed : nullptr, e.tok->size()); // -1: the grammar allows no token here, so the answer ends where it is. An end token is // in the mask only where the grammar can stop (refresh_allowed), so a constrained reply // that completes reports "stop" instead of padding to the budget (engine#1182). @@ -865,6 +865,10 @@ int run_forward_ids(const std::string& model_dir, const std::string& kernels, co if (ids.empty()) throw std::runtime_error("--ids: no token ids"); const std::string model = model_dir + "/model.q4nx"; if (!std::filesystem::exists(model)) throw std::runtime_error(model + " does not exist"); + // The tokenizer's size bounds the printed ids (engine#2017); a container without tokenizer.json keeps the whole row. + int tok_vocab = 0; + if (const std::string tj = model_dir + "/tokenizer.json"; std::filesystem::exists(tj)) + tok_vocab = npu::Tokenizer(tj).size(); // Same device discipline as forward-serve: host-GEMM runs take no lock (engine#1353 -- a host-GEMM run that held it // for 4 h 52 m stopped two lanes' device work), a run that opens the device takes the exclusive one. int lock_fd = -1; @@ -895,10 +899,8 @@ int run_forward_ids(const std::string& model_dir, const std::string& kernels, co " floats for " + std::to_string(m) + " rows)"); const int nv = int(all.size() / size_t(m)); for (int r = 0; r < m; ++r) { - const float* row = all.data() + size_t(r) * size_t(nv); - int best = 0; - for (int v = 1; v < nv; ++v) - if (row[v] > row[best]) best = v; + // Bounded by the tokenizer like the served route (engine#2017): an id past it is a padding position. + const int best = npu::argmax_token(all.data() + size_t(r) * size_t(nv), size_t(nv), tok_vocab); std::printf("%d\n", best); // token ids out, in row order: the id the model chooses at that row } done += m; diff --git a/app/npu_sampling.h b/app/npu_sampling.h index 4e2f41b6f..065005cb6 100644 --- a/app/npu_sampling.h +++ b/app/npu_sampling.h @@ -82,14 +82,25 @@ struct Candidate { int id = 0; }; +// How many leading ids of a logits row are tokens: the tokenizer's vocabulary when it is the +// smaller of the two. The head's width is the container's `vocab_size`, which may exceed the +// tokenizer's (Qwen3-0.6B: 151,936 for 151,669 ids); the ids in between are padding positions with +// no text, and a pick that lands there emits nothing and never ends the reply (engine#1182, +// #2017). `vocab` <= 0 means "no tokenizer to bound by" and keeps the whole row. +inline size_t token_bound(size_t width, int vocab) { + return (vocab > 0 && size_t(vocab) < width) ? size_t(vocab) : width; +} + // Draw one token from `logits`. `allowed`, when given, is a per-token mask (non-zero = // the grammar accepts the token); a mask with nothing set is refused with -1 and the -// caller reports it rather than decoding an unconstrained token. +// caller reports it rather than decoding an unconstrained token. `vocab` is the +// tokenizer's size (token_bound): ids at or past it are never drawn, with or without a mask. inline int sample_token(const std::vector& logits, const SampleParams& p, std::mt19937_64& rng, - const std::vector* allowed = nullptr) { + const std::vector* allowed = nullptr, int vocab = 0) { + const size_t n = token_bound(logits.size(), vocab); std::vector c; - c.reserve(logits.size()); - for (size_t i = 0; i < logits.size(); i++) { + c.reserve(n); + for (size_t i = 0; i < n; i++) { if (allowed && !(*allowed)[i]) continue; c.push_back({logits[i], int(i)}); } @@ -157,6 +168,16 @@ inline int sample_token(const std::vector& logits, const SampleParams& p, return c.back().id; } +// The greedy id of one logits row of `width` floats, bounded like sample_token(): a higher +// logit wins and a tie keeps the lower id. `1bit npu-run` prints this per row. +inline int argmax_token(const float* row, size_t width, int vocab = 0) { + const size_t n = token_bound(width, vocab); + int best = 0; + for (size_t v = 1; v < n; v++) + if (row[v] > row[size_t(best)]) best = int(v); + return best; +} + // One candidate of the raw distribution, as the client reads it: the id and its log-probability. struct Logprob { int id = 0; diff --git a/docs/npu.md b/docs/npu.md index 97e42b0cc..2962883a6 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1841,6 +1841,14 @@ ended itself once its grammar had reached an accepting state — the route puts mask exactly there, so a complete JSON object reports `stop` instead of padding with whitespace to `max_tokens` (engine#1182). +Every pick is bounded by the tokenizer, not only the constrained one. The head is `vocab_size` wide, and +`vocab_size` may exceed the tokenizer's ids: Qwen3's 151,936 against 151,669 (measured on Qwen3-0.6B's +`config.json` and `tokenizer.json`). The ids in between are padding positions with no text, so a token drawn +there emits nothing and never ends the reply. `sample_token()` and `1bit npu-run`'s per-row argmax +(`argmax_token()`) therefore consider only ids below the tokenizer's size (`token_bound()` in +`app/npu_sampling.h`, engine#2017), with or without a grammar. A container with no `tokenizer.json` keeps the +whole row. + The sampler is llama.cpp's, without llama.cpp: `app/npu_sampling.h` repeats the semantics of `third_party/llama.cpp/src/llama-sampler.cpp` (top_k, then top_p, then temperature, then the cumulative-sum draw that `common/sampling.cpp`'s chain ends with) over the raw logits this route diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 56851d2b4..45932a92b 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -49,6 +49,8 @@ using onebit::npu::logprobs_of; using onebit::npu::options_from_body; using onebit::npu::SampleParams; using onebit::npu::sample_token; +using onebit::npu::argmax_token; +using onebit::npu::token_bound; using onebit::npu::stop_holdback; int failures = 0; @@ -1192,7 +1194,36 @@ void test_request_parsing() { } // namespace +// engine#2017: the logits row is `vocab_size` wide, which may exceed the tokenizer (Qwen3-0.6B: 151,936 for 151,669 +// ids; the shape below is the issue's, 32,032 for 32,016). Ids past the tokenizer are padding positions: every real +// logit negative and the padding at exactly 0.0 is the row where an unbounded argmax picks one (engine#1182). +void test_vocab_bound() { + const int V = 32016, W = 32032; + std::vector row(size_t(W), 0.0f); + for (int v = 0; v < V; v++) row[size_t(v)] = -1.0f - float(v % 97) * 0.01f; + row[123] = -0.5f; // the real best + check(argmax_token(row.data(), size_t(W)) >= V, "control: an unbounded argmax lands on a padding id"); + check(argmax_token(row.data(), size_t(W), V) == 123, "the bounded argmax picks the best real id"); + check(sample(row, SampleParams(), 1) >= V, "control: unbounded greedy sample lands on a padding id"); + std::mt19937_64 rng(1); + check(sample_token(row, SampleParams(), rng, nullptr, V) == 123, "bounded greedy sample picks the best real id"); + SampleParams hot; + hot.temperature = 1.0f; + bool all_real = true; + for (uint64_t s = 0; s < 200; s++) { + std::mt19937_64 r(s); + if (sample_token(row, hot, r, nullptr, V) >= V) all_real = false; + } + check(all_real, "a bounded draw at temperature 1 never draws a padding id"); + std::vector mask(size_t(W), 1); + std::mt19937_64 r2(2); + check(sample_token(row, SampleParams(), r2, &mask, V) == 123, "the bound holds with a mask that allows padding"); + check(token_bound(size_t(W), 0) == size_t(W), "no tokenizer keeps the whole row"); + check(token_bound(size_t(V), W) == size_t(V), "a tokenizer larger than the row does not widen it"); +} + int main() { + test_vocab_bound(); test_sampler(); test_sampler_distribution(); test_stop(); From fcc50c1f329602bd024023b60c6898c396dbbc2a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:23:51 -0400 Subject: [PATCH 37/98] amdxdna: a kernel-log report of what TDR recovery did, and a test-only patch that makes a restart fail (Refs #1750) (#2032) tools/amdxdna_tdr_report.py counts, per module load, the timeouts, power-cycle recoveries, throttled recoveries, skipped contexts, failed context restarts (with the step and whether the module failed the parked jobs) and latches. third_party/patches/amdxdna/test-only/restart-fault-injection.patch is outside the numbered series: two module parameters make aie2_hwctx_restart() fail on request, so the failed-restart path of 0010/0013 can be measured with one provoked recovery. Compiled, never loaded. The README has the box's record (18 recoveries, 0 failed restarts) and the owner-side procedure. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 3 + tests/amdxdna_restart_test.sh | 16 + third_party/patches/amdxdna/README.md | 54 ++++ .../test-only/restart-fault-injection.patch | 87 ++++++ tools/amdxdna_tdr_report.py | 289 ++++++++++++++++++ 5 files changed, 449 insertions(+) create mode 100644 third_party/patches/amdxdna/test-only/restart-fault-injection.patch create mode 100755 tools/amdxdna_tdr_report.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 99ec81ca4..f43a670b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -325,6 +325,9 @@ add_python_test(tool_selftest_whisper_encoder_baseline ${CMAKE_SOURCE_DIR}/tools # engine#1780: the DFlash draft package's gate before a loader -- tile-grid sizes from the config, tile-header sanity and # the container's scalars against config.json, on a synthetic package with eight negative controls. Stdlib only. add_python_test(tool_selftest_dflash_draft_info ${CMAKE_SOURCE_DIR}/tools/dflash_draft_info.py --selftest) +# engine#1750: what a TDR recovery did, counted from the kernel log (failed context restarts among them). The self-test +# parses lines in the driver's own formats for three module generations. Stdlib only, no device. +add_python_test(tool_selftest_amdxdna_tdr_report ${CMAKE_SOURCE_DIR}/tools/amdxdna_tdr_report.py --self-test) # engine#2013: the qwen35 (gated linear attention) GGUF -> Q4NX converter. Its own selftest (tile round-trips, the # value-head permutation), and a pack + verify of a small qwen35 GGUF the test writes itself, with the three inputs it # has to refuse. numpy only: no gguf package, no model, no device. diff --git a/tests/amdxdna_restart_test.sh b/tests/amdxdna_restart_test.sh index 94fe2ac1e..460a682ce 100755 --- a/tests/amdxdna_restart_test.sh +++ b/tests/amdxdna_restart_test.sh @@ -52,6 +52,22 @@ for p in "$series"/[0-9][0-9][0-9][0-9]-*.patch; do done echo "ok $n patches apply in order on ${pin:0:12}" +# The test-only fault injection (engine#1750) has to keep applying on top of the series, or the one run it exists +# for cannot be built. It is checked, not applied: the harness below is the carried code. And it must stay out of +# the numbered glob above, which is what a package build takes. +inject=$series/test-only/restart-fault-injection.patch +if ! git -C "$scratch/tree" apply --check "$inject" 2> "$scratch/apply.err"; then + echo "FAIL test-only/restart-fault-injection.patch does not apply on top of the $n carried patches:" + sed 's/^/ /' "$scratch/apply.err" | head -n 5 + exit 1 +fi +echo "ok test-only/restart-fault-injection.patch applies on top of them (checked, not applied)" +if ls "$series"/[0-9][0-9][0-9][0-9]-*.patch | xargs grep -l "TEST ONLY" > "$scratch/leak" 2> /dev/null; then + echo "FAIL a numbered patch carries TEST ONLY code: $(tr '\n' ' ' < "$scratch/leak")" + exit 1 +fi +echo "ok no numbered patch carries TEST ONLY code" + # one function definition: from its header line (column 0, not a prototype) to the first closing brace in column 0 for f in aie2_hwctx_job_pause aie2_hwctx_stop aie2_hwctx_restart aie2_hwctx_job_unpause; do awk -v f="$f" '!on && $0 ~ "^[a-z].*[ *]" f "\\(" && $0 !~ /;[ \t]*$/ { on = 1 } on { print } on && /^}/ { exit }' \ diff --git a/third_party/patches/amdxdna/README.md b/third_party/patches/amdxdna/README.md index cadcfa514..5b42e11db 100644 --- a/third_party/patches/amdxdna/README.md +++ b/third_party/patches/amdxdna/README.md @@ -195,6 +195,60 @@ To load the pair (owner's call; strixhalo, 2026-11-06): the module built from th `7.4.0-rc1-1bit` reports `336751493A77C50DE0B50F7`, and the install has to sign it, `depmod`, and **`update-initramfs`** — the initramfs copy is what boots (2026-11-04: three reboots on the old module without that step). +## Reading what a recovery did: `tools/amdxdna_tdr_report.py` + +The driver logs every step of a TDR recovery, and `tools/amdxdna_tdr_report.py` turns the kernel log into the counts +per module load: timeouts detected, power-cycle recoveries, throttled recoveries, skipped contexts, **failed restarts** +(context, error, the step that failed, and whether the module failed the parked jobs, i.e. carries `0013`), whether the +array latched, and refused context creates by error. It reads `journalctl -k -b` or saved logs, opens no device, and +`--fail-on-failed-restart` makes it a check. + +The record on strixhalo with the module the fleet runs (`F87D0BDF248067862F83BA3`, pin + `0001`-`0009`), one boot, +2026-11-08 23:11 to 2026-11-10 17:40 AST, under the fleet's normal load: + +``` +$ python3 tools/amdxdna_tdr_report.py +module load 1: driver 0.18.0 (2636 amdxdna lines) + timeouts detected 26 + power-cycle recoveries 18 + throttled 8 + skipped contexts 4 + failed restarts 0 + latched (reload needed) 0 + context creates refused 767 (ret -19 x767) +``` + +So in 18 real recoveries no context restart failed: the path `0010` and `0013` change (engine#1750) was not reached +once. That says how often it happens here, not what firmware does when it does. + +## The test-only patch, and the run that would measure `0010`/`0013` + +`test-only/restart-fault-injection.patch` is **not part of the series**. It is outside the numbered files, so +`scripts/mk-kernel-packages.sh` (which takes patches by number) and the series loop in `tests/amdxdna_restart_test.sh` +never pick it up; that test checks both that it still applies on top of the series and that no numbered patch carries +`TEST ONLY` code. It adds two module parameters that make `aie2_hwctx_restart()` fail on request, before or after the +firmware context is created, because a real restart failure needs firmware to refuse a message and cannot be arranged +from user space. + +Verified without the device: applies on `0001`-`0013`; the module compiles against `7.4.0-rc1-1bit` +(`amdxdna.ko` 14,045,256 B, `srcversion 57DF58D0185B9D6E0397B6F`, 0 errors, 0 warnings; `modinfo` lists both +parameters). **Never loaded.** + +The run, which needs a module load and is therefore the owner's: + +1. Build pin + `0001`-`0013` + the test-only patch (`make -C drivers/accel/amdxdna ARCH=x86`), load it on a box nobody + else is using the array on. +2. Start one client that keeps a context open and submits (any `npu-hold` serve). +3. `echo 1 > /sys/module/amdxdna/parameters/restart_inject_count`, then `echo 2 > .../restart_inject_step` + (2 = after the create, the case that left a half-built firmware context), and provoke one recovery (the kernel + lane's `tdr_test` hook, or a real timeout). +4. `python3 tools/amdxdna_tdr_report.py`. Expected with `0013`: one failed restart, with + `firmware context taken away (teardown ret 0), N parked job(s) failed with -ENODEV`, and the client's pending + requests returning at once instead of at their own timeout. `teardown ret` is the answer to the question engine#1750 + put first: whether firmware accepts a destroy for a context it has only half of. The array then latches (a failed + restart in a recovery latches `hw_dead` by design), so reload the shipping module afterwards. +5. Repeat with `restart_inject_step` 1 (create failed: no firmware context to take away). + ## What this directory deliberately does not carry The **0010 debugfs test hook** (`debugfs/accel//tdr_test`, root-only, commands `recover` / `throttle` / diff --git a/third_party/patches/amdxdna/test-only/restart-fault-injection.patch b/third_party/patches/amdxdna/test-only/restart-fault-injection.patch new file mode 100644 index 000000000..452da2593 --- /dev/null +++ b/third_party/patches/amdxdna/test-only/restart-fault-injection.patch @@ -0,0 +1,87 @@ +From: bong-water-water-bong +Subject: [PATCH TEST ONLY] accel/amdxdna: fail a context restart on request + +TEST ONLY. Not part of the carried series, never built into a package or an +image: it sits in test-only/ so that nothing that takes the numbered patches +picks it up. + +aie2_hwctx_restart() fails only when firmware refuses one of its four +messages, which cannot be arranged from user space, so what the driver does +with a context whose restart failed (engine#1750, patches 0010 and 0013) +could not be measured on the array. Two module parameters make the next +restarts fail on purpose: + + restart_inject_step 0 = off + 1 = fail before the firmware context is created + (as if aie2_create_context() had failed) + 2 = fail after it was created + (as if aie2_map_host_buf() had failed) + restart_inject_count how many restarts to fail; each one takes one + +Each injected failure logs "TEST ONLY: injecting a restart failure for + at step " at error level, followed by the driver's own lines for a +failed restart. Applies on top of 0001-0013. +--- +diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c +index 100a07d..55d99a7 100644 +--- a/drivers/accel/amdxdna/aie2_ctx.c ++++ b/drivers/accel/amdxdna/aie2_ctx.c +@@ -34,6 +34,36 @@ uint tdr_timeout_ms = 2000; + module_param(tdr_timeout_ms, uint, 0400); + MODULE_PARM_DESC(tdr_timeout_ms, "TDR (Timeout Detection and Recovery) timeout in milliseconds (0 = disable)"); + ++/* ++ * TEST ONLY (engine#1750), never part of a shipped module: make the next ++ * context restarts fail on purpose, so the failed-restart path can be ++ * measured with one provoked TDR instead of waiting for firmware to ++ * refuse a restart by itself. ++ * ++ * restart_inject_step: 0 = off; 1 = fail before the firmware context is ++ * created (as if aie2_create_context() had failed); 2 = fail after it was ++ * created (as if aie2_map_host_buf() had failed). ++ * restart_inject_count: how many restarts to fail; each injected failure ++ * takes one, and at 0 the injection is off again. ++ */ ++static uint restart_inject_step; ++module_param(restart_inject_step, uint, 0600); ++MODULE_PARM_DESC(restart_inject_step, "TEST ONLY: fail a context restart (1 = before create, 2 = after create)"); ++static uint restart_inject_count; ++module_param(restart_inject_count, uint, 0600); ++MODULE_PARM_DESC(restart_inject_count, "TEST ONLY: number of context restarts restart_inject_step still fails"); ++ ++static bool aie2_restart_inject(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx, uint step) ++{ ++ if (READ_ONCE(restart_inject_step) != step || !READ_ONCE(restart_inject_count)) ++ return false; ++ ++ WRITE_ONCE(restart_inject_count, READ_ONCE(restart_inject_count) - 1); ++ XDNA_ERR(xdna, "TEST ONLY: injecting a restart failure for %s at step %u (%u left)", ++ hwctx->name, step, READ_ONCE(restart_inject_count)); ++ return true; ++} ++ + struct aie2_ctx_health { + struct amdxdna_ctx_health header; + u32 txn_op_idx; +@@ -174,6 +204,12 @@ int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) + goto out; + } + ++ if (aie2_restart_inject(xdna, hwctx, 1)) { ++ ret = -EIO; ++ XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret); ++ goto out; ++ } ++ + ret = aie2_create_context(xdna->dev_handle, hwctx); + if (ret) { + XDNA_ERR(xdna, "Create hwctx failed, ret %d", ret); +@@ -183,6 +219,8 @@ int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx) + ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, + amdxdna_obj_dma_addr(heap), + heap->mem.size); ++ if (!ret && aie2_restart_inject(xdna, hwctx, 2)) ++ ret = -EIO; + if (ret) { + XDNA_ERR(xdna, "Map host buf failed, ret %d", ret); + goto out; diff --git a/tools/amdxdna_tdr_report.py b/tools/amdxdna_tdr_report.py new file mode 100755 index 000000000..386b8948d --- /dev/null +++ b/tools/amdxdna_tdr_report.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""amdxdna_tdr_report.py -- what the amdxdna driver's TDR recovery did, read from the kernel log. + + tools/amdxdna_tdr_report.py this boot's kernel log (journalctl -k -b) + tools/amdxdna_tdr_report.py --boot -1 another boot (journalctl's numbering) + tools/amdxdna_tdr_report.py FILE [FILE...] saved logs (dmesg or journalctl text); - is stdin + tools/amdxdna_tdr_report.py --json ... the same numbers as JSON + tools/amdxdna_tdr_report.py --fail-on-failed-restart ... exit 1 when a context restart failed + tools/amdxdna_tdr_report.py --self-test + +Read-only: it opens no device and changes nothing. Stdlib only. + +Why it exists (engine#1750). A timeout (TDR) makes the driver stop every context, power-cycle the array and +restart the contexts. The question that issue asks is what happens to a context whose restart FAILS, and +the first thing anyone needs is how often that has happened on a box. The driver logs every step, but the +answer was a grep one had to know how to write. This prints it per module load: + + timeouts detected every "TDR timeout detected" + power-cycle recoveries the driver reset the array and walked the contexts + throttled inside the minimum interval between resets; no context was touched + skipped contexts being torn down by their owner (-ESRCH), not a failure + failed restarts " restart failed, ret ", with the context, the error, and the step + that failed when the driver logged one + latched "NPU array is unusable until the driver is reloaded" + context creates refused "Init hwctx failed, ret ", by error + +For a failed restart it also says which behaviour the loaded module has: the carried patch 0013 logs +"no firmware context (teardown ret ), failing parked job(s)" from the restart itself; a module +without it logs only the bare line from the TDR walk, and one without 0010 unpauses the context anyway. + +A module load is a "Initialized amdxdna_accel_driver" line. Lines before the first one in the input are +reported as a load of their own, since a saved log often starts mid-boot. +""" +import argparse +import json +import re +import subprocess +import sys + +LINE = re.compile(r"amdxdna [0-9a-f:.]+: (?:\[drm\] )?(?:\*ERROR\* )?(?P[A-Za-z0-9_]+): (?P.*)$") +LOAD = re.compile(r"Initialized amdxdna_accel_driver (?P\S+)") +FAILED = re.compile(r"^(?P\S+) restart failed, ret (?P-?\d+)" + r"(?:: no firmware context \(teardown ret (?P-?\d+)\), failing (?P\d+) parked job)?") +STEP = re.compile(r"^(Create hwctx failed|Map host buf failed|Add heap \d+ failed|Config cu failed),? ?ret (-?\d+)") +THROTTLED = re.compile(r"^TDR recovery throttled|^NPU power-cycle throttled") +SKIPPED = re.compile(r"^TDR: (\d+) context\(s\) skipped") +LATCHED = re.compile(r"^NPU array is unusable until the driver is reloaded") +NOT_RECOVERED = re.compile(r"^NPU power-cycle did not recover the array: (-?\d+)") +CREATE_REFUSED = re.compile(r"^Init hwctx failed, ret (-?\d+)") + + +def new_load(version): + return {"driver": version, "timeouts": 0, "power_cycles": 0, "throttled": 0, "skipped_contexts": 0, + "failed_restarts": [], "latched": 0, "not_recovered": [], "creates_refused": {}, "lines": 0} + + +def parse(lines): + """-> a list of module loads, each a dict of counts. Lines that are not the driver's are ignored.""" + loads, cur, step = [], None, None + for raw in lines: + m = LOAD.search(raw) + if m: + cur = new_load(m.group("ver")) + loads.append(cur) + step = None + continue + m = LINE.search(raw) + if not m: + continue + if cur is None: + cur = new_load(None) + loads.append(cur) + cur["lines"] += 1 + fn, msg = m.group("fn"), m.group("msg").strip() + if msg == "TDR timeout detected": + cur["timeouts"] += 1 + elif "power-cycling NPU" in msg and fn == "aie2_tdr_recover_all": + # aie2_hw_reset() logs its own "power-cycling NPU" line for the same reset: count the walk's + cur["power_cycles"] += 1 + step = None + elif THROTTLED.search(msg): + cur["throttled"] += 1 + elif SKIPPED.search(msg): + cur["skipped_contexts"] += int(SKIPPED.search(msg).group(1)) + elif LATCHED.search(msg): + cur["latched"] += 1 + elif NOT_RECOVERED.search(msg): + cur["not_recovered"].append(int(NOT_RECOVERED.search(msg).group(1))) + elif CREATE_REFUSED.search(msg): + ret = CREATE_REFUSED.search(msg).group(1) + cur["creates_refused"][ret] = cur["creates_refused"].get(ret, 0) + 1 + elif fn == "aie2_hwctx_restart" and STEP.search(msg): + step = STEP.search(msg).group(1) + elif FAILED.search(msg): + f = FAILED.search(msg) + entry = {"context": f.group("ctx"), "ret": int(f.group("ret")), "step": step, + "logged_by": fn, "teardown_ret": None, "parked_jobs": None} + if f.group("td") is not None: + entry["teardown_ret"], entry["parked_jobs"] = int(f.group("td")), int(f.group("parked")) + prev = cur["failed_restarts"][-1] if cur["failed_restarts"] else None + # with 0013 the restart logs the failure and the TDR walk logs it again: one failure, not two + if (prev and prev["context"] == entry["context"] and prev["ret"] == entry["ret"] + and prev["logged_by"] != fn and not prev.get("paired")): + prev["paired"] = True + if entry["teardown_ret"] is not None: + prev["teardown_ret"], prev["parked_jobs"] = entry["teardown_ret"], entry["parked_jobs"] + else: + cur["failed_restarts"].append(entry) + step = None + for load in loads: + for f in load["failed_restarts"]: + f.pop("paired", None) + f.pop("logged_by", None) + f["module_fails_parked_jobs"] = f["teardown_ret"] is not None # the 0013 line + return loads + + +def render(loads): + out = [] + if not loads: + return "no amdxdna lines in the input\n" + for i, l in enumerate(loads): + head = f"driver {l['driver']}" if l["driver"] else "before the first driver load line in the input" + out.append(f"module load {i + 1}: {head} ({l['lines']} amdxdna lines)") + out.append(f" timeouts detected {l['timeouts']}") + out.append(f" power-cycle recoveries {l['power_cycles']}") + out.append(f" throttled {l['throttled']}") + out.append(f" skipped contexts {l['skipped_contexts']}") + out.append(f" failed restarts {len(l['failed_restarts'])}") + for f in l["failed_restarts"]: + where = f" at '{f['step']}'" if f["step"] else "" + if f["module_fails_parked_jobs"]: + tail = (f"; firmware context taken away (teardown ret {f['teardown_ret']}), " + f"{f['parked_jobs']} parked job(s) failed with -ENODEV (patch 0013)") + else: + tail = "; the module does not say what happened to the parked jobs (no patch 0013)" + out.append(f" {f['context']}: ret {f['ret']}{where}{tail}") + out.append(f" latched (reload needed) {l['latched']}") + if l["not_recovered"]: + out.append(f" power-cycle did not recover the array: {len(l['not_recovered'])} " + f"(ret {', '.join(str(r) for r in l['not_recovered'])})") + if l["creates_refused"]: + by = ", ".join(f"ret {k} x{v}" for k, v in sorted(l["creates_refused"].items(), key=lambda kv: -kv[1])) + out.append(f" context creates refused {sum(l['creates_refused'].values())} ({by})") + return "\n".join(out) + "\n" + + +def read_inputs(args): + if args.files: + for p in args.files: + f = sys.stdin if p == "-" else open(p, errors="replace") + with f: + yield from f + return + cmd = ["journalctl", "-k", "--no-pager", "-b", str(args.boot)] + try: + r = subprocess.run(cmd, capture_output=True, text=True, errors="replace") + except OSError as e: + raise SystemExit(f"cannot run {' '.join(cmd)}: {e}; pass a saved log file instead") + if r.returncode: + raise SystemExit(f"{' '.join(cmd)} exited {r.returncode}: {r.stderr.strip()[:300]}; pass a saved log file instead") + yield from r.stdout.splitlines() + + +# Lines as the driver prints them. The first block is from a box running pin + 0001-0009; the failed +# restarts are written from the format strings in the patched source (0010 alone, then 0013). +P = "Nov 09 04:46:22 host kernel: amdxdna 0000:c6:00.1: [drm] " +SELF = [ + "Nov 08 23:11:19 host kernel: [drm] Initialized amdxdna_accel_driver 0.18.0 for 0000:c6:00.1 on minor 0", + P + "*ERROR* aie2_legacy_tdr_detect: TDR timeout detected", + P + "aie2_tdr_work_func: Device isn't making progress", + P + "*ERROR* aie2_set_cmd_timeout: Firmware timeout state capture:", + P + "*ERROR* aie2_set_cmd_timeout: Context ID: 15", + P + "aie2_tdr_recover_all: Array stopped dispatching, power-cycling NPU to recover", + P + "aie2_hw_reset: NPU firmware unhealthy: power-cycling NPU", + P + "*ERROR* aie2_legacy_tdr_detect: TDR timeout detected", + P + "aie2_tdr_recover_all: TDR recovery throttled: last power-cycle 45560 ms ago (< 60000 ms), retrying on " + "the next watchdog tick; no context touched", + P + "*ERROR* aie2_legacy_tdr_detect: TDR timeout detected", + P + "aie2_tdr_recover_all: Array stopped dispatching, power-cycling NPU to recover", + P + "aie2_tdr_recover_all: TDR: 2 context(s) skipped, being torn down", + P + "*ERROR* aie2_hwctx_init: Alloc hw resource failed, ret -19", + P + "*ERROR* amdxdna_drm_create_hwctx_ioctl: Init hwctx failed, ret -19", + P + "*ERROR* amdxdna_drm_create_hwctx_ioctl: Init hwctx failed, ret -16", + P + "*ERROR* amdxdna_drm_create_hwctx_ioctl: Init hwctx failed, ret -19", + "Nov 09 05:00:00 host kernel: usb 1-1: new high-speed USB device number 4", # not ours + # a second load, a module with 0010 only: the TDR walk logs the failure, and the array latches + "[ 812.100000] [drm] Initialized amdxdna_accel_driver 0.18.0 for 0000:c6:00.1 on minor 0", + "[ 900.000000] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_legacy_tdr_detect: TDR timeout detected", + "[ 900.000100] amdxdna 0000:c6:00.1: [drm] aie2_tdr_recover_all: Firmware unresponsive, power-cycling NPU", + "[ 900.500000] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_hwctx_restart: Map host buf failed, ret -22", + "[ 900.500100] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_tdr_resume_hwctx: hwctx.4242.15 restart failed, ret -22", + "[ 900.500200] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_tdr_recover_all: NPU array is unusable until the driver is " + "reloaded: TDR recovery failed (0, a context could not be restarted); new contexts and submissions are refused " + "with -ENODEV", + # a third load, a module with 0013: the restart logs it (with the jobs), then the walk logs it again + "[ 1500.000000] [drm] Initialized amdxdna_accel_driver 0.18.0 for 0000:c6:00.1 on minor 0", + "[ 1600.000000] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_legacy_tdr_detect: TDR timeout detected", + "[ 1600.000100] amdxdna 0000:c6:00.1: [drm] aie2_tdr_recover_all: Array stopped dispatching, power-cycling NPU to recover", + "[ 1600.400000] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_hwctx_restart: Create hwctx failed, ret -5", + "[ 1600.400100] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_hwctx_restart: hwctx.77.14 restart failed, ret -5: no firmware " + "context (teardown ret 0), failing 3 parked job(s) and later submissions with -ENODEV", + "[ 1600.400200] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_tdr_resume_hwctx: hwctx.77.14 restart failed, ret -5", + "[ 1600.400300] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_hwctx_restart: Config cu failed, ret -110", + "[ 1600.400400] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_hwctx_restart: hwctx.78.13 restart failed, ret -110: no firmware " + "context (teardown ret 0), failing 0 parked job(s) and later submissions with -ENODEV", + "[ 1600.400500] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_tdr_resume_hwctx: hwctx.78.13 restart failed, ret -110", + "[ 1600.400600] amdxdna 0000:c6:00.1: [drm] *ERROR* aie2_tdr_recover_all: NPU power-cycle did not recover the array: -110", +] + + +def self_test(): + fails = [] + + def check(good, what): + if not good: + fails.append(what) + + loads = parse(SELF) + check(len(loads) == 3, f"three module loads, got {len(loads)}") + a, b, c = (loads + [new_load(None)] * 3)[:3] + check(a["driver"] == "0.18.0", "the driver version is read from the load line") + check(a["timeouts"] == 3, f"load 1: 3 timeouts, got {a['timeouts']}") + check(a["power_cycles"] == 2, f"load 1: 2 power-cycles (aie2_hw_reset's own line is not a second one), got {a['power_cycles']}") + check(a["throttled"] == 1 and a["skipped_contexts"] == 2, "load 1: 1 throttled, 2 skipped contexts") + check(a["failed_restarts"] == [] and a["latched"] == 0, "load 1: no failed restart, not latched") + check(a["creates_refused"] == {"-19": 2, "-16": 1}, f"load 1: creates refused by error, got {a['creates_refused']}") + check(a["lines"] == 15, f"load 1: 15 driver lines (the usb line is not one), got {a['lines']}") + fb = b["failed_restarts"] + check(len(fb) == 1 and fb[0]["context"] == "hwctx.4242.15" and fb[0]["ret"] == -22, "load 2: one failed restart, named") + check(fb and fb[0]["step"] == "Map host buf failed", "load 2: the failing step is the driver's own line before it") + check(fb and fb[0]["module_fails_parked_jobs"] is False, "load 2: a bare line means no patch 0013") + check(b["latched"] == 1 and b["power_cycles"] == 1, "load 2: latched, and 'Firmware unresponsive' is a power-cycle too") + fc = c["failed_restarts"] + check(len(fc) == 2, f"load 3: the restart's line and the walk's line are one failure each, got {len(fc)}") + check(len(fc) == 2 and fc[0]["parked_jobs"] == 3 and fc[0]["teardown_ret"] == 0 and fc[0]["step"] == "Create hwctx failed", + "load 3: the 0013 line gives the parked jobs and the teardown result") + check(len(fc) == 2 and fc[1]["context"] == "hwctx.78.13" and fc[1]["ret"] == -110 and fc[1]["step"] == "Config cu failed" + and fc[1]["parked_jobs"] == 0, "load 3: the second context keeps its own step and count") + check(all(f["module_fails_parked_jobs"] for f in fc), "load 3: the module is one with patch 0013") + check(c["not_recovered"] == [-110], "load 3: a power-cycle that did not recover is reported with its error") + text = render(loads) + check("failed restarts 0" in text and "context creates refused 3 (ret -19 x2, ret -16 x1)" in text, "the text for load 1") + check("hwctx.77.14: ret -5 at 'Create hwctx failed'; firmware context taken away (teardown ret 0), 3 parked job(s) " + "failed with -ENODEV (patch 0013)" in text, "the text for a 0013 failure") + check("hwctx.4242.15: ret -22 at 'Map host buf failed'; the module does not say" in text, "the text for a bare failure") + check(parse(["Nov 09 host kernel: usb 1-1: hello"]) == [] and "no amdxdna lines" in render([]), "an input with none of ours") + mid = parse(SELF[1:6]) + check(len(mid) == 1 and mid[0]["driver"] is None and mid[0]["power_cycles"] == 1, "a log that starts mid-boot is still counted") + json.dumps(loads) + for f in fails: + print("FAIL " + f) + print("FAILED" if fails else "amdxdna_tdr_report: ok") + return 1 if fails else 0 + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("files", nargs="*", help="saved kernel logs (- is stdin); default: journalctl -k -b") + p.add_argument("--boot", default="0", help="journalctl boot to read when no file is given (default 0, this boot)") + p.add_argument("--json", action="store_true") + p.add_argument("--fail-on-failed-restart", action="store_true", help="exit 1 when any context restart failed") + p.add_argument("--self-test", action="store_true") + a = p.parse_args() + if a.self_test: + return self_test() + loads = parse(read_inputs(a)) + sys.stdout.write(json.dumps(loads, indent=1) + "\n" if a.json else render(loads)) + if a.fail_on_failed_restart and any(l["failed_restarts"] for l in loads): + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 69f917eaf48d5d8eaa6fb3831613960104fd98b3 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:33:44 -0400 Subject: [PATCH 38/98] tools: hrx_gtt_split.py -- re-measure #856's duplicate from a clean checkout (#2031) engine#856's reproduction named tools/bo_trace.py, which needs root for tracefs and was never committed, so two sweeps found the issue could not be re-measured from main. This tool splits one HRX process's GTT into the loader's model buffer, HRX's packed second copy, KV/state, compute, and the unattributed rest. It reads only lines llama.cpp and HRX already print (-v and GGML_HRX_HOST_WEIGHT_TRACE=1), plus the process's own fdinfo, counting each DRM client once. `run` wraps the command (to go inside a gpu-hold ticket), `parse` re-reads a saved log, and `--self-test` uses real lines from a 4B run. The self-test is registered as a ctest row. docs/hrx.md gets the command and the 4B/27B split. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Opus 5.5 --- CMakeLists.txt | 2 + docs/hrx.md | 28 ++++ tools/hrx_gtt_split.py | 292 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 322 insertions(+) create mode 100755 tools/hrx_gtt_split.py diff --git a/CMakeLists.txt b/CMakeLists.txt index f43a670b8..b817ca27f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -353,6 +353,8 @@ add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test- # the same offset from the data section returns the tensor, from byte 8 it does not. add_python_test(hf_tensor_self ${CMAKE_SOURCE_DIR}/tools/hf_tensor.py --self-test) add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) +# engine#856: one HRX process's GTT split into the loader's copy and HRX's packed second copy, from its own -v log. +add_python_test(hrx_gtt_split_self ${CMAKE_SOURCE_DIR}/tools/hrx_gtt_split.py --self-test) # engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as # it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that # release, and the ordering a candidate needs: previous release < candidate < its release. The diff --git a/docs/hrx.md b/docs/hrx.md index b6e66a63a..184f21fd2 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -836,6 +836,34 @@ repacked-prefill kernel ([llama.cpp #250](https://github.com/1bit-MONSTER/llama. `GGML_HRX_Q6K_PACKED_PREFILL`) stays **opt-in** for the same reason: it is numerically free and speed-neutral, so it buys nothing on its own. +**Measuring the duplicate on your own build.** `tools/hrx_gtt_split.py` splits one process's GTT into the loader's copy and HRX's +packed second copy, using only lines llama.cpp and HRX already print plus the process's own fdinfo. It needs no root and no +tracefs. Inside your ticket: + +``` +gpu-hold -l -t 5 -- tools/hrx_gtt_split.py run --out split-4b -- \ + llama-cli -m Qwen3-4B-Q4_K_M.gguf -ngl 99 -fit off -c 256 -n 16 --temp 0 -no-cnv -p "The capital of France is" +tools/hrx_gtt_split.py parse split-4b/split.log --samples split-4b/gtt.txt # again, from the files it left +``` + +`run` adds `-v`, sets `GGML_HRX_HOST_WEIGHT_TRACE=1`, and samples `drm-total-gtt` with each DRM client counted once. On the +#1351 pin (llama.cpp `40f58efe1`, the same command): + +| part | Qwen3-4B Q4_K_M | Qwen3.8-27B UD-Q4_K_XL | +|---|---:|---:| +| loader's model buffer (HRX) | 2,375.9 MiB | 15,718.5 MiB | +| HRX's packed copies | **795.8 MiB** (38 weights) | **2,631.3 MiB** (66 weights) | +| KV + recurrent state | 36.0 | 165.6 | +| compute + output buffers | 43.2 | 72.1 | +| not attributed | 674.1 | 1,909.5 | +| process peak GTT | 3,925 | 20,497 | +| share of the loader's copy resident twice | 33.5 % | 16.7 % | + +The packed copies are the duplicate. HRX builds one for each weight that a kernel reads in a re-laid-out form; prefill above +a token threshold still reads the loader's copy. "Not attributed" is mostly runtime module memory and copy staging (the +staging buffer is capped at 64 MiB since [llama.cpp #266](https://github.com/1bit-MONSTER/llama.cpp/pull/266)). It is not +weights. This replaces the `bo_trace.py` reproduction in the issue body, which was never committed. + ### Measuring on `HRX0`: what makes a number citable Numbers measured on this device swing far more between *runs* than between *files*, so the harness matters as much as the arm. Everything below is diff --git a/tools/hrx_gtt_split.py b/tools/hrx_gtt_split.py new file mode 100755 index 000000000..f45c6c7e8 --- /dev/null +++ b/tools/hrx_gtt_split.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""hrx_gtt_split: where one HRX process's GTT goes, from its own log (engine#856). + + hrx_gtt_split.py run [--out DIR] [--hz N] -- llama-cli -m M.gguf -ngl 99 ... run it, sample its GTT, print the split + hrx_gtt_split.py parse LOG [--peak-mib N | --samples FILE] the split of a log you already have + hrx_gtt_split.py --self-test + +engine#856 measured the same weights resident twice in GTT. Its reproduction named `tools/bo_trace.py`, which needs +root for tracefs and was never committed, so nobody could re-measure the issue from a clean checkout. Every number in +the split below comes from lines llama.cpp and HRX already print, plus the process's own fdinfo: + + loader's model buffer `load_tensors: HRX0 model buffer size = X MiB` (one per HRX buffer type, summed) + HRX's packed copies the largest `hrx host weights: ... resident=X MiB allocs=N` (GGML_HRX_HOST_WEIGHT_TRACE=1): + the weights HRX re-laid-out for a kernel, held next to the loader's copy + KV / recurrent state `KV buffer size =`, `RS buffer size =` on an HRX buffer type + compute + output `compute buffer size =`, `output buffer size =` on an HRX buffer type + process peak the largest sum of `drm-total-gtt` over the process's amdgpu DRM clients (each client once: + one client can be open on several fds), sampled while it runs + not attributed peak minus the rows above (runtime module memory, staging, transient arenas) + +`duplicate` is packed / model buffer: the share of the loader's copy that is resident a second time. Buffers on a +non-HRX type (`CPU`, `CPU_Mapped`) are host memory, are listed apart and are not in the GTT sum. + +`run` needs `-v` for the buffer lines and adds it, sets GGML_HRX_HOST_WEIGHT_TRACE=1, and writes `/split.log` +(the child's stderr) and `/gtt.txt` (`ms MiB` per sample). It touches the device only through the command you +give it, so run it inside your gpu-hold ticket: `gpu-hold -l L -t 5 -- tools/hrx_gtt_split.py run -- llama-cli ...`. +""" +import argparse +import os +import re +import subprocess +import sys +import tempfile +import time + +BUF = re.compile(r"\b(\S+)\s+(model|KV|RS|compute|output) buffer size =\s*([0-9.]+) MiB") +HOSTW = re.compile(r"hrx host weights: .*?resident=([0-9.]+) MiB allocs=(\d+)") + + +def is_gtt_buft(name): + """HRX0, HRX0_HOST, HRX1 ... are GTT on this APU; CPU / CPU_Mapped / CPU_REPACK are not.""" + return name.startswith("HRX") + + +def parse(text): + """Return the split as a dict; every value is MiB.""" + model = {} # buft -> summed MiB (a buffer type may be split into several buffers) + other = {} # (kind, buft) -> last MiB (a context re-reserves; the last one is what is held) + packed, packed_n, traced = 0.0, 0, False + for line in text.splitlines(): + m = BUF.search(line) + if m: + buft, kind, mib = m.group(1), m.group(2), float(m.group(3)) + if kind == "model": + model[buft] = model.get(buft, 0.0) + mib + else: + other[(kind, buft)] = mib + continue + m = HOSTW.search(line) + if m: + traced = True + if float(m.group(1)) >= packed: + packed, packed_n = float(m.group(1)), int(m.group(2)) + s = { + "model_gtt": sum(v for k, v in model.items() if is_gtt_buft(k)), + "model_host": sum(v for k, v in model.items() if not is_gtt_buft(k)), + "packed": packed if traced else None, + "packed_weights": packed_n, + "state": sum(v for (k, b), v in other.items() if k in ("KV", "RS") and is_gtt_buft(b)), + "compute": sum(v for (k, b), v in other.items() if k in ("compute", "output") and is_gtt_buft(b)), + "model_bufts": model, + } + return s + + +def client_gtt_kib(fdinfo_dir): + """Sum drm-total-gtt over the amdgpu DRM clients behind one process's fds, each client once.""" + seen, total = set(), 0 + try: + fds = os.listdir(fdinfo_dir) + except OSError: + return 0 + for fd in fds: + try: + with open(os.path.join(fdinfo_dir, fd)) as f: + txt = f.read() + except OSError: + continue + if not re.search(r"^drm-driver:\s*amdgpu", txt, re.M): + continue + cid = re.search(r"^drm-client-id:\s*(\d+)", txt, re.M) + if cid: + if cid.group(1) in seen: + continue + seen.add(cid.group(1)) + g = re.search(r"^drm-total-gtt:\s*(\d+) KiB", txt, re.M) or re.search(r"^drm-memory-gtt:\s*(\d+) KiB", txt, re.M) + total += int(g.group(1)) if g else 0 + return total + + +NO_TRACE = "unknown: no `hrx host weights` line (GGML_HRX_HOST_WEIGHT_TRACE off, or no weight was repacked)" + + +def report(s, peak): + rows = [("loader's model buffer (HRX)", s["model_gtt"])] + if s["packed"] is None: + rows.append(("HRX's packed copies", None)) + else: + rows.append(("HRX's packed copies (%d weights)" % s["packed_weights"], s["packed"])) + rows += [("KV + recurrent state", s["state"]), ("compute + output buffers", s["compute"])] + known = sum(v for _, v in rows if v is not None) + out = [] + if not s["model_bufts"]: + out += ["warning: no `model buffer size` line, so this is not a -v log of a model load", ""] + out += ["| part | MiB |", "|---|---:|"] + for name, v in rows: + out.append("| %s | %s |" % (name, NO_TRACE if v is None else "%.1f" % v)) + if peak is not None: + out.append("| not attributed | %.1f |" % (peak - known)) + out.append("| **process peak GTT** | **%.1f** |" % peak) + else: + out.append("| attributed total (no peak given) | %.1f |" % known) + if s["model_host"]: + out.append("") + out.append("host memory, not in the GTT sum: %.1f MiB of model buffers on %s" % ( + s["model_host"], ", ".join(sorted(k for k in s["model_bufts"] if not is_gtt_buft(k))))) + if s["packed"] is not None and s["model_gtt"]: + out.append("duplicate: %.1f of %.1f MiB of the loader's copy is resident a second time (%.1f %%)" % ( + s["packed"], s["model_gtt"], 100.0 * s["packed"] / s["model_gtt"])) + return "\n".join(out) + + +def peak_of_samples(path): + peak = None + with open(path) as f: + for line in f: + p = line.split() + if len(p) >= 2: + v = float(p[1]) + peak = v if peak is None or v > peak else peak + return peak + + +def cmd_run(a): + argv = list(a.command) + if argv and argv[0] == "--": + argv = argv[1:] + if not argv: + sys.exit("hrx_gtt_split: run needs a command after --") + if "-v" not in argv and "--verbose" not in argv: + argv.append("-v") + os.makedirs(a.out, exist_ok=True) + log, samples = os.path.join(a.out, "split.log"), os.path.join(a.out, "gtt.txt") + env = dict(os.environ, GGML_HRX_HOST_WEIGHT_TRACE="1") + t0 = time.monotonic() + with open(log, "w") as err, open(samples, "w") as gt: + p = subprocess.Popen(argv, stderr=err, env=env) + fdinfo = "/proc/%d/fdinfo" % p.pid + peak = 0 + try: + while p.poll() is None: + kib = client_gtt_kib(fdinfo) + peak = max(peak, kib) + gt.write("%d %.1f\n" % (int((time.monotonic() - t0) * 1000), kib / 1024.0)) + time.sleep(1.0 / a.hz) + except KeyboardInterrupt: + p.kill() + p.wait() + raise + with open(log) as f: + s = parse(f.read()) + print("# %s rc=%d log=%s samples=%s" % (" ".join(argv), p.returncode, log, samples)) + if not peak: + print("warning: the process never had an amdgpu DRM client with GTT, so there is no peak to split (not an HRX run?)") + print(report(s, peak / 1024.0 if peak else None)) + return p.returncode + + +def cmd_parse(a): + with open(a.log, errors="replace") as f: + s = parse(f.read()) + peak = a.peak_mib if a.peak_mib is not None else (peak_of_samples(a.samples) if a.samples else None) + print(report(s, peak)) + return 0 + + +# Lines from a real run: Qwen3-4B-Q4_K_M, -ngl 99 -fit off -c 256 -n 16 -v, GGML_HRX_HOST_WEIGHT_TRACE=1, llama.cpp +# 40f58efe1 (the #1351 pin), whose process peaked at 3925 MiB of drm-total-gtt. +FIXTURE_4B = """\ +0.00.378.782 I load_tensors: CPU model buffer size = 304.28 MiB +0.00.378.783 I load_tensors: HRX0 model buffer size = 2375.91 MiB +0.00.402.110 I llama_context: HRX0_HOST output buffer size = 0.58 MiB +0.00.402.300 I llama_kv_cache: HRX0 KV buffer size = 36.00 MiB +0.00.410.001 I sched_reserve: HRX0 compute buffer size = 37.50 MiB +0.00.410.002 I sched_reserve: HRX0_HOST compute buffer size = 5.13 MiB +0.02.369.640 I hrx host weight working set: 304.3 MiB accumulated, 304.3 MiB in this program +0.02.520.878 I hrx host weights: refilled=0 of 1 | hit=0 miss=1 evict=0 resident=304.3 MiB allocs=1 limit=0.0 MiB +0.03.101.877 I hrx host weights: refilled=0 of 36 | hit=1 miss=36 evict=0 resident=784.6 MiB allocs=36 limit=0.0 MiB +0.03.511.020 I hrx host weights: refilled=0 of 3 | hit=2 miss=38 evict=0 resident=795.8 MiB allocs=38 limit=0.0 MiB +0.04.900.100 D ~llama_context: HRX0 compute buffer size is 37.5000 MiB, matches expectation of 37.5000 MiB +""" + + +def self_test(): + fails = [] + + def check(name, got, want): + if (abs(got - want) > 0.01) if isinstance(want, float) else (got != want): + fails.append("%s: got %r, want %r" % (name, got, want)) + + s = parse(FIXTURE_4B) + check("model buffer on HRX", s["model_gtt"], 2375.91) + check("CPU model buffer kept out of GTT", s["model_host"], 304.28) + check("packed = the largest resident, not the first or the last-but-one", s["packed"], 795.8) + check("packed weight count", s["packed_weights"], 38) + check("KV", s["state"], 36.0) + check("compute + output, and the '~llama_context ... size is' echo is not counted twice", s["compute"], 43.21) + r = report(s, 3925.0) + check("not attributed = peak - rows", "| not attributed | 674.1 |" in r, True) + check("duplicate line", "(33.5 %)" in r, True) + + # Synthetic, not from the run: a later program reports less resident (an eviction under GGML_HRX_HOST_WEIGHT_LIMIT_MB). + # The split wants the most that was held at once, so a smaller later line must not replace it. + s = parse(FIXTURE_4B + "0.05.000.000 I hrx host weights: refilled=2 of 2 | hit=0 miss=2 evict=5 resident=512.0 MiB allocs=40 limit=600.0 MiB\n") + check("packed after an eviction keeps the largest", s["packed"], 795.8) + + s = parse(FIXTURE_4B.replace("hrx host weights", "something else")) + check("trace off: packed is unknown, not zero", s["packed"], None) + check("trace off: the report says why", NO_TRACE in report(s, 3925.0), True) + check("not a -v log: the report says so", "warning: no `model buffer size` line" in report(parse("hello\n"), 1.0), True) + + s = parse(FIXTURE_4B + "0.00.378.790 I load_tensors: HRX0 model buffer size = 100.00 MiB\n") + check("a split buffer type is summed", s["model_gtt"], 2475.91) + + with tempfile.TemporaryDirectory() as d: + def fd(n, body): + with open(os.path.join(d, str(n)), "w") as f: + f.write(body) + a = "drm-driver:\tamdgpu\ndrm-client-id:\t7\ndrm-total-gtt:\t1048576 KiB\n" + fd(3, a) + fd(4, a) # same client on a second fd + fd(5, "drm-driver:\tamdgpu\ndrm-client-id:\t9\ndrm-memory-gtt:\t2048 KiB\n") # older kernel key + fd(6, "pos:\t0\nflags:\t02\n") # not a DRM fd + fd(7, "drm-driver:\ti915\ndrm-client-id:\t1\ndrm-total-gtt:\t999 KiB\n") # another driver + check("fdinfo: each amdgpu client once", client_gtt_kib(d), 1048576 + 2048) + p = os.path.join(d, "gtt.txt") + with open(p, "w") as f: + f.write("0 0.0\n200 2674.0\n400 3925.0\n600 3100.0\n") + check("peak of samples", peak_of_samples(p), 3925.0) + + if fails: + for f in fails: + print("hrx_gtt_split self-test FAIL: " + f) + return 1 + print("hrx_gtt_split self-test: ok") + return 0 + + +def main(): + if len(sys.argv) > 1 and sys.argv[1] == "--self-test": + return self_test() + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + r = sub.add_parser("run", help="run a command with the trace on and print its split") + r.add_argument("--out", default="hrx-gtt-split") + r.add_argument("--hz", type=float, default=5.0) + r.add_argument("command", nargs=argparse.REMAINDER) + p = sub.add_parser("parse", help="print the split of an existing -v log") + p.add_argument("log") + g = p.add_mutually_exclusive_group() + g.add_argument("--peak-mib", type=float) + g.add_argument("--samples", help="a `ms MiB` file, as run writes") + a = ap.parse_args() + return cmd_run(a) if a.cmd == "run" else cmd_parse(a) + + +if __name__ == "__main__": + sys.exit(main()) From e6d42eec649a5253689109a0f6a674a9720b9c2a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:55:53 -0400 Subject: [PATCH 39/98] forward-serve: a streamed reply is sent as it is generated, not after (Refs #1785) (#2067) The NPU route built every SSE frame and sent them when the generation had ended, so time to first token was the whole request. A streaming request now generates on its own thread and hands frames to the connection thread through app/stream_bridge.h. Errors before the first frame keep their status codes; a client that leaves stops the generation. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 8 +++ app/forward_serve.cpp | 92 +++++++++++++++++++++--- app/stream_bridge.h | 118 ++++++++++++++++++++++++++++++ docs/npu.md | 9 +++ tests/stream_bridge_test.cpp | 135 +++++++++++++++++++++++++++++++++++ 5 files changed, 354 insertions(+), 8 deletions(-) create mode 100644 app/stream_bridge.h create mode 100644 tests/stream_bridge_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b817ca27f..16cbce9bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1136,6 +1136,14 @@ target_link_libraries(npu_stop_tokens_test PRIVATE nlohmann_json::nlohmann_json) add_test(NAME npu_stop_tokens COMMAND npu_stop_tokens_test) add_dependencies(npu_sampling_test npu_stop_tokens_test) +# The hand-off between the thread that generates a streamed reply and the one that writes it (app/stream_bridge.h, +# engine#1785: the NPU route sent every frame after the generation ended). Two threads and a clock; no device. +add_executable(stream_bridge_test tests/stream_bridge_test.cpp) +target_include_directories(stream_bridge_test PRIVATE ${CMAKE_SOURCE_DIR}/app) +target_link_libraries(stream_bridge_test PRIVATE Threads::Threads) +add_test(NAME stream_bridge COMMAND stream_bridge_test) +add_dependencies(npu_sampling_test stream_bridge_test) + # The plan.json loader (npu/plan/plan.h, engine #799 P1): schema, refusals and the decision fields that # replaced the filename probes. No model, no container, no device. add_executable(npu_elf_args_test tests/npu_elf_args_test.c npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 1fb9db77e..5acc1c741 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -47,6 +47,7 @@ #include "npu_prompt_ids.h" #include "npu_sampling.h" #include "npu_stop_tokens.h" +#include "stream_bridge.h" #include "npu_systemone.h" #include "tokenizer.h" #include "../npu/decision.h" // the d3 readout head (npu/decision.h, issue #653) @@ -193,7 +194,10 @@ int json_int(const std::string& path, const char* key, int def) { return def; } -void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat) { +// `bridge` is where a streamed reply's frames go as they are produced (handle_generate below runs this on its own +// thread for a streaming request); null means the caller wants the whole response in `res`. +void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat, + npu::StreamBridge* bridge) { // One id per response, taken before any step: the dumps of two prompts in one serve must not share names. const int dump_req = e.dump_logits.empty() ? 0 : ++e.dump_seq; json body; @@ -287,7 +291,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res // Returns the finish_reason the request earned: "stop" for an end-of-sequence token, // a stop sequence or a grammar that allows nothing more, "length" when the token // budget ran out. - auto run = [&e, &req, prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req]( + auto run = [&e, &req, prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req, bridge]( const std::function&)>& on_text) { std::string finish = "stop"; // The tokenizer is const and keeps no per-call state, so it runs before the lock. A caller @@ -485,7 +489,8 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res bool stopped_at_eos = false, stopped_at_stop = false; int produced = 0; for (int n = 0; n < limit; n++) { - if (req.is_connection_closed()) break; // the client is gone: free the NPU + // the client is gone: free the NPU (the bridge knows first when a write to it failed) + if (req.is_connection_closed() || (bridge && bridge->cancelled())) break; if (gram) refresh_allowed(); // Raw distribution for a reference comparison: sample_token() takes logits by const // reference and the copy below happens before it runs, so what lands on disk is what @@ -657,7 +662,13 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res chunk_tpl["object"] = "chat.completion.chunk"; chunk_tpl["created"] = created; chunk_tpl["model"] = e.id; + // With a bridge, each frame leaves as soon as it exists (the connection thread is already writing them); + // without one they are collected and sent when the handler returns, as this route always did. auto chunks = std::make_shared>(); + auto send = [&](std::string frame) { + if (bridge) bridge->push(std::move(frame)); + else chunks->push_back(std::move(frame)); + }; // The same splitter as the non-streamed path, fed piece by piece, so a streamed reply // and a non-streamed one cannot disagree about what is reasoning and what is content. npu::AnswerStream stream(family, opts.tools_sent); @@ -675,7 +686,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res // carries no entries. if (!is_reasoning && opts.sample.logprobs && !lp.empty()) chunk["choices"][0]["logprobs"] = logprobs_json(lp); - chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); + send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); } return true; }; @@ -687,7 +698,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res json chunk = chunk_tpl; json delta = is_reasoning ? json{{"reasoning_content", text}} : json{{"content", text}}; chunk["choices"] = json::array({{{"index", 0}, {"delta", delta}, {"finish_reason", nullptr}}}); - chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); + send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); } if (!stream.calls().empty()) { // The calls arrive whole, in one chunk: OpenAI streams argument fragments, and a @@ -703,7 +714,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res json delta{{"tool_calls", calls}}; if (!chunk_tpl.contains("object") || chunk_tpl["object"].is_null()) chunk["object"] = "chat.completion.chunk"; chunk["choices"] = json::array({{{"index", 0}, {"delta", delta}, {"finish_reason", "tool_calls"}}}); - chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); + send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); } // OpenAI's stream ends with one choice that carries an empty delta and the reason the // generation ended; without it a streaming client cannot tell a `length` cut from a @@ -715,7 +726,7 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res chunk["choices"] = json::array({{{"index", 0}, {"delta", json::object()}, {"finish_reason", finish}}}); - chunks->push_back("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); + send("data: " + chunk.dump(-1, ' ', false, json::error_handler_t::replace) + "\n\n"); } } { @@ -723,7 +734,11 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res tail["choices"] = json::array(); tail["usage"] = usage_json(stats); tail["timings"] = timings_json(stats); - chunks->push_back("data: " + tail.dump() + "\n\n"); + send("data: " + tail.dump() + "\n\n"); + } + if (bridge) { + bridge->push("data: [DONE]\n\n"); + return; } auto cursor = std::make_shared(0); res.set_chunked_content_provider("text/event-stream", @@ -752,6 +767,67 @@ void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Res } } +// The route's entry. A request that does not ask for a stream is answered in place. One that does is generated on +// its own thread, so its frames can be written while later tokens are still being produced (app/stream_bridge.h): +// this thread waits for the first frame or the end, and only then decides between an ordinary response -- the 400s +// and 500s a request can earn before it has produced anything keep their status codes -- and a stream. +void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat) { + bool wants_stream = false; + try { + const json body = json::parse(req.body); + wants_stream = body.is_object() && body.contains("stream") && body["stream"].is_boolean() && body["stream"].get(); + } catch (const std::exception&) { + // not JSON: the implementation answers that itself, without a stream + } + if (!wants_stream) { + handle_generate_impl(e, req, res, chat, nullptr); + return; + } + auto bridge = std::make_shared(); + auto early = std::make_shared(); // what the generation answers when it never starts a stream + // `req` outlives this handler: httplib keeps it until the response, content provider included, has been written, + // and the releaser below joins the thread before that. + auto worker = std::make_shared([&e, &req, chat, bridge, early]() { + try { + handle_generate_impl(e, req, *early, chat, bridge.get()); + } catch (const std::exception& ex) { + early->status = 500; + early->set_content(json{{"error", {{"message", ex.what()}}}}.dump(), "application/json"); + } catch (...) { + early->status = 500; + early->set_content(R"({"error":{"message":"generation failed"}})", "application/json"); + } + // A failure after the stream began cannot change the status any more: say so in the stream and end it. + // One that came before any frame pushes nothing, so the handler below still answers it as an ordinary error. + if (early->status >= 400 && bridge->started()) bridge->push("data: " + early->body + "\n\ndata: [DONE]\n\n"); + bridge->finish(); + }); + if (!bridge->wait_first()) { + worker->join(); + res.status = early->status > 0 ? early->status : 500; + res.set_content(early->body, early->has_header("Content-Type") ? early->get_header_value("Content-Type") + : std::string("application/json")); + return; + } + res.set_chunked_content_provider( + "text/event-stream", + [bridge](size_t, httplib::DataSink& sink) { + std::string frame; + while (bridge->next(frame)) { + if (!sink.write(frame.data(), frame.size())) { + bridge->cancel(); // the client went away: the generation stops at its next token + return false; + } + } + sink.done(); + return true; + }, + [bridge, worker](bool) { + bridge->cancel(); + if (worker->joinable()) worker->join(); + }); +} + // The per-question input limit of the decision route: the text path's own 32,768 tokens, or the // backbone's context when that is smaller. Either overflowing is the same failure to the model, so // the smaller one is the limit the request is checked against. diff --git a/app/stream_bridge.h b/app/stream_bridge.h new file mode 100644 index 000000000..cdca732c8 --- /dev/null +++ b/app/stream_bridge.h @@ -0,0 +1,118 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// stream_bridge.h -- hands a streamed reply from the thread that generates it to the thread that writes it. +// +// The NPU route used to build every server-sent-event frame of a streamed reply and send them all after the +// generation had ended, so a client's time to first token was the whole request (engine#1785: 14 chunks, every +// one arriving at 9.668 s of a 9.668 s reply). httplib writes a response body only after the handler returns, +// from a content provider it calls on the connection's thread, so frames can leave while tokens are still being +// produced only if the generation runs on another thread. This is the piece between the two. +// +// producer (the generation) consumer (the connection) +// push(frame) ... push(frame) wait_first() -> a frame is ready, or it ended without one +// finish() next(frame) -> the next frame in order; false when ended and drained +// cancelled() <- the client went away cancel() +// +// The decision that needs wait_first(): a request that fails before its first frame (a prompt over the context, +// a malformed field) must still be answered with its own status code, and a status cannot change once a stream +// has started. So the connection thread waits for the first frame OR the end, and only then chooses between +// "an ordinary error response" and "a stream". +// +// Frames are delivered in the order pushed, each exactly once. finish() may be called once; push() after it is +// ignored. Nothing here knows about HTTP. + +#pragma once + +#include +#include +#include +#include + +namespace onebit::npu { + +class StreamBridge { +public: + // Producer: queue one frame. Returns false when the consumer has cancelled, so the producer can stop early. + bool push(std::string frame) { + std::lock_guard lock(mu_); + if (finished_) return !cancelled_; + frames_.push_back(std::move(frame)); + ++pushed_; + cv_.notify_all(); + return !cancelled_; + } + + // Producer: no more frames. + void finish() { + std::lock_guard lock(mu_); + finished_ = true; + cv_.notify_all(); + } + + // Producer: has the consumer gone? + bool cancelled() const { + std::lock_guard lock(mu_); + return cancelled_; + } + + // Consumer: block until a frame has been pushed or the producer has finished. True when at least one frame + // was ever pushed (a stream should start), false when it ended with none (answer some other way). + bool wait_first() { + std::unique_lock lock(mu_); + cv_.wait(lock, [&] { return pushed_ > 0 || finished_; }); + return pushed_ > 0; + } + + // Consumer: the next frame, blocking while there is none and the producer is still going. False when the + // producer has finished and every frame has been taken, or after cancel(). + bool next(std::string& frame) { + std::unique_lock lock(mu_); + cv_.wait(lock, [&] { return !frames_.empty() || finished_ || cancelled_; }); + if (cancelled_ || frames_.empty()) return false; + frame = std::move(frames_.front()); + frames_.pop_front(); + return true; + } + + // Consumer: stop. The producer sees it at its next push() or cancelled(). + void cancel() { + std::lock_guard lock(mu_); + cancelled_ = true; + cv_.notify_all(); + } + + // Has any frame ever been pushed? (The producer asks this to decide whether a failure can still be an ordinary + // error response or has to be reported inside the stream.) + bool started() const { + std::lock_guard lock(mu_); + return pushed_ > 0; + } + + bool finished() const { + std::lock_guard lock(mu_); + return finished_; + } + +private: + mutable std::mutex mu_; + std::condition_variable cv_; + std::deque frames_; + size_t pushed_ = 0; + bool finished_ = false; + bool cancelled_ = false; +}; + +} // namespace onebit::npu diff --git a/docs/npu.md b/docs/npu.md index 2962883a6..76f584b16 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1520,6 +1520,15 @@ kernels (dense Qwen3: 0.6B, 1.7B and 4B): In Lemonade, the `onebit` recipe's `npu` backend serves the same directories (`--onebit npu`). The model directory holds the weights only; the kernels ship inside the package. +**Streaming.** With `"stream": true` the route sends each piece of the answer as it is produced. The first frame +leaves once the prompt has been processed and the first visible piece exists (a thinking model's opening markup is +held until it can be classified, so that is a token or two after the prompt is done), and the rest follow about +one per token. Before engine#1785 every frame was sent after the generation had ended, so a client's time to first +token was the whole request. A request that fails before producing anything (a prompt over the context, a +malformed field) still gets its own status code, not a 200 stream; one that fails later reports the error as a +last `data:` frame. A client that disconnects stops the generation at the next token, which frees the device for +the next request. The streamed reply and the non-streamed one are the same text. + ### FastFlowLM's NPU2 drops: the families the route does not serve The daily census-watch (engine#588) prints one line per new or changed repo in FastFlowLM's HF diff --git a/tests/stream_bridge_test.cpp b/tests/stream_bridge_test.cpp new file mode 100644 index 000000000..87b4934a5 --- /dev/null +++ b/tests/stream_bridge_test.cpp @@ -0,0 +1,135 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// stream_bridge_test: app/stream_bridge.h, the hand-off between the thread that generates a streamed reply and the +// thread that writes it (engine#1785). No server, no model, no device: two threads and a clock. + +#include "stream_bridge.h" + +#include +#include +#include +#include +#include +#include + +using onebit::npu::StreamBridge; +using clk = std::chrono::steady_clock; + +static int failures = 0; +static void check(bool ok, const std::string& what) { + if (!ok) { + std::printf("FAIL %s\n", what.c_str()); + ++failures; + } +} +static double ms_since(clk::time_point t) { return std::chrono::duration(clk::now() - t).count(); } + +int main() { + // 1. The point of the bridge: a frame is readable when it is pushed, not when the producer finishes. + { + StreamBridge b; + const auto t0 = clk::now(); + std::thread producer([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + b.push("one"); + std::this_thread::sleep_for(std::chrono::milliseconds(400)); + b.push("two"); + b.finish(); + }); + const bool started = b.wait_first(); + const double first_ms = ms_since(t0); + std::string f; + std::vector got; + std::vector at; + while (b.next(f)) { + got.push_back(f); + at.push_back(ms_since(t0)); + } + producer.join(); + check(started, "wait_first reports a stream once a frame exists"); + check(first_ms < 300, "wait_first returns at the first frame, not at the end (" + std::to_string(first_ms) + " ms of ~450)"); + check(got == std::vector({"one", "two"}), "frames arrive in order, each once"); + check(at.size() == 2 && at[0] < 300 && at[1] >= 400, "the first frame is taken well before the second exists"); + check(b.finished() && b.started(), "finished and started are both true at the end"); + } + // 2. A producer that ends without a frame: no stream, so the caller can still answer with a status code. + { + StreamBridge b; + std::thread producer([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + b.finish(); + }); + check(!b.wait_first(), "wait_first is false when the producer ended with no frame"); + std::string f; + check(!b.next(f) && !b.started(), "and there is nothing to read"); + producer.join(); + } + // 3. Frames pushed before anyone waits are kept, and the end is seen only after they are drained. + { + StreamBridge b; + b.push("a"); + b.push("b"); + b.finish(); + b.push("late"); // after finish: ignored + std::string f; + check(b.wait_first() && b.next(f) && f == "a" && b.next(f) && f == "b" && !b.next(f), + "queued frames survive finish() and a push after finish() is dropped"); + } + // 4. cancel(): the consumer stops, a blocked next() returns, and the producer is told. + { + StreamBridge b; + std::atomic producer_saw{false}; + std::thread producer([&] { + b.push("x"); + for (int i = 0; i < 200 && !b.cancelled(); i++) std::this_thread::sleep_for(std::chrono::milliseconds(5)); + producer_saw = b.cancelled(); + check(!b.push("after cancel"), "push reports the cancellation to the producer"); + b.finish(); + }); + std::string f; + check(b.next(f) && f == "x", "the frame before the cancel is delivered"); + std::thread canceller([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(40)); + b.cancel(); + }); + const auto t0 = clk::now(); + check(!b.next(f), "next() returns false once cancelled"); + check(ms_since(t0) < 500, "and a next() that was blocked is released by the cancel"); + canceller.join(); + producer.join(); + check(producer_saw.load(), "the producer sees cancelled()"); + } + // 5. Many frames from a fast producer: none lost, none repeated, order kept. + { + StreamBridge b; + const int n = 20000; + std::thread producer([&] { + for (int i = 0; i < n; i++) b.push(std::to_string(i)); + b.finish(); + }); + std::string f; + int expect = 0; + bool ordered = true; + while (b.next(f)) { + ordered = ordered && f == std::to_string(expect); + ++expect; + } + producer.join(); + check(ordered && expect == n, "20000 frames cross in order with none lost (" + std::to_string(expect) + ")"); + } + if (failures == 0) std::printf("stream_bridge: ok\n"); + return failures == 0 ? 0 : 1; +} From 3151a82c2074935a2717e25a7fe6c1294acae411 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:56:03 -0400 Subject: [PATCH 40/98] npu: the decode-family block gets its own method, body unchanged (engine#803 A2b step one) (#2062) A2b moves the M=16 decode family behind XdnaTarget::open_decode_family(), mirroring open_head (#1478). This is the step that makes that move a copy instead of surgery inside init_impl -- the same step the lm head went through before it became open_head, and the tree's own comment on that block says so. What moved: q4nx_forward.cpp's 'if (want_m16 \&\& !host_gemm_) { ... }' block (157 lines) is now Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, const std::string\& mdir, const std::string\& dir) -- body copied verbatim, de-indented one level, with the three locals it reads threaded in as parameters. Two things deliberately unchanged, because gates read them: * the log strings are byte-for-byte ('[forward] ... (decode)', '[forward] decode family: N role(s) at M=16 ...', the pb16i 'N of M designs have a decode kernel' line) -- the first is grepped by [pb16i-trace] and the second by the window11 gates, so improving a message here breaks a gate that reads it; * the guard shape is preserved: the call site still passes want_m16 (from NPU_DECODE_M16), and the method still prints the 'disabled' line rather than silently returning. Verified: builds (onebit, with the dx add-on and the private runtime), and npu_forward_loader_test passes ('all cases ok'). NOT included, and stated on the issue so it is not assumed: the move into XdnaTarget itself -- the block reaches six forward members (family_tag(), designs_, designs16_, pb16i_, t2s_, host_gemm_) plus these three locals, so the target method needs a parameters struct or accessors, and that is a design decision the next diff has to make rather than a mechanical copy. Co-authored-by: pi-inferzo --- npu/forward/include/q4nx_forward.h | 5 + npu/forward/src/q4nx_forward.cpp | 333 +++++++++++++++-------------- 2 files changed, 180 insertions(+), 158 deletions(-) diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 403481ce6..595be1805 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -234,6 +234,11 @@ class Q4nxNpuForward { // Set only while step_batch_all() runs: the batched epilogue then norms every row and writes M x NV // logits there instead of only the last row's. Null on every other path, so nothing else changes. std::vector* batch_all_out_ = nullptr; + // engine#803 A2b step one: the decode-family block as its own method, body unchanged, so the move into + // XdnaTarget::open_decode_family() is a copy rather than surgery inside init_impl. + bool open_decode_family_local(bool want_m16, xrt::device* dev, const std::string& mdir, + const std::string& dir); + bool init_impl(xrt::device* dev, const char* model_path, const std::string& artifact_dir); bool host_gemm_ = false; diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index b638ad8dd..6a0a0a534 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -426,6 +426,178 @@ bool Q4nxNpuForward::init_host(const char* model_path, const std::string& dir) { return init_impl(nullptr, model_path, dir); } +// engine#803 A2b, step one: the decode family gets its own method with the body unchanged, exactly as the lm +// head did before it moved behind XdnaTarget (open_head). That precedent is why this is a move and not a +// rewrite: the next step is then a copy into the target rather than surgery inside init_impl, and the three +// log lines below are copied byte-for-byte because gates grep them ("[pb16i-trace]", the decode-family +// line, the pb16i refusal) -- "improving" a message here breaks a gate that reads it. +bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, const std::string& mdir, + const std::string& dir) { + if (host_gemm_) return true; // the host arm packs its own weights; no decode family is loaded + if (!want_m16) { + fprintf(stderr, " [forward] decode family: %d role(s) at M=%d, %d on M=%d%s\n", 0, + Q4NX_DESIGN_M_DECODE, ndesigns_, Q4NX_DESIGN_M_PREFILL, " (NPU_DECODE_M16=0: disabled)"); + return true; + } + q4nx_derive_decode_designs(designs_, ndesigns_, designs16_); + // An integer-exact ternary container's decode family is its own kernel set, not the int8 + // M=16 twins: same M=16 activation tile, but the image (and so the tap) is pb16i's or + // t2s's, the ELF name carries that suffix, and the geometry test is the image's + // (pb16i: K%256, N%1024; t2s: K%256, N%32) rather than the int8 twin's tiling. A design + // that is ternary-only skips the derivation entirely: its own entry already IS the decode + // design. + if (pb16i_) { + const std::string ft = family_tag(); + for (int i = 0; i < ndesigns_; i++) { + if (i == xd_->role_split_qhalf() || i == xd_->role_split_kv()) { + designs16_[i] = designs_[i]; + continue; + } + // The wide fused full-attention QKV has no ternary kernel (its int64 half + // needs 448 columns per core); when the split designs exist that role is + // covered by them, so this design simply has no decode kernel. + const Q4nxDesign r16 = designs_[i].role; + if (xd_->role_split_qhalf() >= 0 && (r16 == Q4NX_DESIGN_QKV || r16 == Q4NX_DESIGN_QKV2)) { + designs16_[i].M = 0; + continue; + } + const bool geo_ok = t2s_ ? q4nx_t2s::fits(designs16_[i].K, designs16_[i].N) + : q4nx_pb16i::fits(designs16_[i].K, designs16_[i].N); + if (designs16_[i].M <= 0 || !geo_ok) { + // Say WHY, once per role. A geometry the ternary tile cannot hold is not a + // missing artifact and not an error: the role keeps its M=128 kernel and the + // model is correct, only slower -- so the only symptom is a decode step that + // costs what a prefill step costs, with nothing in the log. That is the same + // "nothing logs the decline" gap the HRX prefill N%16 note names for a + // different dispatcher, and it is cheap to close here. + if (designs16_[i].M > 0 && !geo_ok) + fprintf(stderr, + " [forward] dec %-3s: no %s twin (K=%d N=%d does not fit the tile), " + "staying on M=128\n", + role_tag(designs16_[i].role), t2s_ ? "t2s" : "pb16i", + designs16_[i].K, designs16_[i].N); + designs16_[i].M = 0; + continue; + } + // Build the new name in a copy: snprintf must not read and write the + // same buffer. + std::string n16(designs16_[i].elf_name); + n16.resize(n16.size() - 4); // drop ".elf" + if (n16.size() >= 4 && n16.compare(n16.size() - 4, 4, "_m16") == 0) + n16.resize(n16.size() - 4); // and the m16 twin's suffix + n16 += ft; + // pb16i's decode family carries no `_m16` marker: M=16 is its only design and those + // files already exist under that name on every deployed container. A t2s container's + // M=128 designs keep their int8 names (untagged), so its decode kernels are told + // apart by the marker the arena uses: full_i8_GU_K5120_N17408_t2s_m16.elf. + if (t2s_) n16 += "_m16"; + n16 += ".elf"; + snprintf(designs16_[i].elf_name, Q4NX_ELF_NAME_MAX, "%s", n16.c_str()); + } + } + std::vector m16dirs; + plan_decode_dirs_.clear(); + if (plan_.loaded() && !plan_.decode_dirs.empty()) { + // The plan says which of the two given paths each candidate hangs off, so the runtime + // stops guessing which base a bare directory name meant. + for (const auto& d : plan_.decode_dirs) + plan_decode_dirs_.push_back((d.under_model ? mdir : dir) + "/" + d.path); + for (const auto& p : plan_decode_dirs_) m16dirs.push_back(p); + } else { + m16dirs.push_back(mdir + "/kernels-m16"); + m16dirs.push_back(dir + "/m16"); + m16dirs.push_back(dir); + } + int made = 0, skipped = 0; + for (int i = 0; i < ndesigns_; i++) { + if (designs16_[i].M <= 0) { skipped++; continue; } + std::string elf, kname = "main:seq"; + if (pb16i_ && !pb16i_sealed_.empty()) { + // One ELF for every ternary phase: the file is the same for all roles and + // only the instance name differs, so I8Ctx reuses the one hw_context. + const char* inst = pb16i_instance(designs16_[i].role); + if (!inst) { + fprintf(stderr, " [forward] pb16i: the sealed ELF has no instance for " + "role %s; this role has no decode kernel\n", + role_tag(designs16_[i].role)); + designs16_[i].M = 0; + skipped++; + continue; + } + elf = dir + "/" + pb16i_sealed_; + kname = inst; + if (!file_exists(elf)) { + err_ = plan_.loaded() ? ("plan " + plan_path_ + ": decode.sealed_elf \"" + + pb16i_sealed_ + "\" is not in " + dir + " (missing)") + : ("pb16i: missing sealed ELF " + elf); + return false; + } + } else { + for (const std::string& d : m16dirs) { + std::string cand = d + "/" + designs16_[i].elf_name; + if (file_exists(cand)) { elf = cand; break; } + } + } + if (elf.empty()) { + // A pb16i role without its image kernel must stop the load: it is not + // a slower path, it is a missing one (the int8 weights are not packed + // for these geometries under this layout). + if (pb16i_) { + err_ = plan_.loaded() + ? std::string("plan ") + plan_path_ + ": decode role " + + role_tag(designs16_[i].role) + " needs \"" + designs16_[i].elf_name + + "\" (missing)" + : std::string("pb16i: missing decode ELF ") + designs16_[i].elf_name; + return false; + } + skipped++; + continue; // stay on M=128 + } + I8Ctx* c16 = xd_->make_ctx16(i, designs16_[i]); + if (!c16) { err_ = "cannot create the decode context for role " + std::to_string(i); return false; } + c16->MD = designs16_[i].M; + c16->KD = designs16_[i].K; + c16->ND = designs16_[i].N; + c16->blocked_b = weight_layout_b8_; + c16->pb16i = pb16i_; + c16->t2s = t2s_; + // Which pb16i ELFs leave two int32 planes (the arena's ACC64 family: D, O and + // the full-attention q half). This mirrors the ELF family contract rather + // than inferring it, and load/launch refuse a tensor that needs two planes on + // a one-plane ELF. + if (pb16i_) { + const Q4nxDesign r = designs16_[i].role; + c16->pb16i_planes = + (r == Q4NX_DESIGN_D || r == Q4NX_DESIGN_O || r == Q4NX_DESIGN_QKVFA_Q) ? 2 : 1; + } + if (!c16->init_elf(*dev, elf.c_str(), designs16_[i].M, + designs16_[i].K, designs16_[i].N, 1, kname.c_str())) { + if (pb16i_) { + err_ = "pb16i: init_elf failed for " + elf; + return false; + } + fprintf(stderr, " [forward] M=16 twin failed to load (%s); " + "this role keeps M=128\n", designs16_[i].elf_name); + xd_->drop_ctx16(i); + skipped++; + continue; + } + made++; + fprintf(stderr, " [forward] %-7s M=%d K=%d N=%d %s (decode)\n", + role_tag(designs_[i].role), designs16_[i].M, designs16_[i].K, + designs16_[i].N, elf.c_str()); + } + xd_->note_m16_made(made > 0); + if (pb16i_ && made < ndesigns_ - 0) { + fprintf(stderr, " [forward] %s: %d of %d designs have a decode kernel\n", + t2s_ ? "t2s" : "pb16i", made, ndesigns_); + } + fprintf(stderr, " [forward] decode family: %d role(s) at M=%d, %d on M=%d%s\n", + made, Q4NX_DESIGN_M_DECODE, skipped, Q4NX_DESIGN_M_PREFILL, + want_m16 ? "" : " (NPU_DECODE_M16=0: disabled)"); + return true; +} + bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, const std::string& dir) { if (const char* hg = getenv("NPU_INFER_HOST_GEMM")) { @@ -1258,164 +1430,9 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, xd_->set_m16_check(getenv("NPU_M16_CHECK") != nullptr); const char* m16_env = getenv("NPU_DECODE_M16"); const bool want_m16 = !m16_env || atoi(m16_env) != 0; - if (want_m16 && !host_gemm_) { - q4nx_derive_decode_designs(designs_, ndesigns_, designs16_); - // An integer-exact ternary container's decode family is its own kernel set, not the int8 - // M=16 twins: same M=16 activation tile, but the image (and so the tap) is pb16i's or - // t2s's, the ELF name carries that suffix, and the geometry test is the image's - // (pb16i: K%256, N%1024; t2s: K%256, N%32) rather than the int8 twin's tiling. A design - // that is ternary-only skips the derivation entirely: its own entry already IS the decode - // design. - if (pb16i_) { - const std::string ft = family_tag(); - for (int i = 0; i < ndesigns_; i++) { - if (i == xd_->role_split_qhalf() || i == xd_->role_split_kv()) { - designs16_[i] = designs_[i]; - continue; - } - // The wide fused full-attention QKV has no ternary kernel (its int64 half - // needs 448 columns per core); when the split designs exist that role is - // covered by them, so this design simply has no decode kernel. - const Q4nxDesign r16 = designs_[i].role; - if (xd_->role_split_qhalf() >= 0 && (r16 == Q4NX_DESIGN_QKV || r16 == Q4NX_DESIGN_QKV2)) { - designs16_[i].M = 0; - continue; - } - const bool geo_ok = t2s_ ? q4nx_t2s::fits(designs16_[i].K, designs16_[i].N) - : q4nx_pb16i::fits(designs16_[i].K, designs16_[i].N); - if (designs16_[i].M <= 0 || !geo_ok) { - // Say WHY, once per role. A geometry the ternary tile cannot hold is not a - // missing artifact and not an error: the role keeps its M=128 kernel and the - // model is correct, only slower -- so the only symptom is a decode step that - // costs what a prefill step costs, with nothing in the log. That is the same - // "nothing logs the decline" gap the HRX prefill N%16 note names for a - // different dispatcher, and it is cheap to close here. - if (designs16_[i].M > 0 && !geo_ok) - fprintf(stderr, - " [forward] dec %-3s: no %s twin (K=%d N=%d does not fit the tile), " - "staying on M=128\n", - role_tag(designs16_[i].role), t2s_ ? "t2s" : "pb16i", - designs16_[i].K, designs16_[i].N); - designs16_[i].M = 0; - continue; - } - // Build the new name in a copy: snprintf must not read and write the - // same buffer. - std::string n16(designs16_[i].elf_name); - n16.resize(n16.size() - 4); // drop ".elf" - if (n16.size() >= 4 && n16.compare(n16.size() - 4, 4, "_m16") == 0) - n16.resize(n16.size() - 4); // and the m16 twin's suffix - n16 += ft; - // pb16i's decode family carries no `_m16` marker: M=16 is its only design and those - // files already exist under that name on every deployed container. A t2s container's - // M=128 designs keep their int8 names (untagged), so its decode kernels are told - // apart by the marker the arena uses: full_i8_GU_K5120_N17408_t2s_m16.elf. - if (t2s_) n16 += "_m16"; - n16 += ".elf"; - snprintf(designs16_[i].elf_name, Q4NX_ELF_NAME_MAX, "%s", n16.c_str()); - } - } - std::vector m16dirs; - plan_decode_dirs_.clear(); - if (plan_.loaded() && !plan_.decode_dirs.empty()) { - // The plan says which of the two given paths each candidate hangs off, so the runtime - // stops guessing which base a bare directory name meant. - for (const auto& d : plan_.decode_dirs) - plan_decode_dirs_.push_back((d.under_model ? mdir : dir) + "/" + d.path); - for (const auto& p : plan_decode_dirs_) m16dirs.push_back(p); - } else { - m16dirs.push_back(mdir + "/kernels-m16"); - m16dirs.push_back(dir + "/m16"); - m16dirs.push_back(dir); - } - int made = 0, skipped = 0; - for (int i = 0; i < ndesigns_; i++) { - if (designs16_[i].M <= 0) { skipped++; continue; } - std::string elf, kname = "main:seq"; - if (pb16i_ && !pb16i_sealed_.empty()) { - // One ELF for every ternary phase: the file is the same for all roles and - // only the instance name differs, so I8Ctx reuses the one hw_context. - const char* inst = pb16i_instance(designs16_[i].role); - if (!inst) { - fprintf(stderr, " [forward] pb16i: the sealed ELF has no instance for " - "role %s; this role has no decode kernel\n", - role_tag(designs16_[i].role)); - designs16_[i].M = 0; - skipped++; - continue; - } - elf = dir + "/" + pb16i_sealed_; - kname = inst; - if (!file_exists(elf)) { - err_ = plan_.loaded() ? ("plan " + plan_path_ + ": decode.sealed_elf \"" + - pb16i_sealed_ + "\" is not in " + dir + " (missing)") - : ("pb16i: missing sealed ELF " + elf); - return false; - } - } else { - for (const std::string& d : m16dirs) { - std::string cand = d + "/" + designs16_[i].elf_name; - if (file_exists(cand)) { elf = cand; break; } - } - } - if (elf.empty()) { - // A pb16i role without its image kernel must stop the load: it is not - // a slower path, it is a missing one (the int8 weights are not packed - // for these geometries under this layout). - if (pb16i_) { - err_ = plan_.loaded() - ? std::string("plan ") + plan_path_ + ": decode role " + - role_tag(designs16_[i].role) + " needs \"" + designs16_[i].elf_name + - "\" (missing)" - : std::string("pb16i: missing decode ELF ") + designs16_[i].elf_name; - return false; - } - skipped++; - continue; // stay on M=128 - } - I8Ctx* c16 = xd_->make_ctx16(i, designs16_[i]); - if (!c16) { err_ = "cannot create the decode context for role " + std::to_string(i); return false; } - c16->MD = designs16_[i].M; - c16->KD = designs16_[i].K; - c16->ND = designs16_[i].N; - c16->blocked_b = weight_layout_b8_; - c16->pb16i = pb16i_; - c16->t2s = t2s_; - // Which pb16i ELFs leave two int32 planes (the arena's ACC64 family: D, O and - // the full-attention q half). This mirrors the ELF family contract rather - // than inferring it, and load/launch refuse a tensor that needs two planes on - // a one-plane ELF. - if (pb16i_) { - const Q4nxDesign r = designs16_[i].role; - c16->pb16i_planes = - (r == Q4NX_DESIGN_D || r == Q4NX_DESIGN_O || r == Q4NX_DESIGN_QKVFA_Q) ? 2 : 1; - } - if (!c16->init_elf(*dev, elf.c_str(), designs16_[i].M, - designs16_[i].K, designs16_[i].N, 1, kname.c_str())) { - if (pb16i_) { - err_ = "pb16i: init_elf failed for " + elf; - return false; - } - fprintf(stderr, " [forward] M=16 twin failed to load (%s); " - "this role keeps M=128\n", designs16_[i].elf_name); - xd_->drop_ctx16(i); - skipped++; - continue; - } - made++; - fprintf(stderr, " [forward] %-7s M=%d K=%d N=%d %s (decode)\n", - role_tag(designs_[i].role), designs16_[i].M, designs16_[i].K, - designs16_[i].N, elf.c_str()); - } - xd_->note_m16_made(made > 0); - if (pb16i_ && made < ndesigns_ - 0) { - fprintf(stderr, " [forward] %s: %d of %d designs have a decode kernel\n", - t2s_ ? "t2s" : "pb16i", made, ndesigns_); - } - fprintf(stderr, " [forward] decode family: %d role(s) at M=%d, %d on M=%d%s\n", - made, Q4NX_DESIGN_M_DECODE, skipped, Q4NX_DESIGN_M_PREFILL, - want_m16 ? "" : " (NPU_DECODE_M16=0: disabled)"); - } + // engine#803 A2b (step one): the block moved verbatim into open_decode_family_local(); the call site keeps the + // guard so the "disabled" log line is still printed from one place. + if (!open_decode_family_local(want_m16, dev, mdir, dir)) return false; // The lm head's own context (pb16i ternary images, the int8 ladder, or the q4_1 gemv head). Cut out // into its own method with its body unchanged, so the next step (moving it behind XdnaTarget, as the From de0bf51f81f31354adb16b001b43741a821410a8 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:56:10 -0400 Subject: [PATCH 41/98] gitignore: npu_pack_import_test's q4nx-container fixtures are run output (#2061) test_q4nx_container() writes its fixtures to ./q4nx-container when the test runs with no golden dir. A run from the source root therefore leaves them where `git add -A` stages them: #1986 committed five of them (820 KB), and the copyright gate went red on all five. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Opus 5.5 --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 7925b4d1d..4820db372 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,8 @@ pin-report.json # probe - a red that is the tree's, not the repository's (engine#1564). Ignoring them also stops run output being # committed by accident. .hrx123/logs/ + +# npu_pack_import_test's container-genre fixtures (test_q4nx_container): written to /q4nx-container, and to +# ./q4nx-container when the test runs with no golden dir -- so a run from the source root left them where `git add -A` +# picks them up, and #1986 committed five of them (820 KB) and reddened the copyright gate. Run output, not source. +q4nx-container/ From dd585f2f20d1604eec8d06a85bdf3d6c2cb8ffc1 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:56:17 -0400 Subject: [PATCH 42/98] npu/plan: a fifth OpId cannot arrive without a registry entry (engine#799 rule 1) (#2058) The header says the hole itself: "Without [`count`] a fifth OpId added tomorrow with no entry leaves both green -- the reviewer of this PR named exactly that hole, and it is the hole the rule exists to close". op_coverage() and the test do catch a missing entry, but only when someone runs them, which is after the enum, the registry table and target.h's Op have all been edited by hand. A static_assert beside the enum turns the same omission into a build failure that cannot be skipped, and bumping the number is the decision: adding an op now requires its entry, its known-answer case and the constant to move in one commit. Once the constant is bumped, op_coverage()'s exact-string assertion also forces the gap to be declared rather than left implicit. Verified both directions with a syntax-only compile (no build, disk is under the fleet floor): with four ops -> compiles clean with a fifth op added -> static assertion failed: "a new OpId needs an entry in the registry table, a known-answer case in it, and this number bumped: bumping it is the decision, so a fifth op cannot arrive green by omission (engine#799 rule 1)" Not the plan-load check I first proposed on the issue: a plan names design ROLES, not OpIds (q4nx_pack.h:273 -- the unique role name is not even the display tag, since a qwen3_5 pack derives an attention QKV and a GDN QKVLIN that both print as "QKV"), so enforcing at load would need a hand-written role->op table whose errors refuse VALID plans -- and with all four ops already having entries it would enforce nothing today. The reasoning is on the issue; this commit is the slice that actually closes the stated hole. --- npu/plan/op_registry.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/npu/plan/op_registry.h b/npu/plan/op_registry.h index 307f21ff0..50bbea145 100644 --- a/npu/plan/op_registry.h +++ b/npu/plan/op_registry.h @@ -48,6 +48,17 @@ namespace onebit::npu::plan { // exactly that hole, and it is the hole the rule exists to close. enum class OpId { gemv = 0, gdn_step, attn_step, lm_head, count }; +// engine#799 rule 1, made a compile error rather than a test result. The comment above says it: "Without [`count`] a +// fifth OpId added tomorrow with no entry leaves both green". `op_coverage()` and the test do catch a missing entry -- +// but only when someone runs them, which is after the enum, the table and the target's `Op` have all been edited by +// hand. This turns the same omission into a build failure that cannot be skipped, and bumping the number IS the +// decision: whoever adds an op has to add its entry, its known-answer case and this constant in one commit, or nothing +// compiles. When the number is bumped, `op_coverage()`'s exact-string assertion also forces the gap to be declared +// rather than left implicit. +static_assert(static_cast(OpId::count) == 4, + "a new OpId needs an entry in the registry table, a known-answer case in it, and this number bumped: " + "bumping it is the decision, so a fifth op cannot arrive green by omission (engine#799 rule 1)"); + const char* op_name(OpId op); // One known-answer case. `a` is m x k and `b` is k x n, both row-major; `want` is m x n. From a2f180db1c3065147018868f4a9f576c869629d1 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:56:26 -0400 Subject: [PATCH 43/98] npu-pack: the container genre's geometry, as a pure function with its source named (engine#1786, F1 slice 1) (#2054) * npu-pack: the container genre's geometry, as a pure function with its source named engine#1786 (item F1), slice 1 of 3. #1974 taught the importer to *read* an NPU2 container and to recognise the genre; the plan still says "recognised and reported, not importable", because nothing derives geometry from the container. This is the function that does the deriving, and nothing else yet -- `plan()` is wired to it in slice 2. `container_geometry(meta, config)` takes the container's `__metadata__` and config.json and returns the Dims, the `source` that answered ("dims" / "config.json" / empty), the fields nothing could fill (`missing`) and the fields derived from another (`inferred`). Why the source has to be named: the census on this box (engine#1786) measured three genres over 29 packs -- 13 self-describing with a full `dims` block, 14 gguf-converted with only converter/format/source, and 2 (`d3-edge`, `d3-flash`) with no metadata at all -- and 29 of 29 also carry config.json. So the same field can arrive from two places that can disagree, and a plan that merely holds a number cannot say whether it was declared or read. Order of authority is the container's own statement, then the file beside it. `head_dim` when absent is derived from hidden/heads and reported as **inferred** rather than as declared: that identity is false for every architecture whose q projection is not square. `Q4nxContainer` keeps `meta` for the same reason: the reader already parses the header and was throwing `__metadata__` away. Arms (no pack, no device, no file -- which is what makes the two no-metadata genres pinnable at all): self-describing answers from `dims` and beats a disagreeing config.json; gguf-converted answers from config.json; silent reports all seven required fields missing and invents none; a derivable head_dim is reported as inferred and not also missing. * npu-pack test: the silent-genre arm asserted seven missing fields; there are eight CI caught this arm on its first run: `container_geometry({}, {})` reports eight required fields, not seven, because `rope_theta` is part of the arithmetic (the RoPE base the forward needs) and I had counted the fields from memory. The arm was wrong, the function was right. It asserts the names now rather than only the count, so the next field added to `Dims` fails this arm instead of quietly shrinking what it covers. --------- Co-authored-by: t --- npu/pack_import.cpp | 52 +++++++++++++++++++++++++++++ npu/pack_import.h | 20 +++++++++++ tests/npu_pack_import_test.cpp | 61 ++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 492a7d494..429347cc3 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1904,6 +1904,7 @@ Q4nxContainer read_q4nx_container(const std::string& path) { out.why = path + ": header is not a JSON object"; return out; } + if (j.contains("__metadata__") && j["__metadata__"].is_object()) out.meta = j["__metadata__"]; for (auto it = j.begin(); it != j.end(); ++it) { if (it.key() == "__metadata__") continue; if (!it.value().is_object()) { @@ -3313,6 +3314,57 @@ std::string header_witness_refusal(int vote_hdrs, bool declared_qk) { "neither is being preferred"; } +// engine#1786 (item F1): the container genre's geometry, from the container's own `dims` block and then config.json. +// The order is authority, not preference: `dims` is the pack's own statement about itself, config.json is a file beside +// it that a vendor converter wrote. When the two disagree the container's statement wins and `source` says so -- the +// census measured 13 packs carrying `dims` and 29 of 29 carrying config.json, so a silent disagreement is possible and +// a reader has to be able to see which one answered. +ContainerGeometry container_geometry(const json& meta, const json& config) { + ContainerGeometry g; + const json& d = (meta.is_object() && meta.contains("dims") && meta["dims"].is_object()) ? meta["dims"] : json(); + const json& cfg = config.is_object() ? config : json(); + bool used_dims = false, used_cfg = false; + + // an integer from dims first, then config.json; records which one answered + const auto iv = [&](const char* key, int& dst) -> bool { + if (d.is_object() && d.contains(key) && d[key].is_number_integer()) { dst = d[key].get(); used_dims = true; return true; } + if (cfg.is_object() && cfg.contains(key) && cfg[key].is_number_integer()) { dst = cfg[key].get(); used_cfg = true; return true; } + return false; + }; + const auto fv = [&](const char* key, float& dst) -> bool { + if (d.is_object() && d.contains(key) && d[key].is_number()) { dst = d[key].get(); used_dims = true; return true; } + if (cfg.is_object() && cfg.contains(key) && cfg[key].is_number()) { dst = cfg[key].get(); used_cfg = true; return true; } + return false; + }; + const auto sv = [&](const char* key, std::string& dst) -> bool { + if (d.is_object() && d.contains(key) && d[key].is_string()) { dst = d[key].get(); used_dims = true; return true; } + if (cfg.is_object() && cfg.contains(key) && cfg[key].is_string()) { dst = cfg[key].get(); used_cfg = true; return true; } + return false; + }; + + if (!iv("hidden_size", g.dims.hidden)) g.missing.push_back("hidden_size"); + if (!iv("num_hidden_layers", g.dims.layers)) g.missing.push_back("num_hidden_layers"); + if (!iv("num_attention_heads", g.dims.heads)) g.missing.push_back("num_attention_heads"); + if (!iv("num_key_value_heads", g.dims.kv_heads)) g.missing.push_back("num_key_value_heads"); + if (!iv("vocab_size", g.dims.vocab)) g.missing.push_back("vocab_size"); + if (!iv("intermediate_size", g.dims.intermediate)) g.missing.push_back("intermediate_size"); + if (!iv("head_dim", g.dims.head_dim)) { + // derived, and said to be derived: hidden/heads is only head_dim when the pack does not state one, and it is + // wrong for every architecture whose q projection is not square (GQA with an explicit head_dim is common). + if (g.dims.hidden > 0 && g.dims.heads > 0) { + g.dims.head_dim = g.dims.hidden / g.dims.heads; + g.inferred.push_back("head_dim"); + } else { + g.missing.push_back("head_dim"); + } + } + if (!fv("rope_theta", g.dims.rope_theta)) g.missing.push_back("rope_theta"); + sv("model_type", g.dims.model_type); // not load-bearing for the arithmetic; absent is not missing + + g.source = used_dims ? "dims" : (used_cfg ? "config.json" : ""); + return g; +} + std::string qk_norm_refusal(bool declared, bool found) { if (declared && !found) return "this pack declares use_qk_norm and its per-head q/k gains are not where the block layout says " diff --git a/npu/pack_import.h b/npu/pack_import.h index dc62f27ff..f21dcf51b 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -289,10 +289,30 @@ struct Q4nxContainer { int64_t header_bytes = 0; int64_t payload_bytes = 0; std::vector tensors; // in header order + // The safetensors `__metadata__` object, kept because it is where this genre states its own geometry (engine#1786). + // Three quarters of the packs measured on this box carry a `dims` block with hidden/intermediate/layers/heads/vocab; + // two carry no metadata at all (`d3-edge`, `d3-flash`), which is why the reader keeps it as data and does not + // interpret it here. + nlohmann::json meta; std::string why; // non-empty when the container could not be read; tensors is then empty }; Q4nxContainer read_q4nx_container(const std::string& path); +// engine#1786 (item F1): the container genre's geometry, from the two places a pack can state it, in order of +// authority -- the container's own `dims` block, then config.json. `source` names which one answered, so a plan can +// never present a declared number and a file's number as the same kind of fact; `missing` names the fields neither +// could fill, and `inferred` the ones derived from other fields (head_dim from hidden/heads). +// +// Pure: no file is opened, so the three genres the census measured -- 13 self-describing, 14 gguf-converted, 2 with +// no metadata at all -- can all be pinned without a pack, which is the point of extracting it from `plan()`. +struct ContainerGeometry { + Dims dims; + std::string source; // "dims", "config.json", or empty when neither answered + std::vector missing; // fields no source could fill, by name + std::vector inferred; // fields derived from another field, by name +}; +ContainerGeometry container_geometry(const nlohmann::json& meta, const nlohmann::json& config); + // Reads the pack (model.onnx, its external data, both .fconst headers, config.json). // Throws only when the pack is structurally unreadable. Plan plan(const std::string& pack_dir); diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index e494e8110..2a2bca1d4 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -4765,6 +4765,66 @@ void test_embedding_f32_component(const std::string& root) { fs::remove_all(b); } +// engine#1786 (item F1): the container genre's geometry, pinned as a pure function over the three header shapes the +// census measured on this box -- no pack, no device, no file, which is what makes the two genres with no config.json +// (`d3-edge`, `d3-flash`) pinnable at all. +void test_container_geometry() { + using nlohmann::json; + // 1. self-describing: `dims` answers everything, and config.json must NOT be preferred over it. + { + json meta = {{"dims", {{"hidden_size", 576}, {"intermediate_size", 1536}, {"num_hidden_layers", 30}, + {"num_attention_heads", 9}, {"num_key_value_heads", 3}, {"vocab_size", 49152}, + {"head_dim", 64}, {"rope_theta", 10000.0}, {"model_type", "llama"}}}}; + json cfg = {{"hidden_size", 1024}}; // a deliberate disagreement: the container's own statement wins + const auto g = pi::container_geometry(meta, cfg); + check(g.source == "dims", "self-describing: source is dims"); + check(g.dims.hidden == 576, "self-describing: hidden comes from dims, not config.json"); + check(g.dims.layers == 30 && g.dims.heads == 9 && g.dims.kv_heads == 3 && g.dims.vocab == 49152, + "self-describing: every field dims states is taken"); + check(g.dims.head_dim == 64, "self-describing: a stated head_dim is used as stated"); + check(g.missing.empty(), "self-describing: nothing is missing"); + check(g.inferred.empty(), "self-describing: nothing is inferred"); + } + // 2. gguf-converted: no dims, so config.json answers and the source says so. + { + json meta = {{"converter", "tools/gguf_to_q4nx.py"}, {"format", "q4nx"}, {"source", "x.gguf"}}; + json cfg = {{"hidden_size", 1024}, {"num_hidden_layers", 28}, {"num_attention_heads", 16}, + {"num_key_value_heads", 8}, {"vocab_size", 151936}, {"intermediate_size", 3072}, + {"head_dim", 64}, {"rope_theta", 1000000.0}}; + const auto g = pi::container_geometry(meta, cfg); + check(g.source == "config.json", "gguf-converted: source is config.json"); + check(g.dims.hidden == 1024 && g.dims.vocab == 151936, "gguf-converted: config.json supplies the geometry"); + check(g.missing.empty(), "gguf-converted: nothing is missing when config.json is complete"); + } + // 3. silent: neither answers, so every field is named missing rather than left as a zero that reads like a value. + { + const auto g = pi::container_geometry(json::object(), json()); + check(g.source.empty(), "silent: no source is claimed"); + // eight, and named: the first run of this arm asserted seven and failed in CI -- rope_theta is a required + // field of the arithmetic (it is the RoPE base the forward needs), so the count was wrong and the function + // was right. Asserted by name now, so the next field added to Dims fails this arm instead of shrinking it. + const std::vector want = {"hidden_size", "num_hidden_layers", "num_attention_heads", + "num_key_value_heads", "vocab_size", "intermediate_size", + "head_dim", "rope_theta"}; + check(g.missing.size() == want.size(), "silent: every required field is named missing"); + bool all_named = true; + for (const auto& w : want) + all_named = all_named && std::find(g.missing.begin(), g.missing.end(), w) != g.missing.end(); + check(all_named, "silent: the missing list names each required field"); + check(g.dims.hidden == 0 && g.dims.layers == 0, "silent: no field is invented"); + } + // 4. head_dim absent but derivable: reported as INFERRED, not as declared. + { + json meta = {{"dims", {{"hidden_size", 2048}, {"intermediate_size", 6144}, {"num_hidden_layers", 28}, + {"num_attention_heads", 16}, {"num_key_value_heads", 8}, {"vocab_size", 151936}, + {"rope_theta", 1000000.0}}}}; + const auto g = pi::container_geometry(meta, json()); + check(g.dims.head_dim == 128, "inferred: head_dim is hidden/heads"); + check(g.inferred.size() == 1 && g.inferred[0] == "head_dim", "inferred: the inference is named"); + check(g.missing.size() == 0, "inferred: an inferred field is not also missing"); + } +} + // engine#1786 (item F1): an NPU2 model source is a safetensors container, and this is the reader that decides whether // a pack is that genre. The failure that matters is the silent one -- a wrong header length yields an empty inventory, // which reads as "a pack with no tensors" rather than "a malformed header" -- so each arm asserts the reason, not just @@ -5490,6 +5550,7 @@ int main(int argc, char** argv) { test_attn_payload_off(); test_text_config_geometry(golden_dir.empty() ? "." : golden_dir); test_q4nx_container(golden_dir.empty() ? "." : golden_dir); + test_container_geometry(); test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir); test_rope_parameters_theta(golden_dir.empty() ? "." : golden_dir); test_three_input_matmulnbits(golden_dir.empty() ? "." : golden_dir); From aca74377ac81e534470a6e4b93161c32cde2853c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:56:34 -0400 Subject: [PATCH 44/98] docs/axb35: the rear-USB-A and Type-C halves of engine#823 need different things awake (#2051) Measured on the board 2026-11-10 (BIOS 1.10, rule not installed): the four xHCI functions the keepawake rule matches, plus the two USB4 host routers (0x158d/0x158e) that carry the Type-C ports and that the rule does not match, all in D3cold with the thunderbolt driver bound and four live IRQs. A suspended router with no partner is normal, so this is not a claim that the routers are broken. It is a claim about the gate: a plug into a rear USB-A exercises the shipped rule, a plug into Type-C exercises a path nothing in this repository touches, and the two results must not be read as one. Refs engine#823 Co-authored-by: pi-qdq --- docs/axb35-usb-ports.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/axb35-usb-ports.md b/docs/axb35-usb-ports.md index d369c08ca..06a985d40 100644 --- a/docs/axb35-usb-ports.md +++ b/docs/axb35-usb-ports.md @@ -70,6 +70,34 @@ addressing something a newer kernel had already fixed. Forcing `power/control=on them `D3cold → D0` and their root hubs' ports appear in `/sys/bus/usb/devices`; `--revert` puts them back. So the mechanism works on this kernel; what has not happened is the plug. +## The two halves of this issue have different fixes, and the rule covers one + +Measured on the board 2026-11-10, BIOS 1.10, a dev boot with the rule not installed: + +``` +0000:c5:00.4 0x1587 xhci_hcd D3cold <- rule matches +0000:c7:00.0 0x1588 xhci_hcd D0 <- rule matches +0000:c7:00.3 0x1589 xhci_hcd D3cold <- rule matches +0000:c7:00.4 0x158b xhci_hcd D3cold <- rule matches +0000:c7:00.5 0x158d thunderbolt D3cold <- the rule does NOT match this +0000:c7:00.6 0x158e thunderbolt D3cold <- the rule does NOT match this +``` + +The **rear USB-A** ports hang off the four xHCI functions the rule matches. The **Type-C** ports hang off the two USB4 host routers +(`0x158d`/`0x158e`), which the rule's device list does not contain, and which are in `D3cold` with the `thunderbolt` driver bound and +four live IRQs. So: + +| half of engine#823 | what has to be awake | does the shipped rule cover it | +|---|---|---| +| rear USB-A | one of `0x1587/0x1588/0x1589/0x158b` | **yes** | +| Type-C | `0x158d`/`0x158e` (USB4 host routers) | **no** | + +**A suspended USB4 host router with no partner attached is normal**, so this is not evidence that the routers are broken - it is evidence +that **one plug test cannot test both halves**: a plug into a rear USB-A exercises the rule, a plug into Type-C exercises a path no change +in this repository has touched. Record the router ids with the capture (`tools/xhci_boot_capture.sh` already records `power_state` per +controller; the two `thunderbolt` functions are worth reading the same way) so a "Type-C still dead" result is read as "the router half is +untested" rather than "the keepawake fix failed". + ## What is still owed One device into a rear USB-A or a Type-C port **on a 1bit-OS boot**, checked with `--require-awake` first. If a port is then still From 0073fcd654063bbfd1af910ea90447dbcf208eba Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:56:45 -0400 Subject: [PATCH 45/98] xhci capture: record WHAT is on each connected port, so a plugged drive is distinguishable from the KVM (engine#823) (#2044) The capture counts ports that report `connect`, which is the right primary evidence for #823 -- but on this board the count is ambiguous, and the first run of the new section says exactly why: ``` bus 3 port 2: 1d6b:0104 product=USB Emulation Device manufacturer=JetKVM speed=480Mbps bus 3 port 5: 0e8d:0717 product=Wireless_Device manufacturer=MediaTek Inc. speed=480Mbps bus 4 port 1: dcba:4321 product=PID-KKK manufacturer=MID-SSS speed=5000Mbps ``` Three ports report `connect` and **none of them is a storage device a person plugged in**: a JetKVM's USB-emulation dongle (HID + audio + emulated mass storage), the internal Bluetooth radio, and a device whose vendor/product are `dcba:4321` -- a placeholder vendor id, i.e. emulation again. A reader looking at "3 port(s) with connect" would reasonably conclude the rear/Type-C ports work, which is the opposite of this issue's report. So the capture now writes `connected-ids.txt` from sysfs (idVendor:idProduct, product, manufacturer, speed, per connected port) and prints it in the VERDICT above the per-bus count. The one physical action this issue waits on -- plug a known drive and read the VERDICT -- then has a decisive answer instead of a port count: the drive the owner plugs in appears with its own ids, and anything else in that list is identified as what it is. Verified: `bash -n` clean; a run on this board writes the three lines above; the awk that feeds it resolves `bus 3 port 2` and `bus 4 port 1` from a synthetic `ports.txt` (so the section is not vacuous on a machine without a KVM); nothing in the capture writes outside its outdir. Refs #823, #1606, #1679. Co-authored-by: t --- tools/xhci_boot_capture.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tools/xhci_boot_capture.sh b/tools/xhci_boot_capture.sh index 37aae66d3..7212cc04f 100755 --- a/tools/xhci_boot_capture.sh +++ b/tools/xhci_boot_capture.sh @@ -117,6 +117,25 @@ done lspci -vv 2>/dev/null | awk '/USB controller/{p=1} p{print} p&&/^$/{p=0}' > "$out/lspci-usb.txt" lsusb -t > "$out/lsusb-tree.txt" 2>&1 + +# engine#823: a port with "connect" is only evidence if the reader can tell WHAT is on it. This box carries a JetKVM +# ("USB Emulation Device": HID + audio + mass storage in one composite) and a device whose descriptors read +# `MID-SSS`/`PID-KKK` -- both look like a plugged drive to a port count, and neither answers this issue. So record the +# identity of every connected port, from sysfs, next to the port state: a real drive the owner plugs in is then +# distinguishable by its own idVendor:idProduct and product string rather than by trust. +: > "$out/connected-ids.txt" +while read -r bus port; do + dev="/sys/bus/usb/devices/${bus}-${port}" + [ -d "$dev" ] || continue + vid=$(cat "$dev/idVendor" 2>/dev/null); pid=$(cat "$dev/idProduct" 2>/dev/null) + prod=$(cat "$dev/product" 2>/dev/null); man=$(cat "$dev/manufacturer" 2>/dev/null) + spd=$(cat "$dev/speed" 2>/dev/null) + printf 'bus %s port %s: %s:%s product=%s manufacturer=%s speed=%sMbps\n' \ + "$bus" "$port" "${vid:-????}" "${pid:-????}" "${prod:-?}" "${man:-?}" "${spd:-?}" >> "$out/connected-ids.txt" +done < "$out/connected-ids.txt" dmesg > "$out/dmesg-full.txt" 2>&1 grep -iE 'xhci|usb|acpi.*(usb|UCSI)|pcieport|AER' "$out/dmesg-full.txt" > "$out/dmesg-filtered.txt" 2>&1 @@ -137,6 +156,9 @@ done total=$(wc -l < "$out/controllers.txt") echo " $awake of $total in D0" echo + echo "== what is on each connected port (a plugged drive must be identifiable here, engine#823)" + sed 's/^/ /' "$out/connected-ids.txt" + echo echo "== ports seen with a device attached (connect), per bus" awk '/^== bus/{b=$3} /port [0-9]+:.*connect/{c[b]++} END{for (k in c) printf " bus %s: %s port(s) with connect\n", k, c[k]}' "$out/ports.txt" echo From 0db23586bfa53b72d09347c48c07808ec63af352 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:17:39 -0400 Subject: [PATCH 46/98] npu: the MiniCPM-V 4.7 vision tower on the host, float32, matching the transformers implementation (engine#1788) (#2056) vision::tower_forward() runs a pack's vision component on the CPU: patch embedding, nearest learned positions, 27 blocks with the 2x2 window-attention vit_merger after block insert_layer_id, the last LayerNorm, and the merger to the text model's width. A reference for a later NPU path, not a serving path. Compared with transformers.models.minicpmv4_7 (5.22.2, torch 2.13 CPU) loaded with each published pack's weights on the same input: output max |d| 1.1e-6 (1B, 8x12 grid), 2.4e-6 (1B, 16x20), 9.5e-7 (35B-A3B, 16x20), corr 1.000000000. The reader gains the merge point (config.json insert_layer_id, refused when it is not a block or is stated twice differently). Test: npu_vision_tower (a small LCG-weight tower with that implementation's output as goldens; 8 mutants caught). Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 7 +- docs/npu.md | 40 ++++- npu/vision_pack.cpp | 18 +- npu/vision_pack.h | 3 + npu/vision_tower.cpp | 232 ++++++++++++++++++++++++ npu/vision_tower.h | 54 ++++++ tests/npu_vision_pack_test.cpp | 17 ++ tests/npu_vision_tower_test.cpp | 302 ++++++++++++++++++++++++++++++++ 8 files changed, 668 insertions(+), 5 deletions(-) create mode 100644 npu/vision_tower.cpp create mode 100644 npu/vision_tower.h create mode 100644 tests/npu_vision_tower_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 16cbce9bc..00b96241f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -692,7 +692,7 @@ option(ONEBIT_NPU_HOST "Build the NPU host library and its tests" ${_onebit_npu_ if(ONEBIT_NPU_HOST) find_package(PkgConfig REQUIRED) pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) -add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp +add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) target_include_directories(onebit_npu PUBLIC npu) @@ -1213,6 +1213,11 @@ add_executable(npu_vision_pack_test tests/npu_vision_pack_test.cpp) target_link_libraries(npu_vision_pack_test PRIVATE onebit_npu) add_test(NAME npu_vision_pack COMMAND npu_vision_pack_test) add_dependencies(onebit npu_vision_pack_test) +# The host float32 tower against goldens from the transformers implementation (see the test's header). +add_executable(npu_vision_tower_test tests/npu_vision_tower_test.cpp) +target_link_libraries(npu_vision_tower_test PRIVATE onebit_npu) +add_test(NAME npu_vision_tower COMMAND npu_vision_tower_test) +add_dependencies(onebit npu_vision_tower_test) add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp) target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) diff --git a/docs/npu.md b/docs/npu.md index 76f584b16..1c4396014 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1462,7 +1462,7 @@ tensors 347, all BF16, 1135109120 B of payload, 0 B unaccounted patch embed 14 x 14 pixels x 3 channels -> 1152 positions 4900 (a 70 x 70 grid; image 980) blocks 27 at width 1152, 16 heads x 72, MLP 4304 (4608 stored) -vit_merger attention at 1152, MLP 4608 -> 17216 (17280 stored) -> 1152 +vit_merger after block 6; attention at 1152, MLP 4608 -> 17216 (17280 stored) -> 1152 merger 4608 -> 4608 -> 1024 (the text model's hidden size) source_file mmproj-model-f16.gguf source_sha256 fb9cc047a3cdad5317fd8433a4aeda2d22f00f02ca4bd93b34ad84165b25a900 @@ -1470,8 +1470,7 @@ note config.json states MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE 4304 note config.json states MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE 17216; model.visual.vit_merger.mlp.linear_fc1.weight has 17280 rows, the last 64 all zero: the container pads the model's width, and sizes come from the tensor ``` -This is a reader, not a forward: nothing runs the tower yet. What it settles is what a forward may -rely on. +What the reader settles is what a forward may rely on. - **Sizes come from the tensors.** The geometry is derived from the tensors' own shapes, and every tensor must have the shape the others imply. @@ -1508,6 +1507,41 @@ Pinned by `tests/npu_vision_pack_test.cpp` on crafted containers; `ONEBIT_MINICP the published pack's own numbers. The source comparison above was run by hand and is not a test in the tree. +**The tower on the host (`npu/vision_tower.h`).** `vision::tower_forward(pack, pixels, gh, gw)` runs the +tower in float32 on the CPU: normalised pixels for a `gh × gw` patch grid in, `(gh/4) × (gw/4)` rows at +the text model's width out. It is a reference for whatever runs the tower on the NPU later, not a +serving path; nothing calls it from a serve, and it does no image decoding, resizing or slicing. The +order of operations is the transformers implementation's (`transformers.models.minicpmv4_7`): + +1. patch embedding (a Conv2d whose kernel is its stride), patches row-major; +2. plus one learned position row per patch, picked **nearest** on the 70 × 70 table: row + `floor(y·70/gh)`, column `floor(x·70/gw)`. No interpolation and no rotary term; +3. blocks 0–6: LayerNorm (eps 1e-6), full attention, MLP with the tanh GELU; +4. the `vit_merger`, once, after block `insert_layer_id` (6): attention inside each 2 × 2 window only, + then per window an MLP over the four tokens side by side plus their mean. The grid halves; +5. blocks 7–26 on the halved grid, then `merger.norm`; +6. the merger: 2 × 2 groups again, `pre_norm`, `linear_fc1`, the exact (erf) GELU, `linear_fc2`. + +So both grid extents must be multiples of 4, and anything else is refused. The padded MLPs are read at +the tensor's stride and the model's width. + +Measured against that implementation (transformers 5.22.2, torch 2.13 CPU, float32, eager attention) +loaded with each pack's own weights, same input (`pixel(c, y, x) = sin(0.37c + 0.11y + 0.05x + +0.003xy)`), 2026-11-10: + +| pack | grid | tower before the merger: max \|d\|, corr | output: max \|d\|, corr | +|---|---|---|---| +| 1B | 8 × 12 | 2.9e-5, 1.000000000 | 1.1e-6, 1.000000000 | +| 1B | 16 × 20 | 1.1e-5, 1.000000000 | 2.4e-6, 1.000000000 | +| 35B-A3B | 16 × 20 | 9.5e-6, 1.000000000 | 9.5e-7, 1.000000000 | + +The 16 × 20 grid took 5.5 s and 1.25 GB of memory for this code on one thread; a full 70 × 70 slice +has 61 times the tokens and its attention grows with the square, which is why this is a reference. +`tests/npu_vision_tower_test.cpp` carries a small tower with LCG weights and that implementation's +output for it as goldens, so the arithmetic is checked in CI without a pack; with +`ONEBIT_MINICPM_PACK` and `ONEBIT_MINICPM_ORACLE` set it repeats the comparison above. The reference +run itself needs transformers and torch and is not in the tree. + ## Serving The 1bit NPU engine runs the model on the XDNA 2 NPU. With a release package that carries the NPU diff --git a/npu/vision_pack.cpp b/npu/vision_pack.cpp index 4304b5c69..b89f5878c 100644 --- a/npu/vision_pack.cpp +++ b/npu/vision_pack.cpp @@ -299,6 +299,22 @@ void Pack::open(const std::string& container_path, const std::string& config_jso fail("config.json states the text model's hidden_size " + std::to_string(c["hidden_size"].get()) + " but the merger projects to " + std::to_string(geom_.out_hidden) + ": this tower is not this text model's"); + { + // top level first (the model's own config key), then the vision_config spelling; both present must agree + int top = -1, inner = -1; + const bool have_top = c.contains("insert_layer_id") && c["insert_layer_id"].is_number_integer(); + if (have_top) top = int(c["insert_layer_id"].get()); + const bool have_inner = vc.contains("MINICPM_V_INSERT_LAYER_ID") && vc["MINICPM_V_INSERT_LAYER_ID"].is_number_integer(); + if (have_inner) inner = int(vc["MINICPM_V_INSERT_LAYER_ID"].get()); + if (have_top && have_inner && top != inner) + fail("config.json states insert_layer_id " + std::to_string(top) + " and MINICPM_V_INSERT_LAYER_ID " + + std::to_string(inner) + ": one tower, two merge points"); + if (have_top || have_inner) geom_.insert_layer = have_top ? top : inner; + if ((have_top || have_inner) && (geom_.insert_layer < 0 || geom_.insert_layer >= geom_.layers)) + fail("config.json states insert_layer_id " + std::to_string(geom_.insert_layer) + " but the container holds " + + std::to_string(geom_.layers) + " blocks"); + } + // An MLP's stated width against its tensors. `stems` are the MLPs that share the width (every block, or the // vit_merger alone): fc1 is [width, in] with a [width] bias, fc2 is [out, width]. const auto zero_from = [&](const std::string& name, int64_t rows, int64_t cols, int64_t from, bool by_row) { @@ -382,7 +398,7 @@ int print_info(const std::string& pack_dir) { std::printf("positions %d (a %d x %d grid; image %d)\n", g.pos_rows, g.grid, g.grid, g.grid * g.patch); std::printf("blocks %d at width %d, %d heads x %d, MLP %d (%d stored)\n", g.layers, g.hidden, g.heads, g.head_dim, g.model_intermediate, g.intermediate); - std::printf("vit_merger attention at %d, MLP %d -> %d (%d stored) -> %d\n", g.hidden, g.merge_hidden, + std::printf("vit_merger after block %d; attention at %d, MLP %d -> %d (%d stored) -> %d\n", g.insert_layer, g.hidden, g.merge_hidden, g.model_vit_merger_intermediate, g.vit_merger_intermediate, g.hidden); std::printf("merger %d -> %d -> %d (the text model's hidden size)\n", g.merge_hidden, g.merge_hidden, g.out_hidden); for (const char* k : {"source_file", "source_sha256", "source_dtype", "converter"}) diff --git a/npu/vision_pack.h b/npu/vision_pack.h index 951c16d5f..29472a250 100644 --- a/npu/vision_pack.h +++ b/npu/vision_pack.h @@ -69,6 +69,9 @@ struct Geometry { // and every row past it is zero in every block, else the tensors' own. A forward may stop at these; it must // still index the tensors at `intermediate` / `vit_merger_intermediate`. int model_intermediate = 0, model_vit_merger_intermediate = 0; + // The block after which the vit_merger runs: config.json's `insert_layer_id` (the reference's default, 6, when the + // config does not say). A value that is not a block of this tower is refused. + int insert_layer = 6; }; class Pack { diff --git a/npu/vision_tower.cpp b/npu/vision_tower.cpp new file mode 100644 index 000000000..20d359358 --- /dev/null +++ b/npu/vision_tower.cpp @@ -0,0 +1,232 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "vision_tower.h" + +#include +#include +#include +#include + +namespace onebit::npu::vision { + +namespace { + +// y[n, rows] = x[n, in] * W[rows_stored.., in_stride]^T + b, using the first `rows` rows and the first `in` columns of +// a weight stored `in_stride` wide. The padded MLPs are read through this: the stride is the tensor's, the extent the +// model's. Eight partial sums so the compiler can vectorise without being allowed to reorder a single chain. +void linear(const float* x, int n, int in, const float* w, int in_stride, const float* b, int rows, float* y) { + for (int t = 0; t < n; ++t) { + const float* xr = x + size_t(t) * in; + float* yr = y + size_t(t) * rows; + for (int r = 0; r < rows; ++r) { + const float* wr = w + size_t(r) * in_stride; + float acc[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + int i = 0; + for (; i + 8 <= in; i += 8) + for (int k = 0; k < 8; ++k) acc[k] += xr[i + k] * wr[i + k]; + float s = ((acc[0] + acc[1]) + (acc[2] + acc[3])) + ((acc[4] + acc[5]) + (acc[6] + acc[7])); + for (; i < in; ++i) s += xr[i] * wr[i]; + yr[r] = s + (b ? b[r] : 0.0f); + } + } +} + +// torch.nn.LayerNorm: the population variance, eps inside the root. +void layer_norm(const float* x, int n, int width, const float* w, const float* b, float* y) { + for (int t = 0; t < n; ++t) { + const float* xr = x + size_t(t) * width; + float* yr = y + size_t(t) * width; + double mean = 0; + for (int i = 0; i < width; ++i) mean += xr[i]; + mean /= width; + double var = 0; + for (int i = 0; i < width; ++i) var += (xr[i] - mean) * (xr[i] - mean); + const float inv = float(1.0 / std::sqrt(var / width + 1e-6)); + for (int i = 0; i < width; ++i) yr[i] = float(xr[i] - mean) * inv * w[i] + b[i]; + } +} + +inline float gelu_tanh(float v) { + return 0.5f * v * (1.0f + std::tanh(0.7978845608028654f * (v + 0.044715f * v * v * v))); +} +inline float gelu_erf(float v) { return 0.5f * v * (1.0f + std::erf(v * 0.7071067811865476f)); } + +struct Attn { + std::vector qkv_w, qkv_b, proj_w, proj_b; +}; +Attn load_attn(const Pack& p, const std::string& stem) { + return {p.f32(stem + ".attn.qkv.weight"), p.f32(stem + ".attn.qkv.bias"), p.f32(stem + ".attn.proj.weight"), + p.f32(stem + ".attn.proj.bias")}; +} + +// out[n, H] = proj(attention(x)), where tokens attend only inside consecutive groups of `group` tokens (group == n is +// full attention). qkv is [3H, H]: q, k, v stacked in that order, heads contiguous inside each. +void attention(const float* x, int n, int H, int heads, int head_dim, int group, const Attn& a, float* out) { + std::vector qkv(size_t(n) * 3 * H), ctx(size_t(n) * H), prob(static_cast(group)); + linear(x, n, H, a.qkv_w.data(), H, a.qkv_b.data(), 3 * H, qkv.data()); + const float scale = 1.0f / std::sqrt(float(head_dim)); + for (int g0 = 0; g0 < n; g0 += group) { + for (int h = 0; h < heads; ++h) { + for (int i = 0; i < group; ++i) { + const float* q = qkv.data() + size_t(g0 + i) * 3 * H + size_t(h) * head_dim; + float mx = -INFINITY; + for (int j = 0; j < group; ++j) { + const float* k = qkv.data() + size_t(g0 + j) * 3 * H + H + size_t(h) * head_dim; + float acc[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + int d = 0; + for (; d + 8 <= head_dim; d += 8) + for (int u = 0; u < 8; ++u) acc[u] += q[d + u] * k[d + u]; + float s = ((acc[0] + acc[1]) + (acc[2] + acc[3])) + ((acc[4] + acc[5]) + (acc[6] + acc[7])); + for (; d < head_dim; ++d) s += q[d] * k[d]; + prob[size_t(j)] = s * scale; + if (prob[size_t(j)] > mx) mx = prob[size_t(j)]; + } + double sum = 0; + for (int j = 0; j < group; ++j) { + prob[size_t(j)] = std::exp(prob[size_t(j)] - mx); + sum += prob[size_t(j)]; + } + float* o = ctx.data() + size_t(g0 + i) * H + size_t(h) * head_dim; + std::memset(o, 0, sizeof(float) * size_t(head_dim)); + for (int j = 0; j < group; ++j) { + const float pj = float(prob[size_t(j)] / sum); + const float* v = qkv.data() + size_t(g0 + j) * 3 * H + 2 * size_t(H) + size_t(h) * head_dim; + for (int d = 0; d < head_dim; ++d) o[d] += pj * v[d]; + } + } + } + } + linear(ctx.data(), n, H, a.proj_w.data(), H, a.proj_b.data(), H, out); +} + +// fc2(act(fc1(x))): `mid` rows of fc1 and `mid` columns of an fc2 stored `mid_stored` wide. +template +void mlp(const float* x, int n, int in, const std::vector& w1, const std::vector& b1, int mid, + const std::vector& w2, const std::vector& b2, int mid_stored, int out_w, Act act, float* out) { + std::vector h(size_t(n) * mid); + linear(x, n, in, w1.data(), in, b1.data(), mid, h.data()); + for (float& v : h) v = act(v); + linear(h.data(), n, mid, w2.data(), mid_stored, b2.data(), out_w, out); +} + +// The four tokens of each 2x2 window of a gh x gw row-major grid, side by side: out[(gh/2)*(gw/2), 4*H], windows +// row-major, inside a window (0,0) (0,1) (1,0) (1,1). This is both the vit_merger's window order and the merger's view. +std::vector windows(const float* x, int gh, int gw, int H) { + std::vector o(size_t(gh) * gw * H); + size_t at = 0; + for (int wy = 0; wy < gh / 2; ++wy) + for (int wx = 0; wx < gw / 2; ++wx) + for (int dy = 0; dy < 2; ++dy) + for (int dx = 0; dx < 2; ++dx) { + std::memcpy(o.data() + at, x + (size_t(2 * wy + dy) * gw + size_t(2 * wx + dx)) * H, sizeof(float) * size_t(H)); + at += size_t(H); + } + return o; +} + +} // namespace + +std::vector tower_forward(const Pack& p, const float* pixels, int gh, int gw, std::vector* tower) { + const Geometry& g = p.geometry(); + const std::string pre = "model.visual."; + if (g.heads <= 0 || g.head_dim <= 0) + throw std::runtime_error(p.path() + ": the head count is only in config.json, and none was given"); + if (g.insert_layer < 0 || g.insert_layer >= g.layers) + throw std::runtime_error(p.path() + ": the vit_merger runs after block " + std::to_string(g.insert_layer) + + ", which this tower of " + std::to_string(g.layers) + " blocks does not have"); + if (!pixels || gh <= 0 || gw <= 0 || gh % 4 != 0 || gw % 4 != 0) + throw std::runtime_error(p.path() + ": a " + std::to_string(gh) + " x " + std::to_string(gw) + + " patch grid cannot be merged twice by 2 x 2 (both extents must be multiples of 4)"); + if (gh > 4096 || gw > 4096) throw std::runtime_error(p.path() + ": the patch grid is larger than any image this tower takes"); + const int H = g.hidden, P = g.patch, C = g.channels, M = g.merge_hidden; + int n = gh * gw; + + // 1 + 2: patches (c, dy, dx) in the weight's own order, then the nearest position row + std::vector x(size_t(n) * H); + { + const std::vector w = p.f32(pre + "patch_embed.proj.weight"), b = p.f32(pre + "patch_embed.proj.bias"); + const std::vector pos = p.f32(pre + "pos_embed.weight"); + const int pd = C * P * P, W = gw * P; + std::vector patch(static_cast(pd)); + for (int y = 0; y < gh; ++y) + for (int xx = 0; xx < gw; ++xx) { + size_t at = 0; + for (int c = 0; c < C; ++c) + for (int dy = 0; dy < P; ++dy) + for (int dx = 0; dx < P; ++dx) + patch[at++] = pixels[(size_t(c) * gh * P + size_t(y * P + dy)) * W + size_t(xx * P + dx)]; + float* row = x.data() + (size_t(y) * gw + xx) * H; + linear(patch.data(), 1, pd, w.data(), pd, b.data(), H, row); + const float* pr = pos.data() + size_t(nearest_position(y, xx, gh, gw, g.grid)) * H; + for (int i = 0; i < H; ++i) row[i] += pr[i]; + } + } + + std::vector t(size_t(n) * H), u(size_t(n) * H); + const auto tanh_act = [](float v) { return gelu_tanh(v); }; + for (int layer = 0; layer < g.layers; ++layer) { + const std::string b = pre + "blocks." + std::to_string(layer); + // 3 / 5: one block + layer_norm(x.data(), n, H, p.f32(b + ".norm1.weight").data(), p.f32(b + ".norm1.bias").data(), t.data()); + attention(t.data(), n, H, g.heads, g.head_dim, n, load_attn(p, b), u.data()); + for (size_t i = 0; i < size_t(n) * H; ++i) x[i] += u[i]; + layer_norm(x.data(), n, H, p.f32(b + ".norm2.weight").data(), p.f32(b + ".norm2.bias").data(), t.data()); + mlp(t.data(), n, H, p.f32(b + ".mlp.linear_fc1.weight"), p.f32(b + ".mlp.linear_fc1.bias"), g.model_intermediate, + p.f32(b + ".mlp.linear_fc2.weight"), p.f32(b + ".mlp.linear_fc2.bias"), g.intermediate, H, tanh_act, u.data()); + for (size_t i = 0; i < size_t(n) * H; ++i) x[i] += u[i]; + + if (layer != g.insert_layer) continue; + // 4: the vit_merger + const std::string v = pre + "vit_merger"; + layer_norm(x.data(), n, H, p.f32(v + ".norm1.weight").data(), p.f32(v + ".norm1.bias").data(), t.data()); + const std::vector normed = windows(t.data(), gh, gw, H); + std::vector res = windows(x.data(), gh, gw, H); // the residual, in window order too + attention(normed.data(), n, H, g.heads, g.head_dim, 4, load_attn(p, v), u.data()); + for (size_t i = 0; i < size_t(n) * H; ++i) res[i] += u[i]; // (n/4 windows) x (4 tokens side by side) + const int nw = n / 4; + std::vector flat(size_t(nw) * M), y(size_t(nw) * H); + layer_norm(res.data(), nw, M, p.f32(v + ".pre_norm.weight").data(), p.f32(v + ".pre_norm.bias").data(), flat.data()); + mlp(flat.data(), nw, M, p.f32(v + ".mlp.linear_fc1.weight"), p.f32(v + ".mlp.linear_fc1.bias"), + g.model_vit_merger_intermediate, p.f32(v + ".mlp.linear_fc2.weight"), p.f32(v + ".mlp.linear_fc2.bias"), + g.vit_merger_intermediate, H, tanh_act, y.data()); + for (int wdw = 0; wdw < nw; ++wdw) + for (int i = 0; i < H; ++i) { + const float* r = res.data() + size_t(wdw) * M; + y[size_t(wdw) * H + i] += 0.25f * (r[i] + r[H + i] + r[2 * H + i] + r[3 * H + i]); + } + x.swap(y); + gh /= 2; + gw /= 2; + n = nw; + } + + // 5: the tower's last LayerNorm + std::vector last(size_t(n) * H); + layer_norm(x.data(), n, H, p.f32(pre + "merger.norm.weight").data(), p.f32(pre + "merger.norm.bias").data(), last.data()); + if (tower) *tower = last; + + // 6: the merger + const int no = n / 4; + const std::vector grouped = windows(last.data(), gh, gw, H); + std::vector normed(size_t(no) * M), out(size_t(no) * g.out_hidden); + layer_norm(grouped.data(), no, M, p.f32(pre + "merger.pre_norm.weight").data(), p.f32(pre + "merger.pre_norm.bias").data(), + normed.data()); + mlp(normed.data(), no, M, p.f32(pre + "merger.linear_fc1.weight"), p.f32(pre + "merger.linear_fc1.bias"), M, + p.f32(pre + "merger.linear_fc2.weight"), p.f32(pre + "merger.linear_fc2.bias"), M, g.out_hidden, + [](float v) { return gelu_erf(v); }, out.data()); + return out; +} + +} // namespace onebit::npu::vision diff --git a/npu/vision_tower.h b/npu/vision_tower.h new file mode 100644 index 000000000..79cbe2167 --- /dev/null +++ b/npu/vision_tower.h @@ -0,0 +1,54 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The MiniCPM-V 4.7 vision tower, host float32 (engine#1788): pixels in, the rows the text model receives out. +// +// A reference, not a serving path: plain loops, one image, no device. It exists so that whatever later runs this +// tower on the NPU has numbers to be compared with, and it is itself compared with the transformers implementation +// (transformers.models.minicpmv4_7, 5.22.2) on a published pack's weights: docs/npu.md has the figures. +// +// The order of operations, each line read from that implementation and pinned by tests/npu_vision_tower_test.cpp: +// 1. patch embedding: a Conv2d whose kernel is its stride, so one matrix product per patch, patches row-major; +// 2. + a learned position row per patch, picked NEAREST on the pack's square table: row floor(y * side / gh), +// column floor(x * side / gw). No interpolation. (The reference buckets y/gh in float32 against k/side; the integer +// form gives the same index for every grid height and width from 1 to 140 at side 70, measured.) +// 3. blocks 0..insert_layer: x += proj(attention(LN1(x))), x += fc2(gelu_tanh(fc1(LN2(x)))). Full attention over +// the image, no rotary term, scale 1/sqrt(head_dim), LayerNorm eps 1e-6; +// 4. the vit_merger, once, after block insert_layer: LN1, attention INSIDE each 2x2 window only, the residual, then +// per window fc2(gelu_tanh(fc1(pre_norm(the four tokens side by side)))) + the mean of the four. The grid halves; +// 5. the remaining blocks on the halved grid, then one LayerNorm (the pack's `merger.norm`); +// 6. the merger: 2x2 groups again, pre_norm over the four tokens side by side, fc1, GELU (the exact erf form here, +// not the tanh approximation the blocks use), fc2 to the text model's width. The grid halves again. +// So a gh x gw patch grid gives (gh/4) * (gw/4) rows, row-major, and both extents must be multiples of 4. +#pragma once + +#include "vision_pack.h" + +#include + +namespace onebit::npu::vision { + +// `pixels` is (channels, gh*patch, gw*patch) float32, already rescaled and normalised. Returns (gh/4 * gw/4, +// out_hidden). When `tower` is given it receives step 5's output, (gh/2 * gw/2, hidden): the tower before the merger. +// Throws std::runtime_error when the grid is not a multiple of 4 in both extents, is empty, or the pack's geometry +// lacks what only config.json states (the head count). +std::vector tower_forward(const Pack& pack, const float* pixels, int gh, int gw, std::vector* tower = nullptr); + +// Step 2's table row for patch (y, x) of a gh x gw grid on a side x side table. +inline int nearest_position(int y, int x, int gh, int gw, int side) { + return int((int64_t(y) * side) / gh) * side + int((int64_t(x) * side) / gw); +} + +} // namespace onebit::npu::vision diff --git a/tests/npu_vision_pack_test.cpp b/tests/npu_vision_pack_test.cpp index 9d2fbeb5c..fd260df74 100644 --- a/tests/npu_vision_pack_test.cpp +++ b/tests/npu_vision_pack_test.cpp @@ -378,6 +378,22 @@ void test_refusals() { refused("config: another patch size", with("PATCH_SIZE", 4), {"PATCH_SIZE 4"}); refused("config: an image the position table does not cover", with("IMAGE_SIZE", 12), {"IMAGE_SIZE 12", "4 x 4"}); refused("config: another position count", with("VISION_NUM_POSITION_EMBEDDINGS", 25), {"NUM_POSITION_EMBEDDINGS 25", "16 rows"}); + // the vit_merger's place: read from the config, refused when it is not a block or is stated twice differently + { + json c = config_of(s); + c["insert_layer_id"] = 1; + try { + const vis::Pack p(kPath, c.dump()); + check(p.geometry().insert_layer == 1, "config: insert_layer_id is read"); + } catch (const std::exception& e) { + check(false, std::string("a config with insert_layer_id loads (threw: ") + e.what() + ")"); + } + c["insert_layer_id"] = 2; + refused("config: a merge point past the last block", c.dump(), {"insert_layer_id 2", "2 blocks"}); + c["insert_layer_id"] = 0; + c["vision_config"]["MINICPM_V_INSERT_LAYER_ID"] = 1; + refused("config: two merge points", c.dump(), {"insert_layer_id 0", "MINICPM_V_INSERT_LAYER_ID 1"}); + } { json c = config_of(s); c["hidden_size"] = 12; @@ -425,6 +441,7 @@ void test_published_pack(const std::string& dir) { check(g.model_intermediate == 4304 && g.model_vit_merger_intermediate == 17216, "published pack: the model's MLP widths are 4304 and 17216, the rest verified zero in all 27 blocks"); check(g.merge_hidden == 4608, "published pack: mergers at 4 x 1152"); + check(g.insert_layer == 6, "published pack: the vit_merger runs after block 6"); check(p.notes().size() == 2, "published pack: both padded widths are noted"); // The 1B pack's container names its source; the 35B-A3B's carries no __metadata__ at all. Both are accepted, // and a source hash that is present has to look like one. diff --git a/tests/npu_vision_tower_test.cpp b/tests/npu_vision_tower_test.cpp new file mode 100644 index 000000000..fcb71588a --- /dev/null +++ b/tests/npu_vision_tower_test.cpp @@ -0,0 +1,302 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_vision_tower_test +// +// The host float32 MiniCPM-V 4.7 vision tower (npu/vision_tower.h, engine#1788) against the transformers +// implementation (transformers.models.minicpmv4_7, 5.22.2, torch 2.13 CPU, float32). +// +// The fixture is a small tower written here: width 16 (2 heads x 8), 2-pixel patches, a 4 x 4 position table, 8 blocks +// with the vit_merger after block 6, MLPs 12 wide stored 16 wide (zero padded, as the published packs store theirs), +// projecting to 12. Its weights come from a 32-bit LCG, so the same file can be rebuilt anywhere; the goldens below are +// what the transformers classes return for that file and the input pixel(c, y, x) = sin(0.37c + 0.11y + 0.05x + +// 0.003xy) on a 4 x 8 patch grid. Nothing here was produced by the code under test. +// +// ONEBIT_VISION_KEEP_FIXTURE=1 leaves the container and its config.json in the working directory (that is how the +// goldens were produced). ONEBIT_MINICPM_PACK= with ONEBIT_MINICPM_ORACLE= compares a published pack's output with a reference run of the same input. +#include "../npu/vision_tower.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using nlohmann::json; +namespace vis = onebit::npu::vision; + +int g_fail = 0, g_arms = 0; +void check(bool ok, const std::string& what) { + ++g_arms; + if (!ok) { + ++g_fail; + std::printf("FAIL %s\n", what.c_str()); + } +} + +const int H = 16, C = 3, P = 2, SIDE = 4, L = 8, I = 12, I_STORED = 16, VI = 48, VI_STORED = 64, O = 12; +const int GH = 4, GW = 8; + +uint32_t g_lcg = 20261110u; +float lcg_unit() { // in [-1, 1) + g_lcg = g_lcg * 1664525u + 1013904223u; + return float(g_lcg >> 8) / 8388608.0f - 1.0f; +} +uint16_t bf16(float v) { + uint32_t b; + std::memcpy(&b, &v, 4); + return uint16_t((b + 0x8000u) >> 16); +} + +struct Writer { + json table = json::object(); + std::vector payload; + // kind 0: a matrix or bias, 0.3 * u (bias 0.1 * u); kind 1: a norm weight, 1 + 0.2 * u. + // Elements at or past `live` along `axis` are zero: the container's padding. + void add(const std::string& name, std::vector shape, int kind, int axis = -1, int64_t live = 0) { + uint64_t n = 1; + for (int64_t d : shape) n *= uint64_t(d); + const uint64_t off = payload.size(); + const int64_t cols = shape.size() > 1 ? shape[1] : 1; + for (uint64_t e = 0; e < n; ++e) { + const bool is_bias = shape.size() == 1 && kind == 0; + float v = kind == 1 ? 1.0f + 0.2f * lcg_unit() : (is_bias ? 0.1f : 0.3f) * lcg_unit(); + if (axis == 0 && int64_t(e) / cols >= live) v = 0.0f; + if (axis == 1 && int64_t(e) % cols >= live) v = 0.0f; + const uint16_t h = bf16(v); + payload.push_back(uint8_t(h & 0xff)); + payload.push_back(uint8_t(h >> 8)); + } + table["model.visual." + name] = {{"dtype", "BF16"}, {"shape", shape}, {"data_offsets", {off, off + n * 2}}}; + } + void linear(const std::string& stem, int64_t rows, int64_t cols) { + add(stem + ".weight", {rows, cols}, 0); + add(stem + ".bias", {rows}, 0); + } + void norm(const std::string& stem, int64_t n) { + add(stem + ".weight", {n}, 1); + add(stem + ".bias", {n}, 0); + } + void padded_mlp(const std::string& stem, int64_t in, int64_t live, int64_t stored, int64_t out) { + add(stem + ".linear_fc1.weight", {stored, in}, 0, 0, live); + add(stem + ".linear_fc1.bias", {stored}, 0, 0, live); + add(stem + ".linear_fc2.weight", {out, stored}, 0, 1, live); + add(stem + ".linear_fc2.bias", {out}, 0); + } +}; + +const char* const kContainer = "npu_vision_tower_test.q4nx"; +const char* const kConfig = "npu_vision_tower_test.config.json"; + +std::string write_fixture() { + g_lcg = 20261110u; + Writer w; + w.add("patch_embed.proj.weight", {H, C, P, P}, 0); + w.add("patch_embed.proj.bias", {H}, 0); + w.add("pos_embed.weight", {SIDE * SIDE, H}, 0); + for (int i = 0; i < L; ++i) { + const std::string b = "blocks." + std::to_string(i) + "."; + w.norm(b + "norm1", H); + w.linear(b + "attn.qkv", 3 * H, H); + w.linear(b + "attn.proj", H, H); + w.norm(b + "norm2", H); + w.padded_mlp(b + "mlp", H, I, I_STORED, H); + } + w.norm("vit_merger.norm1", H); + w.linear("vit_merger.attn.qkv", 3 * H, H); + w.linear("vit_merger.attn.proj", H, H); + w.norm("vit_merger.pre_norm", 4 * H); + w.padded_mlp("vit_merger.mlp", 4 * H, VI, VI_STORED, H); + w.norm("merger.norm", H); + w.norm("merger.pre_norm", 4 * H); + w.linear("merger.linear_fc1", 4 * H, 4 * H); + w.linear("merger.linear_fc2", O, 4 * H); + const std::string head = w.table.dump(); + const uint64_t len = head.size(); + std::ofstream f(kContainer, std::ios::binary | std::ios::trunc); + f.write(reinterpret_cast(&len), 8); + f.write(head.data(), std::streamsize(head.size())); + f.write(reinterpret_cast(w.payload.data()), std::streamsize(w.payload.size())); + + json vc; + vc["MINICPM_V_PATCH_SIZE"] = P; + vc["MINICPM_V_IMAGE_SIZE"] = SIDE * P; + vc["MINICPM_V_VISION_EMBED_DIM"] = H; + vc["MINICPM_V_VISION_NUM_HEADS"] = 2; + vc["MINICPM_V_VISION_HEAD_DIM"] = 8; + vc["MINICPM_V_VISION_MLP_INTERMEDIATE_SIZE"] = I; + vc["MINICPM_V_VISION_NUM_POSITION_EMBEDDINGS"] = SIDE * SIDE; + vc["MINICPM_V_VISION_NUM_LAYERS"] = L; + vc["MINICPM_V_VISION_LAYER_NORM_EPSILON"] = 1e-6; + vc["MINICPM_V_VIT_MERGER_INTERMEDIATE_SIZE"] = VI; + json c; + c["hidden_size"] = O; + c["insert_layer_id"] = 6; + c["vision_config"] = vc; + const std::string text = c.dump(); + std::ofstream cf(kConfig); + cf << text; + return text; +} + +std::vector test_pixels(int gh, int gw, int patch) { + const int hp = gh * patch, wp = gw * patch; + std::vector px(size_t(3) * hp * wp); + for (int c = 0; c < 3; ++c) + for (int y = 0; y < hp; ++y) + for (int x = 0; x < wp; ++x) + px[(size_t(c) * hp + y) * wp + x] = float(std::sin(0.37 * c + 0.11 * y + 0.05 * x + 0.003 * double(x) * y)); + return px; +} + +// GOLDENS_BEGIN (transformers 5.22.2, torch 2.13.0+cpu, float32, eager attention) +const float kGoldenOut[2 * O] = { + 1.66476488f, -0.92423868f, 0.45880169f, 0.54087150f, 0.91351569f, -2.78720903f, + 1.69952393f, 0.54694951f, -1.22599459f, 2.31425953f, 0.93287897f, 0.15626144f, + 1.68985367f, -0.91641307f, 0.10820520f, 0.28901398f, 0.99814737f, -2.51858306f, + 1.39385486f, 0.41435477f, -1.09378004f, 2.35683727f, 0.77005690f, 0.12177789f, +}; +const double kGoldenTowerSum = -0.859797; +// GOLDENS_END + +double max_abs_diff(const std::vector& a, const float* b, size_t n) { + double m = a.size() == n ? 0.0 : 1e30; + for (size_t i = 0; i < n && i < a.size(); ++i) m = std::fmax(m, std::fabs(double(a[i]) - double(b[i]))); + return m; +} + +void test_fixture(const std::string& cfg) { + try { + const vis::Pack p(kContainer, cfg); + check(p.geometry().model_intermediate == I && p.geometry().intermediate == I_STORED && p.geometry().insert_layer == 6, + "fixture: MLPs 12 wide stored 16 wide, vit_merger after block 6"); + const std::vector px = test_pixels(GH, GW, P); + std::vector tower; + const std::vector out = vis::tower_forward(p, px.data(), GH, GW, &tower); + check(out.size() == size_t(2 * O) && tower.size() == size_t(GH / 2) * (GW / 2) * H, + "fixture: a 4 x 8 grid gives 8 tower rows and 2 output rows"); + const double d = max_abs_diff(out, kGoldenOut, size_t(2 * O)); + double sum = 0; + for (float v : tower) sum += v; + std::printf(" fixture: max |out - reference| = %.3g; tower sum %.6f (reference %.6f)\n", d, sum, kGoldenTowerSum); + std::printf(" out[0,:6] %.8f %.8f %.8f %.8f %.8f %.8f\n", out[0], out[1], out[2], out[3], out[4], out[5]); + check(d < 2e-5, "fixture: every output value is the reference's within 2e-5"); + check(std::fabs(sum - kGoldenTowerSum) < 2e-3, "fixture: the tower before the merger sums to the reference's"); + + // grids the two 2 x 2 merges cannot take + for (const auto& [gh, gw] : std::vector>{{4, 6}, {2, 8}, {0, 8}, {4, -4}}) { + std::string why; + try { vis::tower_forward(p, px.data(), gh, gw); } catch (const std::exception& e) { why = e.what(); } + check(why.find("multiples of 4") != std::string::npos, + "a " + std::to_string(gh) + " x " + std::to_string(gw) + " grid is refused by name"); + } + } catch (const std::exception& e) { + check(false, std::string("the fixture runs (threw: ") + e.what() + ")"); + } + // without a config there is no head count, and the forward says so instead of guessing one + try { + const vis::Pack p(kContainer, ""); + const std::vector px = test_pixels(GH, GW, P); + std::string why; + try { vis::tower_forward(p, px.data(), GH, GW); } catch (const std::exception& e) { why = e.what(); } + check(why.find("head count") != std::string::npos, "no config: the forward refuses for want of a head count"); + } catch (const std::exception& e) { + check(false, std::string("the fixture opens without a config (threw: ") + e.what() + ")"); + } +} + +// The reference buckets y / gh (float32) against k / side; the integer form must give the same row and column. These +// are the reference's own indices for a 70-wide table (printed from get_vision_nearest_position_ids). +void test_nearest_position() { + check(vis::nearest_position(0, 0, 8, 12, 70) == 0, "nearest position: the first patch is table row 0"); + check(vis::nearest_position(7, 11, 8, 12, 70) == 61 * 70 + 64, "nearest position: (7, 11) of 8 x 12 is (61, 64)"); + check(vis::nearest_position(3, 5, 8, 12, 70) == 26 * 70 + 29, "nearest position: (3, 5) of 8 x 12 is (26, 29)"); + check(vis::nearest_position(69, 69, 70, 70, 70) == 69 * 70 + 69, "nearest position: a full grid maps onto itself"); + check(vis::nearest_position(139, 0, 140, 4, 70) == 69 * 70, "nearest position: a grid taller than the table shares rows"); +} + +std::vector read_f32(const std::string& path) { + std::ifstream f(path, std::ios::binary | std::ios::ate); + if (!f) return {}; + std::vector v(size_t(f.tellg()) / 4); + f.seekg(0); + f.read(reinterpret_cast(v.data()), std::streamsize(v.size() * 4)); + return v; +} + +void test_published(const std::string& pack_dir, const std::string& oracle) { + try { + int gh = 0, gw = 0; + { std::ifstream g(oracle + "/grid.txt"); g >> gh >> gw; } + const std::vector px = read_f32(oracle + "/pixels.f32"), want = read_f32(oracle + "/out.f32"), + want_tower = read_f32(oracle + "/tower.f32"); + const vis::Pack p(pack_dir); + check(gh > 0 && gw > 0 && px.size() == size_t(3) * gh * gw * p.geometry().patch * p.geometry().patch, + "published pack: the reference input is this grid's pixels"); + std::vector tower; + const std::vector out = vis::tower_forward(p, px.data(), gh, gw, &tower); + const auto stats = [](const std::vector& a, const std::vector& b, double* mx, double* corr, double* mean_abs) { + double sa = 0, sb = 0, saa = 0, sbb = 0, sab = 0; + *mx = a.size() == b.size() ? 0 : 1e30; + *mean_abs = 0; + const size_t n = a.size() < b.size() ? a.size() : b.size(); + for (size_t i = 0; i < n; ++i) { + *mx = std::fmax(*mx, std::fabs(double(a[i]) - b[i])); + sa += a[i]; sb += b[i]; saa += double(a[i]) * a[i]; sbb += double(b[i]) * b[i]; sab += double(a[i]) * b[i]; + *mean_abs += std::fabs(double(b[i])); + } + *mean_abs /= double(n ? n : 1); + const double va = saa - sa * sa / double(n), vb = sbb - sb * sb / double(n); + *corr = (sab - sa * sb / double(n)) / std::sqrt(va * vb); + }; + double mx, corr, mean_abs; + stats(tower, want_tower, &mx, &corr, &mean_abs); + std::printf(" published pack, %d x %d grid: tower %zu values, max |d| %.3g, corr %.9f, reference mean |v| %.4f\n", gh, gw, + tower.size(), mx, corr, mean_abs); + check(tower.size() == want_tower.size() && corr > 0.99999, "published pack: the tower before the merger matches the reference"); + stats(out, want, &mx, &corr, &mean_abs); + std::printf(" published pack, %d x %d grid: output %zu values, max |d| %.3g, corr %.9f, reference mean |v| %.4f\n", gh, gw, + out.size(), mx, corr, mean_abs); + check(out.size() == want.size() && corr > 0.99999 && mx < 5e-3, "published pack: the output matches the reference"); + } catch (const std::exception& e) { + check(false, std::string("the published pack runs (threw: ") + e.what() + ")"); + } +} + +} // namespace + +int main() { + const std::string cfg = write_fixture(); + test_fixture(cfg); + test_nearest_position(); + if (!std::getenv("ONEBIT_VISION_KEEP_FIXTURE")) { + std::remove(kContainer); + std::remove(kConfig); + } + const char* pack = std::getenv("ONEBIT_MINICPM_PACK"); + const char* oracle = std::getenv("ONEBIT_MINICPM_ORACLE"); + if (pack && *pack && oracle && *oracle) test_published(pack, oracle); + else std::printf(" (ONEBIT_MINICPM_PACK / ONEBIT_MINICPM_ORACLE not set: the published-pack arm did not run)\n"); + std::printf("vision tower: %d checks, %d failed\n", g_arms, g_fail); + return g_fail ? 1 : 0; +} From 1af5725bca09c363e6e43598fe916075a8391322 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:27:14 -0400 Subject: [PATCH 47/98] npu: the DFlash draft block's acceptance arithmetic, with the measured counts as known answers (refs #1780) (#2053) A speculative round offers k drafted positions behind one verify pass and yields the tokens the target accepted plus the one the pass commits past them, so tokens = accepted + bonus and tokens/round = (accepted + bonus)/rounds. The round costs cost_steps ordinary decode steps, so a draft pays only while tokens/round > cost_steps -- 1.05 for a one-pass round and 2.05 for the two-pass mtp::run shape, which is why the same acceptance passes one shape and fails the other. Header-only and host-only: no container, no device, no NPU runtime. The known answers are the OGA route's own counts (docs/oga-drafter-path.md): k=4 over 54 rounds, and k=7 over two replays. Each row reconciles exactly as tokens = accepted + bonus, and the gap from the ideal 1 + accepted/round is exactly shortfall/rounds where the shortfall is the 2, 2 and 4 sequence-ending rounds -- so the arithmetic reports why a run lands below its ideal instead of averaging the difference away. Registered at top level with add_dependencies(onebit dflash_accept_test): an add_test alone leaves the target unbuilt on this runner (engine#1858/#1930/#1931). Co-authored-by: t --- CMakeLists.txt | 8 +++ npu/forward/include/dflash_accept.h | 72 ++++++++++++++++++++++++++ tests/dflash_accept_test.cpp | 80 +++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 npu/forward/include/dflash_accept.h create mode 100644 tests/dflash_accept_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 00b96241f..00d1d719a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1441,6 +1441,14 @@ add_dependencies(onebit npu_answer_parser_test) add_executable(batch_check_compare_test tests/batch_check_compare_test.cpp) target_include_directories(batch_check_compare_test PRIVATE npu/forward/include) add_test(NAME batch_check_compare COMMAND batch_check_compare_test) + +# engine#1780: the DFlash draft block's acceptance arithmetic -- tokens = accepted + bonus, the break-even rule, and the +# published OGA-route counts as known answers. Header-only and host-only: no container, no device, no NPU runtime, so it +# runs wherever this file is built. Registered at top level with the same hook as the row above, for the same reason. +add_executable(dflash_accept_test tests/dflash_accept_test.cpp) +target_include_directories(dflash_accept_test PRIVATE npu/forward/include) +add_test(NAME dflash_accept COMMAND dflash_accept_test) +add_dependencies(onebit dflash_accept_test) # direction B (tools/ci/ci_targets_check.py): a target ci.yml's --target list does not name is "never built", and ci.yml # is a workflow file no lane can patch. The documented hook is the one onebit_npu_forward_syntax uses. add_dependencies(onebit batch_check_compare_test) diff --git a/npu/forward/include/dflash_accept.h b/npu/forward/include/dflash_accept.h new file mode 100644 index 000000000..b53018ed1 --- /dev/null +++ b/npu/forward/include/dflash_accept.h @@ -0,0 +1,72 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#1780: what a DFlash draft block has to clear before it is worth running. +// +// A speculative round offers `k` drafted positions behind one verify pass and yields the positions the target accepted +// plus the one token the verify pass commits past them: `tokens = accepted + bonus`. With `rounds` rounds that is +// +// tokens_per_round = (accepted + bonus) / rounds +// +// and the round costs `cost_steps` ordinary decode steps, so the draft pays only while `tokens_per_round > cost_steps`. +// A one-pass round (verify only) costs ~1.05 steps and a two-pass round (`mtp::run`: verify, then the step that consumes +// the committed token) costs ~2.05, both measured on the OGA route (docs/oga-drafter-path.md). That is why the same +// acceptance can be a pass in one round shape and a fail in the other, and why this is a function rather than a number. +// +// The `bonus` is not always one per round: when the sequence ends, the final round(s) commit no extra token. Measured on +// the OGA route at k=4 and k=7, the shortfall is 2, 2 and 4 rounds of the 54/46/47 -- which is exactly the difference +// between the ideal `1 + accepted/round` and the published 2.963 / 3.478 / 3.404. Carrying `bonus` explicitly keeps that +// visible instead of hiding it in a rounding difference. +#ifndef ONEBIT_NPU_DFLASH_ACCEPT_H +#define ONEBIT_NPU_DFLASH_ACCEPT_H + +#include + +namespace onebit { +namespace dflash { + +/* Accepted positions per offered draft, the draft's own quality: 108/216 = 0.5 at k=4 on the OGA route. */ +inline double per_position_acceptance(std::int64_t accepted, std::int64_t offered) { + if (offered <= 0) return 0.0; + return static_cast(accepted) / static_cast(offered); +} + +/* Tokens a round yields: the accepted positions plus the tokens the verify pass commits past them. */ +inline double tokens_per_round(std::int64_t accepted, std::int64_t bonus, std::int64_t rounds) { + if (rounds <= 0) return 0.0; + return static_cast(accepted + bonus) / static_cast(rounds); +} + +/* The ideal form, `1 + accepted/round`, which assumes every round earns its bonus. Only for comparison: a measured + * round whose sequence ended early is lower, and the difference is the shortfall below, not noise. */ +inline double ideal_tokens_per_round(std::int64_t accepted, std::int64_t rounds) { + if (rounds <= 0) return 0.0; + return 1.0 + static_cast(accepted) / static_cast(rounds); +} + +/* Rounds that committed no extra token -- the sequence-ending rounds. 0 for an ideal run. */ +inline std::int64_t bonus_shortfall(std::int64_t rounds, std::int64_t bonus) { + return rounds > bonus ? rounds - bonus : 0; +} + +/* The break-even rule. `cost_steps` is the round's cost in ordinary decode steps (1.05 one pass, 2.05 two). */ +inline bool pays(double tokens_per_round_value, double cost_steps) { + return tokens_per_round_value > cost_steps; +} + +} // namespace dflash +} // namespace onebit + +#endif // ONEBIT_NPU_DFLASH_ACCEPT_H diff --git a/tests/dflash_accept_test.cpp b/tests/dflash_accept_test.cpp new file mode 100644 index 000000000..43361a8cd --- /dev/null +++ b/tests/dflash_accept_test.cpp @@ -0,0 +1,80 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#1780: known answers for the draft block's acceptance arithmetic, taken from the OGA-route measurement rather +// than invented. The three rows are docs/oga-drafter-path.md's counts: k=4 over 54 rounds, and k=7 over two replays. +// Host-only: no container, no device, no NPU runtime -- so it runs wherever this file is built. +#include "dflash_accept.h" + +#include +#include +#include + +namespace df = onebit::dflash; + +static int failures = 0; + +static void check(bool ok, const std::string& what) { + if (!ok) { + ++failures; + std::printf("FAIL %s\n", what.c_str()); + } +} + +static bool near(double a, double b) { return std::fabs(a - b) < 5e-4; } + +int main() { + /* The three measured rows. `tokens` is what the tool printed; `accepted` is the drafts the target took. */ + struct Row { int k; std::int64_t offered, rounds, accepted, tokens; double want_train; }; + const Row rows[] = { + {4, 216, 54, 108, 160, 2.963}, + {7, 322, 46, 116, 160, 3.478}, + {7, 329, 47, 117, 160, 3.404}, + }; + for (const Row& r : rows) { + const std::int64_t bonus = r.tokens - r.accepted; + const double tpr = df::tokens_per_round(r.accepted, bonus, r.rounds); + check(near(tpr, r.want_train), + "accept: k=" + std::to_string(r.k) + " reproduces the published tokens/round (" + std::to_string(tpr) + ")"); + /* tokens = accepted + bonus is exact, so the shortfall is exactly the rounds that ended the sequence. */ + check(r.accepted + bonus == r.tokens, "accept: tokens == accepted + bonus (the decomposition is exact)"); + const std::int64_t shortfall = df::bonus_shortfall(r.rounds, bonus); + const double ideal = df::ideal_tokens_per_round(r.accepted, r.rounds); + check(near(ideal - static_cast(shortfall) / static_cast(r.rounds), tpr), + "accept: the gap from the ideal form is exactly shortfall/rounds, not rounding"); + check(shortfall > 0 && shortfall <= 4, + "accept: and the shortfall is a handful of sequence-ending rounds (" + std::to_string(shortfall) + ")"); + } + + /* Per-position acceptance, the draft's own quality: 0.5 at k=4, ~0.36 at k=7. */ + check(near(df::per_position_acceptance(108, 216), 0.5), "accept: 108 of 216 offers is 0.5 per position"); + check(near(df::per_position_acceptance(116, 322), 0.3602), "accept: 116 of 322 is 0.3602"); + + /* The break-even rule, and the case that makes it a function: the same acceptance passes one round shape and + * fails the other. 2.963 tokens/round against 1.05 steps pays; against 2.05 it barely pays; a draft at 1.5 does not. */ + check(df::pays(3.000, 1.05), "accept: 3.0 tokens/round pays for a one-pass round (1.05 steps)"); + check(df::pays(3.000, 2.05), "accept: 3.0 tokens/round also pays for a two-pass round (2.05 steps)"); + check(!df::pays(1.500, 2.05), "accept: 1.5 tokens/round does NOT pay for a two-pass round -- the distinction"); + check(df::pays(1.500, 1.05), "accept: but it does pay for a one-pass round"); + /* A drafter that accepts nothing is the identity, 1.0 token/round, and must never be reported as a win. */ + check(near(df::tokens_per_round(0, 54, 54), 1.0), "accept: a drafter that accepts nothing gives 1.0 token/round"); + check(!df::pays(1.0, 1.0), "accept: and 1.0 does not pay for a round that costs a step (the rule is strict)"); + /* Degenerate inputs are answers, not crashes: a forward can be handed a zero-length run. */ + check(df::tokens_per_round(0, 0, 0) == 0.0 && df::per_position_acceptance(0, 0) == 0.0 && df::bonus_shortfall(0, 0) == 0, + "accept: zero rounds, zero offers and zero bonus are answers rather than division by zero"); + + if (failures == 0) std::printf("dflash accept: all checks passed\n"); + return failures == 0 ? 0 : 1; +} From 72ba575a5385e89b99fb1cdf1d1e69d05a941814 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:38:49 -0400 Subject: [PATCH 48/98] npu: load a DFlash draft package as a model source: config + container -> a checked, decodable model (Refs #1780) (#2048) npu/dflash_draft.{h,cpp}: reads the DFlash config.json, opens the Q4NX container with the engine's reader, checks every tensor against the config's geometry, and hands out views that decode one tensor at a time through the forward's q4_1 tile decoder. Refuses by name: a missing or mis-sized tensor, an unexpected dtype label, a scalar that disagrees with the config, an unexplained tensor, DFlash2, a weight path outside the package. The fusion matrix is stored as [hidden/32][taps*hidden][32], not row-major; measured on the Bolt draft and asserted with a control. tests/dflash_draft_test.cpp: synthetic package (known answers at every element, 16 refusals), plus an optional arm on a real package. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 10 + docs/amd-drafters.md | 22 ++ npu/dflash_draft.cpp | 275 +++++++++++++++++++++++ npu/dflash_draft.h | 113 ++++++++++ tests/dflash_draft_test.cpp | 436 ++++++++++++++++++++++++++++++++++++ 5 files changed, 856 insertions(+) create mode 100644 npu/dflash_draft.cpp create mode 100644 npu/dflash_draft.h create mode 100644 tests/dflash_draft_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 00d1d719a..a0b3ce73b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1240,6 +1240,16 @@ endif() # q4nx_pack.c is listed here rather than reached through onebit_npu: the integer host arm's checks # (engine#1808) call q4nx_host_gemm_i8*() directly, and those live in that file. add_executable(npu_pack_test tests/npu_pack_test.cpp npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c) +# A DFlash draft package as a model source (engine#1780): config + container -> a checked model whose tensors decode +# one at a time. The loader is compiled here and not into onebit_npu because it needs the q4_1 tile decoder, which +# lives with the forward (npu/forward/src/q4nx_dequant.c); it moves there with the draft's forward. Synthetic package, +# so no model, no device; a real package is an optional argument (-DONEBIT_DFLASH_DRAFT=). +add_executable(dflash_draft_test tests/dflash_draft_test.cpp npu/dflash_draft.cpp npu/forward/src/q4nx_dequant.c) +target_include_directories(dflash_draft_test PRIVATE npu npu/forward/include) +target_link_libraries(dflash_draft_test PRIVATE onebit_npu) +add_dependencies(onebit dflash_draft_test) +set(ONEBIT_DFLASH_DRAFT "" CACHE PATH "A DFlash draft package directory for dflash_draft_test's real-package arm") +add_test(NAME dflash_draft COMMAND dflash_draft_test ${ONEBIT_DFLASH_DRAFT}) add_executable(flm_head_import tools/flm_head_import.cpp) target_link_libraries(flm_head_import PRIVATE onebit_npu) add_executable(npu_flm_head_import_test tests/npu_flm_head_import_test.cpp) diff --git a/docs/amd-drafters.md b/docs/amd-drafters.md index 699edf009..a7a7235d1 100644 --- a/docs/amd-drafters.md +++ b/docs/amd-drafters.md @@ -116,6 +116,28 @@ the config, or a tensor it cannot explain (the second draft family, `DFlash2Draf convolution tensors this tool does not know) each fail by name. `--selftest` runs those controls on a synthetic package (ctest `tool_selftest_dflash_draft_info`). +### The loader (`npu/dflash_draft.h`) + +`onebit::npu::dflash::load_draft(dir)` is the C++ half of that gate: it reads the DFlash `config.json`, opens the +container with the engine's own Q4NX reader, checks every tensor against the geometry the config gives, and returns +a model whose tensors are views into the mapped file. Nothing is decoded at load (the Bolt draft's projections are +327,680,000 weights, 1.3 GB as f32); `decode_projection()`, `decode_gain()` and `decode_fusion()` produce one tensor +at a time, through the same q4_1 tile decoder the forward uses. It throws, naming the tensor or key, on a missing or +mis-sized tensor, a dtype label it does not expect, a container scalar that disagrees with the config, a tensor it +cannot explain, another architecture (`DFlash2DraftModel`), or a weight path that leaves the package. + +Two facts about the container that the loader depends on, both measured on the Bolt draft: + +| fact | evidence | +|---|---| +| `model.fc.weight` is **not** row-major. Its declared shape `[80, 12800, 32]` is the element order: 32-row blocks, column by column, the row inside the block last. | Magnitudes that share an input column correlate about five times more strongly than magnitudes that share an output row. Decoded in the declared order the four neighbour figures are consistent (next row 0.107, 32 rows on 0.106; next column 0.022, 32 columns on 0.021). Read as plain row-major they contradict each other (next row 0.019 but 32 rows on 0.107; next column 0.105 but 32 columns on 0.024). `dflash_draft_test` asserts the first and uses the second as its control. | +| A norm gain may be negative. | Three post-attention norms each hold one gain just below zero (-0.0009 to -0.0024) among 2,560. As bf16 that is a gain crossing zero; as half-float the same element would be -0.8 among values of 1.3 to 2.4, which is one more reason the bf16 reading is the right one. | + +`dflash_draft_test` builds a synthetic package (every projection encoded tile by tile from a formula and compared at +every element, the fusion matrix checked for row-major order, sixteen refusals each matched on its message) and needs +no model; given a package directory (`-DONEBIT_DFLASH_DRAFT=` or `ONEBIT_TEST_DFLASH_DRAFT`) it also loads and +decodes the real one. The loader is not wired into `forward-serve` yet: there is no forward for it to feed. + ## PARD-2's repos are weights-only — borrow the target's tokenizer `amd/PARD2-*` ship **`README.md`, `config.json`, `model.safetensors`, `warp_model.bin`** and nothing else: **no tokenizer diff --git a/npu/dflash_draft.cpp b/npu/dflash_draft.cpp new file mode 100644 index 000000000..85a0cec8b --- /dev/null +++ b/npu/dflash_draft.cpp @@ -0,0 +1,275 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// dflash_draft.cpp -- see dflash_draft.h. + +#include "dflash_draft.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include "file_map.h" +#include "forward/include/q4nx_dequant.h" +#include "pack_import.h" + +namespace onebit::npu::dflash { + +namespace { + +using nlohmann::json; + +[[noreturn]] void fail(const std::string& what) { throw std::runtime_error("dflash draft: " + what); } + +float bf16(const uint8_t* p) { + uint16_t h; + std::memcpy(&h, p, 2); + const uint32_t bits = uint32_t(h) << 16; + float f; + std::memcpy(&f, &bits, 4); + return f; +} + +int need_int(const json& c, const char* key, const std::string& path) { + if (!c.contains(key)) fail(path + ": no '" + key + "'"); + if (!c[key].is_number_integer()) fail(path + ": '" + key + "' is not an integer"); + const int64_t v = c[key].get(); + if (v <= 0 || v > (int64_t(1) << 30)) fail(path + ": '" + key + "' is " + std::to_string(v)); + return int(v); +} + +float need_float(const json& c, const char* key, const std::string& path) { + if (!c.contains(key)) fail(path + ": no '" + key + "'"); + if (!c[key].is_number()) fail(path + ": '" + key + "' is not a number"); + const double v = c[key].get(); + if (!std::isfinite(v) || v <= 0.0) fail(path + ": '" + key + "' is " + std::to_string(v)); + return float(v); +} + +bool close_to(double a, double b) { return std::fabs(a - b) <= 1e-5 * std::max(1.0, std::fabs(b)); } + +} // namespace + +DraftConfig read_draft_config(const std::string& package_dir) { + const std::string path = package_dir + "/config.json"; + std::ifstream in(path); + if (!in) fail("cannot open " + path); + json c; + try { + in >> c; + } catch (const std::exception& e) { + fail(path + ": not JSON (" + e.what() + ")"); + } + if (!c.is_object()) fail(path + ": not a JSON object"); + const std::string type = c.value("model_type", json("")).is_string() ? c.value("model_type", std::string()) : ""; + if (type != "dflash") fail(path + ": model_type is '" + type + "', not 'dflash'"); + // The architecture decides which tensors exist. This loader knows one; a package naming another is a different + // model (DFlash2 adds a selector and a convolution), and reading it with these rules would drop those silently. + if (c.contains("architectures")) { + if (!c["architectures"].is_array() || c["architectures"].empty() || !c["architectures"][0].is_string()) + fail(path + ": 'architectures' is not a list of names"); + const std::string arch = c["architectures"][0].get(); + if (arch != "DFlashDraftModel") fail(path + ": architecture '" + arch + "' is not read by this loader (DFlashDraftModel is)"); + } + DraftConfig o; + o.hidden = need_int(c, "hidden_size", path); + o.intermediate = need_int(c, "intermediate_size", path); + o.layers = need_int(c, "num_hidden_layers", path); + o.heads = need_int(c, "num_attention_heads", path); + o.kv_heads = need_int(c, "num_key_value_heads", path); + o.head_dim = need_int(c, "head_dim", path); + o.block_size = need_int(c, "block_size", path); + o.sliding_window = need_int(c, "sliding_window", path); + o.rms_eps = need_float(c, "rms_norm_eps", path); + o.rope_theta = need_float(c, "rope_theta", path); + o.embedding_scale = need_float(c, "embedding_scale", path); + if (!c.contains("mask_token_id") || !c["mask_token_id"].is_number_integer() || c["mask_token_id"].get() < 0) + fail(path + ": no usable 'mask_token_id'"); + o.mask_token_id = c["mask_token_id"].get(); + if (!c.contains("target_layer_ids") || !c["target_layer_ids"].is_array() || c["target_layer_ids"].empty()) + fail(path + ": no 'target_layer_ids'"); + int prev = -1; + for (const auto& v : c["target_layer_ids"]) { + if (!v.is_number_integer() || v.get() < 0 || v.get() > 4096) + fail(path + ": 'target_layer_ids' holds something that is not a layer index"); + const int id = int(v.get()); + if (id <= prev) fail(path + ": 'target_layer_ids' is not strictly increasing at " + std::to_string(id)); + o.target_layer_ids.push_back(id); + prev = id; + } + if (!c.contains("draft_weight") || !c["draft_weight"].is_string() || c["draft_weight"].get().empty()) + fail(path + ": no 'draft_weight'"); + o.draft_weight = c["draft_weight"].get(); + if (o.draft_weight.find('/') != std::string::npos || o.draft_weight.find("..") != std::string::npos) + fail(path + ": 'draft_weight' must be a file name inside the package, got '" + o.draft_weight + "'"); + if (c.contains("causal")) { + if (!c["causal"].is_boolean()) fail(path + ": 'causal' is not a boolean"); + o.causal = c["causal"].get(); + } + if (c.contains("base_model") && c["base_model"].is_string()) o.base_model = c["base_model"].get(); + if (o.kv_heads > o.heads || o.heads % o.kv_heads != 0) + fail(path + ": " + std::to_string(o.heads) + " heads cannot share " + std::to_string(o.kv_heads) + " key/value heads"); + if (o.block_size < 2) fail(path + ": block_size " + std::to_string(o.block_size) + " drafts nothing"); + return o; +} + +DraftModel load_draft(const std::string& package_dir) { + DraftModel m; + m.cfg = read_draft_config(package_dir); + const DraftConfig& c = m.cfg; + const std::string path = package_dir + "/" + c.draft_weight; + + const pack_import::Q4nxContainer table = pack_import::read_q4nx_container(path); + if (!table.why.empty()) fail(table.why); + size_t size = 0; + std::string why; + const uint8_t* base = map_file(path, size, &why); + if (base == nullptr) fail(why); + m.keep = std::shared_ptr(base, [size](const void* p) { unmap_file(static_cast(p), size); }); + m.container_bytes = size; + m.tensors = int(table.tensors.size()); + const size_t payload = 8 + size_t(table.header_bytes); + + std::map by_name; + for (const auto& t : table.tensors) by_name[t.name] = &t; + std::set used; + + // Every tensor is taken through here, so "inside the file" and "accounted for" are checked once, for all of them. + const auto take = [&](const std::string& name, const char* dtype, size_t want_bytes, int rows, int cols) { + const auto it = by_name.find(name); + if (it == by_name.end()) fail(path + ": no tensor '" + name + "'"); + const pack_import::Q4nxTensor& t = *it->second; + if (t.dtype != dtype) fail(path + ": '" + name + "' is declared " + t.dtype + ", expected " + dtype); + if (t.begin < 0 || t.end < t.begin || payload + size_t(t.end) > size) + fail(path + ": '" + name + "' lies outside the file"); + const size_t have = size_t(t.end - t.begin); + if (have != want_bytes) + fail(path + ": '" + name + "' holds " + std::to_string(have) + " B, and " + std::to_string(rows) + " x " + + std::to_string(cols) + " needs " + std::to_string(want_bytes)); + used.insert(name); + TensorView v; + v.name = name; + v.data = base + payload + size_t(t.begin); + v.bytes = have; + v.rows = rows; + v.cols = cols; + return v; + }; + const auto projection = [&](const std::string& name, int n, int k) { + if (n % 32 != 0 || k % 256 != 0) + fail(path + ": '" + name + "' would be " + std::to_string(n) + " x " + std::to_string(k) + + ", which is not a whole 32 x 256 tile grid"); + return take(name, "I8", q4nx_tensor_bytes(n, k), n, k); + }; + const auto gain = [&](const std::string& name, int width) { return take(name, "BF16", size_t(width) * 2, 1, width); }; + + // The container repeats the interface scalars; a package whose two halves disagree is not one model. + const auto scalar = [&](const std::string& name, const char* dtype, size_t count) { + return take(name, dtype, count * 4, 1, int(count)).data; + }; + const auto i32_at = [](const uint8_t* p, size_t i) { int32_t v; std::memcpy(&v, p + i * 4, 4); return v; }; + const auto f32_at = [](const uint8_t* p) { float v; std::memcpy(&v, p, 4); return v; }; + const auto agree = [&](bool same, const std::string& name, const std::string& got, const std::string& want) { + if (!same) fail(path + ": '" + name + "' is " + got + " and config.json says " + want); + }; + { + const uint8_t* p = scalar("dflash.block_size", "I32", 1); + agree(i32_at(p, 0) == c.block_size, "dflash.block_size", std::to_string(i32_at(p, 0)), std::to_string(c.block_size)); + p = scalar("dflash.sliding_window", "I32", 1); + agree(i32_at(p, 0) == c.sliding_window, "dflash.sliding_window", std::to_string(i32_at(p, 0)), std::to_string(c.sliding_window)); + p = scalar("dflash.target_layer_ids", "I32", size_t(c.taps())); + for (int i = 0; i < c.taps(); ++i) + agree(i32_at(p, size_t(i)) == c.target_layer_ids[size_t(i)], "dflash.target_layer_ids[" + std::to_string(i) + "]", + std::to_string(i32_at(p, size_t(i))), std::to_string(c.target_layer_ids[size_t(i)])); + p = scalar("dflash.rope_theta", "F32", 1); + agree(close_to(f32_at(p), c.rope_theta), "dflash.rope_theta", std::to_string(f32_at(p)), std::to_string(c.rope_theta)); + p = scalar("dflash.rms_eps", "F32", 1); + agree(close_to(f32_at(p) * 1e6, double(c.rms_eps) * 1e6), "dflash.rms_eps", std::to_string(f32_at(p)), std::to_string(c.rms_eps)); + p = scalar("dflash.embedding_scale", "F32", 1); + agree(close_to(f32_at(p), c.embedding_scale), "dflash.embedding_scale", std::to_string(f32_at(p)), std::to_string(c.embedding_scale)); + } + + // The fusion matrix: hidden x (taps * hidden), stored in 32-row blocks (see decode_fusion), so hidden must fill them + // and the declared shape must be that blocking -- a plain [hidden, taps*hidden] label would mean another element order. + if (c.hidden % 32 != 0) fail(path + ": hidden_size " + std::to_string(c.hidden) + " does not fill 32-row blocks"); + const int fused = c.taps() * c.hidden; + m.fc = take("model.fc.weight", "BF16", size_t(c.hidden) * size_t(fused) * 2, c.hidden, fused); + { + const std::vector want = {c.hidden / 32, fused, 32}; + if (by_name["model.fc.weight"]->shape != want) + fail(path + ": 'model.fc.weight' is not declared [hidden/32, taps*hidden, 32], so its element order is not the one this loader undoes"); + } + m.hidden_norm = gain("model.hidden_norm.weight", c.hidden); + m.norm = gain("model.norm.weight", c.hidden); + for (int l = 0; l < c.layers; ++l) { + const std::string p = "model.layers." + std::to_string(l) + "."; + DraftLayer L; + L.q = projection(p + "self_attn.q_proj.weight", c.q_dim(), c.hidden); + L.k = projection(p + "self_attn.k_proj.weight", c.kv_dim(), c.hidden); + L.v = projection(p + "self_attn.v_proj.weight", c.kv_dim(), c.hidden); + L.o = projection(p + "self_attn.o_proj.weight", c.hidden, c.q_dim()); + L.gate = projection(p + "mlp.gate_proj.weight", c.intermediate, c.hidden); + L.up = projection(p + "mlp.up_proj.weight", c.intermediate, c.hidden); + L.down = projection(p + "mlp.down_proj.weight", c.hidden, c.intermediate); + L.input_norm = gain(p + "input_layernorm.weight", c.hidden); + L.post_norm = gain(p + "post_attention_layernorm.weight", c.hidden); + L.q_norm = gain(p + "self_attn.q_norm.weight", c.head_dim); + L.k_norm = gain(p + "self_attn.k_norm.weight", c.head_dim); + m.layers.push_back(L); + } + for (const auto& t : table.tensors) + if (used.count(t.name) == 0) + fail(path + ": tensor '" + t.name + "' is not part of a DFlashDraftModel (" + std::to_string(table.tensors.size() - used.size()) + + " unexplained): this is another architecture"); + return m; +} + +std::vector decode_projection(const TensorView& t) { + if (!t.ok()) fail("decode_projection of an empty view"); + std::vector out(size_t(t.rows) * size_t(t.cols)); + // Not the size-dispatching q4nx_dequant_tensor(): a projection here is a tile grid by contract, and a grid whose + // byte count happened to equal rows*cols*2 must not be read as plain 2-byte floats. + if (t.bytes != q4nx_tensor_bytes(t.rows, t.cols) || + q4nx_dequant_tensor_chunked(t.data, t.bytes, t.rows, t.cols, 5120, out.data()) != 0) + fail("'" + t.name + "' does not decode as a " + std::to_string(t.rows) + " x " + std::to_string(t.cols) + " q4_1 tile grid"); + return out; +} + +std::vector decode_gain(const TensorView& t) { + if (!t.ok() || t.bytes != size_t(t.cols) * 2) fail("decode_gain of '" + t.name + "': not 2 bytes per element"); + std::vector out(size_t(t.cols)); + for (int i = 0; i < t.cols; ++i) out[size_t(i)] = bf16(t.data + size_t(i) * 2); + return out; +} + +std::vector decode_fusion(const TensorView& t) { + if (!t.ok() || t.rows % 32 != 0 || t.bytes != size_t(t.rows) * size_t(t.cols) * 2) + fail("decode_fusion of '" + t.name + "': not hidden x fused at 2 bytes per element"); + const size_t cols = size_t(t.cols); + std::vector out(size_t(t.rows) * cols); + for (size_t a = 0; a < size_t(t.rows) / 32; ++a) + for (size_t k = 0; k < cols; ++k) { + const uint8_t* p = t.data + ((a * cols + k) * 32) * 2; + for (size_t r = 0; r < 32; ++r) out[(a * 32 + r) * cols + k] = bf16(p + r * 2); + } + return out; +} + +} // namespace onebit::npu::dflash diff --git a/npu/dflash_draft.h b/npu/dflash_draft.h new file mode 100644 index 000000000..d5e72ad51 --- /dev/null +++ b/npu/dflash_draft.h @@ -0,0 +1,113 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// dflash_draft.h -- load a DFlash draft package as a model source (engine#1780, the drafts half of F2). +// +// A draft is a two-file source: `config.json` in the DFlash schema (model_type "dflash": the draft's own geometry, the +// block size, the mask token, the target layers it reads) and a Q4NX container with its weights. The container is in +// forms this tree already decodes (tools/dflash_draft_info.py is the gate that established it), so this is a loader +// and not a format: it checks every tensor against the geometry the config gives, and hands each one out decoded. +// +// What it guarantees, and refuses by name otherwise: +// - the config is the DFlash schema with every key the loader needs; another architecture (DFlash2, with selector +// and convolution tensors) is refused rather than half-read; +// - every projection of every layer is present and is exactly the q4_1 tile grid of the N x K the config implies; +// - every norm and the fusion matrix are present at 2 bytes per element in the declared shape; +// - the container's own scalars agree with config.json; +// - no tensor is left unexplained: a tensor this loader does not know means an architecture it does not know. +// +// Nothing is decoded at load. A draft's projections are 1.3 GB as f32, so the model holds views into the mapped file +// and decode_*() produces one tensor at a time; the caller decides what to keep. +// +// Not here: the draft's forward (five non-causal layers over a masked block), the target-side taps, and anything on +// the device. This is the step before them. + +#pragma once + +#include +#include +#include +#include +#include + +namespace onebit::npu::dflash { + +struct DraftConfig { + int hidden = 0; // hidden_size: the draft's width, and the target model's + int intermediate = 0; // intermediate_size + int layers = 0; // num_hidden_layers + int heads = 0; // num_attention_heads + int kv_heads = 0; // num_key_value_heads + int head_dim = 0; + int block_size = 0; // positions per drafted block (the first is the last accepted token) + int64_t mask_token_id = -1; + std::vector target_layer_ids; // the target layers whose hidden states the draft reads, in fusion order + int sliding_window = 0; + float rms_eps = 0.0f; + float rope_theta = 0.0f; + float embedding_scale = 0.0f; + bool causal = false; + std::string draft_weight; // the container's file name, relative to the package + std::string base_model; // the target this draft was made for, as the package names it (may be empty) + + int taps() const { return int(target_layer_ids.size()); } + int q_dim() const { return heads * head_dim; } + int kv_dim() const { return kv_heads * head_dim; } +}; + +// One tensor of the container: where its bytes are and the matrix they describe. +struct TensorView { + std::string name; + const uint8_t* data = nullptr; + size_t bytes = 0; + int rows = 0; // N (out); 1 for a vector + int cols = 0; // K (in), or the vector's length + bool ok() const { return data != nullptr && bytes != 0; } +}; + +struct DraftLayer { + TensorView q, k, v, o, gate, up, down; // q4_1 tile grids, N x K + TensorView input_norm, post_norm, q_norm, k_norm; // 2-byte gains +}; + +struct DraftModel { + DraftConfig cfg; + TensorView fc; // hidden x (taps * hidden): fuses the target layers' hidden states + TensorView hidden_norm; // applied to the fused features + TensorView norm; // the final norm + std::vector layers; + size_t container_bytes = 0; + int tensors = 0; // tensors in the container, all of them accounted for + std::shared_ptr keep; // owns the mapping every view points into +}; + +// config.json of a draft package. Throws std::runtime_error naming the key or the value that is wrong. +DraftConfig read_draft_config(const std::string& package_dir); + +// The whole package: config, container, every check above. Throws std::runtime_error with the tensor and the reason. +DraftModel load_draft(const std::string& package_dir); + +// A projection as f32, row-major [rows][cols] (out[n * cols + k] multiplies input k into output n). +std::vector decode_projection(const TensorView& t); + +// A norm's gains as f32. +std::vector decode_gain(const TensorView& t); + +// The fusion matrix as f32, row-major [hidden][taps * hidden]. The container stores it as [hidden/32][taps*hidden][32] +// -- 32-row blocks, column by column, the row inside the block last -- and that is the order this undoes: +// out[(a*32 + r) * cols + k] = stored[(a * cols + k) * 32 + r]. +std::vector decode_fusion(const TensorView& t); + +} // namespace onebit::npu::dflash diff --git a/tests/dflash_draft_test.cpp b/tests/dflash_draft_test.cpp new file mode 100644 index 000000000..52735cd82 --- /dev/null +++ b/tests/dflash_draft_test.cpp @@ -0,0 +1,436 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// dflash_draft_test.cpp -- the DFlash draft loader (npu/dflash_draft.h, engine#1780). +// +// Two halves: +// 1. a synthetic package written here, small enough to build in a moment, with known answers: every projection is +// encoded tile by tile from a formula and must decode to that formula's values; the fusion matrix must come back +// in row-major order; and each way a package can be wrong is refused with a message naming the thing; +// 2. the real Bolt draft, when a path is given (argv[1] or ONEBIT_TEST_DFLASH_DRAFT): every tensor loads and decodes +// finite, one at a time, and the fusion matrix's element order is checked against a property of the weights +// themselves (see fusion_order_holds). Without a path this half prints SKIP and the test still passes. +// +// No device, no kernels, no forward. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dflash_draft.h" + +namespace { + +namespace fs = std::filesystem; +using namespace onebit::npu::dflash; + +int g_failures = 0; +int g_checks = 0; + +void check(bool cond, const std::string& what) { + ++g_checks; + if (!cond) { + std::printf("FAIL %s\n", what.c_str()); + ++g_failures; + } +} + +uint16_t to_bf16(float f) { // exact for the values used here (few mantissa bits) + uint32_t bits; + std::memcpy(&bits, &f, 4); + return uint16_t(bits >> 16); +} +void put16(std::string& s, uint16_t v) { s.append(reinterpret_cast(&v), 2); } +void put32(std::string& s, const void* p) { s.append(static_cast(p), 4); } + +// The synthetic weights, as formulas a decoder has to reproduce. +int code_of(int salt, int n, int k) { return (n * 7 + k * 3 + salt) & 15; } +float scale_of(int salt, int n, int group) { return 0.0078125f * float(1 + ((n + group + salt) % 4)); } // 1/128 .. 4/128 +float min_of(int salt, int n, int group) { return -0.0625f * float(1 + ((n * 3 + group + salt) % 5)); } +float weight_of(int salt, int n, int k) { + return float(code_of(salt, n, k)) * scale_of(salt, n, k / 32) + min_of(salt, n, k / 32); +} +float fusion_of(int n, int k) { return 0.03125f * float((n * 5 + k * 11) % 64 - 32); } +float gain_of(int salt, int i) { return 1.0f + 0.125f * float((i + salt) % 8); } + +// One N x K projection as a q4_1 tile grid: tiles row-major, each 5120 B = 256 bf16 scales (index group*32 + row), +// 256 bf16 minimums (same index), then 4096 B of 4-bit codes (two 16-row halves x 8 groups, 256 B each; inside one, +// byte 8*j + row/2 for column j of the group, even row in the low nibble). +std::string encode_projection(int salt, int N, int K) { + std::string out; + for (int tr = 0; tr < N / 32; ++tr) + for (int tc = 0; tc < K / 256; ++tc) { + std::string tile(5120, '\0'); + for (int g = 0; g < 8; ++g) + for (int r = 0; r < 32; ++r) { + const int n = tr * 32 + r, group = tc * 8 + g; + const uint16_t s = to_bf16(scale_of(salt, n, group)), z = to_bf16(min_of(salt, n, group)); + std::memcpy(&tile[size_t(2 * (g * 32 + r))], &s, 2); + std::memcpy(&tile[size_t(512 + 2 * (g * 32 + r))], &z, 2); + } + for (int r = 0; r < 32; ++r) + for (int c = 0; c < 256; ++c) { + const int half = r / 16, rr = r % 16, g = c / 32, j = c % 32; + const int code = code_of(salt, tr * 32 + r, tc * 256 + c); + unsigned char& b = reinterpret_cast(tile[size_t(1024 + (half * 8 + g) * 256 + 8 * j + rr / 2)]); + b = (rr % 2 == 0) ? (unsigned char)((b & 0xF0) | code) : (unsigned char)((b & 0x0F) | (code << 4)); + } + out += tile; + } + return out; +} + +struct Geometry { + int hidden = 256, intermediate = 512, layers = 2, heads = 4, kv_heads = 2, head_dim = 64, block_size = 8; + std::vector taps = {1, 3}; +}; + +struct Entry { + std::string name, dtype, shape, bytes; +}; + +// The package's tensors in container order. `edit` may rewrite the list (drop one, add one, damage one). +std::vector tensors_of(const Geometry& g) { + std::vector t; + const auto scalar_f = [&](const std::string& n, float v) { std::string b; put32(b, &v); t.push_back({n, "F32", "[1]", b}); }; + const auto scalar_i = [&](const std::string& n, int32_t v) { std::string b; put32(b, &v); t.push_back({n, "I32", "[1]", b}); }; + scalar_f("dflash.embedding_scale", 16.0f); + scalar_f("dflash.rms_eps", 1e-6f); + scalar_f("dflash.rope_theta", 1000000.0f); + scalar_i("dflash.block_size", g.block_size); + scalar_i("dflash.sliding_window", 1024); + { + std::string b; + for (int id : g.taps) { const int32_t v = id; put32(b, &v); } + t.push_back({"dflash.target_layer_ids", "I32", "[" + std::to_string(g.taps.size()) + "]", b}); + } + const int fused = int(g.taps.size()) * g.hidden; + { + std::string b; // [hidden/32][fused][32] + for (int a = 0; a < g.hidden / 32; ++a) + for (int k = 0; k < fused; ++k) + for (int r = 0; r < 32; ++r) put16(b, to_bf16(fusion_of(a * 32 + r, k))); + t.push_back({"model.fc.weight", "BF16", + "[" + std::to_string(g.hidden / 32) + "," + std::to_string(fused) + ",32]", b}); + } + const auto gain = [&](const std::string& n, int salt, int width) { + std::string b; + for (int i = 0; i < width; ++i) put16(b, to_bf16(gain_of(salt, i))); + t.push_back({n, "BF16", "[" + std::to_string(width) + "]", b}); + }; + const auto proj = [&](const std::string& n, int salt, int N, int K) { + const std::string b = encode_projection(salt, N, K); + t.push_back({n, "I8", "[" + std::to_string(b.size() / 5120) + ",5120]", b}); + }; + gain("model.hidden_norm.weight", 1, g.hidden); + const int q = g.heads * g.head_dim, kv = g.kv_heads * g.head_dim; + for (int l = 0; l < g.layers; ++l) { + const std::string p = "model.layers." + std::to_string(l) + "."; + const int s = 10 * (l + 1); + gain(p + "input_layernorm.weight", s + 1, g.hidden); + gain(p + "post_attention_layernorm.weight", s + 2, g.hidden); + gain(p + "self_attn.k_norm.weight", s + 3, g.head_dim); + gain(p + "self_attn.q_norm.weight", s + 4, g.head_dim); + proj(p + "self_attn.q_proj.weight", s + 1, q, g.hidden); + proj(p + "self_attn.k_proj.weight", s + 2, kv, g.hidden); + proj(p + "self_attn.v_proj.weight", s + 3, kv, g.hidden); + proj(p + "self_attn.o_proj.weight", s + 4, g.hidden, q); + proj(p + "mlp.gate_proj.weight", s + 5, g.intermediate, g.hidden); + proj(p + "mlp.up_proj.weight", s + 6, g.intermediate, g.hidden); + proj(p + "mlp.down_proj.weight", s + 7, g.hidden, g.intermediate); + } + gain("model.norm.weight", 2, g.hidden); + return t; +} + +std::string config_of(const Geometry& g) { + std::string taps; + for (size_t i = 0; i < g.taps.size(); ++i) taps += (i ? "," : "") + std::to_string(g.taps[i]); + return "{\"model_type\":\"dflash\",\"architectures\":[\"DFlashDraftModel\"],\"base_model\":\"synthetic\"," + "\"draft_weight\":\"dflash_weight.q4nx\",\"block_size\":" + std::to_string(g.block_size) + + ",\"mask_token_id\":4,\"target_layer_ids\":[" + taps + "],\"num_hidden_layers\":" + std::to_string(g.layers) + + ",\"hidden_size\":" + std::to_string(g.hidden) + ",\"intermediate_size\":" + std::to_string(g.intermediate) + + ",\"num_attention_heads\":" + std::to_string(g.heads) + ",\"num_key_value_heads\":" + std::to_string(g.kv_heads) + + ",\"head_dim\":" + std::to_string(g.head_dim) + + ",\"rms_norm_eps\":1e-06,\"rope_theta\":1000000.0,\"sliding_window\":1024,\"embedding_scale\":16.0,\"causal\":false}"; +} + +void write_package(const fs::path& dir, const std::string& config, const std::vector& tensors, size_t drop_tail = 0) { + fs::create_directories(dir); + std::ofstream(dir / "config.json") << config; + std::string header = "{\"__metadata__\":{}", payload; + for (const Entry& e : tensors) { + header += ",\"" + e.name + "\":{\"dtype\":\"" + e.dtype + "\",\"shape\":" + e.shape + ",\"data_offsets\":[" + + std::to_string(payload.size()) + "," + std::to_string(payload.size() + e.bytes.size()) + "]}"; + payload += e.bytes; + } + header += "}"; + const uint64_t n = header.size(); + std::ofstream f(dir / "dflash_weight.q4nx", std::ios::binary); + f.write(reinterpret_cast(&n), 8); + f.write(header.data(), std::streamsize(header.size())); + f.write(payload.data(), std::streamsize(payload.size() - drop_tail)); +} + +std::string replaced(std::string s, const std::string& from, const std::string& to) { + const size_t at = s.find(from); + if (at == std::string::npos) { + std::printf("FAIL test bug: '%s' is not in the config\n", from.c_str()); + ++g_failures; + return s; + } + return s.replace(at, from.size(), to); +} + +// load_draft() must throw, and the message must name `needle`. +void refused(const fs::path& dir, const std::string& needle, const std::string& what) { + try { + (void)load_draft(dir.string()); + check(false, what + ": accepted"); + } catch (const std::runtime_error& e) { + const bool named = std::string(e.what()).find(needle) != std::string::npos; + check(named, what + ": refused, but the message does not say '" + needle + "': " + e.what()); + } +} + +void test_synthetic(const fs::path& root) { + const Geometry g; + const std::vector good = tensors_of(g); + const std::string cfg = config_of(g); + + { // the package as it should be + write_package(root / "good", cfg, good); + DraftModel m; + try { + m = load_draft((root / "good").string()); + } catch (const std::exception& e) { + check(false, std::string("the synthetic package loads: ") + e.what()); + return; + } + check(m.cfg.hidden == 256 && m.cfg.intermediate == 512 && m.cfg.layers == 2 && m.cfg.heads == 4 && m.cfg.kv_heads == 2 && + m.cfg.head_dim == 64 && m.cfg.block_size == 8 && m.cfg.mask_token_id == 4 && !m.cfg.causal, + "the config's geometry is read"); + check(m.cfg.target_layer_ids == std::vector({1, 3}) && m.cfg.taps() == 2 && m.cfg.base_model == "synthetic", + "and its target layers, in order"); + check(m.layers.size() == 2 && m.tensors == int(good.size()) && m.tensors == 6 + 3 + 2 * 11, "every tensor is accounted for"); + check(m.fc.rows == 256 && m.fc.cols == 512, "the fusion matrix is hidden x (taps * hidden)"); + + const auto same = [&](const TensorView& v, int salt, int N, int K, const std::string& name) { + check(v.rows == N && v.cols == K, name + " is " + std::to_string(N) + " x " + std::to_string(K)); + const std::vector w = decode_projection(v); + size_t bad = 0; + for (int n = 0; n < N; ++n) + for (int k = 0; k < K; ++k) + if (w[size_t(n) * size_t(K) + size_t(k)] != weight_of(salt, n, k)) ++bad; + check(w.size() == size_t(N) * size_t(K) && bad == 0, + name + " decodes to the encoded values at every element (" + std::to_string(bad) + " differ)"); + }; + for (int l = 0; l < 2; ++l) { + const DraftLayer& L = m.layers[size_t(l)]; + const int s = 10 * (l + 1); + const std::string p = "layer " + std::to_string(l) + " "; + same(L.q, s + 1, 256, 256, p + "q_proj"); + same(L.k, s + 2, 128, 256, p + "k_proj"); + same(L.v, s + 3, 128, 256, p + "v_proj"); + same(L.o, s + 4, 256, 256, p + "o_proj"); + same(L.gate, s + 5, 512, 256, p + "gate_proj"); + same(L.up, s + 6, 512, 256, p + "up_proj"); + same(L.down, s + 7, 256, 512, p + "down_proj"); + const std::vector in = decode_gain(L.input_norm), qn = decode_gain(L.q_norm), kn = decode_gain(L.k_norm), + post = decode_gain(L.post_norm); + bool ok = in.size() == 256 && post.size() == 256 && qn.size() == 64 && kn.size() == 64; + for (size_t i = 0; ok && i < in.size(); ++i) ok = in[i] == gain_of(s + 1, int(i)) && post[i] == gain_of(s + 2, int(i)); + for (size_t i = 0; ok && i < qn.size(); ++i) ok = kn[i] == gain_of(s + 3, int(i)) && qn[i] == gain_of(s + 4, int(i)); + check(ok, p + "norm gains decode to the encoded values, each under its own name"); + } + const std::vector fc = decode_fusion(m.fc); + size_t bad = 0; + for (int n = 0; n < 256; ++n) + for (int k = 0; k < 512; ++k) + if (fc[size_t(n) * 512 + size_t(k)] != fusion_of(n, k)) ++bad; + check(fc.size() == 256u * 512u && bad == 0, "the fusion matrix comes back row-major (" + std::to_string(bad) + " elements differ)"); + // and the stored order really is a different one: read plainly, element (1, 0) is not what is stored second + check(fusion_of(1, 0) != fusion_of(0, 1), "the fixture distinguishes the two element orders"); + const std::vector hn = decode_gain(m.hidden_norm), fn = decode_gain(m.norm); + check(hn[3] == gain_of(1, 3) && fn[3] == gain_of(2, 3) && hn[3] != fn[3], "hidden_norm and the final norm are not swapped"); + } + + // What must be refused, each by name. + const auto with = [&](const std::string& name, const std::function&)>& edit, const std::string& config) { + std::vector t = good; + edit(t); + write_package(root / name, config, t); + return root / name; + }; + const auto none = [](std::vector&) {}; + const auto find = [](std::vector& t, const std::string& n) -> Entry& { + for (Entry& e : t) + if (e.name == n) return e; + std::printf("FAIL test bug: no tensor %s\n", n.c_str()); + ++g_failures; + return t[0]; + }; + refused(with("missing", [&](std::vector& t) { + for (size_t i = 0; i < t.size(); ++i) + if (t[i].name == "model.layers.1.mlp.up_proj.weight") { t.erase(t.begin() + long(i)); break; } + }, cfg), + "no tensor 'model.layers.1.mlp.up_proj.weight'", "a missing projection"); + refused(with("short", [&](std::vector& t) { find(t, "model.layers.0.self_attn.k_proj.weight").bytes.resize(5120); }, cfg), + "'model.layers.0.self_attn.k_proj.weight' holds 5120 B", "a projection with too few tiles"); + refused(with("extra", [&](std::vector& t) { t.push_back({"model.selector.weight", "BF16", "[4]", std::string(8, '\0')}); }, cfg), + "'model.selector.weight' is not part of a DFlashDraftModel", "a tensor of another architecture"); + refused(with("dtype", [&](std::vector& t) { find(t, "model.norm.weight").dtype = "F16"; }, cfg), + "'model.norm.weight' is declared F16", "a gain under another dtype label"); + refused(with("plainfc", [&](std::vector& t) { find(t, "model.fc.weight").shape = "[256,512]"; }, cfg), + "'model.fc.weight' is not declared [hidden/32, taps*hidden, 32]", "a fusion matrix declared in another element order"); + refused(with("block", [&](std::vector& t) { const int32_t v = 16; std::memcpy(find(t, "dflash.block_size").bytes.data(), &v, 4); }, cfg), + "'dflash.block_size' is 16 and config.json says 8", "a container that disagrees with its config on the block size"); + refused(with("taps", [&](std::vector& t) { const int32_t v = 2; std::memcpy(find(t, "dflash.target_layer_ids").bytes.data() + 4, &v, 4); }, cfg), + "'dflash.target_layer_ids[1]' is 2 and config.json says 3", "a container that disagrees on a target layer"); + refused(with("dflash2", none, replaced(cfg, "DFlashDraftModel", "DFlash2DraftModel")), + "architecture 'DFlash2DraftModel' is not read by this loader", "a DFlash2 package"); + refused(with("type", none, replaced(cfg, "\"model_type\":\"dflash\"", "\"model_type\":\"qwen3\"")), + "model_type is 'qwen3'", "a config that is not a draft's"); + refused(with("nokey", none, replaced(cfg, "\"head_dim\":64,", "")), "no 'head_dim'", "a config without head_dim"); + refused(with("order", none, replaced(cfg, "[1,3]", "[3,1]")), "'target_layer_ids' is not strictly increasing", "target layers out of order"); + refused(with("path", none, replaced(cfg, "\"dflash_weight.q4nx\"", "\"../good/dflash_weight.q4nx\"")), + "'draft_weight' must be a file name inside the package", "a weight path that leaves the package"); + refused(with("grid", none, replaced(cfg, "\"intermediate_size\":512", "\"intermediate_size\":500")), + "is not a whole 32 x 256 tile grid", "a geometry that does not fill whole tiles"); + refused(with("heads", none, replaced(cfg, "\"num_key_value_heads\":2", "\"num_key_value_heads\":3")), + "4 heads cannot share 3 key/value heads", "key/value heads that do not divide the heads"); + { // the last tensor's bytes are not all there + write_package(root / "cut", cfg, good, 100); + refused(root / "cut", "", "a container cut short of its last tensor"); + } + { + fs::create_directories(root / "empty"); + refused(root / "empty", "cannot open", "a directory with no config.json"); + } +} + +// The fusion matrix's stored element order, checked on the weights themselves. Magnitudes that share an input column +// are correlated far more strongly than magnitudes that share an output row (each column multiplies one feature of a +// target layer, and features differ in scale). In the right order that holds whichever neighbours are compared: the next +// row and the row 32 away agree with each other, and so do the next column and the column 32 away. Read in the plain +// row-major order instead, "the next column" is really the same column and "32 columns away" is not, so the two +// column figures disagree by several times. The assertion is that consistency, not a threshold on one number. +bool fusion_order_holds(const std::vector& w, int rows, int cols, double out[4]) { + std::vector lw(w.size()); + double mean = 0.0; + for (size_t i = 0; i < w.size(); ++i) { + lw[i] = std::log(std::fabs(w[i]) + 1e-6f); + mean += lw[i]; + } + mean /= double(w.size()); + double var = 0.0; + for (float v : lw) var += (double(v) - mean) * (double(v) - mean); + var /= double(w.size()); + const auto corr = [&](int dn, int dk) { + double acc = 0.0; + size_t count = 0; + for (int n = 0; n + dn < rows; n += 3) + for (int k = 0; k + dk < cols; k += 7) { + acc += (double(lw[size_t(n) * size_t(cols) + size_t(k)]) - mean) * + (double(lw[size_t(n + dn) * size_t(cols) + size_t(k + dk)]) - mean); + ++count; + } + return acc / double(count) / var; + }; + out[0] = corr(1, 0); // next row, same column + out[1] = corr(32, 0); // 32 rows on, same column + out[2] = corr(0, 1); // same row, next column + out[3] = corr(0, 32); // same row, 32 columns on + const auto near = [](double a, double b) { return std::fabs(a - b) <= 0.5 * std::max(std::fabs(a), std::fabs(b)); }; + return near(out[0], out[1]) && near(out[2], out[3]) && out[0] > 2.0 * std::max(out[2], out[3]); +} + +void test_real(const std::string& dir) { + DraftModel m; + try { + m = load_draft(dir); + } catch (const std::exception& e) { + check(false, std::string("the real package loads: ") + e.what()); + return; + } + const DraftConfig& c = m.cfg; + std::printf("real package: %d tensors, %zu bytes; %d layers, hidden %d, intermediate %d, %d heads / %d kv x %d, block %d, taps", + m.tensors, m.container_bytes, c.layers, c.hidden, c.intermediate, c.heads, c.kv_heads, c.head_dim, c.block_size); + for (int id : c.target_layer_ids) std::printf(" %d", id); + std::printf("\n"); + check(int(m.layers.size()) == c.layers && m.tensors == 6 + 3 + 11 * c.layers, "every tensor of the real package is accounted for"); + size_t elements = 0; + for (const DraftLayer& L : m.layers) { + for (const TensorView* v : {&L.q, &L.k, &L.v, &L.o, &L.gate, &L.up, &L.down}) { + const std::vector w = decode_projection(*v); // one at a time: the set is 1.3 GB as f32 + size_t nonfinite = 0; + double sum2 = 0.0; + for (float x : w) { + if (!std::isfinite(x)) ++nonfinite; + else sum2 += double(x) * double(x); + } + const double rms = std::sqrt(sum2 / double(w.size())); + check(nonfinite == 0 && rms > 1e-4 && rms < 1.0, + v->name + " decodes finite with a weight-like rms (" + std::to_string(rms) + ", " + std::to_string(nonfinite) + " non-finite)"); + elements += w.size(); + } + for (const TensorView* v : {&L.input_norm, &L.post_norm, &L.q_norm, &L.k_norm}) { + // Finite and bounded, not "positive": three of the Bolt draft's post-attention norms hold one gain just + // below zero (-0.0009 .. -0.0024), which is what a learned gain crossing zero looks like. + bool ok = true; + for (float x : decode_gain(*v)) ok = ok && std::isfinite(x) && std::fabs(x) < 100.0f; + check(ok, v->name + " is finite and bounded"); + } + } + std::printf("real package: %zu projection weights decoded\n", elements); + const std::vector fc = decode_fusion(m.fc); + double r[4]; + const bool right = fusion_order_holds(fc, m.fc.rows, m.fc.cols, r); + std::printf("fusion matrix, as decoded: next row %.4f, 32 rows on %.4f | next column %.4f, 32 columns on %.4f\n", r[0], r[1], r[2], r[3]); + check(right, "the fusion matrix's decoded order is self-consistent: column-sharing neighbours agree, row-sharing neighbours agree"); + // the control: the same bytes read as plain row-major must NOT pass, or the check above proves nothing + std::vector plain(fc.size()); + for (size_t i = 0; i < plain.size(); ++i) { + uint16_t h; + std::memcpy(&h, m.fc.data + i * 2, 2); + const uint32_t bits = uint32_t(h) << 16; + std::memcpy(&plain[i], &bits, 4); + } + const bool wrong = fusion_order_holds(plain, m.fc.rows, m.fc.cols, r); + std::printf("fusion matrix, read plainly: next row %.4f, 32 rows on %.4f | next column %.4f, 32 columns on %.4f\n", r[0], r[1], r[2], r[3]); + check(!wrong, "and the plain row-major reading of the same bytes is not"); +} + +} // namespace + +int main(int argc, char** argv) { + const fs::path root = fs::temp_directory_path() / ("dflash-draft-test-" + std::to_string(::getpid())); + fs::remove_all(root); + test_synthetic(root); + fs::remove_all(root); + const char* real = (argc > 1 && argv[1][0] != '\0') ? argv[1] : std::getenv("ONEBIT_TEST_DFLASH_DRAFT"); + if (real != nullptr && *real != '\0') test_real(real); + else std::printf("SKIP the real package: pass its directory, or set ONEBIT_TEST_DFLASH_DRAFT\n"); + if (g_failures == 0) std::printf("dflash_draft: ok (%d checks)\n", g_checks); + return g_failures == 0 ? 0 : 1; +} From d4dc5e7ce1a05bfdcea6363e007183030b4ce7da Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:50:26 -0400 Subject: [PATCH 49/98] bump-hrx: the failure report carries the failing step and the conflicting files (Refs #1945) (#2045) * bump-hrx: the failure report carries the failing step and the conflicting files #1945's own closing line is the reason this exists: bump-hrx failed on every scheduled run from 2026-10-01 to 2026-10-05 without anyone noticing. The workflow now files an issue so it cannot be silent, and it dedupes while one is open -- both verified, both left alone. What it did NOT do is say what broke: the body was a static template with the run URL and instructions, so the reader was told where to look and never what to find. The failing step's name, the conflicting paths and the SHAs were all in the log the run already had. So the body construction moves into scripts/bump-hrx-report.sh, stdout = the issue body, which makes it testable without running the workflow (this lane cannot edit .github/workflows -- no `workflow` scope; the one-line call site is left as a stated change for whoever has it). Three cases the report must keep apart, because conflating any two of them is how a report stops being read: * files in conflict -- git spells this two ways (CONFLICT (content): Merge conflict in , and CONFLICT (modify/delete): deleted in ...); a report that knows only the first silently lists half the conflict, which the second arm caught during development; * a failure that is NOT a conflict -- must say so rather than print an empty file block, since an empty list reads as "no problem found"; * a log that could not be read -- must say THAT, not "no conflicts", because "nothing conflicted" and "nobody looked" are different facts and only one of them is actionable. Eleven arms in tests/bump_hrx_report_test.sh, verified in both directions: an empty --log, a repeated path (listed once), the modify/delete spelling, a non-conflict log, no log at all, and the usage contract (--run required, unknown flag exits 2). Refs #1945. * bump-hrx-report: the full Apache notice in the header (copyright.py --check) * tests: register bump_hrx_report, so the ratchet that caught it can also run it (engine#1945) --------- Co-authored-by: bong-water-water-bong --- CMakeLists.txt | 1 + scripts/bump-hrx-report.sh | 78 +++++++++++++++++++++++++++++++++++ tests/bump_hrx_report_test.sh | 65 +++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100755 scripts/bump-hrx-report.sh create mode 100755 tests/bump_hrx_report_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index a0b3ce73b..a90354cea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -200,6 +200,7 @@ add_test(NAME serve_hangup COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_hangup_test.s # engine#1945: scripts/bump-hrx-sync.sh tells AMD's own commit from a line AMD rebased apart from a commit that # is not AMD's. Local git repositories only: no network, no GPU. add_test(NAME bump_hrx_sync COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_sync_test.sh ${CMAKE_SOURCE_DIR}/scripts/bump-hrx-sync.sh) +add_test(NAME bump_hrx_report COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_report_test.sh ${CMAKE_SOURCE_DIR}/scripts/bump-hrx-report.sh) add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $) # serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU. add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $) diff --git a/scripts/bump-hrx-report.sh b/scripts/bump-hrx-report.sh new file mode 100755 index 000000000..a75443953 --- /dev/null +++ b/scripts/bump-hrx-report.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# bump-hrx-report.sh -- build the body of the issue a failed scheduled sync files. +# +# Why this exists (engine#1945): the report step already files an issue so a broken sync cannot sit unnoticed +# for five days, but its body was a static template -- the run URL and instructions, never the failing step or +# the files in conflict. So the reader was told where to look and not what broke. This turns the log the run +# already has into the body, and lives in scripts/ so it is testable without running the workflow. +# +# usage: bump-hrx-report.sh --run [--step ] [--log |-] [--attempt ] +# stdout = the issue body; rc 0 unless an argument is missing. +# --log - reads stdin. No --log means "the log was not available": the body says so rather than claiming +# there were no conflicts, which is the difference between "nothing conflicted" and "nobody looked". +set -uo pipefail + +run="" step="" log="" attempt="" +while [ $# -gt 0 ]; do + case "$1" in + --run) run=${2:?--run needs a URL}; shift 2 ;; + --step) step=${2:?--step needs a name}; shift 2 ;; + --log) log=${2:?--log needs a path or -}; shift 2 ;; + --attempt) attempt=${2:?--attempt needs a number}; shift 2 ;; + -h|--help) sed -n '3,17p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "bump-hrx-report: unknown argument: $1" >&2; exit 2 ;; + esac +done +[ -n "$run" ] || { echo "bump-hrx-report: --run is required" >&2; exit 2; } + +text="" +if [ "$log" = "-" ]; then + text=$(cat) +elif [ -n "$log" ]; then + if [ -r "$log" ]; then text=$(cat "$log"); else + echo "bump-hrx-report: cannot read log: $log" >&2; exit 2 + fi +fi + +# git names every conflicting path in a line of the form "CONFLICT (content): Merge conflict in "; +# "CONFLICT (modify/delete)" and friends use the same prefix. Paths can repeat across retries, so dedupe. +# git spells the same event two ways, and a report that knows only one of them silently lists half the conflict: +# CONFLICT (content): Merge conflict in +# CONFLICT (modify/delete): deleted in and modified in +# CONFLICT (rename/delete): renamed ... (path is the source side) +conflicts=$(printf '%s\n' "$text" \ + | sed -n -e 's/.*CONFLICT ([^)]*): Merge conflict in \(.*\)$/\1/p' \ + -e 's/.*CONFLICT ([^)]*): \(.*\) \(deleted\|renamed\) in .*/\1/p' \ + | sort -u) + +printf 'The bump-hrx run failed: %s\n' "$run" +if [ -n "$step" ]; then + printf '\n**Failing step:** `%s`' "$step" + [ -n "$attempt" ] && printf ' (attempt %s)' "$attempt" + printf '\n' +fi + +if [ -n "$conflicts" ]; then + printf '\n**Conflicting files** (from the log):\n\n```\n%s\n```\n' "$conflicts" + printf '\nResolve these by hand on `1bit/hrx-vulkan-patched` or on `1bit-MONSTER/hrx-system` `1bit/main`, then re-run the workflow.\n' +elif [ -z "$log" ]; then + printf '\n**The log was not available to this report**, so the conflicting files are not listed here. Open the run above and read the failing step.\n' +else + printf '\n**No conflicting files were named in the log** -- this failure is not a merge conflict. Open the run above and read the failing step.\n' +fi + +printf '\nWhy this report exists: bump-hrx failed on every scheduled run from 2026-10-01 to 2026-10-05 without anyone\nnoticing, and the HRX pin silently froze. A failed run files this issue; it is deduped while open.\n' diff --git a/tests/bump_hrx_report_test.sh b/tests/bump_hrx_report_test.sh new file mode 100755 index 000000000..52d8cca8e --- /dev/null +++ b/tests/bump_hrx_report_test.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# bump_hrx_report_test.sh -- arms for scripts/bump-hrx-report.sh (engine#1945). +# +# The three cases a failure report must distinguish, because conflating any two of them is how a report stops +# being read: files in conflict, a failure that is not a conflict, and a report that could not read the log. +# +# usage: bump_hrx_report_test.sh +set -uo pipefail + +S=${1:?usage: bump_hrx_report_test.sh } +[ -x "$S" ] || { echo "not executable: $S"; exit 1; } +fail=0 +ok() { echo "ok $1"; } +bad() { echo "FAIL $1"; fail=1; } + +# 1. A merge conflict: git's two shapes (content and modify/delete), one path repeated as a retry would. +got=$("$S" --run https://example/runs/1 --log - --step "Sync the fork and merge AMD's pin into our line" <<'LOG' +Auto-merging ggml/src/ggml-hrx/ggml-hrx.cpp +CONFLICT (content): Merge conflict in ggml/src/ggml-hrx/ggml-hrx.cpp +CONFLICT (modify/delete): drivers/fake.c deleted in HEAD and modified in abc123 +CONFLICT (content): Merge conflict in ggml/src/ggml-hrx/ggml-hrx.cpp +LOG +) +[[ $got == *"ggml/src/ggml-hrx/ggml-hrx.cpp"* ]] && ok "a conflict names the file" || bad "a conflict names the file" +[[ $got == *"drivers/fake.c"* ]] && ok "the modify/delete form is caught too" || bad "the modify/delete form is caught too" +[[ $(printf '%s' "$got" | grep -c 'ggml-hrx/ggml-hrx.cpp') -ge 1 ]] || bad "path present" +n=$(printf '%s\n' "$got" | sed -n '/^```$/,/^```$/p' | grep -c 'ggml/src/ggml-hrx/ggml-hrx.cpp') +[ "$n" = 1 ] && ok "a repeated path is listed once" || bad "a repeated path is listed once (got $n)" +[[ $got == *"Failing step"*"Sync the fork"* ]] && ok "the failing step is in the body" || bad "the failing step is in the body" + +# 2. A failure that is NOT a conflict must say so -- an empty list would read as "no problem found". +got=$("$S" --run https://example/runs/2 --log - --step "Merge AMD's hrx-system pin into our line" <<'LOG' +error: could not apply 1a2b3c4... some commit +fatal: Exiting because of an unresolved dispute +LOG +) +[[ $got == *"not a merge conflict"* ]] && ok "a non-conflict failure says so" || bad "a non-conflict failure says so" +[[ $got != *'```'* ]] && ok "and prints no empty file block" || bad "and prints no empty file block" + +# 3. No log at all: the body must still render and must not claim there were no conflicts. +got=$("$S" --run https://example/runs/3 --step "Sync") +[[ $got == *"log was not available"* ]] && ok "a missing log is stated, not implied" || bad "a missing log is stated, not implied" +[[ $got != *"not a merge conflict"* ]] && ok "a missing log is not reported as a non-conflict" || bad "a missing log is not reported as a non-conflict" +[[ $got == *"https://example/runs/3"* ]] && ok "the run URL is always present" || bad "the run URL is always present" + +# 4. Contract: --run is required, an unknown flag is a usage error, not a silent empty body. +"$S" > /dev/null 2>&1; [ $? = 2 ] && ok "a missing --run exits 2" || bad "a missing --run exits 2" +"$S" --run x --nope > /dev/null 2>&1; [ $? = 2 ] && ok "an unknown argument exits 2" || bad "an unknown argument exits 2" + +[ $fail = 0 ] && echo "all bump-hrx-report arms passed" +exit $fail From f9dcbc40e72b6acb026699a1f82009230acc6d3e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:00:40 -0400 Subject: [PATCH 50/98] npu-pack: the 1.9 canonical copy is written under a temporary name and renamed into place (Refs #1985) (#2043) The canonical Token .fconst lives at a path every import of the same pack shares, was written in place with O_TRUNC, and is reused on length. A second import that started while the first was writing saw a short file, truncated it and wrote again; the first then read its own copy while it was being refilled. On Qwen3-1.7B, three `npu-pack info` 0.15 s apart on clean main: 4 of 24 runs reported 280 refused tensors, qk_norm false, tiles_measured 0, exit 0 -- the numbers of engine#1985, from one binary on one pack. write_canonical_copy() writes beside the copy and renames. The temporary name starts with the scratch stem, so the age sweep removes one a killed run left, and not with a pack prefix, so another import's sweep cannot delete a copy that is being written. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/npu.md | 9 ++++ npu/pack_import.cpp | 51 +++++++++++++++--- npu/pack_import.h | 10 ++++ tests/npu_pack_import_test.cpp | 99 ++++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 6 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 1c4396014..934c80a45 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1135,6 +1135,15 @@ working directory's files are not the importer's to clean. The 6 hours are what shared directory, so the number stays above the longest import; it is not the knob for disk or tmpfs pressure. The test suite sets one per run and removes it at exit. +The cache directory is shared: every import of the same pack on the box uses the same file name. The copy is therefore +written under a temporary name beside it (`1bit-fconst19tmp---`) and renamed into place, so an import finds +either no copy or a complete one (engine#1985). Before that the copy was written in place, and two imports of one pack +started a fraction of a second apart could leave the first reading a file the second had just truncated. The result was a +wrong layout reading with exit 0: `Qwen3-1.7B` reported 280 refused tensors, `qk_norm` false and 0 measured tiles in 4 of +24 runs of three imports 0.15 s apart, and 0 of 144 with the rename. A temporary file left by a killed run is removed by +the 6-hour rule above. A binary older than this change still writes in place, so a box where old and new binaries import +the same pack at the same moment can still show it. + ### Linking the kernel set the container will be served with A model directory carries its own NPU kernels in `npu/` (see "A model directory"), and for an imported diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 429347cc3..1e9f5a357 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -32,6 +33,7 @@ #include #include #include +#include namespace onebit::npu::pack_import { namespace { @@ -782,12 +784,8 @@ std::string canonical_token_fconst(const std::string& path, int hidden, int head } out.insert(out.end(), fc.map + tail_off, fc.map + fc.size); unmap_file(fc.map, fc.size); - { - std::ofstream os(work, std::ios::binary | std::ios::trunc); - if (!os) throw std::runtime_error(work + ": cannot write the canonical copy of the 1.9 .fconst"); - os.write(reinterpret_cast(out.data()), std::streamsize(out.size())); - if (!os) throw std::runtime_error(work + ": short write of the canonical copy of the 1.9 .fconst"); - } + // engine#1985: never in place. The path is shared by every import of this pack on the box, see write_canonical_copy(). + write_canonical_copy(work, out.data(), out.size()); return work; } @@ -6133,6 +6131,47 @@ int run(int argc, char** argv) { } +// engine#1985: the temporary name a canonical copy is written under before it is renamed onto `work`. Same directory +// (rename is only atomic inside one filesystem). The name starts with the scratch stem `1bit-fconst19`, so the age rule +// of evict_stale_canonical_copies() removes one a killed run left behind, and continues with `tmp-`, so it can never +// match a pack's `1bit-fconst19v-...` prefix: another import's sweep must not delete a copy that is being written. +std::string canonical_copy_temp_name(const std::string& work) { + static std::atomic serial{0}; + const fs::path p(work); + const std::string base = "1bit-fconst19tmp-" + std::to_string(static_cast(::getpid())) + "-" + + std::to_string(serial.fetch_add(1)) + "-" + p.filename().string(); + return (p.parent_path() / base).string(); +} + +// engine#1985: write the canonical copy under a private name and rename it onto `work`. The copy used to be written in +// place with O_TRUNC, at a path every import of the same pack shares, and reuse is decided by length -- so a second +// import that started while the first was still writing saw a short file, truncated it and began again, and the first +// then read its own copy while it was being refilled. Measured on Qwen3-1.7B with three imports 0.15 s apart: 4 of 24 +// runs read "0 in-block offsets", refused all 280 attention tensors and exited 0, on a pack a lone import reads with 0 +// gaps. With a rename a reader finds no copy or a complete one, and two writers each install the same bytes. +void write_canonical_copy(const std::string& work, const uint8_t* data, size_t len) { + const std::string tmp = canonical_copy_temp_name(work); + { + std::ofstream os(tmp, std::ios::binary | std::ios::trunc); + if (!os) throw std::runtime_error(tmp + ": cannot write the canonical copy of the 1.9 .fconst"); + os.write(reinterpret_cast(data), std::streamsize(len)); + os.flush(); + if (!os) { + os.close(); + std::error_code rm; + fs::remove(tmp, rm); + throw std::runtime_error(tmp + ": short write of the canonical copy of the 1.9 .fconst"); + } + } + std::error_code ec; + fs::rename(tmp, work, ec); + if (ec) { + std::error_code rm; + fs::remove(tmp, rm); + throw std::runtime_error(work + ": cannot move the canonical copy of the 1.9 .fconst into place (" + ec.message() + ")"); + } +} + // Public (engine#1662): see the declaration in pack_import.h. Lives outside the anonymous namespace because the test // calls it directly with crafted files, so the decision that guards a ~291 MB rewrite is pinned without a 1.9 pack. bool canonical_copy_reusable(const std::string& work, int64_t expected_len) { diff --git a/npu/pack_import.h b/npu/pack_import.h index f21dcf51b..183052ae1 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -323,6 +323,16 @@ Plan plan(const std::string& pack_dir); // and a directory on the scratch path must all come back false. bool canonical_copy_reusable(const std::string& work, int64_t expected_len); +// engine#1985: how the canonical copy reaches `work`. It is written under canonical_copy_temp_name(work) -- a private name +// in the same directory -- and renamed into place, so a concurrent import of the same pack finds no copy or a complete one, +// never one that is being written. Writing in place let a second import truncate the file the first was about to read. +// Throws std::runtime_error when the copy cannot be written or moved; the temporary file is removed on either failure. +// Exposed so the rule has an arm that needs no pack (threads writing one path while a reader checks every complete-length +// file it finds), and so the temporary name's two properties are pinned: a pack's sweep prefix never matches it, and the +// age rule does. +std::string canonical_copy_temp_name(const std::string& work); +void write_canonical_copy(const std::string& work, const uint8_t* data, size_t len); + // Removes the canonical Token .fconst copies in `dir` that nothing will reuse (engine#1770). Reuse writes a new copy when the // source changes rather than overwriting, so without this the scratch directory grows one multi-hundred-MB file per change; on // this box 699 files / 8.7 GiB of tmpfs. Three rules: diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 2a2bca1d4..30f4d6605 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -44,6 +44,8 @@ #include #include +#include +#include #include #include #include @@ -836,6 +838,102 @@ void test_external_import_success(const std::string& root) { // ".qweight.empty: no data", which reports a placeholder as though the bytes were absent rather than elsewhere. // Crafted, so this needs no 1.9 pack: a graph with one resolvable tensor, one MatMulNBits whose qweight input is a // `.empty` placeholder with no sibling, and a model.bin next to it. +// engine#1985: the canonical copy lives at a path every import of the same pack shares, and it used to be written in place. +// A second import that started mid-write saw a short file, truncated it and wrote again, and the first then read its own +// copy while it was being refilled: Qwen3-1.7B came back with 280 refused tensors and exit 0 in 4 of 24 runs of three +// imports 0.15 s apart. The arms: the bytes arrive and no temporary file stays; a failed write leaves nothing; the +// temporary name is out of reach of another import's prefix sweep and inside the age sweep; and the race itself, with +// threads doing what an import does (reuse the copy if it is complete, else write it, then read it). +void test_write_canonical_copy() { + const fs::path dir = fs::temp_directory_path() / ("1bit-atomic-" + std::to_string(::getpid())); + std::error_code ec; + fs::remove_all(dir, ec); + fs::create_directories(dir); + const std::string prefix = "1bit-fconst19v1-7-2048-16-28-"; + const std::string name = prefix + "957724736-1234.fconst"; + const std::string work = (dir / name).string(); + const size_t n = size_t(4) << 20; + std::vector body(n); + for (size_t i = 0; i < n; ++i) body[i] = uint8_t((i * 2654435761u) >> 24) | 1; // never 0: a refilled hole reads as 0 + auto same = [&](const std::string& f) { + std::ifstream is(f, std::ios::binary); + if (!is) return false; + std::vector got(n + 1); + is.read(reinterpret_cast(got.data()), std::streamsize(got.size())); + return size_t(is.gcount()) == n && std::memcmp(got.data(), body.data(), n) == 0; + }; + auto entries = [&] { + int k = 0; + for (const auto& e : fs::directory_iterator(dir)) { (void)e; ++k; } + return k; + }; + + pi::write_canonical_copy(work, body.data(), n); + check(same(work), "atomic copy: the bytes written are the bytes at the path"); + check(entries() == 1, "atomic copy: no temporary file is left beside the copy"); + std::vector other(body); + other[n / 2] ^= 0xff; + pi::write_canonical_copy(work, other.data(), n); + check(!same(work) && entries() == 1, "atomic copy: a second write replaces the copy, again leaving one file"); + pi::write_canonical_copy(work, body.data(), n); + + bool threw = false; + try { + pi::write_canonical_copy((dir / "no-such-dir" / name).string(), body.data(), n); + } catch (const std::runtime_error&) { threw = true; } + check(threw && entries() == 1, "atomic copy: a copy that cannot be written throws and leaves nothing behind"); + + // the temporary name: beside the copy, not the copy, unique per call, invisible to the prefix sweep, visible to the age sweep + const std::string t1 = pi::canonical_copy_temp_name(work), t2 = pi::canonical_copy_temp_name(work); + const std::string b1 = fs::path(t1).filename().string(); + check(fs::path(t1).parent_path() == fs::path(work).parent_path(), "atomic copy: the temporary file is in the copy's directory"); + check(t1 != work && t1 != t2, "atomic copy: the temporary name is not the copy's and is new on every call"); + check(b1.compare(0, prefix.size(), prefix) != 0 && b1.compare(0, 13, "1bit-fconst19") == 0, + "atomic copy: the temporary name starts with the scratch stem and not with the pack's prefix"); + { std::ofstream o(t1, std::ios::binary); o << "being written"; } + pi::evict_stale_canonical_copies(dir.string(), name, prefix, std::string(), 3600); + check(fs::exists(t1) && fs::exists(work), "atomic copy: another import's sweep does not remove a copy that is being written"); + fs::last_write_time(t1, fs::file_time_type::clock::now() - std::chrono::hours(2)); + pi::evict_stale_canonical_copies(dir.string(), name, prefix, std::string(), 3600); + check(!fs::exists(t1) && fs::exists(work), "atomic copy: a temporary file a killed run left is removed by the age rule"); + + // the race: every thread is an import. It reuses a complete-length copy or writes one, then reads the path, as + // canonical_token_fconst()'s caller does. One thread also removes the copy now and then, so the cycle restarts. + fs::remove(work, ec); + std::atomic bad{0}, read_ok{0}, wrote{0}; + auto importer = [&](int id) { + for (int r = 0; r < 24; ++r) { + if (!pi::canonical_copy_reusable(work, int64_t(n))) { + try { pi::write_canonical_copy(work, body.data(), n); ++wrote; } catch (const std::runtime_error&) { ++bad; } + } + std::error_code e2; + // An open that fails is the remover having been there, not a bad copy; a copy that OPENS must be whole. + if (std::ifstream(work, std::ios::binary).good()) { + std::ifstream is(work, std::ios::binary); + std::vector got(n + 1); + if (is) { + is.read(reinterpret_cast(got.data()), std::streamsize(got.size())); + if (size_t(is.gcount()) == n && std::memcmp(got.data(), body.data(), n) == 0) ++read_ok; + else ++bad; + } + } + if (id == 0 && r % 4 == 3) fs::remove(work, e2); + } + }; + std::vector pool; + for (int i = 0; i < 6; ++i) pool.emplace_back(importer, i); + for (auto& t : pool) t.join(); + check(bad.load() == 0, "atomic copy: no import ever reads a copy another import is still writing (engine#1985)"); + check(read_ok.load() > 0 && wrote.load() > 1, "atomic copy: the race arm really wrote more than once and read copies back"); + int leftover = 0; + for (const auto& e : fs::directory_iterator(dir)) + if (e.path().filename().string() != name) ++leftover; + check(leftover == 0, "atomic copy: the race leaves no temporary files"); + fs::remove_all(dir, ec); + std::printf(" atomic canonical copy: 11 arms (bytes, replace, failed write, temporary name x4, %d racing writes / %d reads, no leftovers)\n", + wrote.load(), read_ok.load()); +} + // engine#1662: the ~291 MB canonical Token .fconst rewrite is skipped when the copy on the scratch path already holds // exactly the bytes this run would write. The decision is a length test against a length that is a function of the source // alone, so what can go wrong is the test being too willing -- a truncated copy from an interrupted run, a leftover @@ -5534,6 +5632,7 @@ int main(int argc, char** argv) { test_external_container_names(); test_external_packed_genre(golden_dir.empty() ? "." : golden_dir); test_external_import_success(golden_dir.empty() ? "." : golden_dir); + test_write_canonical_copy(); test_canonical_copy_reusable(); test_evict_stale_canonical_copies(); test_sweep_fconst_scratch_dirs(); test_sweep_is_wired(golden_dir.empty() ? "." : golden_dir); From b2f17e29120cb8a6e51f7d59814a3d28c3fba945 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:10:17 -0400 Subject: [PATCH 51/98] npu: a sealed set can carry a wide prefill family, named by the pack or the environment: 28.3-30.8 -> 33.9-35.9 tok/s to first token with a 32-row set (Refs #1990) (#2042) * npu: a sealed set can carry a wide prefill family (NPU_WIDE_SEALED_ELF, off by default): 28.3-30.8 -> 35.0-35.9 tok/s to first token with a 32-row set (Refs #1990) A sealed pb16i/t2s set is built for 16 rows per launch, so a prefill chunk is 16 rows. NPU_WIDE_SEALED_ELF names a second sealed ELF with the same instances built for NPU_WIDE_ROWS rows (default 32). When it loads for every role that has a 16-row kernel, the chunk width becomes that row count and launches of more than 16 rows take the wide kernel; decode and chunks of up to 16 rows stay on the 16-row family. All roles or none, and a launch carrying more rows than its kernel holds is refused by name. The three rules (wide_family_complete, chunk_rows, wide_takes) live in batch_width.h and are pinned without a device by 14 new arms in tests/npu_pack_test.cpp. Experimental: no pack ships a wide ELF and the plan has no key for one, so it is an environment switch. Co-Authored-By: Claude Fable 5.2 * npu: a pack can name its wide sealed set (q4nx__sealed_wide_elf); the environment overrides or switches it off (Refs #1990) config.json keys q4nx_t2s_sealed_wide_elf / q4nx_pb16i_sealed_wide_elf (a file in the kernels dir) and q4nx_sealed_wide_rows (default 32). NPU_WIDE_SEALED_ELF= overrides, =0 switches the pack's family off. A named file that is missing or does not load is logged and the pass stays at 16 rows. wide_elf_path() holds the choice and has 7 arms in tests/npu_pack_test.cpp. Measured through the pack key on bonsai2-27b-t2s2: first token 28.7-29.0 tok/s off, 33.9-34.4 tok/s on, same text, oracle MATCH. Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/npu.md | 29 +++++++++++++++++ npu/forward/include/batch_width.h | 44 +++++++++++++++++++++++++ npu/forward/include/q4nx_forward.h | 3 ++ npu/forward/src/q4nx_forward.cpp | 52 ++++++++++++++++++++++++++++++ npu/target/xdna/xdna_target.cpp | 50 ++++++++++++++++++++++++++-- npu/target/xdna/xdna_target.h | 13 ++++++++ tests/npu_pack_test.cpp | 42 ++++++++++++++++++++++++ 7 files changed, 230 insertions(+), 3 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 934c80a45..a297da858 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -538,6 +538,35 @@ second pair is not quoted as a speedup. `NPU_DEQ_LDEXP=2` runs both forms and co `tests/npu_pb16i_layout_test.cpp` holds the two forms to the same bits for one and two planes, the edges of the exponent range and exponents outside it. The `[bl]` line names the term as `deq`. +**A sealed set may carry a wide family for prefill (#1990).** The served sealed sets are built for 16 +rows per launch, so a prefill chunk is 16 rows and a 536-token prompt is 34 blocks of 416 launches. A pack +may name a second sealed ELF with the same instances built for more rows: `q4nx_t2s_sealed_wide_elf` (or +`q4nx_pb16i_sealed_wide_elf`) in `config.json`, a file in the kernels directory beside the 16-row one, and +`q4nx_sealed_wide_rows` for its row count (default 32, allowed 17..128). No pack ships one yet, so nothing +changes for an existing pack. `NPU_WIDE_SEALED_ELF=` overrides the pack's file, `NPU_WIDE_SEALED_ELF=0` +switches a pack's wide family off, and `NPU_WIDE_ROWS` overrides the row count. + +When the wide ELF loads for every role that has a 16-row kernel, the chunk width becomes its row count and a +launch of more than 16 rows takes the wide kernel; decode steps and chunks of up to 16 rows stay on the +16-row family. It is all roles or none: the chunk width is one number for the pass, so a set missing one +role is not adopted. A named file that is missing or does not load never stops the serve: the log says why +(`[forward] wide family NOT adopted (...)`) and chunks stay at 16 rows. The weight images are shared, only +the activation and result buffers differ. A launch that carries more rows than its kernel holds is refused +by name. The rules (`wide_family_complete`, `chunk_rows`, `wide_takes`, `wide_elf_path` in +`npu/forward/include/batch_width.h`) are pinned without a device by `tests/npu_pack_test.cpp`. + +Measured on `bonsai2-27b-t2s2` with a 32-row set, one `npu-hold` ticket per row of the table, one binary, +arms off / on / off / on, three 510-536-token prompts per arm (`prompt_tokens / wall`, `max_tokens 1`): + +| how the wide set was named | first token, wide off | first token, wide on | 16-row block median | 32-row block median | +|---|---|---|---|---| +| `NPU_WIDE_SEALED_ELF` | 28.3-29.8, 30.5-30.8 tok/s | **35.6-35.9, 35.0-35.4 tok/s** | 534.2, 509.3 ms | 871.0, 881.5 ms | +| the pack's `config.json` key | 28.7-28.9, 28.7-29.0 tok/s | **33.9-34.2, 34.2-34.4 tok/s** | 541.2, 541.4 ms | 914.0, 908.3 ms | + +Two 16-row blocks against one 32-row block is 30-31 rows/s against 35-37. The 8-token greedy text is +identical with the family off and on, and `NPU_BATCH_CHECK=1` on the wide path reports MATCH on every chunk +it checks (32-, 4- and 1-row) in both tickets. Only 32 rows has been measured. + **A t2s container wants a sealed set for the same reason a pb16i one does.** The layout is not what makes a phase change expensive: the **hw_context** is. Measured on this box in-loop on 2026-11-02, a t2s decode family shipped as eight per-role `_t2s2_m16` ELFs paid **+2.0-2.3 ms on every launch, in every role** — more diff --git a/npu/forward/include/batch_width.h b/npu/forward/include/batch_width.h index 9901849a9..3365c7ff0 100644 --- a/npu/forward/include/batch_width.h +++ b/npu/forward/include/batch_width.h @@ -25,6 +25,8 @@ // pinned by a host test with no device, no ELF and no runtime. #pragma once +#include + namespace onebit { namespace npu { @@ -42,5 +44,47 @@ inline int batch_width_min(const int* widths, int n, int fallback) { return w > 0 ? w : (fallback > 0 ? fallback : 1); } +// --- a wide family beside the 16-row one (engine#1990) --------------------------------------------------------- +// +// A sealed set may carry the same roles twice: built for `narrow` rows (decode and short chunks) and for `wide` rows +// (prefill chunks). The three questions below are the ones that can be wrong in the dangerous direction, so they +// live here with the minimum above and are pinned by the same host test. + +// Is every role that has a narrow kernel also covered by a wide kernel of exactly `wide` rows? The chunk width is +// one number for the whole pass, so a single role without a wide kernel means the wide family cannot be used at +// all: its launches would carry more rows than that role's buffers hold. `wide` must exceed every narrow width. +inline bool wide_family_complete(const int* narrow_widths, const int* wide_widths, int n, int wide) { + if (!narrow_widths || !wide_widths || n <= 0 || wide <= 0) return false; + int roles = 0; + for (int i = 0; i < n; ++i) { + if (narrow_widths[i] <= 0) { + if (wide_widths[i] > 0) return false; // a wide kernel for a role the set does not run + continue; + } + if (wide_widths[i] != wide || wide <= narrow_widths[i]) return false; + ++roles; + } + return roles > 0; +} + +// The chunk width of a pass: the wide family's rows when one was adopted (0 = none), else the narrow minimum. +inline int chunk_rows(int narrow, int wide) { return wide > narrow ? wide : narrow; } + +// Does a launch of `rows` rows go to the wide family? Only when it does not fit the narrow one and does fit the +// wide one: a wide launch costs more than a narrow one whatever it carries, so a decode step never takes it, and a +// launch past `wide` rows is never routed here (the caller refuses it by name). +inline bool wide_takes(int rows, int narrow, int wide) { return wide > narrow && rows > narrow && rows <= wide; } + +// Which file is the wide sealed ELF, if any? The environment wins over the pack so a set can be tried, or switched +// off, without editing a pack: `env` "0" means no wide family even when the pack names one, any other non-empty +// value is the path as given. Otherwise the pack's name is used: as given when absolute, else inside `kernels_dir` +// (where the 16-row sealed ELF lives). Empty answer = no wide family. `env` may be null. +inline std::string wide_elf_path(const char* env, const std::string& pack_name, const std::string& kernels_dir) { + if (env && *env) return (env[0] == '0' && env[1] == 0) ? std::string() : std::string(env); + if (pack_name.empty()) return std::string(); + if (pack_name[0] == '/' || kernels_dir.empty()) return pack_name; + return kernels_dir + "/" + pack_name; +} + } // namespace npu } // namespace onebit diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index 595be1805..aac96f583 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -369,6 +369,9 @@ class Q4nxNpuForward { // hw_contexts. Config `q4nx_pb16i_sealed_elf` names it in the kernels dir; without // it the per-role `_pb16i.elf` files are used as before. std::string pb16i_sealed_; + std::string wide_sealed_; // q4nx__sealed_wide_elf: a wide prefill family (engine#1990) + int wide_sealed_rows_ = 32; // q4nx_sealed_wide_rows + std::string sealed_dir_; // the kernels dir the 16-row sealed ELF was found in // ---- the NPU target (engine#799 P4) ---- // Everything that talks to a device lives behind this: the ELF contexts (M=128 prefill and the M=16 // decode twin), the phase-selected launch, the packed-weight BO cache, the pb16i sidecar images, the diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 6a0a0a534..61480b018 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -24,6 +24,7 @@ // Numerics follow tools/q4nx_forward_ref.py so the two can be compared // element-wise. #include "q4nx_forward.h" +#include "batch_width.h" #include "q4nx_state_planes.h" // the plane-copy half of a snapshot, device-free and unit-tested #include "model/op_dispatch.h" // the optional-op fallback seam (engine#803 A3) #include @@ -525,6 +526,7 @@ bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, c continue; } elf = dir + "/" + pb16i_sealed_; + sealed_dir_ = dir; kname = inst; if (!file_exists(elf)) { err_ = plan_.loaded() ? ("plan " + plan_path_ + ": decode.sealed_elf \"" + @@ -595,6 +597,52 @@ bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, c fprintf(stderr, " [forward] decode family: %d role(s) at M=%d, %d on M=%d%s\n", made, Q4NX_DESIGN_M_DECODE, skipped, Q4NX_DESIGN_M_PREFILL, want_m16 ? "" : " (NPU_DECODE_M16=0: disabled)"); + // engine#1990: a wide prefill family for a sealed pb16i/t2s set: a second sealed ELF carrying the same + // instances built for more rows per launch (default 32), so a chunk of more than 16 rows runs one launch per + // role instead of two. The pack names it (config `q4nx__sealed_wide_elf`, rows in + // `q4nx_sealed_wide_rows`); NPU_WIDE_SEALED_ELF overrides the file ("0" = off) and NPU_WIDE_ROWS the rows. + // All roles or none -- max_batch_rows() is the chunk width, so one role without a wide kernel would be + // launched past its buffers. A named file that is missing or does not load is logged and the pass stays at + // 16 rows: a pack must never stop serving over an optional speed-up. + const std::string wide_path = + onebit::npu::wide_elf_path(getenv("NPU_WIDE_SEALED_ELF"), wide_sealed_, sealed_dir_); + if (const char* wide_elf = wide_path.c_str(); + !wide_path.empty() && pb16i_ && !pb16i_sealed_.empty() && made > 0) { + const char* wr = getenv("NPU_WIDE_ROWS"); + const int rows = wr && *wr ? atoi(wr) : wide_sealed_rows_; + int wide = 0; + std::string why; + if (rows <= Q4NX_DESIGN_M_DECODE || rows > Q4NX_DESIGN_M_PREFILL) why = "NPU_WIDE_ROWS must be 17..128"; + else if (!file_exists(wide_elf)) why = std::string("no such file: ") + wide_elf; + for (int i = 0; why.empty() && i < ndesigns_; i++) { + I8Ctx* c16 = xd_->ctx16(i); + if (!c16) continue; + const char* inst = pb16i_instance(designs16_[i].role); + I8Ctx* cw = inst ? xd_->make_ctx_wide(i) : nullptr; + if (!cw) { why = std::string("no wide context for role ") + role_tag(designs16_[i].role); break; } + cw->MD = rows; + cw->KD = c16->KD; + cw->ND = c16->ND; + cw->blocked_b = c16->blocked_b; + cw->pb16i = c16->pb16i; + cw->t2s = c16->t2s; + cw->pb16i_planes = c16->pb16i_planes; + if (!cw->init_elf(*dev, wide_elf, rows, c16->KD, c16->ND, 1, inst)) { + why = std::string("init_elf failed for instance ") + inst; + break; + } + wide++; + } + if (why.empty() && wide > 0 && !xd_->adopt_wide(rows)) why = "a role with a decode context has no wide one"; + if (why.empty() && wide > 0) { + fprintf(stderr, " [forward] wide family: %d role(s) at M=%d from %s (chunks of 17..%d rows)\n", + wide, rows, wide_elf, rows); + } else { + xd_->drop_wide(); + fprintf(stderr, " [forward] wide family NOT adopted (%s); chunks stay at %d rows\n", + why.empty() ? "no role has a decode context" : why.c_str(), Q4NX_DESIGN_M_DECODE); + } + } return true; } @@ -686,6 +734,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // pb16i ELF (which would read 4,608-B tiles out of 2,176-B images). pb16i_sealed_ = read_config_string( cfg_path, t2s_ ? "\"q4nx_t2s_sealed_elf\"" : "\"q4nx_pb16i_sealed_elf\"", ""); + // engine#1990: an optional second sealed ELF, the same instances built for more rows per launch. + wide_sealed_ = read_config_string( + cfg_path, t2s_ ? "\"q4nx_t2s_sealed_wide_elf\"" : "\"q4nx_pb16i_sealed_wide_elf\"", ""); + wide_sealed_rows_ = q4nx_config_int(cfg_path.c_str(), "q4nx_sealed_wide_rows", 32); // A folded container also declares the fold: `q4nx_hadamard` names the sign table and // `q4nx_hadamard_block` the Hadamard block. Without it the sidecar images would be // read in the stored (rotated) basis against un-rotated activations. diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 5864d12e2..9ca41c244 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -295,7 +295,10 @@ int XdnaTarget::max_rows() const { if (pb16i_) for (int i = 0; i < n; i++) if (ctx16_[i]) { widths16[i] = ctx16_[i]->MD; any16 = true; } - if (any16) return onebit::npu::batch_width_min(widths16, n, 1); + if (any16) { + // engine#1990: with a wide family adopted for every role, a chunk may be as wide as that family. + return onebit::npu::chunk_rows(onebit::npu::batch_width_min(widths16, n, 1), wide_rows_); + } int widths[Q4NX_MAX_DESIGNS] = {0}; bool any_ctx = false; for (int i = 0; i < n; i++) @@ -315,6 +318,8 @@ int XdnaTarget::max_rows() const { int XdnaTarget::rows_for(int role, int want) const { if (role < 0 || role >= Q4NX_MAX_DESIGNS) return want; if (m16_ && want >= 1 && want <= 16 && ctx16_[role]) return ctx16_[role]->MD; + if (pb16i_ && ctxw_[role] && onebit::npu::wide_takes(want, Q4NX_DESIGN_M_DECODE, wide_rows_)) + return ctxw_[role]->MD; // engine#1990 return ctx_[role] ? ctx_[role]->MD : want; } @@ -339,6 +344,32 @@ I8Ctx* XdnaTarget::ctx16(int role) const { if (role < 0 || role >= Q4NX_MAX_DESIGNS) return nullptr; return ctx16_[role].get(); } +I8Ctx* XdnaTarget::make_ctx_wide(int role) { + if (role < 0 || role >= Q4NX_MAX_DESIGNS) return nullptr; + ctxw_[role] = std::make_unique(); + return ctxw_[role].get(); +} +I8Ctx* XdnaTarget::ctx_wide(int role) const { + if (role < 0 || role >= Q4NX_MAX_DESIGNS) return nullptr; + return ctxw_[role].get(); +} +bool XdnaTarget::adopt_wide(int rows) { + int narrow[Q4NX_MAX_DESIGNS] = {0}, wide[Q4NX_MAX_DESIGNS] = {0}; + for (int i = 0; i < Q4NX_MAX_DESIGNS; i++) { + if (ctx16_[i]) narrow[i] = ctx16_[i]->MD; + if (ctxw_[i]) wide[i] = ctxw_[i]->MD; + } + if (!pb16i_ || !onebit::npu::wide_family_complete(narrow, wide, Q4NX_MAX_DESIGNS, rows)) { + drop_wide(); + return false; + } + wide_rows_ = rows; + return true; +} +void XdnaTarget::drop_wide() { + for (auto& c : ctxw_) c.reset(); + wide_rows_ = 0; +} void XdnaTarget::drop_ctx16(int role) { if (role >= 0 && role < Q4NX_MAX_DESIGNS) ctx16_[role].reset(); } @@ -363,7 +394,8 @@ void XdnaTarget::note_head(int M, double dpk, double dql, double dwt, double drb int XdnaTarget::design_index(I8Ctx* c) const { if (!c) return -1; for (int i = 0; i < ndesigns_ && i < Q4NX_MAX_DESIGNS; i++) - if (ctx_[i].get() == c || (ctx16_[i] && ctx16_[i].get() == c)) return i; + if (ctx_[i].get() == c || (ctx16_[i] && ctx16_[i].get() == c) || (ctxw_[i] && ctxw_[i].get() == c)) + return i; return -1; } @@ -1011,11 +1043,23 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: // Rows > 1 is prefill and keeps the M=128 kernel (one launch per chunk); a // single row here is a decode step and gets the M=16 twin, exactly like // npu_gemm(). - I8Ctx& kctx = *ctx_for(&ctx, M, ar.decode_twin); + I8Ctx* kctxp = ctx_for(&ctx, M, ar.decode_twin); + // engine#1990: more rows than the 16-row family holds go to the wide family when the set has one. The weight + // BO, its e_ref and its mode are the launch's (layer, key) slot, not the context's, so both families read the + // same image; only the row count of the activation and result buffers differs. + if (pb16i_ && ar.role >= 0 && ar.role < Q4NX_MAX_DESIGNS && ctxw_[ar.role] && + onebit::npu::wide_takes(M, Q4NX_DESIGN_M_DECODE, wide_rows_)) + kctxp = ctxw_[ar.role].get(); + I8Ctx& kctx = *kctxp; const bool prof = prof_; const auto tp = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; if (pb16i_ && kctx.pb16i) { out.assign((size_t)M * (size_t)N, 0.0f); // sized before every refusal path below + if (M > kctx.MD) { + // Never launch more rows than the kernel's buffers hold: that reads and writes past them (engine#1372). + err = "pb16i: " + std::to_string(M) + " rows on a " + std::to_string(kctx.MD) + "-row kernel"; + return; + } if (!sc) { err = "pb16i: no sidecar tag for this projection"; return; } PackedWeight* slot = nullptr; if (weight_cache_ && wkey >= 0 && ar.layer >= 0 && ar.layer < n_layers_ && dev_) { diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h index 3ba8ae2ce..687289d98 100644 --- a/npu/target/xdna/xdna_target.h +++ b/npu/target/xdna/xdna_target.h @@ -178,6 +178,17 @@ class XdnaTarget final : public NpuTarget { std::unique_ptr& head_ctx_ref(); void note_m16_made(bool any) { m16_ = m16_ || any; } void drop_ctx16(int role); + // engine#1990: a pb16i/t2s set's WIDE family -- the same roles built for more than 16 rows per launch, one + // sealed hw_context like the decode family. A launch of more than 16 rows takes it; decode and short chunks + // keep the 16-row family (a wide launch costs more than a 16-row one whatever it carries, so one row must + // never run on it). adopt_wide() takes the family only if EVERY role that has a decode context has a wide + // one of exactly `rows` rows (batch_width.h: wide_family_complete); otherwise it drops every wide context + // and answers false. max_rows() is the chunk width, and a chunk has to be launchable on all of its roles. + I8Ctx* make_ctx_wide(int role); + I8Ctx* ctx_wide(int role) const; + void drop_wide(); + bool adopt_wide(int rows); + int wide_rows() const { return wide_rows_; } // The context a launch of `rows` rows on `role` would use: the M=16 twin when one was adopted and // the row count fits, otherwise the M=128 one. The forward needs this where it launches a GEMM // itself (the LM head) instead of going through gemv(). @@ -219,6 +230,8 @@ class XdnaTarget final : public NpuTarget { std::unique_ptr ctx_[Q4NX_MAX_DESIGNS]; Q4nxDesignGeom designs16_[Q4NX_MAX_DESIGNS]; std::unique_ptr ctx16_[Q4NX_MAX_DESIGNS]; + std::unique_ptr ctxw_[Q4NX_MAX_DESIGNS]; // the wide pb16i/t2s family (engine#1990) + int wide_rows_ = 0; // 0 = no wide family std::unique_ptr ctx_head_; bool m16_ = false; bool m16_check_ = false; diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index e8f354a95..879e43ec6 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -112,8 +112,50 @@ static void batch_width_arms() { std::printf(" batch width: 7 arms (min not first, empty slots, fallback)\n"); } +/* engine#1990: a sealed set may carry a wide family beside the 16-row one. The rules that decide whether it is used, + * how wide a chunk is and which launches take it are pure, and wrong in the dangerous direction means more rows in a + * launch than a kernel's buffers hold. */ +static void wide_family_arms() { + using onebit::npu::chunk_rows; + using onebit::npu::wide_family_complete; + using onebit::npu::wide_takes; + const int n16[] = {16, 16, 0, 16}; + const int w32[] = {32, 32, 0, 32}; + check(wide_family_complete(n16, w32, 4, 32), "wide family: every decode role has a 32-row kernel -> complete"); + const int hole[] = {32, 0, 0, 32}; + check(!wide_family_complete(n16, hole, 4, 32), "wide family: one decode role without a wide kernel -> not complete"); + const int mixed[] = {32, 64, 0, 32}; + check(!wide_family_complete(n16, mixed, 4, 32), "wide family: a role built for another row count -> not complete"); + const int extra[] = {32, 32, 32, 32}; + check(!wide_family_complete(n16, extra, 4, 32), "wide family: a wide kernel for a role with no decode one -> refused"); + const int w16[] = {16, 16, 0, 16}; + check(!wide_family_complete(n16, w16, 4, 16), "wide family: 'wide' must be wider than the decode family"); + const int none[] = {0, 0, 0, 0}; + check(!wide_family_complete(none, none, 4, 32), "wide family: no role at all is not a family"); + check(!wide_family_complete(nullptr, w32, 4, 32) && !wide_family_complete(n16, w32, 0, 32) && + !wide_family_complete(n16, w32, 4, 0), "wide family: null, empty and zero-row inputs are not complete"); + check(chunk_rows(16, 0) == 16, "chunk rows: no wide family keeps the decode width"); + check(chunk_rows(16, 32) == 32, "chunk rows: an adopted wide family widens the chunk"); + check(chunk_rows(16, 8) == 16, "chunk rows: a 'wide' value below the decode width never narrows the chunk"); + check(!wide_takes(1, 16, 32) && !wide_takes(16, 16, 32), "wide takes: 1 and 16 rows stay on the decode family"); + check(wide_takes(17, 16, 32) && wide_takes(32, 16, 32), "wide takes: 17 and 32 rows go to the wide family"); + check(!wide_takes(33, 16, 32), "wide takes: 33 rows are not routed to a 32-row kernel"); + check(!wide_takes(17, 16, 0) && !wide_takes(17, 16, 16), "wide takes: no family, or one no wider than decode, takes nothing"); + using onebit::npu::wide_elf_path; + check(wide_elf_path(nullptr, "", "/k").empty() && wide_elf_path("", "", "/k").empty(), + "wide elf: neither the environment nor the pack names one -> none"); + check(wide_elf_path(nullptr, "W.elf", "/k") == "/k/W.elf", "wide elf: the pack's name is looked up in the kernels dir"); + check(wide_elf_path("", "W.elf", "/k") == "/k/W.elf", "wide elf: an empty variable is not an override"); + check(wide_elf_path(nullptr, "/abs/W.elf", "/k") == "/abs/W.elf", "wide elf: an absolute pack name is used as given"); + check(wide_elf_path("/e/X.elf", "W.elf", "/k") == "/e/X.elf", "wide elf: the environment wins over the pack"); + check(wide_elf_path("0", "W.elf", "/k").empty(), "wide elf: NPU_WIDE_SEALED_ELF=0 switches a pack's wide family off"); + check(wide_elf_path("0x.elf", "", "/k") == "0x.elf", "wide elf: only the exact value 0 means off"); + std::printf(" wide family: 21 arms (complete or not, chunk width, which launch takes it, which file)\n"); +} + int main(int argc, char** argv) { batch_width_arms(); + wide_family_arms(); quant_int4_arms(); for (int G : {8, 9, 16}) { for (int n : {G, 3 * G}) { // whole groups: a ragged tail is not a permutation From e731df91f793dccdba340268eac53de5e7209fe1 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:33:52 -0400 Subject: [PATCH 52/98] npu-pack: the shape rule for a 1.8 external-data initializer, pinned before the reader exists (engine#1503) (#2087) Those packs zero every dims field, so a byte length has to become a shape -- and a shape inferred wrongly imports cleanly and serves garbage (the failure this thread has recorded three times: #1874, #1888, #1993). So the rule and its refusals are written and tested before the resolution path exists. norm/bias -> [elements] only when the length is one of the model's declared sizes weight -> [k, elements/k] only when exactly ONE declared size divides it; zero or two divisors both refuse, naming the length and the candidates 8 arms, 2 accepts and 6 refusals, compiled and run standalone (g++ -std=c++17 -Inpu): 8/8, rc 0. Registered with add_dependencies(onebit ...) so ci_targets sees the target being built. Two errors of my own the arms caught, both kept in the comments: '.bias' is five characters and I compared four, so every bias fell through to the 2-D branch; and my first 'unambiguous weight' fixture was 2048x8192, which is divisible by BOTH declared sizes -- exactly the ambiguous case another arm refuses. Co-authored-by: pi-rlk --- CMakeLists.txt | 7 ++ npu/pack_initializer_shape.h | 107 ++++++++++++++++++++++++++ tests/pack_initializer_shape_test.cpp | 90 ++++++++++++++++++++++ 3 files changed, 204 insertions(+) create mode 100644 npu/pack_initializer_shape.h create mode 100644 tests/pack_initializer_shape_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index a90354cea..2e18dcbef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -431,6 +431,13 @@ add_test(NAME serve_watchdog COMMAND serve_watchdog_test) # engine#1781: the comparator behind NPU_BATCH_CHECK -- pure arithmetic over two float arrays, so it runs with no model, # no device and no runtime. It is a header, so the test includes it directly and needs no link libraries. add_executable(npu_sampling_test tests/npu_sampling_test.cpp) +# The shape rule for a 1.8 external-data initializer (npu/pack_initializer_shape.h, engine#1503's graph-initializer +# reader). Those packs zero every `dims`, so a byte length has to become a shape -- and a shape inferred wrongly imports +# cleanly and serves garbage, so the refusals are the arms worth pinning. Header-only, no parser, no pack, no device. +add_executable(pack_initializer_shape_test tests/pack_initializer_shape_test.cpp) +target_include_directories(pack_initializer_shape_test PRIVATE npu) +add_test(NAME pack_initializer_shape COMMAND pack_initializer_shape_test) +add_dependencies(onebit pack_initializer_shape_test) target_include_directories(npu_sampling_test PRIVATE app) target_link_libraries(npu_sampling_test PRIVATE nlohmann_json::nlohmann_json) # The NPU route's bounded load retry (app/npu_load_retry.h, engine #1010, docs/npu.md "When a load follows an evict"): diff --git a/npu/pack_initializer_shape.h b/npu/pack_initializer_shape.h new file mode 100644 index 000000000..13d7df2c5 --- /dev/null +++ b/npu/pack_initializer_shape.h @@ -0,0 +1,107 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// pack_initializer_shape.h -- the SHAPE rule for a 1.8-generation initializer read out of external data +// (engine#1503, the graph-initializer reader). Header-only and dependency-free, so the rule is testable with no +// ONNX parser, no pack and no device -- which is the point: the resolution path must not be written until this rule can +// be shown to refuse the cases it cannot decide. +// +// Why the rule is the hard part and not the reading: those packs zero every `dims` field on the initializers, so all +// that survives is `external_data { offset, length }` against the sibling `.data` file -- a LENGTH, not a shape. A shape +// guessed wrongly imports cleanly and serves garbage, which is the failure mode this thread has recorded three times +// (#1874's unsigned zero points, #1888's wrong quantisation rule, #1993's untied head). So the rule refuses whenever the +// length is consistent with more than one shape, and names what it saw. +#pragma once + +#include +#include +#include + +namespace onebit::npu { + +// A resolved initializer shape. ndim is 1 for the norms and biases, 2 for a weight; d1 is unused at ndim == 1. +struct InitShape { + bool ok = false; + int ndim = 0; + int64_t d0 = 0; + int64_t d1 = 0; + std::string why; // set on refusal: what was measured, in numbers +}; + +// Names that are 1-D by convention in this genre. The refusal this reader fixes names exactly these two: +// `model.layers.N.input_layernorm.weight` and `model.layers.N.post_attention_layernorm.weight`. +inline bool is_1d_by_name(const std::string& name) { + if (name.size() >= 5 && name.compare(name.size() - 5, 5, ".bias") == 0) return true; + return name.find("norm") != std::string::npos || name.find("Norm") != std::string::npos; +} + +// Resolve one initializer's shape from its byte length and the model's known sizes (hidden, intermediate, vocab, ...). +// +// * a length that is not a whole number of elements is refused (the file and the element type disagree); +// * a 1-D name (norm/bias) takes [elements] ONLY when that length is one of the known sizes -- a norm is exactly one +// of them, and a length that matches none is not a norm; +// * a weight takes [k, elements/k] ONLY when exactly one known size divides it. Zero divisors or two are both a +// refusal: two would mean two plausible weight shapes, which is the guess this rule exists to avoid. +inline InitShape resolve_init_shape(const std::string& name, int64_t length, int elem_bytes, + const std::vector& known) { + InitShape s; + if (length <= 0 || elem_bytes <= 0) { + s.why = "a non-positive length (" + std::to_string(length) + " B) or element size (" + + std::to_string(elem_bytes) + " B)"; + return s; + } + if (length % elem_bytes != 0) { + s.why = std::to_string(length) + " B is not a whole number of " + std::to_string(elem_bytes) + "-byte elements"; + return s; + } + const int64_t elems = length / elem_bytes; + if (is_1d_by_name(name)) { + for (const int64_t k : known) + if (k > 0 && k == elems) { + s.ok = true; + s.ndim = 1; + s.d0 = elems; + return s; + } + s.why = "a norm/bias of " + std::to_string(elems) + " elements matches none of the model's declared sizes"; + return s; + } + int64_t found = 0; + int ndiv = 0; + for (const int64_t k : known) + if (k > 0 && elems % k == 0) { + ++ndiv; + found = k; + } + if (ndiv == 0) { + s.why = "a weight of " + std::to_string(elems) + " elements has no declared dimension that divides it"; + return s; + } + if (ndiv > 1) { + std::string all; + for (const int64_t k : known) + if (k > 0 && elems % k == 0) all += (all.empty() ? "" : ", ") + std::to_string(k); + s.why = "a weight of " + std::to_string(elems) + " elements is consistent with several declared dimensions (" + + all + "), so the shape cannot be decided from the length"; + return s; + } + s.ok = true; + s.ndim = 2; + s.d0 = found; + s.d1 = elems / found; + return s; +} + +} // namespace onebit::npu diff --git a/tests/pack_initializer_shape_test.cpp b/tests/pack_initializer_shape_test.cpp new file mode 100644 index 000000000..1d2476c8a --- /dev/null +++ b/tests/pack_initializer_shape_test.cpp @@ -0,0 +1,90 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// pack_initializer_shape_test -- the shape rule for a 1.8 external-data initializer (engine#1503). No ONNX parser, no +// pack, no device: the rule is a function of (name, byte length, element size, the model's declared sizes), so every arm +// is a hand-computed value. The refusals matter more than the accepts -- a shape inferred wrongly imports cleanly and +// serves garbage, so the cases that must NOT resolve are the ones worth pinning. +#include "pack_initializer_shape.h" + +#include +#include + +using onebit::npu::InitShape; +using onebit::npu::resolve_init_shape; + +static int failures = 0; +static void check(bool ok, const char* what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what); + if (!ok) ++failures; +} + +int main() { + // The measured pack: Llama-3.2-1B 1.8 hybrid, hidden 2048, IM 8192, vocab 128256, bf16 elements. + const std::vector known{2048, 8192, 128256}; + const int bf16 = 2; + + // (i) the refusal this reader exists to fix: a layernorm weight is 2048 bf16 = 4096 B, one of the known sizes. + { + const InitShape s = resolve_init_shape("model.layers.0.input_layernorm.weight", 2048 * bf16, bf16, known); + check(s.ok && s.ndim == 1 && s.d0 == 2048, "(i) a layernorm weight resolves to [2048]"); + } + // (ii) a 2-D weight whose dimension is unambiguous: 2048 x 3 elements -- only 2048 divides it among the known + // sizes, so the shape is decided. (2048 x 8192 would NOT be unambiguous: both are declared sizes, which is arm + // (v) below, and my first version of this arm asserted the opposite and was wrong.) + { + const InitShape s = resolve_init_shape("model.layers.0.mlp.down_proj.weight", 2048 * 3LL * bf16, bf16, known); + check(s.ok && s.ndim == 2 && s.d0 == 2048 && s.d1 == 3, "(ii) a weight only one declared size divides resolves"); + } + // (iii) a length that is not a whole number of elements is refused, and says so with both numbers. + { + const InitShape s = resolve_init_shape("model.layers.0.mlp.down_proj.weight", 4097, bf16, known); + check(!s.ok && s.why.find("4097") != std::string::npos && s.why.find("2-byte") != std::string::npos, + "(iii) an odd byte length is refused, naming the length and the element size"); + } + // (iv) a norm whose length matches none of the declared sizes is refused: it is not a norm of this model. + { + const InitShape s = resolve_init_shape("model.layers.0.input_layernorm.weight", 3000 * bf16, bf16, known); + check(!s.ok && s.why.find("matches none") != std::string::npos, "(iv) a norm of an undeclared size is refused"); + } + // (v) the case the rule exists for: a weight consistent with TWO declared dimensions must refuse, not pick one. + // 2048*8192 elements is divisible by both 2048 and 8192, so a weight of that length is ambiguous. + { + const InitShape s = resolve_init_shape("model.layers.0.some.weight", 2048 * 8192LL * bf16, bf16, known); + check(!s.ok && s.why.find("several declared dimensions") != std::string::npos, + "(v) a weight consistent with two dimensions is REFUSED as ambiguous, naming them"); + } + // (vi) a weight with no dividing declared dimension is refused. + { + const InitShape s = resolve_init_shape("model.layers.0.some.weight", 1000 * bf16, bf16, known); + check(!s.ok && s.why.find("no declared dimension") != std::string::npos, + "(vi) a weight with no dividing dimension is refused"); + } + // (vii) a bias is 1-D by name and takes a known size; a bias of an undeclared size is refused like a norm. + { + const InitShape a = resolve_init_shape("model.layers.0.attn.q_proj.bias", 2048 * bf16, bf16, known); + const InitShape b = resolve_init_shape("model.layers.0.attn.q_proj.bias", 777 * bf16, bf16, known); + check(a.ok && a.ndim == 1 && a.d0 == 2048 && !b.ok, "(vii) a bias is 1-D like a norm, and refuses an undeclared size"); + } + // (viii) a non-positive length or element size is refused rather than divided by zero. + { + const InitShape a = resolve_init_shape("x.norm.weight", 0, bf16, known); + const InitShape b = resolve_init_shape("x.norm.weight", 4096, 0, known); + check(!a.ok && !b.ok, "(viii) a zero length or zero element size is refused, not divided by"); + } + + if (failures == 0) std::printf("pack initializer shape: 8 arms pass\n"); + return failures ? 1 : 0; +} From fa86a09329838e429f4b40264a30f139fba4d88c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:33:58 -0400 Subject: [PATCH 53/98] =?UTF-8?q?forward-serve:=20--prefix-pin=20pins=20th?= =?UTF-8?q?e=20end=20of=20the=20system=20turn=20on=20the=20first=20request?= =?UTF-8?q?=20=E2=80=94=20second=20question=209.1=E2=80=939.5=20s=20?= =?UTF-8?q?=E2=86=92=201.4=20s=20to=20first=20token=20(Refs=20#1785)=20(#2?= =?UTF-8?q?085)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * forward-serve: --prefix-pin pins the end of the system turn on the first request (Refs #1785) Comparing with the previous request found the shared part only on the second request, so the third was the first warm one, and it could run a few tokens into the question. A chat request with a leading system message now renders those messages alone and pins where the full prompt stops agreeing with that (PrefixCache::pin_at). Co-Authored-By: Claude Fable 5.2 * forward-serve: the early pin renders a probe user message after the system turn (Refs #1785) The Qwen3.5 template refuses to render a system message alone, so on the rendered-template path the early pin was silently not made and the second question stayed cold. npu::pin_probe_messages() adds one user message whose text no question starts with; the route logs when no early pin can be made. Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- app/forward_serve.cpp | 36 ++++++++++++++++++++++++++++++++++-- app/npu_chatml.h | 25 +++++++++++++++++++++++++ app/npu_prefix_cache.h | 18 +++++++++++++++--- docs/npu.md | 25 ++++++++++++++++++++++++- tests/npu_chatml_test.cpp | 28 ++++++++++++++++++++++++++++ tests/npu_sampling_test.cpp | 33 +++++++++++++++++++++++++++++++++ 6 files changed, 159 insertions(+), 6 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 5acc1c741..4a43ab585 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -216,6 +216,12 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // from `ids`. Do not log `prompt` as the request's text without handling this case first. std::vector prompt_ids; bool prompt_is_ids = false; + // --prefix-pin: the leading system messages and a probe user message (npu::pin_probe_messages), rendered. Whatever + // of that the full prompt starts with is the part every question under this system prompt shares, so it can be pinned on + // the FIRST request (PrefixCache::pin_at) -- comparing with the previous request only finds it on the second, + // and can run a few tokens into the question when two questions open with the same words. Empty = the + // request has no system message, or the route is not pinning. + std::string system_prompt; try { if (chat) { if (!e.chat_ok) throw std::runtime_error(e.chat_refusal); @@ -232,8 +238,25 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib extra["enable_thinking"] = npu::thinking_from_body(body); prompt = npu::render_chat_template(e.tmpl, ordered.at("messages"), /*add_generation_prompt=*/true, extra, e.bos_text, e.eos_text); + if (e.prefix_pin) { + // A template that cannot render the probe is no reason to refuse the request: it only means + // there is no early pin, and the log says so. + try { + const nlohmann::ordered_json probe = npu::pin_probe_messages(ordered.at("messages")); + if (!probe.empty()) + system_prompt = npu::render_chat_template(e.tmpl, probe, /*add_generation_prompt=*/false, extra, + e.bos_text, e.eos_text); + } catch (const std::exception& ex) { + system_prompt.clear(); + std::fprintf(stderr, "[prefix] no early pin: the template did not render the system turn (%s)\n", ex.what()); + } + } } else { prompt = npu::chatml(body.at("messages"), npu::thinking_from_body(body)); + if (e.prefix_pin) { + const json probe = npu::pin_probe_messages(body.at("messages")); + if (!probe.empty()) system_prompt = npu::chatml(probe, npu::thinking_from_body(body)); + } } } else { prompt_is_ids = npu::prompt_ids_from_body(body, e.tok->size(), prompt_ids); @@ -291,7 +314,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // Returns the finish_reason the request earned: "stop" for an end-of-sequence token, // a stop sequence or a grammar that allows nothing more, "length" when the token // budget ran out. - auto run = [&e, &req, prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req, bridge]( + auto run = [&e, &req, prompt, system_prompt, prompt_ids, prompt_is_ids, max_tokens, chat, &opts, gram, &stats, dump_req, bridge]( const std::function&)>& on_text) { std::string finish = "stop"; // The tokenizer is const and keeps no per-call state, so it runs before the lock. A caller @@ -358,7 +381,16 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib int pin_cut = -1; size_t pin_tokens = 0; if (e.prefix_pin && e.prefix && e.prefix->enabled()) { - pin_tokens = e.prefix->pin_point(e.prev_ids, ids); + // The end of the system turn first: it is known on the first request and it is the right place. Only + // a request with no system message falls back to what it shares with the previous request. + if (!system_prompt.empty()) { + std::vector lead = e.tok->encode(system_prompt); + if (chat && e.add_bos && e.bos_id >= 0 && (lead.empty() || lead.front() != e.bos_id)) + lead.insert(lead.begin(), e.bos_id); + pin_tokens = e.prefix->pin_at(ids, npu::shared_prefix_len(lead, ids)); + } else { + pin_tokens = e.prefix->pin_point(e.prev_ids, ids); + } if (pin_tokens > 0 && int(pin_tokens) - 1 >= resume) pin_cut = int(pin_tokens) - 1; } { diff --git a/app/npu_chatml.h b/app/npu_chatml.h index ed8094357..7ec367fa1 100644 --- a/app/npu_chatml.h +++ b/app/npu_chatml.h @@ -61,6 +61,31 @@ inline std::string chatml(const nlohmann::json& messages, bool thinking = true) return p; } +// The messages a chat route renders to find where a request's system turn ends (--prefix-pin, engine#1785): the +// request's leading system messages, then one user message whose text no question starts with. The rendering of +// these and the rendering of the real request agree up to the first token of the user's text and differ there, so +// their common prefix is the part every question under this system prompt shares. +// +// The user message is there because a template may refuse to render without one. Measured: the Qwen3.5 template +// rendered alone on its system message throws, and the early pin was silently not made on that container. +// +// Empty array when the request does not open with a system message: there is nothing to pin early. +template +Json pin_probe_messages(const Json& messages) { + Json lead = Json::array(); + if (!messages.is_array()) return lead; + for (const auto& m : messages) { + if (!m.is_object() || !m.contains("role") || !m["role"].is_string() || m["role"].template get() != "system") break; + lead.push_back(m); + } + if (lead.empty()) return lead; + Json probe = Json::object(); + probe["role"] = "user"; + probe["content"] = "\xC2\xA4\xC2\xA4 pin probe \xC2\xA4\xC2\xA4"; + lead.push_back(probe); + return lead; +} + // Whether the container's tokenizer can represent the prompt above. `chatml()` is rendered for // every container (#832), and a tokenizer without the anchors spells `<|im_start|>` as ordinary // pieces, so the model is prompted with text its own template never produces (#963: on diff --git a/app/npu_prefix_cache.h b/app/npu_prefix_cache.h index b66562b1c..0412a676d 100644 --- a/app/npu_prefix_cache.h +++ b/app/npu_prefix_cache.h @@ -37,6 +37,7 @@ // What they share is the part before the first difference, so that part gets a slot of its own: // pin_point(prev, ids) how many leading tokens this request shares with the previous one, when that // is worth a slot and no slot covers it already +// pin_at(ids, n) the same decision for a point the caller knows already (the end of the system turn) // pin(prefix, bytes) keep the state at that point. One pinned slot at a time; the budget's eviction // and a longer prompt's replacement both leave it alone, a new pin() replaces it. // The rule reads token ids only, so it needs no knowledge of the template or of what a system prompt is. @@ -186,14 +187,25 @@ class PrefixCache { size_t pin_point(const std::vector& prev, const std::vector& ids) const { if (!enabled()) return 0; const size_t shared = shared_prefix_len(prev, ids); - if (shared == prev.size() || shared == ids.size()) return 0; + if (shared == prev.size()) return 0; + return pin_at(ids, shared); + } + + // The same decision for a point the caller already knows: `n` leading tokens of `ids` that every request of + // this kind will share -- the end of the system turn, which a chat route can find on the FIRST request by + // rendering the system messages alone, instead of waiting for a second request to compare with. Returns `n` + // when the state there is worth pinning, else 0: 0 when `n` is the whole prompt or more (nothing follows it, + // so insert() keeps it), when it adds fewer than prefix-min-gain tokens over the slot that already covers + // this request (the pin itself, on every request after the one that made it), and when the cache is off. + size_t pin_at(const std::vector& ids, size_t n) const { + if (!enabled() || n == 0 || n >= ids.size()) return 0; size_t covered = 0; // the longest live slot that is a prefix of this request for (const PrefixSlot& s : slots_) { if (!s.live || s.ids.size() > ids.size() || s.ids.size() <= covered) continue; if (std::equal(s.ids.begin(), s.ids.end(), ids.begin())) covered = s.ids.size(); } - if (shared <= covered || shared - covered < size_t(opt_.min_gain)) return 0; - return shared; + if (n <= covered || n - covered < size_t(opt_.min_gain)) return 0; + return n; } // Keep `prefix` (the first pin_point() tokens of a request) as the pinned slot. Returns the slot the diff --git a/docs/npu.md b/docs/npu.md index a297da858..38fb948d2 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1784,10 +1784,12 @@ The slot table (`app/npu_prefix_cache.h`) has a slot kind for the part such requ | call | what it decides | |---|---| | `pin_point(prev, ids)` | how many leading tokens this request shares with the previous one. 0 when one prompt is a prefix of the other (a repeat or a continued conversation, which the ordinary slots already keep), when the shared part adds fewer than `--prefix-min-gain` tokens over a slot that already covers the request, or when the cache is off | +| `pin_at(ids, n)` | the same decision for a point the caller already knows: `n` leading tokens every such request shares. 0 when `n` is the whole prompt, when a slot already covers it, or when it is under `--prefix-min-gain` | | `pin(prefix, bytes)` | keep the state at that point. One pinned slot at a time: a new pin replaces it; eviction for room and a longer prompt's replacement both leave it alone | | `lookup(ids)` | unchanged, and reports `pinned` when the hit is the pinned slot | -It reads token ids only, so it does not depend on the template or on which message is the system prompt. +The table itself reads token ids only, so it does not depend on the template. Where the pin goes is the caller's +choice between two rules, below. **`--prefix-pin`** (on `1bit serve --device npu` and on `forward-serve`; off by default) turns it on. The forward keeps the previous request's prompt ids, and when `pin_point()` is non-zero it stops the prefill one @@ -1812,6 +1814,27 @@ The default-cache arm's times are not in the table: the ticket ended during its three cache decisions (from the server log) are known. A request's time here is the whole HTTP call, prefill plus 16 generated tokens, not TTFT. +**Pinned on the first request (engine#1785).** That run needed two requests to find the shared part, so the +third was the first warm one, and when two questions opened with the same words the pin ran a few tokens into the +question, which a question opening differently could not use. A chat request with a leading system message now +gives the point directly: the route renders the leading system messages followed by a probe user message (the +container's template, or ChatML) and pins where the full prompt stops agreeing with that, on the first request +(`pin_at`). The probe user message is there because a template may refuse to render a system turn alone (the +Qwen3.5 template does); if a template cannot render even that, the request has no early pin and the log says so +(`[prefix] no early pin: …`). Only a request with no +system message still uses the comparison with the previous request (`pin_point`). On FastFlowLM's +`Qwen3.5-2B-NPU2` container under `--flash`, a 1,016-token prompt, time to first token (two server loads): + +| request | before | after | +|---|---|---| +| first question | 8.8 s | 10.6 / 11.1 s (busier box; makes the pin: 1,003 tokens) | +| **second question** | 9.1–9.5 s (made the pin: 1,009 tokens) | **1.39 / 1.38 s** (1,002 of 1,017 cached) | +| third question | 1.06 s | 1.40 / 1.35 s | + +The "before" column was taken with two other changes that make decode about a third faster on this model, the +"after" column without them and at twice the box load, so compare the rows' shape (which request is the first warm +one), not the third row's two figures. + **`--flash`** (`1bit serve --device npu --flash`) is the single-turn profile as one word: `--ctx-size 1024` plus `--prefix-pin`. It refuses what contradicts it (`--ctx-size` other than 1024, `--prefix-slots 0`, a device other than `npu`) instead of quietly winning or losing. diff --git a/tests/npu_chatml_test.cpp b/tests/npu_chatml_test.cpp index 16b1526f3..d43909570 100644 --- a/tests/npu_chatml_test.cpp +++ b/tests/npu_chatml_test.cpp @@ -126,6 +126,34 @@ int main(int argc, char** argv) { check(!without.encode(chatml(msgs, true)).empty(), "the refusal is not about whether the prompt tokenizes at all -- it does, into the wrong ids"); + // pin_probe_messages (engine#1785): what the route renders to find the end of the system turn + { + using onebit::npu::pin_probe_messages; + const nlohmann::json req = nlohmann::json::parse( + R"([{"role":"system","content":"Be brief."},{"role":"user","content":"What is item 3?"}])"); + const nlohmann::json probe = pin_probe_messages(req); + check(probe.size() == 2 && probe[0] == req[0] && probe[1]["role"] == "user", + "the probe is the system message and one user message (a template may refuse to render without one)"); + check(probe[1]["content"] != req[1]["content"], "whose text is not the question"); + // the two renderings agree through the user turn's opening and part at the first character of its text + const std::string full = chatml(req, true), lead = chatml(probe, true); + size_t same = 0; + while (same < full.size() && same < lead.size() && full[same] == lead[same]) ++same; + const std::string expect = "<|im_start|>system\nBe brief.<|im_end|>\n<|im_start|>user\n"; + check(full.compare(0, same, expect) == 0 && same == expect.size(), + "the common prefix of the two ChatML renderings is the system turn plus the user turn's opening"); + const nlohmann::json two = nlohmann::json::parse( + R"([{"role":"system","content":"A"},{"role":"system","content":"B"},{"role":"user","content":"q"},{"role":"system","content":"late"}])"); + check(pin_probe_messages(two).size() == 3, "every LEADING system message is kept, a later one is not"); + check(pin_probe_messages(nlohmann::json::parse(R"([{"role":"user","content":"q"}])")).empty(), + "a request that does not open with a system message has no probe"); + check(pin_probe_messages(nlohmann::json::parse(R"([{"role":7},{"content":"x"}])")).empty() && + pin_probe_messages(nlohmann::json::parse("{}")).empty(), + "malformed messages give no probe instead of throwing"); + const nlohmann::ordered_json oreq = nlohmann::ordered_json::parse(R"([{"role":"system","content":"S"},{"role":"user","content":"q"}])"); + check(pin_probe_messages(oreq).size() == 2, "the same for the ordered JSON the template path uses"); + } + if (failures == 0) std::printf("npu_chatml: ok\n"); return failures == 0 ? 0 : 1; } diff --git a/tests/npu_sampling_test.cpp b/tests/npu_sampling_test.cpp index 45932a92b..e791743f6 100644 --- a/tests/npu_sampling_test.cpp +++ b/tests/npu_sampling_test.cpp @@ -529,6 +529,39 @@ void test_prefix_pin() { check(PrefixCache(off).pin_point(q1, q2) == 0, "an off cache pins nothing"); check(PrefixCache(off).pin(q1, 10) == -1, "and refuses a pin"); } + { // pin_at: the end of the system turn is known on the FIRST request, so the second one is already warm + PrefixCache c(o); + check(c.pin_at(q1, 900) == 900, "the first request under a 900-token system prompt pins its 900 tokens"); + check(c.pin_point({}, q1) == 0, "where comparing with a previous request has nothing to offer yet"); + std::string why; + const int pin = c.pin(std::vector(q1.begin(), q1.begin() + 900), 2000, &why); + c.insert(q1, 1000); + const auto d = c.lookup(q2); + check(pin >= 0 && d.slot == pin && d.cached_tokens == 900 && d.pinned, + "so the SECOND question resumes from the pin: 900 of its 930 tokens"); + check(c.pin_at(q2, 900) == 0 && c.pin_at(q3, 900) == 0, "and later requests do not pin the same point again"); + // what is not worth a pin, or is not one + check(c.pin_at(q1, 0) == 0, "no system turn, no pin"); + check(c.pin_at(q1, q1.size()) == 0 && c.pin_at(q1, q1.size() + 5) == 0, "a point at or past the end of the prompt is insert()'s"); + PrefixCache fresh(o); + check(fresh.pin_at(q1, 127) == 0 && fresh.pin_at(q1, 128) == 128, "a system turn shorter than prefix-min-gain is not pinned"); + PrefixOptions off = o; + off.slots = 0; + check(PrefixCache(off).pin_at(q1, 900) == 0, "an off cache pins nothing"); + // a pin that ran into the question (two questions opening alike) does not cover a question that opens + // differently, and the system turn's own end then replaces it + PrefixCache over(o); + std::vector a1 = prompt(900, 5000, 40), a2 = prompt(900, 5000, 40), b1 = prompt(900, 8000, 40); + a2[905] = 4242; // same first five question tokens, then different + const size_t ran_over = over.pin_point(a1, a2); + check(ran_over == 905, "comparing two alike-opening questions pins five tokens into the question"); + over.pin(std::vector(a2.begin(), a2.begin() + long(ran_over)), 2000); + check(over.lookup(b1).cached_tokens == 0, "which a differently opening question cannot use"); + check(over.pin_at(b1, 900) == 900, "the system turn's end is still worth pinning for it"); + const int repin = over.pin(std::vector(b1.begin(), b1.begin() + 900), 2000); + check(repin >= 0 && over.lookup(b1).cached_tokens == 900 && over.lookup(a1).cached_tokens == 900, + "and once pinned there, every question under that system prompt resumes from it"); + } { // the pin outlives what evicts and replaces everything else PrefixOptions b = o; b.slots = 4; // one more than the three snapshots the budget holds, so a slot is always free From 8079720050db3284d1a8ddc7979b928553904ee2 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:34:06 -0400 Subject: [PATCH 54/98] ci: the runner health check reads the restart policy, not just whether the runner is alive (engine#1504) (#2082) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state checks answer "is it accepting jobs now". They cannot see a unit whose next OOM kill ends it for good, because that is a configuration property and the state is healthy right up to the kill. Measured on strixhalo 2026-11-11, this script before the change: PASS …strixhalo-bench.service active/running, last result success, 0 restart(s) systemctl show -p Restart -p OOMPolicy -p MemoryMax …strixhalo-bench.service Restart=no OOMPolicy=stop MemoryMax=infinity So it passed the unit carrying exactly the defect the issue describes. docs/ci-runners.md already specifies the policy and the reason for each line: Restart=always because runsvc.sh exits 0 after a kill (on-failure never fires), OOMPolicy=continue because "stop" lets an OOM kill of a JOB stop the RUNNER, and a MemoryMax ceiling so the cgroup hits its own limit before the box's OOM killer chooses. policy_verdict() checks those three per unit and prints its own line. On this box it now reports three units that need the installer: strixhalo-bench (Restart=no, OOMPolicy=stop, no ceiling), strixhalo-org (same), and strixhalo-engine (policy right, ceiling missing). Nothing is restarted or reconfigured here — the installer is a sudo action and whether to restart a runner is a decision, not a check. Self-test grows from 9 to 15 cases: the live bench unit's exact triple, on-failure, stop, an infinite ceiling, and an unreadable policy. Refs #1504 Co-authored-by: agent --- scripts/ci-runner-health.sh | 47 +++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/scripts/ci-runner-health.sh b/scripts/ci-runner-health.sh index 86a46c96b..20f47c874 100755 --- a/scripts/ci-runner-health.sh +++ b/scripts/ci-runner-health.sh @@ -34,6 +34,32 @@ # (no such units on this host, so a silent pass would be the defect). set -uo pipefail +# -> one verdict line. The state checks above answer "is it alive", which is what the +# issue named; this answers "will it come back", which is what actually kept both runners dead for 3h45m (engine#1504). +# A unit with Restart=no/OOMPolicy=stop PASSes every state check right up to the OOM kill that ends it for good, so the +# configuration has to be read, not inferred from ActiveState. Expected values are docs/ci-runners.md's, and the reason +# for each is one line there (runsvc.sh exits 0 after a kill, so on-failure never fires; OOMPolicy=stop lets an OOM kill +# of a JOB stop the RUNNER). +policy_verdict() { + _rst=${1:-} _oom=${2:-} _mmax=${3:-} + if [ -z "$_rst" ]; then + printf 'FAIL no restart policy could be read for this runner, so whether it would come back after an OOM kill is unknown' + return 0 + fi + _bad="" + # always, not on-failure: runsvc.sh reports success when it is killed, so on-failure never fires. + [ "$_rst" = always ] || _bad="$_bad Restart=$_rst(want always);" + [ "${_oom:-}" = continue ] || _bad="$_bad OOMPolicy=${_oom:-?}(want continue);" + # an infinite ceiling means the cgroup never hits its own limit, so the box's OOM killer chooses - and the bench runner + # is the unit whose jobs are memory-heavy by definition. + [ -n "${_mmax:-}" ] && [ "$_mmax" != infinity ] || _bad="$_bad MemoryMax=${_mmax:-?}(want a ceiling);" + if [ -n "$_bad" ]; then + printf 'FAIL the restart policy cannot restart this runner after an OOM kill:%s one OOM kill and the unit stops for good, while jobs read as pending' "$_bad" + return 0 + fi + printf 'PASS Restart=always, OOMPolicy=continue, MemoryMax=%s - an OOM kill costs a job, not the runner' "$_mmax" +} + # -> one verdict line runner_verdict() { _st=${1:-} _sub=${2:-} _res=${3:-} _nr=${4:-} @@ -75,7 +101,15 @@ self_test() { else printf 'FAIL the restart window should be named in the message\n'; rc=1; fi # a restart policy that works shows NRestarts>0 and result success - which is a PASS with the count visible chk PASS "$(runner_verdict active running success 3)" - [ "$rc" = 0 ] && echo "ci-runner-health self-test: ok (9 cases)" || echo "ci-runner-health self-test: FAIL" + # engine#1504: the state checks cannot see a unit that is one OOM kill away from being gone for good, which is why the + # bench runner PASSed this script while carrying Restart=no/OOMPolicy=stop. + chk PASS "$(policy_verdict always continue 51539607552)" + chk FAIL "$(policy_verdict no stop infinity)" # the live strixhalo-bench unit on 2026-11-11 + chk FAIL "$(policy_verdict on-failure continue 51539607552)" # on-failure never fires: runsvc.sh exits 0 + chk FAIL "$(policy_verdict always stop 51539607552)" # an OOM kill of a JOB stops the RUNNER + chk FAIL "$(policy_verdict always continue infinity)" # no cgroup ceiling: the box's OOM killer chooses + chk FAIL "$(policy_verdict '' '' '')" # nothing readable is not a pass + [ "$rc" = 0 ] && echo "ci-runner-health self-test: ok (15 cases)" || echo "ci-runner-health self-test: FAIL" return $rc } @@ -93,15 +127,24 @@ fi rc=0 for u in "${units[@]}"; do - line=$("$SC" show "$u" -p ActiveState -p SubState -p Result -p NRestarts 2>/dev/null) + line=$("$SC" show "$u" -p ActiveState -p SubState -p Result -p NRestarts -p Restart -p OOMPolicy -p MemoryMax 2>/dev/null) st=$(printf '%s\n' "$line" | sed -n 's/^ActiveState=//p') sub=$(printf '%s\n' "$line" | sed -n 's/^SubState=//p') res=$(printf '%s\n' "$line" | sed -n 's/^Result=//p') nr=$(printf '%s\n' "$line" | sed -n 's/^NRestarts=//p') + rst=$(printf '%s\n' "$line" | sed -n 's/^Restart=//p') + oom=$(printf '%s\n' "$line" | sed -n 's/^OOMPolicy=//p') + mmax=$(printf '%s\n' "$line" | sed -n 's/^MemoryMax=//p') v=$(runner_verdict "$st" "$sub" "$res" "$nr") case "$v" in PASS*) printf 'PASS %-52s %s\n' "$u" "${v#PASS }" ;; *) printf 'FAIL %-52s %s\n' "$u" "${v#FAIL }"; rc=1 ;; esac + # engine#1504: alive now is not the same as after the next OOM kill, so the policy is its own line. + p=$(policy_verdict "$rst" "$oom" "$mmax") + case "$p" in + PASS*) printf 'PASS %-52s %s\n' "" "${p#PASS }" ;; + *) printf 'FAIL %-52s %s\n' "" "${p#FAIL }"; rc=1 ;; + esac done exit "$rc" From 0c64350fd822c90f883f3cec5b543667dbe6d433 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:34:13 -0400 Subject: [PATCH 55/98] serve: let the operator choose the load mode, so #1668's prefill lever is reachable (#2081) engine#1668 measured `-lm none` at +17.5 % prefill, -1.6 % decode, ~604 MiB more GTT, and documented the mechanism and the "do not quote a prefill delta across it" rule (docs/bench.md). What it could not do was be *selected* where prefill costs time-to-first-token: `serve` passed `--load-mode mmap` unconditionally in the child's argv, and an explicit flag beats the child's own LLAMA_ARG_LOAD_MODE, so the environment could never win. The flag is now passed only when the operator has not chosen a mode: if (std::getenv("LLAMA_ARG_LOAD_MODE") == nullptr) argv.insert(... {"--load-mode", "mmap"}); so the default is byte-identical (mmap when nothing is set) and LLAMA_ARG_LOAD_MODE=none reaches the child through the mechanism llama.cpp already has (common/arg.cpp:2581 accepts none/mmap/mlock/mmap+mlock/dio and .set_env'd it), rather than through a second variable invented here. The extra GTT is the price of asking: three more ~306 MiB tensors become device-resident, which is why this stays an override rather than becoming the default on a box where the pool budget is what runs out first (engine#1711). docs/bench.md gains the "how it is selected in a server" paragraph under the measurement, so the lever and its cost are read together. Not built here: this box is at 167 G, under the 175 G floor this lane builds under. Checked statically instead -- is already included (serve.cpp:113) and the file already uses getenv 38 times, the value is validated by the child's own parser, and the default path is unchanged by construction (the branch is skipped when the variable is unset). fixes #1668 Co-authored-by: bong-water-water-bong --- app/serve.cpp | 11 ++++++++++- docs/bench.md | 7 +++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app/serve.cpp b/app/serve.cpp index 69f3ec145..65cc89962 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -1151,7 +1151,16 @@ Launch launch_for(const Options& o, const std::string& device, int child_port) { // the experts stay file-backed: without these, llama.cpp copies every one into RAM // (a pinned host buffer, or a CPU repack), which is what streaming avoids // (per_layer_token_embd: Qwen3.8-Flash-Next's 26.8 GiB table, only ever row-gathered) - argv.insert(argv.end(), {"-ot", "exps=CPU,per_layer_token_embd=CPU", "--no-host", "--no-repack", "--load-mode", "mmap"}); + argv.insert(argv.end(), {"-ot", "exps=CPU,per_layer_token_embd=CPU", "--no-host", "--no-repack"}); + // engine#1668: `-lm none` is +17.5 % prefill at -1.6 % decode for ~604 MiB more GTT (docs/bench.md), + // and prefill is what a served model pays for TTFT -- but this site hard-coded `--load-mode mmap`, + // and an explicit flag beats the child's own LLAMA_ARG_LOAD_MODE, so the measured lever could not be + // selected in a server at all. Default unchanged (mmap when the operator has not asked for something + // else); setting LLAMA_ARG_LOAD_MODE=none in the environment now reaches the child, with the GTT cost + // that implies documented beside the measurement. + if (std::getenv("LLAMA_ARG_LOAD_MODE") == nullptr) { + argv.insert(argv.end(), {"--load-mode", "mmap"}); + } #endif } if (device == "hrx") { diff --git a/docs/bench.md b/docs/bench.md index 9233a739a..9dff89c8a 100644 --- a/docs/bench.md +++ b/docs/bench.md @@ -69,6 +69,13 @@ bytes rather than slack: three more ~306 MiB tensors become device-resident (3 quote a prefill delta measured across it. `-lm none` is not a lever to ship on the strength of one measurement — it costs GTT on a box where GTT is the scarce resource, and the honest statement is the mechanism above, not the percentage. +**How it is selected in a server (engine#1668).** The measurement above was taken with `llama-bench`/`llama-server` directly, and +`1bit serve` hard-coded `--load-mode mmap` for its child, so the lever could not be reached where prefill actually costs +time-to-first-token. It now passes that flag **only when the operator has not chosen a mode**: set `LLAMA_ARG_LOAD_MODE=none` +(the child's own variable, so there is one mechanism rather than two) and the flag is omitted, letting the environment value +ride. The default is unchanged, and the 604 MiB of GTT the `-lm none` row records is the price of asking — a per-model cost on +a box where the pool budget is the scarce resource (engine#1711), which is why this is an override and not a new default. + ## What is measured - **Prompt:** one long prompt (default: the first 8,000 characters of `docs/*.md`, about 1,800 From 0dbb593a1970b6b73e85240c7352b44f9d462629 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:34:18 -0400 Subject: [PATCH 56/98] tests/census_watch: derive "reviewed classes are documented" from significant.json (#2078) The arm asserted the rule for a hand-written three-name dict, so the file it is about could grow to 37 classes with no row in docs/arch-gaps.md and the test stayed green. It now loops over the registry's reviewed classes, with a floor so an empty load cannot pass vacuously, and the failing line names the classes that are missing a row. That change found one entry already in breach: DeepseekV4FlashSuperExpertForCausalLM (added by engine#1922) was the single undocumented class out of 37, so it gains the row the contract asks for, with the reason it is not an alias of deepseek_v41 (AW-SECD-compressed weights, which nothing at the pin decodes). Both arms were checked by mutation: without the doc row the check fails naming the class, and with an empty registry it fails on the floor rather than passing. Co-authored-by: bong-water-water-bong --- docs/arch-gaps.md | 12 ++++++++++++ tests/census_watch_test.py | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/arch-gaps.md b/docs/arch-gaps.md index 8f7d5fdb5..aa1d2cb0a 100644 --- a/docs/arch-gaps.md +++ b/docs/arch-gaps.md @@ -1075,3 +1075,15 @@ which are retried on every run; and the `!! FLM` lines. Those keep the 2026-11-0 and `smolvla` as above, and `!! NPU FLM` is engine #588. This run's issue window also includes the `gpt-oss` (`GPT-OSS-20B`, `GPT-OSS-Safeguard-20b`) and `lfm2` (`LFM2-1.2B`, `LFM2-2.6B`) Q4NX packs. Their mapping is HRX-only too, so they belong to #588, not to a registry edit. + +## 2026-11-10: `DeepseekV4FlashSuperExpertForCausalLM` — reviewed, and recorded here + +Found by the check this page's contract implies and `tests/census_watch_test.py` now enforces: every class in +`registry/significant.json` has to be cited here, and this one was added there (engine#1922) without a row, so it was +the single undocumented entry out of 37. + +* `DeepseekV4FlashSuperExpertForCausalLM` (`model_type` `deepseek_v4_flash_superexpert`) — DeepSeek V4.1 Flash + SuperExpert, **published as remote code and with AW-SECD-compressed weights** (8.5 GB, claimed 0.962 retention); + MLA-style geometry (`kv_lora_rank` 512, `q_lora_rank` 1536, `qk_rope_head_dim` 64, `v_head_dim` 128, 32 heads / + 16 KV, IM 4096). **Not an alias of `deepseek_v41`**: nothing at the pin decodes AW-SECD weights, so mapping it to + that architecture would name a backend that cannot read the file. Reported, not alerted (`"watch": "report"`). diff --git a/tests/census_watch_test.py b/tests/census_watch_test.py index 04f04b906..69270cde7 100644 --- a/tests/census_watch_test.py +++ b/tests/census_watch_test.py @@ -311,6 +311,16 @@ def fetch(url): check("reviewed non-alias classes are documented", all(cls in gap_docs for cls in reviewed_gaps) and "not an alias" in gap_docs) +# engine#2014 review: the list above is hand-written and names three classes, so it keeps covering those three +# while the file it is about grows (37 entries today) -- a fourth, fifth and thirty-seventh class can be added +# with no row in docs/arch-gaps.md and this file stays green. significant.json's own `about` says its classes +# are documented there, so derive the list from the file: a missing row then fails here instead of resting on +# three names that nobody will remember to extend. The floor stops the loop passing vacuously if the file were +# ever loaded empty -- a check that iterates nothing must not report success. +undocumented = sorted(cls for cls in sig if cls not in gap_docs) +check(f"every one of the {len(sig)} reviewed classes is cited in docs/arch-gaps.md" + + (f" -- missing: {', '.join(undocumented)}" if undocumented else ""), + len(sig) >= 30 and not undocumented) # 14. A reviewed class marked `"watch": "report"` is listed but does not alert (engine#381). check("GrugMoeForCausalLM is reviewed as report-only", From 59d342d21ba21983df46c71660ea98fe2663ec7b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:34:26 -0400 Subject: [PATCH 57/98] ci: a comment cancels the PR's pr-agent review, because both enter one group (#2071) `pr-agent` triggers on `pull_request` and on `issue_comment`, and both land in the same concurrency group (`pr-agent-`) with `cancel-in-progress: true`. So every comment on a PR - a review verdict, an `/ask`, a status note - cancels the review of the head it is commenting on, and the job restarts from the beginning on a single serialised runner. Measured 2026-11-10 on PR #2007 (`27a9788`): the review ran 19:21:26 -> cancelled 19:30:31, with `build`, `hrx_build` and CodeQL green and every READY verdict posted; the PR sat with `pr_agent: COMPLETED/CANCELLED` while the runner queue held other work. That is the "red on pr_agent alone" the issue thread (#1993) reports for #1633, #1638, #1675 and #2007. Group by event name as well as number: a `synchronize` still cancels the review of the head it replaced (a newer head makes the older review moot), and a comment no longer cancels a push-triggered review. Verified on the expression: push -> `pr-agent-2007-pull_request`, comment -> `pr-agent-2007-issue_comment`, newer push -> same group as the earlier push. Refs #1993. Co-authored-by: bong-water-water-bong --- .github/workflows/pr-agent.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-agent.yml b/.github/workflows/pr-agent.yml index 6ff716d87..013a9f52b 100644 --- a/.github/workflows/pr-agent.yml +++ b/.github/workflows/pr-agent.yml @@ -38,7 +38,12 @@ jobs: runs-on: [self-hosted, pr-agent] # on the job, not the workflow: a comment that skips the job (a bot's, say) must not cancel a review concurrency: - group: pr-agent-${{ github.event.pull_request.number || github.event.issue.number }} + # A comment and a push are different jobs on one serialised model runner: an `/ask` must not cancel the review of + # the PR it is asked about, and a `synchronize` must cancel the review of the head it replaced. Grouping by event + # name keeps push-cancels-push (right: a newer head makes the older review moot) without letting a busy thread + # starve it. Measured 2026-11-10: PR #2007's review ran 19:21:26 -> cancelled 19:30:31, and `issue_comment: + # [created]` puts every comment on the PR into this same group with `cancel-in-progress: true`. + group: pr-agent-${{ github.event.pull_request.number || github.event.issue.number }}-${{ github.event_name }} cancel-in-progress: true timeout-minutes: 30 # same-repo PRs, and comments by the owner, members or collaborators on a pull request From f308fa9b7a4e6d25d399cd28e4eb9f605999b026 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:34:32 -0400 Subject: [PATCH 58/98] npu-pack: plan() fills a container's geometry and names the source that answered (#2070) engine#1786 (item F1), slice 2 of 3 -- stacked on slice 1 (#2054), which added container_geometry(). This wires it into the container-genre arm of plan(), so a recognised NPU2 container now carries its geometry instead of only its tensor count. Plan gains container_geometry_source / _missing / _inferred, and container_note names the source inline ("geometry from dims", "geometry from config.json", or "the geometry is not known here"), plus any field derived rather than declared and any field no source states. config.json is read through load_geometry() rather than a private parse, so this genre and every other agree about what config.json means (the text_config lift and the top-level-wins rule); container_geometry() still prefers the container's own `dims` over it. Still not importable, and the note says so: what is missing now is the role map (which tensor is which projection), not the geometry. That is slice 3. Arms: dims answers and beats a disagreeing config.json; config.json answers when there is no dims; with neither, the plan reports every required field missing and says the geometry is unknown rather than reporting zeros as facts. The arm removes its fixture directories on every exit -- the defect #2024 fixes for its neighbour is not repeated here. Co-authored-by: t --- npu/pack_import.cpp | 28 ++++++++++++-- npu/pack_import.h | 6 +++ tests/npu_pack_import_test.cpp | 68 ++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 1e9f5a357..d1e844bc7 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3389,11 +3389,33 @@ Plan plan(const std::string& pack_dir) { p.container_tensors = int64_t(c.tensors.size()); p.container_payload_bytes = c.payload_bytes; for (const auto& t : c.tensors) p.container_dtypes[t.dtype]++; + // slice 2: the geometry, and the name of the source that answered. config.json is read through the ordinary + // loader so this genre and every other one agree about what config.json means (text_config lift, top-level + // wins) -- the container's own `dims` still takes precedence in container_geometry(). + std::string cfg_source; + const json cfg = load_geometry(pack_dir, &cfg_source); + const ContainerGeometry g = container_geometry(c.meta, cfg); + p.dims = g.dims; + p.container_geometry_source = g.source; + p.container_geometry_missing = g.missing; + p.container_geometry_inferred = g.inferred; + std::string geom = g.source.empty() + ? "the container declares no `dims` and no config.json answered, so its geometry is not known here" + : "geometry from " + g.source; + if (!g.inferred.empty()) { + geom += " ("; + for (size_t i = 0; i < g.inferred.size(); ++i) geom += (i ? ", " : "") + g.inferred[i]; + geom += " inferred)"; + } + if (!g.missing.empty()) { + geom += "; no source states "; + for (size_t i = 0; i < g.missing.size(); ++i) geom += (i ? ", " : "") + g.missing[i]; + } p.container_note = c.why.empty() ? "the pack is a safetensors container (model.q4nx): " + std::to_string(c.tensors.size()) + - " tensors, " + std::to_string(c.payload_bytes) + " B of payload, no graph. Reading it as a" - " weights source needs the container-genre plan (engine#1786, item F1); until then this pack is" - " recognised and reported, not importable." + " tensors, " + std::to_string(c.payload_bytes) + " B of payload, no graph; " + geom + + ". Reading it as a weights source needs the container-genre plan (engine#1786, item F1); until" + " then this pack is recognised and reported, not importable." : "model.q4nx is present but unreadable: " + c.why; return p; } diff --git a/npu/pack_import.h b/npu/pack_import.h index 183052ae1..513025dbf 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -222,6 +222,12 @@ struct Plan { int64_t container_payload_bytes = 0; std::map container_dtypes; std::string container_note; // what the container genre still needs, empty for a non-container pack + // engine#1786 (item F1), slice 2: WHERE the container's geometry came from, and what it could not reach. + // A bare number in `dims` cannot say whether the pack declared it or a file beside it did, and those two are free to + // disagree (13 of the 29 packs measured carry a `dims` block; all 29 carry config.json). Empty for a non-container pack. + std::string container_geometry_source; // "dims", "config.json", or empty when neither answered + std::vector container_geometry_missing; // fields no source could fill, by name + std::vector container_geometry_inferred; // fields derived from another field, by name int64_t graph_matmuls = -1; // MatMulNBits nodes in the main graph int64_t graph_real_weights = 0; // of those, how many point at an initializer with dims (i.e. real weights) int64_t graph_external = 0; // initializers that declare external data (location + offset) diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 30f4d6605..57e3e8862 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -4923,6 +4923,73 @@ void test_container_geometry() { } } +// engine#1786 (item F1), slice 2: `plan()` reports WHERE a container's geometry came from. The three genres the census +// measured are the three cases: `dims` answers (and beats a disagreeing config.json), config.json answers when there is +// no `dims`, and neither answering is reported as unknown rather than as a zero geometry. +void test_container_plan(const std::string& root) { + namespace fs = std::filesystem; + // Under ctest `root` is the source tree's tests/golden, so these three directories are removed on every way out: + // the arm is new, and leaving fixtures behind is the defect #2024 fixes for its neighbour -- not repeating it here. + struct Cleanup { + fs::path base; + ~Cleanup() { + for (const char* n : {"cg-dims", "cg-cfg", "cg-silent"}) { + std::error_code e; + fs::remove_all(base / n, e); + } + } + } cleanup{fs::path(root)}; + const auto write_pack = [&](const std::string& name, const std::string& meta_json, const std::string& config) { + const fs::path dir = fs::path(root) / name; + std::error_code ec; + fs::remove_all(dir, ec); + fs::create_directories(dir, ec); + std::string hdr = "{"; + if (!meta_json.empty()) hdr += "\"__metadata__\":" + meta_json + ","; + hdr += "\"model.embed_tokens.weight\":{\"dtype\":\"BF16\",\"shape\":[4,2],\"data_offsets\":[0,16]}}"; + std::string out(8, '\0'); + const int64_t n = int64_t(hdr.size()); + for (int i = 0; i < 8; ++i) out[size_t(i)] = char((n >> (8 * i)) & 0xFF); + out += hdr; + out.append(16, '\x22'); + std::ofstream(dir / "model.q4nx", std::ios::binary).write(out.data(), std::streamsize(out.size())); + if (!config.empty()) std::ofstream(dir / "config.json") << config; + return dir.string(); + }; + const std::string dims = "{\"dims\":{\"hidden_size\":576,\"intermediate_size\":1536,\"num_hidden_layers\":30," + "\"num_attention_heads\":9,\"num_key_value_heads\":3,\"vocab_size\":49152," + "\"head_dim\":64,\"rope_theta\":10000.0,\"model_type\":\"llama\"}}"; + + // 1. self-describing, and a config.json that disagrees: the container's own statement wins and is named. + { + const auto pl = pi::plan(write_pack("cg-dims", dims, "{\"hidden_size\": 1024}")); + check(pl.container_geometry_source == "dims", "plan: a self-describing container reports source=dims"); + check(pl.dims.hidden == 576, "plan: dims.hidden comes from the container, not the disagreeing config.json"); + check(pl.dims.layers == 30 && pl.dims.vocab == 49152, "plan: the container's geometry reaches the plan"); + check(pl.container_geometry_missing.empty(), "plan: nothing is reported missing for a complete container"); + check(pl.container_note.find("geometry from dims") != std::string::npos, + "plan: the note says which source answered"); + } + // 2. gguf-converted: no dims, so config.json answers -- and the source says so. + { + const auto pl = pi::plan(write_pack("cg-cfg", "{\"converter\":\"tools/gguf_to_q4nx.py\"}", + "{\"hidden_size\":1024,\"num_hidden_layers\":28," + "\"num_attention_heads\":16,\"num_key_value_heads\":8," + "\"vocab_size\":151936,\"intermediate_size\":3072," + "\"head_dim\":64,\"rope_theta\":1000000.0}")); + check(pl.container_geometry_source == "config.json", "plan: a converted container reports source=config.json"); + check(pl.dims.hidden == 1024 && pl.dims.layers == 28, "plan: config.json supplies the geometry"); + } + // 3. silent: neither answers, so the plan says the geometry is unknown instead of reporting zeros as facts. + { + const auto pl = pi::plan(write_pack("cg-silent", "", "")); + check(pl.container_geometry_source.empty(), "plan: the silent genre claims no source"); + check(pl.container_geometry_missing.size() == 8, "plan: every required field is reported missing, by name"); + check(pl.container_note.find("geometry is not known here") != std::string::npos, + "plan: the note says the geometry is unknown rather than printing a zero geometry"); + } +} + // engine#1786 (item F1): an NPU2 model source is a safetensors container, and this is the reader that decides whether // a pack is that genre. The failure that matters is the silent one -- a wrong header length yields an empty inventory, // which reads as "a pack with no tensors" rather than "a malformed header" -- so each arm asserts the reason, not just @@ -5650,6 +5717,7 @@ int main(int argc, char** argv) { test_text_config_geometry(golden_dir.empty() ? "." : golden_dir); test_q4nx_container(golden_dir.empty() ? "." : golden_dir); test_container_geometry(); + test_container_plan(golden_dir.empty() ? "." : golden_dir); test_hybrid_pack_import(golden_dir.empty() ? "." : golden_dir); test_rope_parameters_theta(golden_dir.empty() ? "." : golden_dir); test_three_input_matmulnbits(golden_dir.empty() ? "." : golden_dir); From 17884875b168291d2c1f20593af8e1f64fbb24d4 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:46:37 -0400 Subject: [PATCH 59/98] tools: hrx_dispatch_profile.py, time per HRX dispatch from the backend's serial trace (engine #679) (#2084) The profiling slice of #679 was blocked on "no binary on this box emits per-dispatch timing". The backend already has a serial mode (GGML_HRX_DEBUG_SERIAL_EXECUTION=1) that waits for the device after every dispatch and logs the launch and the completion with the kernel id, but no time and no name. The tool sets the switch, time-stamps the lines as they arrive, sums per command program and kernel, and names the kernels from the pinned fork's manifests (the catalog id is FNV-1a over family and name). It needs no rebuild and works on the pinned build. docs/hrx.md: how to run it, a measured table (Qwen3-4B, 512 against 8,192 tokens of prompt), and what the numbers are not (the serial mode is slower than a normal run). ctest: hrx_dispatch_profile_self. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 2 + docs/hrx.md | 28 +++++ tools/hrx_dispatch_profile.py | 224 ++++++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100755 tools/hrx_dispatch_profile.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e18dcbef..728e7efae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -356,6 +356,8 @@ add_python_test(hf_tensor_self ${CMAKE_SOURCE_DIR}/tools/hf_tensor.py --self-tes add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) # engine#856: one HRX process's GTT split into the loader's copy and HRX's packed second copy, from its own -v log. add_python_test(hrx_gtt_split_self ${CMAKE_SOURCE_DIR}/tools/hrx_gtt_split.py --self-test) +# engine#679: time per HRX dispatch from the backend's serial trace (the id hash and the per-kernel sums). +add_python_test(hrx_dispatch_profile_self ${CMAKE_SOURCE_DIR}/tools/hrx_dispatch_profile.py --self-test) # engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as # it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that # release, and the ordering a candidate needs: previous release < candidate < its release. The diff --git a/docs/hrx.md b/docs/hrx.md index 184f21fd2..7870a0af9 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -864,6 +864,34 @@ a token threshold still reads the loader's copy. "Not attributed" is mostly runt staging buffer is capped at 64 MiB since [llama.cpp #266](https://github.com/1bit-MONSTER/llama.cpp/pull/266)). It is not weights. This replaces the `bo_trace.py` reproduction in the issue body, which was never committed. +**Which kernel a graph's time is in.** `tools/hrx_dispatch_profile.py` gives the time per HRX dispatch, summed per kernel, +on any HRX build (it needs no rebuild). The backend's serial mode (`GGML_HRX_DEBUG_SERIAL_EXECUTION=1`) waits for the device +after every dispatch and prints a line at launch and at completion; the tool sets the switch, time-stamps those lines and +names the kernels from the pinned fork's manifests. Inside your ticket: + +``` +gpu-hold -l -t 10 -- tools/hrx_dispatch_profile.py run --log prof.log --json prof.json -- \ + llama-bench -m Qwen3-4B-Q4_K_M.gguf -ngl 99 -p 8192 -n 16 -r 1 +tools/hrx_dispatch_profile.py parse prof.log --top 20 # again, from the trace it kept +``` + +It prints one table per command program (a graph shape: prefill batch, decode step). Measured on the pin `2293f95ea`, +Qwen3-4B Q4_K_M, the prefill program of a 512-token prompt against the most frequent prefill program of an 8,192-token +prompt (12 of its 16 batches of 512), one run each, 2026-11-11: + +| kernel | 512-token prompt, ms each | share | 8,192-token prompt, ms each | share | +|---|---:|---:|---:|---:| +| `ggml_flash_attention_f32_f16_wmma` | 0.371 | 7.5% | 2.150 | 49.3% | +| `ggml_mul_mat_swiglu_q4_k_f16_wmma_prefill_wave32` | 1.714 | 33.6% | 1.728 | 19.3% | +| `ggml_mul_mat_q4_k_f16_wmma_prefill_wave32` | 0.425 | 34.2% | 0.393 | 18.0% | +| `ggml_mul_mat_q6_k_f16_wmma_prefill_wave32` | 0.635 | 12.4% | 0.632 | 7.0% | + +The matmuls cost the same at any context; the attention kernel is the part that grows. + +What the numbers are and are not: each is the wall time from launch to device-idle with one dispatch in flight, so it holds +one wake-up of the waiting thread and the run is slower than a normal one (the same command: pp512 2,977 -> 2,500 tok/s, +tg16 67 -> 36, pp8192 1,191 -> 1,167). Use it to rank kernels and to see what grows with the context, not as a rate. + ### Measuring on `HRX0`: what makes a number citable Numbers measured on this device swing far more between *runs* than between *files*, so the harness matters as much as the arm. Everything below is diff --git a/tools/hrx_dispatch_profile.py b/tools/hrx_dispatch_profile.py new file mode 100755 index 000000000..ec7edbb37 --- /dev/null +++ b/tools/hrx_dispatch_profile.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""hrx_dispatch_profile: time per HRX dispatch, from the backend's own serial trace (engine#679). + + hrx_dispatch_profile.py run [--log FILE] [--top N] [--json FILE] -- llama-bench -m M.gguf -ngl 99 ... + hrx_dispatch_profile.py parse LOG [--top N] [--json FILE] + hrx_dispatch_profile.py --self-test + +The HRX backend has a serial mode, GGML_HRX_DEBUG_SERIAL_EXECUTION=1: it waits for the device after every dispatch +and prints one line when the dispatch is launched and one when the device has finished it. The lines carry the +kernel's id but no time and no name. `run` sets that switch, stamps every trace line with a monotonic time as it +arrives, and adds the two up per kernel; the names come from the kernel manifests of the pinned llama.cpp fork. + +What the numbers are: the wall time from "dispatch launched" to "device idle", one dispatch at a time. That is the +kernel's device time plus one wake-up of the waiting thread, so a sum over a graph is larger than the same graph in +a normal run, where dispatches are queued back to back. Use it to see WHICH kernel a graph's time is in and how +that changes with the context length, not as a tok/s figure. Run the command under gpu-hold like any device run. +""" +import argparse +import collections +import glob +import json +import os +import re +import subprocess +import sys +import time + +TRACE_PREFIX = "hrx debug serial: " +DEFAULT_MANIFESTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "third_party", "llama.cpp", "ggml", + "src", "ggml-hrx", "kernel-corpus", "kernels") +# manifests that do not name a family per export use the directory's catalog family +DIRECTORY_FAMILY = {"qwen_moe": "qwen3_moe", "qwen": "qwen", "hrx": "hrx", "loom-libs": "loom_libs"} + + +def kernel_id(family, name): + """kernel_catalog_id() of the fork's kernel-corpus-catalog.h: FNV-1a over family, a zero byte, name.""" + mask, prime = (1 << 64) - 1, 1099511628211 + value = 1469598103934665603 + for byte in family.encode(): + value = ((value ^ byte) * prime) & mask + value = (value * prime) & mask + for byte in name.encode(): + value = ((value ^ byte) * prime) & mask + return value + + +def load_kernel_names(manifest_dir): + names = {} + for path in sorted(glob.glob(os.path.join(manifest_dir, "*", "manifest.json"))): + directory = os.path.basename(os.path.dirname(path)) + with open(path) as handle: + manifest = json.load(handle) + for export in manifest.get("exports", []): + family = export.get("family") or DIRECTORY_FAMILY.get(directory, directory) + names[kernel_id(family, export["name"])] = family + ":" + export["name"] + return names + + +def parse_fields(text): + return dict(re.findall(r"(\w+)=(\S+)", text)) + + +def profile(lines): + """lines: iterable of (time_ns, trace text without the prefix). Returns the per-program, per-kernel totals.""" + programs = collections.OrderedDict() + open_dispatch = {} + for time_ns, text in lines: + event, _, rest = text.partition(" ") + fields = parse_fields(rest) + phase = fields.get("phase") + if "kernel_id" not in fields or "index" not in fields: + continue + key = (fields.get("list"), fields.get("list_commands"), fields["index"]) + if event == "begin" and phase == "command-dispatch": + open_dispatch[key] = [time_ns, None] + elif event == "end" and phase == "command-dispatch" and key in open_dispatch: + open_dispatch[key][1] = time_ns + elif event == "sync-end" and phase == "command-post-dispatch" and key in open_dispatch: + begin_ns, launched_ns = open_dispatch.pop(key) + program = programs.setdefault((fields.get("list"), int(fields.get("list_commands", 0))), {}) + row = program.setdefault(int(fields["kernel_id"]), {"dispatches": 0, "total_ns": 0, "launch_ns": 0}) + row["dispatches"] += 1 + row["total_ns"] += time_ns - begin_ns + row["launch_ns"] += (launched_ns if launched_ns is not None else begin_ns) - begin_ns + return programs + + +def report(programs, names, top, out=sys.stdout): + result = [] + for (list_kind, command_count), kernels in programs.items(): + total_ns = sum(row["total_ns"] for row in kernels.values()) + dispatches = sum(row["dispatches"] for row in kernels.values()) + runs = dispatches / command_count if command_count else 0.0 + print(f"\nprogram list={list_kind} commands={command_count}: {dispatches} dispatches " + f"({runs:.1f} runs), {total_ns / 1e6:.1f} ms in total", file=out) + print(f" {'ms':>10} {'share':>6} {'count':>7} {'ms each':>9} {'launch ms':>10} kernel", file=out) + rows = sorted(kernels.items(), key=lambda item: -item[1]["total_ns"]) + entry = {"list": list_kind, "commands": command_count, "dispatches": dispatches, "total_ms": total_ns / 1e6, + "kernels": []} + for index, (kid, row) in enumerate(rows): + name = names.get(kid, str(kid)) + share = row["total_ns"] / total_ns if total_ns else 0.0 + entry["kernels"].append({"kernel": name, "dispatches": row["dispatches"], "ms": row["total_ns"] / 1e6, + "launch_ms": row["launch_ns"] / 1e6, "share": share}) + if index < top: + print(f" {row['total_ns'] / 1e6:10.2f} {share:6.1%} {row['dispatches']:7d} " + f"{row['total_ns'] / 1e6 / row['dispatches']:9.3f} {row['launch_ns'] / 1e6:10.2f} {name}", + file=out) + if len(rows) > top: + print(f" ... {len(rows) - top} more kernels (--top)", file=out) + result.append(entry) + if not result: + print("no dispatch was traced: is the binary an HRX build, and did the run reach the device?", file=out) + return result + + +def read_log(path): + with open(path) as handle: + for line in handle: + stamp, _, text = line.rstrip("\n").partition(" ") + if stamp.isdigit() and text.startswith(TRACE_PREFIX): + yield int(stamp), text[len(TRACE_PREFIX):] + + +def run_command(command, log_path): + env = dict(os.environ, GGML_HRX_DEBUG_SERIAL_EXECUTION="1") + lines = [] + log = open(log_path, "w") if log_path else None + process = subprocess.Popen(command, env=env, stderr=subprocess.PIPE, text=True, errors="replace", bufsize=1) + try: + for line in process.stderr: + now = time.monotonic_ns() + if line.startswith(TRACE_PREFIX): + lines.append((now, line[len(TRACE_PREFIX):].rstrip("\n"))) + if log: + log.write(f"{now} {line}") + else: + sys.stderr.write(line) + finally: + process.wait() + if log: + log.close() + return lines, process.returncode + + +def self_test(): + assert kernel_id("loom_libs", "ggml_mul_mat_f32_f32_wmma") == 5916100870017593149, "the catalog id hash changed" + detail = "list=main index={i} list_commands=2 ordinal={i} kind=Kernel kernel_id={k} bindings=3" + trace = [] + clock = 1000 + for run in range(2): + for index, (kid, launch, device) in enumerate([(7, 10, 90), (9, 5, 20)]): + text = detail.format(i=index, k=kid) + trace.append((clock, "sync-begin phase=command-pre-dispatch " + text)) + trace.append((clock, "sync-end phase=command-pre-dispatch " + text)) + trace.append((clock, "begin phase=command-dispatch " + text)) + trace.append((clock + launch, "end phase=command-dispatch " + text)) + trace.append((clock + launch, "sync-begin phase=command-post-dispatch " + text)) + trace.append((clock + launch + device, "sync-end phase=command-post-dispatch " + text)) + clock += launch + device + 1000 # host time between dispatches is not charged to a kernel + trace.append((clock, "begin phase=command-list list=main list_commands=2 main_commands=2")) + programs = profile(trace) + assert list(programs) == [("main", 2)], programs + assert programs[("main", 2)][7] == {"dispatches": 2, "total_ns": 200, "launch_ns": 20}, programs + assert programs[("main", 2)][9] == {"dispatches": 2, "total_ns": 50, "launch_ns": 10}, programs + with open(os.devnull, "w") as sink: + result = report(programs, {7: "f:slow", 9: "f:fast"}, 1, out=sink) + assert report({}, {}, 5, out=sink) == [] + assert [k["kernel"] for k in result[0]["kernels"]] == ["f:slow", "f:fast"], result + assert abs(result[0]["kernels"][0]["share"] - 0.8) < 1e-9, result + print("hrx_dispatch_profile self-test: OK") + return 0 + + +def main(): + if "--self-test" in sys.argv[1:]: + return self_test() + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="mode", required=True) + for mode in ("run", "parse"): + p = sub.add_parser(mode) + p.add_argument("--top", type=int, default=12, help="kernels printed per program (default 12)") + p.add_argument("--json", help="write the full table here") + p.add_argument("--manifests", default=DEFAULT_MANIFESTS, help="the fork's kernel-corpus/kernels directory") + if mode == "run": + p.add_argument("--log", help="keep the time-stamped trace here (on disk, not /tmp)") + p.add_argument("command", nargs=argparse.REMAINDER, help="-- CMD ARGS...") + else: + p.add_argument("log") + args = parser.parse_args() + names = load_kernel_names(args.manifests) + if not names: + print(f"note: no kernel manifest under {args.manifests}; kernels are shown by id", file=sys.stderr) + code = 0 + if args.mode == "run": + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + parser.error("run needs a command after --") + lines, code = run_command(command, args.log) + else: + lines = list(read_log(args.log)) + result = report(profile(lines), names, args.top) + if args.json: + with open(args.json, "w") as handle: + json.dump(result, handle, indent=1) + return code if code else (0 if result else 1) + + +if __name__ == "__main__": + sys.exit(main()) From 540e1fd81c9371fade86340882dbd7de90461533 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:56:57 -0400 Subject: [PATCH 60/98] npu: a MiniCPM-V 4.7 image from bytes to the tower's input: plan, torch-exact resize, slices (engine#1788) (#2066) * npu: the MiniCPM-V 4.7 image plan: sizes, slice grid and text rows, matching the reference processor (engine#1788) vision::plan_image(height, width) is the arithmetic of the model's image processor with no pixel touched: the whole view, the slice grid (up to 9), the resize the slices are cut from, and the rows each piece becomes after the tower's two 2x2 merges. Python's round-half-even and int() truncation are followed exactly. Test: npu_vision_plan, against transformers' MiniCPMV4_6ImageProcessor 5.22.2 for 3,481 image sizes (one hash over every number, 16 plans written out). Five mutants caught; a sixth was equivalent and its claim removed from the header. Co-Authored-By: Claude Fable 5.2 * npu: MiniCPM-V 4.7 image bytes to the tower's input: torch's uint8 bicubic bit for bit, slices, normalise (engine#1788) vision::resize_bicubic_u8 reproduces torch.nn.functional.interpolate(uint8, bicubic, antialias) exactly (11 cases held as hashes of the reference's bytes); vision::preprocess returns the plan's pieces as normalised planes, whole view first. End to end on the 1B pack (460x470 synthetic image, 3 pieces, 196 rows) against the reference plan + torch resize + transformers tower: max |d| 7.5e-6, corr 1.000000000. 110 s on one thread: a reference, not a serving path. Test: npu_vision_image (42 checks; 6 mutants caught; +3 end to end with ONEBIT_MINICPM_PACK / ONEBIT_MINICPM_E2E). Co-Authored-By: Claude Fable 5.2 * npu: the image plan says it never crops, the constants are pinned on known bytes, adversarial sizes (review of #2066) Review asks at 44b5438: (1) say and test what happens to sizes that are not multiples of anything: resized whole, never refused, never cropped; the slices tile the refine size exactly. (2) state the constants and pin one on known bytes: a one-colour (0, 255, 128) image comes out as planes of exactly -1, +1 and 0.5/127.5, in R, G, B order. (3) adversarial sizes: odd and off-by-one sizes, the largest accepted sizes (the reference's own answers for 2^20 strips), and one past the limit on each axis refused. Docs say the plan touches no file and no pixel, and does not bound its result. Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 12 +- docs/npu.md | 52 ++++++++ npu/vision_image.cpp | 154 ++++++++++++++++++++++ npu/vision_image.h | 51 ++++++++ npu/vision_plan.h | 156 ++++++++++++++++++++++ tests/npu_vision_image_test.cpp | 220 ++++++++++++++++++++++++++++++++ tests/npu_vision_plan_test.cpp | 150 ++++++++++++++++++++++ 7 files changed, 794 insertions(+), 1 deletion(-) create mode 100644 npu/vision_image.cpp create mode 100644 npu/vision_image.h create mode 100644 npu/vision_plan.h create mode 100644 tests/npu_vision_image_test.cpp create mode 100644 tests/npu_vision_plan_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 728e7efae..3f60faae3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -702,7 +702,7 @@ option(ONEBIT_NPU_HOST "Build the NPU host library and its tests" ${_onebit_npu_ if(ONEBIT_NPU_HOST) find_package(PkgConfig REQUIRED) pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8) -add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp +add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp) target_include_directories(onebit_npu PUBLIC npu) @@ -1228,6 +1228,16 @@ add_executable(npu_vision_tower_test tests/npu_vision_tower_test.cpp) target_link_libraries(npu_vision_tower_test PRIVATE onebit_npu) add_test(NAME npu_vision_tower COMMAND npu_vision_tower_test) add_dependencies(onebit npu_vision_tower_test) +# The image plan (sizes, slice grid, text rows) against the reference image processor's answers for 3,481 sizes. +add_executable(npu_vision_plan_test tests/npu_vision_plan_test.cpp) +target_include_directories(npu_vision_plan_test PRIVATE npu) +add_test(NAME npu_vision_plan COMMAND npu_vision_plan_test) +add_dependencies(onebit npu_vision_plan_test) +# The resize (bit for bit against torch's uint8 antialiased bicubic) and the pieces handed to the tower. +add_executable(npu_vision_image_test tests/npu_vision_image_test.cpp) +target_link_libraries(npu_vision_image_test PRIVATE onebit_npu) +add_test(NAME npu_vision_image COMMAND npu_vision_image_test) +add_dependencies(onebit npu_vision_image_test) add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp) target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) diff --git a/docs/npu.md b/docs/npu.md index 38fb948d2..3ace1d57f 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1580,6 +1580,58 @@ output for it as goldens, so the arithmetic is checked in CI without a pack; wit `ONEBIT_MINICPM_PACK` and `ONEBIT_MINICPM_ORACLE` set it repeats the comparison above. The reference run itself needs transformers and torch and is not in the tree. +**Where an image goes before the tower (`npu/vision_plan.h`).** `vision::plan_image(height, width)` is +the arithmetic of the model's image processor (transformers' `MiniCPMV4_6ImageProcessor`, which 4.7 +loads), with no pixel touched: the model looks at an image whole, resized to about 448 × 448 pixels, +and, when the image is larger than that, again as a grid of up to 9 slices cut from a second, larger +resize. Every piece is a multiple of 56 pixels (4 patches) in both extents, because the tower merges +2 × 2 twice, and becomes `(height/56) × (width/56)` rows on the text side. + +| image | whole view | slices | text rows | +|---|---|---|---| +| 448 × 448 or smaller (scaled up) | 448 × 448 | none | 64 | +| 480 × 640 | 392 × 504 | 1 × 2 of 560 × 392 | 63 + 2 × 70 | +| 1080 × 1920 | 336 × 616 | 2 × 4 of 448 × 448 | 66 + 8 × 64 | +| 3024 × 4032 | 392 × 504 | 3 × 3 of 392 × 504 | 63 + 9 × 63 | + +(heights first.) So one 448-pixel slice is 64 rows, not the 1,225 that 980 / 14 / 2 squared suggests: the +position table is 70 × 70, but no piece the processor produces is that large unless the image is a very +long strip. The plan is checked against the reference's own answers for 3,481 image sizes in +`tests/npu_vision_plan_test.cpp`. Nothing is cropped or padded and no size is refused for not being a +multiple of anything: every image is resized whole (385 × 384 becomes 448 × 448, so the aspect ratio moves +a little, as it does in the model's own processor), and the slices tile the refine size exactly. The plan +does not bound its result: a 1,048,576 × 1 strip gives the reference's answer, a piece 458,752 pixels +tall, and it is the caller that decides how many rows to accept. `plan_image()` reads no file and touches +no pixel, which is what lets it be checked anywhere; keep it that way. + +**From bytes to the tower's input (`npu/vision_image.h`).** `vision::preprocess(rgb, height, width, plan)` +resizes an RGB byte image to the plan's whole view and, when it is sliced, to the refine size, cuts the +slices, and returns each piece as `(3, height, width)` float32 with value `(byte − 127.5) / 127.5` +(rescale by 1/255, then mean 0.5 and std 0.5, the same for each channel; R, G, B order in and out; byte +0 is exactly −1 and byte 255 exactly +1): the whole view first, then the slices row by row. The resize is bicubic with antialiasing on bytes and +reproduces `torch.nn.functional.interpolate(uint8, mode="bicubic", antialias=True)` (torch 2.13) bit +for bit: the a = −0.5 cubic stretched by the scale when shrinking, integer weights at the largest +precision that still fits int16 (PIL's fixed 22 bits differs by one in about 1 % of bytes), width +first and then height. `tests/npu_vision_image_test.cpp` holds the reference's bytes for eleven +resizes as hashes. + +End to end on the 1B pack, 2026-11-10: a 460 × 470 synthetic image through plan, resize, pieces and +`tower_forward()`, against the reference's plan, torch's resize and the transformers tower on the same +image: + +| piece | patches | rows | max \|d\| | corr | +|---|---|---|---|---| +| whole view 448 × 448 | 32 × 32 | 64 | 6.4e-6 | 1.000000000 | +| slice 0, 616 × 336 | 44 × 24 | 66 | 7.5e-6 | 1.000000000 | +| slice 1, 616 × 336 | 44 × 24 | 66 | 6.7e-6 | 1.000000000 | + +That run took 110 s on one thread, which is the host reference's cost and the reason it is not a +serving path. Two things are not established: the reference side called torch's `interpolate` +directly, on the understanding that this is what the processor's torchvision resize does for byte +images on a CPU build with AVX2 (torchvision is not on this box, so that link is read, not run); and +no real photograph or image decoder has been through it. The slice separators and placeholder rows on +the text side are not written. + ## Serving The 1bit NPU engine runs the model on the XDNA 2 NPU. With a release package that carries the NPU diff --git a/npu/vision_image.cpp b/npu/vision_image.cpp new file mode 100644 index 000000000..bb6dbdce5 --- /dev/null +++ b/npu/vision_image.cpp @@ -0,0 +1,154 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "vision_image.h" + +#include +#include +#include + +namespace onebit::npu::vision { + +namespace { + +double cubic(double x) { + const double a = -0.5; + if (x < 0) x = -x; + if (x < 1.0) return ((a + 2.0) * x - (a + 3.0)) * x * x + 1.0; + if (x < 2.0) return (((x - 5.0) * x + 8.0) * x - 4.0) * a; + return 0.0; +} + +struct Taps { + std::vector first; // per output index: the first input index + std::vector count; // and how many taps + std::vector weight; // count taps each, `stride` apart + int stride = 0, precision = 0; +}; + +Taps taps_for(int in, int out) { + const double scale = double(in) / double(out); + const double fs = scale > 1.0 ? scale : 1.0, support = 2.0 * fs, inv = 1.0 / fs; + Taps t; + t.stride = int(std::ceil(support)) * 2 + 1; + t.first.resize(size_t(out)); + t.count.resize(size_t(out)); + std::vector w(size_t(out) * t.stride, 0.0); + double wmax = 0.0; + for (int i = 0; i < out; ++i) { + const double center = scale * (i + 0.5); + int lo = int(center - support + 0.5); + if (lo < 0) lo = 0; + int hi = int(center + support + 0.5); + if (hi > in) hi = in; + const int n = hi - lo; + double total = 0.0; + for (int j = 0; j < n; ++j) { + w[size_t(i) * t.stride + j] = cubic((j + lo - center + 0.5) * inv); + total += w[size_t(i) * t.stride + j]; + } + for (int j = 0; j < n; ++j) { + if (total != 0.0) w[size_t(i) * t.stride + j] /= total; + if (w[size_t(i) * t.stride + j] > wmax) wmax = w[size_t(i) * t.stride + j]; + } + t.first[size_t(i)] = lo; + t.count[size_t(i)] = n; + } + // the largest precision at which the biggest weight still fits a signed 16-bit integer + for (int p = 0; p < 22; ++p) { + if (int64_t(0.5 + wmax * double(int64_t(1) << (p + 1))) >= (1 << 15)) break; + t.precision = p + 1; + } + t.weight.resize(w.size()); + for (size_t k = 0; k < w.size(); ++k) { + const double v = w[k] * double(int64_t(1) << t.precision); + t.weight[k] = int32_t(v < 0 ? -0.5 + v : 0.5 + v); + } + return t; +} + +inline uint8_t clip8(int64_t v) { return uint8_t(v < 0 ? 0 : (v > 255 ? 255 : v)); } + +} // namespace + +std::vector resize_bicubic_u8(const uint8_t* rgb, int height, int width, int out_height, int out_width) { + if (!rgb || height <= 0 || width <= 0 || out_height <= 0 || out_width <= 0 || out_height > (1 << 16) || out_width > (1 << 16)) + throw std::runtime_error("cannot resize a " + std::to_string(height) + " x " + std::to_string(width) + " image to " + + std::to_string(out_height) + " x " + std::to_string(out_width)); + // width first + std::vector mid; + const uint8_t* src = rgb; + if (out_width != width) { + const Taps t = taps_for(width, out_width); + const int64_t half = t.precision > 0 ? int64_t(1) << (t.precision - 1) : 0; + mid.resize(size_t(height) * out_width * 3); + for (int y = 0; y < height; ++y) + for (int x = 0; x < out_width; ++x) { + const int32_t* w = t.weight.data() + size_t(x) * t.stride; + const uint8_t* p = rgb + (size_t(y) * width + size_t(t.first[size_t(x)])) * 3; + int64_t acc[3] = {half, half, half}; + for (int j = 0; j < t.count[size_t(x)]; ++j) + for (int c = 0; c < 3; ++c) acc[c] += int64_t(p[size_t(j) * 3 + c]) * w[j]; + for (int c = 0; c < 3; ++c) mid[(size_t(y) * out_width + x) * 3 + c] = clip8(acc[c] >> t.precision); + } + src = mid.data(); + } + if (out_height == height) return src == rgb ? std::vector(rgb, rgb + size_t(height) * width * 3) : mid; + // then height + const Taps t = taps_for(height, out_height); + const int64_t half = t.precision > 0 ? int64_t(1) << (t.precision - 1) : 0; + std::vector out(size_t(out_height) * out_width * 3); + const size_t row = size_t(out_width) * 3; + for (int y = 0; y < out_height; ++y) { + const int32_t* w = t.weight.data() + size_t(y) * t.stride; + for (size_t k = 0; k < row; ++k) { + int64_t acc = half; + for (int j = 0; j < t.count[size_t(y)]; ++j) acc += int64_t(src[(size_t(t.first[size_t(y)]) + j) * row + k]) * w[j]; + out[size_t(y) * row + k] = clip8(acc >> t.precision); + } + } + return out; +} + +std::vector preprocess(const uint8_t* rgb, int height, int width, const ImagePlan& plan) { + if (plan.source.height <= 0 || plan.source.width <= 0 || plan.source.grid_h <= 0 || plan.source.grid_w <= 0) + throw std::runtime_error("the image plan is empty"); + // one piece: the (h, w) window of an interleaved image at (y0, x0), as normalised planes + const auto planes = [](const std::vector& img, int img_w, int y0, int x0, const PiecePlan& q) { + Piece p; + p.grid_h = q.grid_h; + p.grid_w = q.grid_w; + p.pixels.resize(size_t(3) * q.height * q.width); + for (int c = 0; c < 3; ++c) + for (int y = 0; y < q.height; ++y) + for (int x = 0; x < q.width; ++x) + p.pixels[(size_t(c) * q.height + y) * q.width + x] = + (float(img[(size_t(y0 + y) * img_w + size_t(x0 + x)) * 3 + c]) - 127.5f) / 127.5f; + return p; + }; + std::vector out; + out.push_back(planes(resize_bicubic_u8(rgb, height, width, plan.source.height, plan.source.width), plan.source.width, 0, 0, + plan.source)); + if (plan.slice_rows > 0) { + if (plan.refine_height != plan.slice.height * plan.slice_rows || plan.refine_width != plan.slice.width * plan.slice_cols) + throw std::runtime_error("the image plan's slices do not tile its refine size"); + const std::vector refine = resize_bicubic_u8(rgb, height, width, plan.refine_height, plan.refine_width); + for (int r = 0; r < plan.slice_rows; ++r) + for (int c = 0; c < plan.slice_cols; ++c) + out.push_back(planes(refine, plan.refine_width, r * plan.slice.height, c * plan.slice.width, plan.slice)); + } + return out; +} + +} // namespace onebit::npu::vision diff --git a/npu/vision_image.h b/npu/vision_image.h new file mode 100644 index 000000000..02a811ded --- /dev/null +++ b/npu/vision_image.h @@ -0,0 +1,51 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// A MiniCPM-V 4.7 image from RGB bytes to what the vision tower takes (engine#1788): resize, slice, normalise. +// +// plan_image() (vision_plan.h) says which sizes; this does the pixels. The pieces come out in the order the reference +// processor feeds the tower: the whole view first, then the slices row by row. Each is (3, height, width) float32, +// value (byte - 127.5) / 127.5, i.e. rescale by 1/255 then mean 0.5 and std 0.5 (preprocessor_config.json), the same +// for all three channels. Input and planes are in R, G, B order: byte 0 is exactly -1, byte 255 exactly +1. +// +// The resize is bicubic with antialiasing on BYTES, and it is torch's, bit for bit: torch.nn.functional.interpolate( +// uint8, mode="bicubic", antialias=True) at torch 2.13. That is integer arithmetic, so it can be exact, and three things +// about it are load bearing (tests/npu_vision_image_test.cpp has the reference's bytes): +// - the kernel is the a = -0.5 cubic, stretched by the scale when shrinking, normalised per output pixel; +// - weights become integers at the largest precision whose biggest weight still fits int16 (NOT PIL's fixed 22 bits: +// that differs from torch by one in about 1 % of bytes); +// - width first, then height, with the intermediate image rounded to bytes in between. A pass whose size does not +// change is skipped. +#pragma once + +#include "vision_plan.h" + +#include +#include + +namespace onebit::npu::vision { + +// `rgb` is (height, width, 3) interleaved bytes; the result is (out_height, out_width, 3). Throws on empty sizes. +std::vector resize_bicubic_u8(const uint8_t* rgb, int height, int width, int out_height, int out_width); + +struct Piece { + int grid_h = 0, grid_w = 0; // patches: what tower_forward() takes as gh, gw + std::vector pixels; // (3, grid_h * patch, grid_w * patch) +}; + +// Every piece of `plan` (which must be plan_image(height, width, ...) for this image), source first. +std::vector preprocess(const uint8_t* rgb, int height, int width, const ImagePlan& plan); + +} // namespace onebit::npu::vision diff --git a/npu/vision_plan.h b/npu/vision_plan.h new file mode 100644 index 000000000..873d2542c --- /dev/null +++ b/npu/vision_plan.h @@ -0,0 +1,156 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Where a MiniCPM-V 4.7 image goes before the vision tower sees it (engine#1788): the sizes, the slice grid, and how +// many rows each piece becomes on the text side. Arithmetic only; no pixel is touched here. +// +// The model looks at an image twice: once whole, resized to about `scale_resolution`^2 pixels, and, when the image is +// larger than that, again as a rows x cols grid of slices cut from a second, larger resize. Every piece is a multiple +// of 4 patches in both extents, because the tower merges 2 x 2 twice (npu/vision_tower.h), and becomes +// (height/patch/4) * (width/patch/4) rows. +// +// This is transformers' MiniCPMV4_6ImageProcessor (the image processor MiniCPM-V 4.7 loads), followed step by step, +// including the places where the obvious arithmetic gives a different answer: +// - `round()` there is Python's: halves go to the EVEN neighbour (2.5 -> 2), which std::round does not do; +// - `int()` truncates, and the width is computed from the already truncated height; +// - the slice grid tries one piece fewer and one more than area / scale^2, and a tie goes to more rows; +// - an image that is not sliced is scaled UP to the resolution; one that is sliced has its whole view scaled down only. +// (The per-slice size, refine / grid, is an exact division: the refine size was just rounded to a multiple of the grid.) +// tests/npu_vision_plan_test.cpp holds that implementation's answers for 3,481 image sizes. +// +// Two properties a caller may rely on, both tested: +// - NOTHING IS CROPPED OR PADDED. No image size is refused for not being a multiple of anything: every size is +// resized, whole, to sizes that are multiples of 4 patches, and the slices tile the refine size with no remainder. +// The aspect ratio therefore moves a little (385 x 384 becomes 448 x 448); that is the model's own behaviour. +// - this header touches no file, no pixel and no platform: sizes in, sizes out. Keep it that way; it is what lets the +// plan be checked anywhere. +// It does not bound the result. A very long strip gives the reference's answer, which can be a piece hundreds of +// thousands of pixels long: the caller decides how many rows (total_rows()) it will accept. +#pragma once + +#include +#include +#include +#include +#include + +namespace onebit::npu::vision { + +struct PiecePlan { + int height = 0, width = 0; // pixels the piece is resized (source) or cut (slice) to + int grid_h = 0, grid_w = 0; // patches + int rows = 0; // rows on the text side: grid_h/4 * grid_w/4 +}; + +struct ImagePlan { + PiecePlan source; // the whole image + int slice_rows = 0, slice_cols = 0; // 0 x 0: the image is not sliced + int refine_height = 0, refine_width = 0; // the resize the slices are cut from (slice_rows x slice_cols equal pieces) + PiecePlan slice; // every slice: they are all the same size + // Everything the tower is run on, in the order the reference feeds it: the source, then the slices row by row. + int pieces() const { return 1 + slice_rows * slice_cols; } + int total_rows() const { return source.rows + slice_rows * slice_cols * slice.rows; } +}; + +namespace plan_detail { + +// Python's round(): to nearest, halves to even. The default floating-point rounding mode is that rule. +inline int64_t py_round(double v) { return int64_t(std::nearbyint(v)); } + +// max(round(length / divisor) * divisor, divisor) +inline int64_t ensure_divide(double length, int64_t divisor) { + const int64_t r = py_round(length / double(divisor)) * divisor; + return r > divisor ? r : divisor; +} + +inline void best_resize(double height, double width, int scale, int patch, bool allow_upscale, int64_t* out_h, int64_t* out_w) { + if (height * width > double(scale) * double(scale) || allow_upscale) { + const double aspect = width / height; + const double h = double(int64_t(double(scale) / std::sqrt(aspect))); // int(): truncation + height = h; + width = double(int64_t(h * aspect)); + } + const int64_t unit = int64_t(patch) * 4; // two 2 x 2 merges + *out_w = ensure_divide(width, unit); + *out_h = ensure_divide(height, unit); +} + +} // namespace plan_detail + +// The plan for a height x width image. The defaults are the model's (preprocessor_config.json: max_slice_nums 9; the +// image processor's scale_resolution 448; the tower's 14-pixel patch). Throws on a size that is not an image or that +// the arithmetic cannot hold. +inline ImagePlan plan_image(int height, int width, int max_slice_nums = 9, int scale_resolution = 448, int patch = 14) { + using namespace plan_detail; + if (height <= 0 || width <= 0 || height > (1 << 20) || width > (1 << 20)) + throw std::runtime_error("an image of " + std::to_string(height) + " x " + std::to_string(width) + " pixels cannot be planned"); + if (max_slice_nums < 1 || max_slice_nums > 64 || scale_resolution < 4 * patch || scale_resolution > 8192 || patch < 1) + throw std::runtime_error("the image plan's parameters are out of range"); + ImagePlan p; + const double H = height, W = width; + + // the slice grid: the factorisation of about (area / scale^2) pieces whose shape is closest to the image's + const double log_ratio = std::log(W / H); + const double ratio = W * H / (double(scale_resolution) * double(scale_resolution)); + const int64_t ceil_ratio = int64_t(std::ceil(ratio)); + const int multiple = int(ceil_ratio < max_slice_nums ? ceil_ratio : max_slice_nums); + if (multiple > 1) { + int best_rows = 1, best_cols = 1; + double min_error = INFINITY; + for (int n = multiple - 1; n <= multiple + 1; ++n) { + if (n == 1 || n > max_slice_nums) continue; + for (int rows = 1; rows <= n; ++rows) { + if (n % rows != 0) continue; + const int cols = n / rows; + const double error = std::fabs(log_ratio - std::log(double(cols) / double(rows))); + if (error < min_error || (error == min_error && rows > best_rows)) { + best_rows = rows; + best_cols = cols; + min_error = error; + } + } + } + p.slice_rows = best_rows; + p.slice_cols = best_cols; + } + const bool sliced = p.slice_rows > 0; + + const auto piece = [&](int64_t h, int64_t w) { + if (h > (int64_t(1) << 24) || w > (int64_t(1) << 24)) throw std::runtime_error("the image plan does not fit"); + PiecePlan q; + q.height = int(h); + q.width = int(w); + q.grid_h = q.height / patch; + q.grid_w = q.width / patch; + q.rows = (q.grid_h / 4) * (q.grid_w / 4); + return q; + }; + int64_t sh = 0, sw = 0; + best_resize(H, W, scale_resolution, patch, !sliced, &sh, &sw); // a small image is scaled UP to the resolution + p.source = piece(sh, sw); + + if (sliced) { + const int64_t refine_w = ensure_divide(W, p.slice_cols), refine_h = ensure_divide(H, p.slice_rows); + int64_t bh = 0, bw = 0; + best_resize(double(refine_h) / double(p.slice_rows), double(refine_w) / double(p.slice_cols), scale_resolution, patch, true, + &bh, &bw); + p.slice = piece(bh, bw); + p.refine_height = p.slice.height * p.slice_rows; + p.refine_width = p.slice.width * p.slice_cols; + } + return p; +} + +} // namespace onebit::npu::vision diff --git a/tests/npu_vision_image_test.cpp b/tests/npu_vision_image_test.cpp new file mode 100644 index 000000000..6fc28fad9 --- /dev/null +++ b/tests/npu_vision_image_test.cpp @@ -0,0 +1,220 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_vision_image_test +// +// The image path before the tower (npu/vision_image.h, engine#1788). +// +// resize_bicubic_u8 against torch 2.13's torch.nn.functional.interpolate(uint8, mode="bicubic", antialias=True): for +// each case the image below is resized by both, and the reference's bytes are held here as an FNV-1a hash of the whole +// output plus three sample bytes. The image is integer arithmetic, so it is the same picture in any language: +// byte(y, x, c) = (x*(7+c) + y*(13-c) + ((x*y) % 31)*5 + c*29 + ((x/9 + y/5) % 2)*90) & 255 +// preprocess(): the pieces, their order, and that each value is (byte - 127.5) / 127.5 of the resized image. +#include "../npu/vision_image.h" +#include "../npu/vision_tower.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace vis = onebit::npu::vision; + +int g_fail = 0, g_arms = 0; +void check(bool ok, const std::string& what) { + ++g_arms; + if (!ok) { + ++g_fail; + std::printf("FAIL %s\n", what.c_str()); + } +} + +std::vector image(int h, int w) { + std::vector v(size_t(h) * w * 3); + for (int y = 0; y < h; ++y) + for (int x = 0; x < w; ++x) + for (int c = 0; c < 3; ++c) + v[(size_t(y) * w + x) * 3 + c] = + uint8_t((x * (7 + c) + y * (13 - c) + ((x * y) % 31) * 5 + c * 29 + ((x / 9 + y / 5) % 2) * 90) & 255); + return v; +} + +uint64_t fnv(const std::vector& b) { + uint64_t h = 0xcbf29ce484222325ULL; + for (uint8_t v : b) h = (h ^ v) * 0x100000001b3ULL; + return h; +} + +// {height, width, out height, out width, hash of the reference's output, {byte (0,0,0), byte (oh/2, ow/3, 1), byte (oh-1, ow-1, 2)}} +struct Case { + int h, w, oh, ow; + uint64_t hash; + int sample[3]; +}; +const Case kCases[] = { + {37, 53, 56, 112, 0x183132e60e013115ULL, {0, 52, 214}}, // up in both + {480, 640, 392, 504, 0xa3482dec42670b72ULL, {3, 169, 175}}, // down in both: a 480 x 640 image's whole view + {97, 211, 448, 448, 0x71de32691a84dd1fULL, {0, 202, 255}}, // up in one, up more in the other + {300, 200, 168, 112, 0xd94cee2b35fef077ULL, {9, 201, 197}}, + {64, 64, 56, 56, 0x909ccbe6090e462dULL, {2, 161, 119}}, // a slight shrink + {1000, 30, 224, 56, 0x1a51666a837eb165ULL, {25, 137, 139}}, // down 4.5x in one, up in the other + {50, 50, 50, 112, 0x5bf628c4db3c6e45ULL, {0, 71, 79}}, // height unchanged: that pass is skipped + {56, 56, 56, 56, 0x97d7e9db2a2df6b3ULL, {0, 127, 58}}, // nothing to do: the image comes back + {3, 5, 56, 56, 0xfe99d14c4912f969ULL, {0, 56, 162}}, // fewer input pixels than taps + {1, 1, 448, 448, 0x9053309dc853d325ULL, {0, 29, 58}}, + {480, 640, 560, 784, 0x4c864ee63453d9d4ULL, {0, 91, 158}}, // the same image's refine size (1 x 2 slices) +}; + +} // namespace + +int main() { + for (const Case& c : kCases) { + const std::vector in = image(c.h, c.w); + const std::vector out = vis::resize_bicubic_u8(in.data(), c.h, c.w, c.oh, c.ow); + const std::string name = std::to_string(c.h) + " x " + std::to_string(c.w) + " -> " + std::to_string(c.oh) + " x " + std::to_string(c.ow); + const bool sized = out.size() == size_t(c.oh) * c.ow * 3; + check(sized, name + ": the output has the asked size"); + if (!sized) continue; + const int s0 = out[0], s1 = out[(size_t(c.oh / 2) * c.ow + size_t(c.ow / 3)) * 3 + 1], + s2 = out[(size_t(c.oh - 1) * c.ow + size_t(c.ow - 1)) * 3 + 2]; + check(s0 == c.sample[0] && s1 == c.sample[1] && s2 == c.sample[2], + name + ": three sample bytes are the reference's (" + std::to_string(s0) + ", " + std::to_string(s1) + ", " + std::to_string(s2) + ")"); + check(fnv(out) == c.hash, name + ": every byte is the reference's"); + } + { + std::string why; + const std::vector in = image(4, 4); + try { vis::resize_bicubic_u8(in.data(), 4, 4, 0, 8); } catch (const std::exception& e) { why = e.what(); } + check(why.find("cannot resize") != std::string::npos, "an empty output size is refused"); + } + + // preprocess: a 480 x 640 image is one whole view (392 x 504) and 1 x 2 slices of 560 x 392 cut from 560 x 784 + { + const int h = 480, w = 640; + const std::vector in = image(h, w); + const vis::ImagePlan plan = vis::plan_image(h, w); + const std::vector pieces = vis::preprocess(in.data(), h, w, plan); + check(pieces.size() == 3 && plan.slice_rows == 1 && plan.slice_cols == 2, "480 x 640: the whole view and two slices"); + if (pieces.size() == 3) { + check(pieces[0].grid_h == 28 && pieces[0].grid_w == 36 && pieces[0].pixels.size() == size_t(3) * 392 * 504, + "the whole view comes first, 28 x 36 patches"); + check(pieces[1].grid_h == 40 && pieces[1].grid_w == 28 && pieces[2].pixels.size() == size_t(3) * 560 * 392, + "then the slices, 40 x 28 patches each"); + const std::vector whole = vis::resize_bicubic_u8(in.data(), h, w, 392, 504); + const std::vector refine = vis::resize_bicubic_u8(in.data(), h, w, 560, 784); + bool ok0 = true, ok1 = true, ok2 = true; + for (int c = 0; c < 3; ++c) + for (int y = 0; y < 392; y += 7) + for (int x = 0; x < 504; x += 5) + ok0 = ok0 && pieces[0].pixels[(size_t(c) * 392 + y) * 504 + x] == + (float(whole[(size_t(y) * 504 + x) * 3 + c]) - 127.5f) / 127.5f; + for (int c = 0; c < 3; ++c) + for (int y = 0; y < 560; y += 7) + for (int x = 0; x < 392; x += 5) { + ok1 = ok1 && pieces[1].pixels[(size_t(c) * 560 + y) * 392 + x] == + (float(refine[(size_t(y) * 784 + x) * 3 + c]) - 127.5f) / 127.5f; + ok2 = ok2 && pieces[2].pixels[(size_t(c) * 560 + y) * 392 + x] == + (float(refine[(size_t(y) * 784 + 392 + x) * 3 + c]) - 127.5f) / 127.5f; + } + check(ok0, "the whole view is the resized image as planes, (byte - 127.5) / 127.5"); + check(ok1, "slice 0 is the left half of the refine image"); + check(ok2, "slice 1 is the right half of the refine image"); + float lo = 2, hi = -2; + for (float v : pieces[0].pixels) { lo = v < lo ? v : lo; hi = v > hi ? v : hi; } + check(lo >= -1.0f && hi <= 1.0f && lo < -0.9f && hi > 0.9f, "values span [-1, 1]"); + } + } + // The constants, on bytes whose answer is known without a reference: an image that is one colour, (0, 255, 128) in + // R, G, B order. Channel order, the 0..255 -> [-1, 1] range and the mean/std all show here: a swapped channel + // moves the planes, a 0..1 range leaves every value near -1, and a 1/128 divisor misses +1. + { + std::vector in(size_t(60) * 90 * 3); + for (size_t i = 0; i < in.size(); i += 3) { in[i] = 0; in[i + 1] = 255; in[i + 2] = 128; } + const std::vector big = vis::resize_bicubic_u8(in.data(), 60, 90, 448, 672); + bool flat = true; + for (size_t i = 0; i < big.size(); i += 3) flat = flat && big[i] == 0 && big[i + 1] == 255 && big[i + 2] == 128; + check(flat, "a one-colour image is the same colour after the resize, every byte"); + const vis::ImagePlan plan = vis::plan_image(60, 90); + const std::vector pieces = vis::preprocess(in.data(), 60, 90, plan); + bool r = !pieces.empty(), g = r, b = r; + if (!pieces.empty()) { + const size_t plane = pieces[0].pixels.size() / 3; + for (size_t i = 0; i < plane; ++i) { + r = r && pieces[0].pixels[i] == -1.0f; + g = g && pieces[0].pixels[plane + i] == 1.0f; + b = b && pieces[0].pixels[2 * plane + i] == (128.0f - 127.5f) / 127.5f; + } + } + check(r, "plane 0 is red: byte 0 is exactly -1"); + check(g, "plane 1 is green: byte 255 is exactly +1"); + check(b, "plane 2 is blue: byte 128 is 0.5 / 127.5"); + } + // a small image: scaled up to one 448 x 448 view, no slices + { + const std::vector in = image(100, 100); + const std::vector pieces = vis::preprocess(in.data(), 100, 100, vis::plan_image(100, 100)); + check(pieces.size() == 1 && pieces[0].grid_h == 32 && pieces[0].grid_w == 32, "100 x 100: one 32 x 32-patch view"); + } + // End to end on a published pack: bytes -> plan -> resize -> pieces -> tower, against a reference run of the same + // image (the reference's plan, torch's resize, the transformers tower). ONEBIT_MINICPM_E2E holds image.txt + // ("height width") and out_.f32 for each piece. + const char* pack = std::getenv("ONEBIT_MINICPM_PACK"); + const char* e2e = std::getenv("ONEBIT_MINICPM_E2E"); + if (pack && *pack && e2e && *e2e) { + try { + int h = 0, w = 0; + { std::ifstream f(std::string(e2e) + "/image.txt"); f >> h >> w; } + const std::vector in = image(h, w); + const vis::ImagePlan plan = vis::plan_image(h, w); + const std::vector pieces = vis::preprocess(in.data(), h, w, plan); + const vis::Pack p(pack); + std::printf(" %d x %d image: %zu piece(s), %d text rows\n", h, w, pieces.size(), plan.total_rows()); + for (size_t k = 0; k < pieces.size(); ++k) { + const std::vector out = vis::tower_forward(p, pieces[k].pixels.data(), pieces[k].grid_h, pieces[k].grid_w); + std::vector want; + { + std::ifstream f(std::string(e2e) + "/out_" + std::to_string(k) + ".f32", std::ios::binary | std::ios::ate); + if (f) { + want.resize(size_t(f.tellg()) / 4); + f.seekg(0); + f.read(reinterpret_cast(want.data()), std::streamsize(want.size() * 4)); + } + } + double mx = 0, sa = 0, sb = 0, saa = 0, sbb = 0, sab = 0; + const size_t n = out.size() < want.size() ? out.size() : want.size(); + for (size_t i = 0; i < n; ++i) { + mx = std::fmax(mx, std::fabs(double(out[i]) - want[i])); + sa += out[i]; sb += want[i]; saa += double(out[i]) * out[i]; sbb += double(want[i]) * want[i]; sab += double(out[i]) * want[i]; + } + const double corr = n ? (sab - sa * sb / double(n)) / std::sqrt((saa - sa * sa / double(n)) * (sbb - sb * sb / double(n))) : 0; + std::printf(" piece %zu (%d x %d patches): %zu values, max |d| %.3g, corr %.9f\n", k, pieces[k].grid_h, pieces[k].grid_w, + out.size(), mx, corr); + check(out.size() == want.size() && !out.empty() && corr > 0.99999 && mx < 5e-3, + "published pack, piece " + std::to_string(k) + ": the rows are the reference's"); + } + } catch (const std::exception& e) { + check(false, std::string("the published pack runs end to end (threw: ") + e.what() + ")"); + } + } else { + std::printf(" (ONEBIT_MINICPM_PACK / ONEBIT_MINICPM_E2E not set: the end-to-end arm did not run)\n"); + } + std::printf("vision image: %d checks, %d failed\n", g_arms, g_fail); + return g_fail ? 1 : 0; +} diff --git a/tests/npu_vision_plan_test.cpp b/tests/npu_vision_plan_test.cpp new file mode 100644 index 000000000..ca34e38af --- /dev/null +++ b/tests/npu_vision_plan_test.cpp @@ -0,0 +1,150 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_vision_plan_test +// +// The image plan (npu/vision_plan.h, engine#1788) against transformers' MiniCPMV4_6ImageProcessor 5.22.2: the methods +// find_best_resize, get_refine_size and get_sliced_grid, called as the processor's _preprocess calls them, for every +// pair of the 59 sizes below (3,481 images). The eight numbers of each plan are folded into one FNV-1a hash, compared +// with the hash the reference produces for the same walk; sixteen plans are also written out so a failure says where. +#include "../npu/vision_plan.h" + +#include +#include +#include +#include + +namespace { + +namespace vis = onebit::npu::vision; + +int g_fail = 0, g_arms = 0; +void check(bool ok, const std::string& what) { + ++g_arms; + if (!ok) { + ++g_fail; + std::printf("FAIL %s\n", what.c_str()); + } +} + +const int kSizes[] = {1, 2, 3, 7, 13, 14, 27, 28, 55, 56, 57, 100, 111, 224, 300, + 336, 392, 419, 420, 447, 448, 449, 476, 500, 503, 504, 600, 640, 672, 700, + 720, 768, 800, 896, 900, 960, 1000, 1024, 1080, 1200, 1280, 1344, 1440, 1536, 1600, + 1792, 1920, 2000, 2048, 2160, 2560, 3000, 3024, 3840, 4032, 4096, 5000, 6000, 8000}; + +// {height, width, {source h, source w, slice rows, slice cols, refine h, refine w, slice h, slice w}}, from the reference +struct Row { + int h, w, plan[8]; +}; +const Row kRows[] = { + {448, 448, {448, 448, 0, 0, 0, 0, 0, 0}}, + {1, 1, {448, 448, 0, 0, 0, 0, 0, 0}}, + {100, 100, {448, 448, 0, 0, 0, 0, 0, 0}}, + {449, 448, {448, 448, 2, 1, 672, 616, 336, 616}}, + {480, 640, {392, 504, 1, 2, 560, 784, 560, 392}}, + {720, 1280, {336, 616, 2, 3, 784, 1512, 392, 504}}, + {1080, 1920, {336, 616, 2, 4, 896, 1792, 448, 448}}, + {1920, 1080, {616, 336, 4, 2, 1792, 896, 448, 448}}, + {3024, 4032, {392, 504, 3, 3, 1176, 1512, 392, 504}}, + {224, 2000, {168, 1344, 1, 4, 280, 2688, 280, 672}}, + {5000, 300, {1848, 112, 9, 1, 5544, 336, 616, 336}}, + {896, 896, {448, 448, 2, 2, 896, 896, 448, 448}}, + {1344, 1344, {448, 448, 3, 3, 1344, 1344, 448, 448}}, + {600, 800, {392, 504, 2, 2, 784, 1008, 392, 504}}, + {8000, 8000, {448, 448, 3, 3, 1344, 1344, 448, 448}}, + {14, 8000, {56, 10304, 0, 0, 0, 0, 0, 0}}, + // sizes that are not multiples of anything: resized, never refused, never cropped + {385, 384, {448, 448, 0, 0, 0, 0, 0, 0}}, + {447, 449, {448, 448, 0, 0, 0, 0, 0, 0}}, + {449, 449, {448, 448, 2, 1, 672, 616, 336, 616}}, + {1, 2, {336, 616, 0, 0, 0, 0, 0, 0}}, + // the largest sizes the plan accepts. The strips are the reference's answers too, absurd as they are (a 56-pixel + // wide piece 458,752 pixels tall): the plan is arithmetic, and it is the caller that bounds total_rows(). + {1048576, 1048576, {448, 448, 3, 3, 1344, 1344, 448, 448}}, + {1, 1048576, {56, 56, 1, 7, 56, 1048600, 56, 149800}}, + {1048576, 1, {458752, 56, 7, 1, 1213632, 56, 173376, 56}}, + {1048576, 3, {264880, 56, 9, 1, 794808, 56, 88312, 56}}, +}; +const uint64_t kReferenceHash = 0x8480087394bf204fULL; // the walk below, by the reference +const int kReferenceMaxRows = 716; // the most text rows any of the 3,481 plans gives + +void eight(const vis::ImagePlan& p, int out[8]) { + out[0] = p.source.height; + out[1] = p.source.width; + out[2] = p.slice_rows; + out[3] = p.slice_cols; + out[4] = p.refine_height; + out[5] = p.refine_width; + out[6] = p.slice.height; + out[7] = p.slice.width; +} + +} // namespace + +int main() { + for (const Row& r : kRows) { + int got[8]; + eight(vis::plan_image(r.h, r.w), got); + bool same = true; + for (int i = 0; i < 8; ++i) same = same && got[i] == r.plan[i]; + check(same, std::to_string(r.h) + " x " + std::to_string(r.w) + ": source " + std::to_string(got[0]) + " x " + + std::to_string(got[1]) + ", grid " + std::to_string(got[2]) + " x " + std::to_string(got[3]) + ", refine " + + std::to_string(got[4]) + " x " + std::to_string(got[5]) + ", slice " + std::to_string(got[6]) + " x " + + std::to_string(got[7]) + " is the reference's plan"); + } + + uint64_t h = 0xcbf29ce484222325ULL; + int max_rows = 0, pairs = 0; + bool shapes_ok = true, tiles_ok = true; + for (int height : kSizes) + for (int width : kSizes) { + const vis::ImagePlan p = vis::plan_image(height, width); + int v[8]; + eight(p, v); + for (int x : v) + for (int b = 0; b < 4; ++b) h = (h ^ uint64_t((uint32_t(x) >> (8 * b)) & 0xff)) * 0x100000001b3ULL; + if (p.total_rows() > max_rows) max_rows = p.total_rows(); + ++pairs; + // what the tower needs of every piece: both patch extents multiples of 4, and at least one row + const auto ok = [](const vis::PiecePlan& q) { + return q.grid_h > 0 && q.grid_w > 0 && q.grid_h % 4 == 0 && q.grid_w % 4 == 0 && q.height == q.grid_h * 14 && + q.width == q.grid_w * 14 && q.rows == q.grid_h * q.grid_w / 16; + }; + shapes_ok = shapes_ok && ok(p.source) && (p.slice_rows == 0 || ok(p.slice)) && p.slice_rows * p.slice_cols <= 9 && + p.pieces() == 1 + p.slice_rows * p.slice_cols; + // nothing is cropped or padded: the slices tile the refine size exactly, with no remainder on either axis + tiles_ok = tiles_ok && p.refine_height == p.slice.height * p.slice_rows && p.refine_width == p.slice.width * p.slice_cols; + } + std::printf(" %d image sizes: hash 0x%016llx (reference 0x%016llx), most text rows %d\n", pairs, (unsigned long long)h, + (unsigned long long)kReferenceHash, max_rows); + check(h == kReferenceHash, "all 3,481 plans are the reference's (one hash over every number)"); + check(max_rows == kReferenceMaxRows, "the largest plan gives the reference's row count"); + check(shapes_ok, "every piece of every plan is a multiple of 4 patches in both extents, and at most 9 slices"); + check(tiles_ok, "the slices of every plan tile its refine size exactly: no remainder to crop or pad"); + + { + const vis::ImagePlan p = vis::plan_image(1080, 1920); + check(p.pieces() == 9 && p.source.rows == 6 * 11 && p.slice.rows == 64 && p.total_rows() == 66 + 8 * 64, + "1080 x 1920: one 24 x 44 source (66 rows) and eight 32 x 32 slices (64 rows each), 578 rows"); + } + // one past the largest size on each axis, and the non-sizes + for (const auto& [hh, ww] : std::vector>{{0, 100}, {100, 0}, {-1, 5}, {1 << 21, 10}, {(1 << 20) + 1, 448}, {448, (1 << 20) + 1}}) { + std::string why; + try { vis::plan_image(hh, ww); } catch (const std::exception& e) { why = e.what(); } + check(why.find("cannot be planned") != std::string::npos, std::to_string(hh) + " x " + std::to_string(ww) + " is refused"); + } + std::printf("vision plan: %d checks, %d failed\n", g_arms, g_fail); + return g_fail ? 1 : 0; +} From 3725f12e880fd889515e104d30d8ec0f1edbfa5a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:08:33 -0400 Subject: [PATCH 61/98] forward-serve: the tokenizer's eos_token and the ChatML turn end are stop ids too (Refs #1785, #1786) (#2064) FastFlowLM's Qwen3.5-2B container names <|endoftext|> in config.json and <|im_end|> in tokenizer_config.json; the model ends a chat turn with the second, which was served as text with one more token after it. npu::stop_token_ids(): config.json's eos_token_id, then the tokenizer's eos_token, then the chat route's own turn end when it renders ChatML. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- app/forward_serve.cpp | 15 ++++++++++ app/npu_stop_tokens.h | 55 ++++++++++++++++++++++++++++++++++ docs/npu.md | 6 ++++ tests/npu_stop_tokens_test.cpp | 42 ++++++++++++++++++++++++++ 4 files changed, 118 insertions(+) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 4a43ab585..0ead20865 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -1088,6 +1088,7 @@ int run_forward_serve(int argc, char** argv) { e.bos_id = json_int(cfg, "bos_token_id", -1); e.add_bos = json_int(cfg, "add_bos_token", 0) != 0; e.eos = npu::eos_token_ids_from_config(cfg); + const size_t eos_from_config = e.eos.size(); e.endoftext = e.tok->token_id("<|endoftext|>"); // q4nx_config_int finds the key at any depth (a text_config's too). e.max_context = q4nx_config_int(cfg.c_str(), "max_position_embeddings", 0); @@ -1143,6 +1144,20 @@ int run_forward_serve(int argc, char** argv) { (specials.empty() ? std::string() : " (model specials: " + specials + ")"); std::fprintf(stderr, "1bit forward-serve: chat route disabled: %s\n", e.chat_refusal.c_str()); } + // What ends a generation is more than config.json's eos_token_id (npu::stop_token_ids): the tokenizer's own + // eos_token, and, when this route closes its prompt turns with ChatML's marker, that marker. Decided here, + // after the template, because whether the route renders ChatML is known only now. + { + const bool chatml_route = !can_render && e.chat_ok; + e.eos = npu::stop_token_ids(npu::read_json_or_null(cfg), npu::read_json_or_null(dir + "/tokenizer_config.json"), + chatml_route ? std::string("<|im_end|>") : std::string(), + [&](const std::string& text) { return e.tok->token_id(text); }); + std::string named; + for (size_t i = 0; i < e.eos.size(); i++) + named += std::string(i ? ", " : "") + std::to_string(e.eos[i]) + " " + e.tok->decode(e.eos[i]) + + (i < eos_from_config ? "" : " (tokenizer / chat turn end)"); + std::fprintf(stderr, "1bit forward-serve: stops on %s\n", named.empty() ? "<|endoftext|> only" : named.c_str()); + } // The template variables `{{ bos_token }}`/`{{ eos_token }}` render: the strings the tokenizer // actually has (a null bos_token in tokenizer_config.json is "", and the engine's -1 is the same // fact), not a second read of the JSON that could disagree with the tokenizer. diff --git a/app/npu_stop_tokens.h b/app/npu_stop_tokens.h index aa910004c..d3fe3c3f7 100644 --- a/app/npu_stop_tokens.h +++ b/app/npu_stop_tokens.h @@ -56,6 +56,61 @@ inline std::vector eos_token_ids(const nlohmann::json& cfg) { return ids; } +/// tokenizer_config.json's `eos_token` as text: a plain string, or the AddedToken object form +/// `{"content": "<|im_end|>", ...}`. Empty when the key is absent, null or another shape. +inline std::string eos_token_text(const nlohmann::json& tokenizer_config) { + if (!tokenizer_config.is_object() || !tokenizer_config.contains("eos_token")) return {}; + const nlohmann::json& e = tokenizer_config["eos_token"]; + if (e.is_string()) return e.get(); + if (e.is_object() && e.contains("content") && e["content"].is_string()) return e["content"].get(); + return {}; +} + +/// Every id that ends a generation on a container, in the order they were found and without repeats: +/// +/// 1. config.json's `eos_token_id` (one integer or a list): what eos_token_ids() reads; +/// 2. tokenizer_config.json's `eos_token`, looked up in the tokenizer. The two files are written by +/// different tools and need not agree. FastFlowLM's Qwen3.5-2B container says `eos_token_id: 248044` +/// (<|endoftext|>) in config.json and `eos_token: "<|im_end|>"` (248046) in tokenizer_config.json, and the +/// model ends a chat turn with the second. With only the first, the answer came back as +/// "Item 3 is the 3D printer.<|im_end|>\n": the turn end served as text and one more token generated +/// after it (engine#1785's row on that container); +/// 3. `chat_turn_end`, when the caller's chat route renders a prompt whose turns end with a marker of its +/// own (ChatML's <|im_end|>): a prompt we close with that marker is answered by a turn that ends with it, +/// whatever the two files say. Pass an empty string when the route has no such marker. +/// +/// `id_of` maps a token's text to its id, or a negative number when the tokenizer has no such token; a name it +/// does not know adds nothing. +template +std::vector stop_token_ids(const nlohmann::json& config, const nlohmann::json& tokenizer_config, + const std::string& chat_turn_end, IdOf&& id_of) { + std::vector ids = eos_token_ids(config); + const auto add = [&](const std::string& text) { + if (text.empty()) return; + const int id = id_of(text); + if (id < 0) return; + for (int have : ids) + if (have == id) return; + ids.push_back(id); + }; + add(eos_token_text(tokenizer_config)); + add(chat_turn_end); + return ids; +} + +/// A JSON file, or null when it is missing or malformed: both stop-id sources are optional. +inline nlohmann::json read_json_or_null(const std::string& path) { + std::ifstream f(path); + if (!f) return nullptr; + nlohmann::json j; + try { + f >> j; + } catch (const std::exception&) { + return nullptr; + } + return j; +} + /// eos_token_ids() for a container's config.json. Unreadable file -> empty. inline std::vector eos_token_ids_from_config(const std::string& cfg_path) { std::ifstream f(cfg_path); diff --git a/docs/npu.md b/docs/npu.md index 3ace1d57f..71ac34ff1 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -68,6 +68,12 @@ curl -s http://127.0.0.1:8000/v1/chat/completions -H 'Content-Type: application/ npu/ the model's NPU kernels ``` +**What ends an answer.** The serve stops on every id in `config.json`'s `eos_token_id` (one integer or a list), on +`<|endoftext|>` when the tokenizer has it, on `tokenizer_config.json`'s `eos_token` when that file is beside the +tokenizer, and, when the chat route falls back to ChatML, on `<|im_end|>`. The two files need not agree: FastFlowLM's +`Qwen3.5-2B-NPU2` container names `<|endoftext|>` in the first and `<|im_end|>` in the second, and the model ends a +chat turn with the second. The load prints the list (`forward-serve: stops on …`). + `1bit serve -m ` serves such a directory on the NPU behind the OpenAI-compatible API ([serve.md](serve.md)); inside Lemonade, that is what its `onebit` backend runs (`--onebit npu`). diff --git a/tests/npu_stop_tokens_test.cpp b/tests/npu_stop_tokens_test.cpp index 3779d786f..6ff5c20ae 100644 --- a/tests/npu_stop_tokens_test.cpp +++ b/tests/npu_stop_tokens_test.cpp @@ -79,6 +79,48 @@ int main() { "unreadable config"); std::remove(cfg.c_str()); + // 6. the tokenizer's eos_token and the chat route's turn end (engine#1785, FastFlowLM's Qwen3.5-2B container): + // config.json names <|endoftext|>, tokenizer_config.json names <|im_end|>, and the model ends its turn with + // the second, which was served as text. + { + using onebit::npu::eos_token_text; + using onebit::npu::stop_token_ids; + const auto id_of = [](const std::string& t) { + if (t == "<|endoftext|>") return 248044; + if (t == "<|im_end|>") return 248046; + if (t == "") return 106; + return -1; + }; + const json cfg = json::parse(R"({"eos_token_id": 248044})"); + const json tok = json::parse(R"({"eos_token": "<|im_end|>"})"); + check(eos_token_text(tok) == "<|im_end|>", "eos_token as a plain string"); + check(eos_token_text(json::parse(R"({"eos_token": {"content": "<|im_end|>", "lstrip": false}})")) == "<|im_end|>", + "eos_token as an AddedToken object"); + check(eos_token_text(json::parse(R"({"eos_token": null})")).empty() && eos_token_text(json::parse("{}")).empty() && + eos_token_text(json::parse(R"({"eos_token": 7})")).empty() && eos_token_text(json(nullptr)).empty(), + "a missing, null or non-text eos_token is no token"); + check_ids(stop_token_ids(cfg, tok, "", id_of), {248044, 248046}, + "the tokenizer's eos_token is a stop id beside config.json's (the token that was served as text)"); + check_ids(stop_token_ids(cfg, json(nullptr), "", id_of), {248044}, "with no tokenizer_config the list is config.json's, unchanged"); + check_ids(stop_token_ids(cfg, json(nullptr), "<|im_end|>", id_of), {248044, 248046}, + "a route that closes its turns with <|im_end|> stops on it even when neither file names it"); + check_ids(stop_token_ids(cfg, tok, "<|im_end|>", id_of), {248044, 248046}, "named twice, listed once"); + check_ids(stop_token_ids(json::parse(R"({"eos_token_id": [248046, 248044]})"), tok, "<|im_end|>", id_of), {248046, 248044}, + "config.json's order is kept and nothing is repeated"); + check_ids(stop_token_ids(cfg, json::parse(R"({"eos_token": "<|eot_id|>"})"), "<|nope|>", id_of), {248044}, + "a name the tokenizer does not have adds nothing"); + check_ids(stop_token_ids(json::parse("{}"), json::parse(R"({"eos_token": ""})"), "", id_of), {106}, + "with no eos_token_id at all, the tokenizer's eos_token is the list"); + check_ids(stop_token_ids(json(nullptr), json(nullptr), "", id_of), {}, "nothing readable, nothing added"); + const std::string f = write_cfg("tokcfg", R"({"eos_token": "<|im_end|>"})"); + check(eos_token_text(onebit::npu::read_json_or_null(f)) == "<|im_end|>", "read from a file"); + check(onebit::npu::read_json_or_null("/tmp/npu_stop_tokens_does_not_exist.json").is_null(), "a missing file is null"); + const std::string bad = write_cfg("badjson", "{ not json"); + check(onebit::npu::read_json_or_null(bad).is_null(), "a malformed file is null"); + std::remove(f.c_str()); + std::remove(bad.c_str()); + } + if (failures == 0) std::printf("all stop-token checks passed\n"); return failures == 0 ? 0 : 1; } From c4826e6e14af01b7b8b4524d8381c60a80f5b398 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:30:41 -0400 Subject: [PATCH 62/98] tools: hrx_dispatch_profile.py names unfinished dispatches, and its self-test survives python -O (engine #679) (#2096) Follow-up to #2084, from its reviews: - a dispatch that was launched and never finished (killed run, cut trace) was dropped without a word; it is now listed under the tables with its kernel, and stated as not counted - the self-test used assert, which python -O strips: a broken arm passed. It uses a check() helper now - the output states the clock and the unit; --help documents the kept trace's line format - `parse` on a file that is not a trace says so (lines read, none a trace line) instead of asking about the build - --manifests can be repeated, for a build with extra kernel directories whose kernels showed as numeric ids - docs/hrx.md: shares are compared inside one program only (the serial mode slows decode more than prefill) Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/hrx.md | 3 + tools/hrx_dispatch_profile.py | 125 +++++++++++++++++++++++++++------- 2 files changed, 102 insertions(+), 26 deletions(-) diff --git a/docs/hrx.md b/docs/hrx.md index 7870a0af9..cf6ab2ac7 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -891,6 +891,9 @@ The matmuls cost the same at any context; the attention kernel is the part that What the numbers are and are not: each is the wall time from launch to device-idle with one dispatch in flight, so it holds one wake-up of the waiting thread and the run is slower than a normal one (the same command: pp512 2,977 -> 2,500 tok/s, tg16 67 -> 36, pp8192 1,191 -> 1,167). Use it to rank kernels and to see what grows with the context, not as a rate. +Decode is slowed much more than prefill, so compare shares inside one program and never between programs. A dispatch that +was launched and never finished (a ticket that ran out, a cut trace) is listed under the tables and is not in them. A build +with extra kernel directories shows those kernels as numeric ids; name them with a second `--manifests DIR`. ### Measuring on `HRX0`: what makes a number citable diff --git a/tools/hrx_dispatch_profile.py b/tools/hrx_dispatch_profile.py index ec7edbb37..026137865 100755 --- a/tools/hrx_dispatch_profile.py +++ b/tools/hrx_dispatch_profile.py @@ -27,7 +27,19 @@ What the numbers are: the wall time from "dispatch launched" to "device idle", one dispatch at a time. That is the kernel's device time plus one wake-up of the waiting thread, so a sum over a graph is larger than the same graph in a normal run, where dispatches are queued back to back. Use it to see WHICH kernel a graph's time is in and how -that changes with the context length, not as a tok/s figure. Run the command under gpu-hold like any device run. +that changes with the context length, not as a tok/s figure. The serial mode slows decode much more than prefill, +so compare shares inside one program, never between programs. Run the command under gpu-hold like any device run. + +Times are milliseconds from time.monotonic_ns() (CLOCK_MONOTONIC, nanosecond steps), read when the trace line arrives. + +A kept trace (`run --log`, the input of `parse`) is one line per trace line, the arrival time in ns and the line: + hrx debug serial: begin phase=command-dispatch list=main index=0 list_commands=612 ordinal=0 kind=Kernel kernel_id= bindings=3 + hrx debug serial: end phase=command-dispatch ... the launch returned + hrx debug serial: sync-end phase=command-post-dispatch ... the device finished it +A dispatch is the three lines with the same list, list_commands and index. Lines of other phases are ignored. + +Kernels of a build with extra kernel directories are not in the fork's manifests and show as a numeric id; give +their directory with a second --manifests. """ import argparse import collections @@ -75,7 +87,8 @@ def parse_fields(text): def profile(lines): - """lines: iterable of (time_ns, trace text without the prefix). Returns the per-program, per-kernel totals.""" + """lines: iterable of (time_ns, trace text without the prefix). + Returns (per-program per-kernel totals, kernel ids of the dispatches that were launched and never finished).""" programs = collections.OrderedDict() open_dispatch = {} for time_ns, text in lines: @@ -86,21 +99,24 @@ def profile(lines): continue key = (fields.get("list"), fields.get("list_commands"), fields["index"]) if event == "begin" and phase == "command-dispatch": - open_dispatch[key] = [time_ns, None] + open_dispatch[key] = [time_ns, None, int(fields["kernel_id"])] elif event == "end" and phase == "command-dispatch" and key in open_dispatch: open_dispatch[key][1] = time_ns elif event == "sync-end" and phase == "command-post-dispatch" and key in open_dispatch: - begin_ns, launched_ns = open_dispatch.pop(key) + begin_ns, launched_ns, _ = open_dispatch.pop(key) program = programs.setdefault((fields.get("list"), int(fields.get("list_commands", 0))), {}) row = program.setdefault(int(fields["kernel_id"]), {"dispatches": 0, "total_ns": 0, "launch_ns": 0}) row["dispatches"] += 1 row["total_ns"] += time_ns - begin_ns row["launch_ns"] += (launched_ns if launched_ns is not None else begin_ns) - begin_ns - return programs + return programs, [entry[2] for entry in open_dispatch.values()] -def report(programs, names, top, out=sys.stdout): +def report(programs, names, top, out=sys.stdout, unfinished=(), lines_read=None): result = [] + if programs: + print("times: ms, time.monotonic_ns() at the arrival of each trace line; launch to device-idle, serial mode", + file=out) for (list_kind, command_count), kernels in programs.items(): total_ns = sum(row["total_ns"] for row in kernels.values()) dispatches = sum(row["dispatches"] for row in kernels.values()) @@ -123,17 +139,29 @@ def report(programs, names, top, out=sys.stdout): if len(rows) > top: print(f" ... {len(rows) - top} more kernels (--top)", file=out) result.append(entry) + if unfinished: + kernels = sorted({names.get(kid, str(kid)) for kid in unfinished}) + print(f"\nnote: {len(unfinished)} dispatch(es) were launched and never finished (the run was killed or the " + f"trace is cut). Their time is NOT in the tables: {', '.join(kernels)}", file=out) if not result: - print("no dispatch was traced: is the binary an HRX build, and did the run reach the device?", file=out) + if lines_read is not None and lines_read[1] == 0: + print(f"no dispatch was traced: {lines_read[0]} line(s) read, none is a time-stamped " + f"'{TRACE_PREFIX.strip()}' line. Is this a trace kept by `run --log`?", file=out) + else: + print("no dispatch was traced: is the binary an HRX build, and did the run reach the device?", file=out) return result def read_log(path): - with open(path) as handle: + """Returns (trace lines, lines read).""" + lines, read = [], 0 + with open(path, errors="replace") as handle: for line in handle: + read += 1 stamp, _, text = line.rstrip("\n").partition(" ") if stamp.isdigit() and text.startswith(TRACE_PREFIX): - yield int(stamp), text[len(TRACE_PREFIX):] + lines.append((int(stamp), text[len(TRACE_PREFIX):])) + return lines, read def run_command(command, log_path): @@ -158,7 +186,13 @@ def run_command(command, log_path): def self_test(): - assert kernel_id("loom_libs", "ggml_mul_mat_f32_f32_wmma") == 5916100870017593149, "the catalog id hash changed" + fails = [] + + def check(name, got, want): + if got != want: + fails.append("%s: got %r, want %r" % (name, got, want)) + + check("catalog id hash", kernel_id("loom_libs", "ggml_mul_mat_f32_f32_wmma"), 5916100870017593149) detail = "list=main index={i} list_commands=2 ordinal={i} kind=Kernel kernel_id={k} bindings=3" trace = [] clock = 1000 @@ -173,17 +207,48 @@ def self_test(): trace.append((clock + launch + device, "sync-end phase=command-post-dispatch " + text)) clock += launch + device + 1000 # host time between dispatches is not charged to a kernel trace.append((clock, "begin phase=command-list list=main list_commands=2 main_commands=2")) - programs = profile(trace) - assert list(programs) == [("main", 2)], programs - assert programs[("main", 2)][7] == {"dispatches": 2, "total_ns": 200, "launch_ns": 20}, programs - assert programs[("main", 2)][9] == {"dispatches": 2, "total_ns": 50, "launch_ns": 10}, programs - with open(os.devnull, "w") as sink: - result = report(programs, {7: "f:slow", 9: "f:fast"}, 1, out=sink) - assert report({}, {}, 5, out=sink) == [] - assert [k["kernel"] for k in result[0]["kernels"]] == ["f:slow", "f:fast"], result - assert abs(result[0]["kernels"][0]["share"] - 0.8) < 1e-9, result - print("hrx_dispatch_profile self-test: OK") - return 0 + programs, unfinished = profile(trace) + check("programs", list(programs), [("main", 2)]) + check("slow kernel", programs.get(("main", 2), {}).get(7), {"dispatches": 2, "total_ns": 200, "launch_ns": 20}) + check("fast kernel", programs.get(("main", 2), {}).get(9), {"dispatches": 2, "total_ns": 50, "launch_ns": 10}) + check("all finished", unfinished, []) + + class Sink: + def __init__(self): + self.text = "" + + def write(self, text): + self.text += text + + sink = Sink() + result = report(programs, {7: "f:slow", 9: "f:fast"}, 1, out=sink) + check("order", [k["kernel"] for k in result[0]["kernels"]] if result else None, ["f:slow", "f:fast"]) + check("share", round(result[0]["kernels"][0]["share"], 9) if result else None, 0.8) + check("units in the header", "time.monotonic_ns()" in sink.text, True) + + # a dispatch that is launched and never finished is named, not dropped + cut = trace[:6] + [(9000, "begin phase=command-dispatch " + detail.format(i=1, k=9)), + (9005, "end phase=command-dispatch " + detail.format(i=1, k=9))] + programs, unfinished = profile(cut) + check("unfinished dispatch", unfinished, [9]) + check("finished one still counted", programs.get(("main", 2), {}).get(7, {}).get("dispatches"), 1) + sink = Sink() + report(programs, {9: "f:fast"}, 5, out=sink, unfinished=unfinished) + check("unfinished is reported", "1 dispatch(es) were launched and never finished" in sink.text + and "f:fast" in sink.text, True) + + # nothing traced: a file that is not a trace is told apart from a run that reached no device + sink = Sink() + check("empty result", report({}, {}, 5, out=sink, lines_read=(3, 0)), []) + check("wrong file message", "3 line(s) read, none is a time-stamped" in sink.text, True) + sink = Sink() + report({}, {}, 5, out=sink, lines_read=(3, 2)) + check("no device message", "is the binary an HRX build" in sink.text, True) + + for fail in fails: + print("FAIL " + fail) + print("hrx_dispatch_profile self-test: %s" % ("FAILED" if fails else "OK")) + return 1 if fails else 0 def main(): @@ -195,25 +260,33 @@ def main(): p = sub.add_parser(mode) p.add_argument("--top", type=int, default=12, help="kernels printed per program (default 12)") p.add_argument("--json", help="write the full table here") - p.add_argument("--manifests", default=DEFAULT_MANIFESTS, help="the fork's kernel-corpus/kernels directory") + p.add_argument("--manifests", action="append", + help="a kernel-corpus/kernels directory; repeat it for a build with extra kernel directories " + "(default: the pinned fork's)") if mode == "run": p.add_argument("--log", help="keep the time-stamped trace here (on disk, not /tmp)") p.add_argument("command", nargs=argparse.REMAINDER, help="-- CMD ARGS...") else: p.add_argument("log") args = parser.parse_args() - names = load_kernel_names(args.manifests) + names = {} + for directory in args.manifests or [DEFAULT_MANIFESTS]: + names.update(load_kernel_names(directory)) if not names: - print(f"note: no kernel manifest under {args.manifests}; kernels are shown by id", file=sys.stderr) + print(f"note: no kernel manifest under {args.manifests or DEFAULT_MANIFESTS}; kernels are shown by id", + file=sys.stderr) code = 0 + lines_read = None if args.mode == "run": command = args.command[1:] if args.command[:1] == ["--"] else args.command if not command: parser.error("run needs a command after --") lines, code = run_command(command, args.log) else: - lines = list(read_log(args.log)) - result = report(profile(lines), names, args.top) + lines, read = read_log(args.log) + lines_read = (read, len(lines)) + programs, unfinished = profile(lines) + result = report(programs, names, args.top, unfinished=unfinished, lines_read=lines_read) if args.json: with open(args.json, "w") as handle: json.dump(result, handle, indent=1) From 0b88efa59ec8bad556869e01f883e85826678ccc Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:30:47 -0400 Subject: [PATCH 63/98] weekly: the runner preflight names a restart-policy failure as one (refs #1504) (#2094) Since #2082, ci-runner-health.sh also fails a unit that is up but whose restart policy would leave it down after the next OOM kill. weekly.sh still summarised every such failure as "a runner unit is not accepting jobs". On strixhalo tonight all three runners are active/running, and only their policy fails. The behaviour is unchanged: the preflight still refuses. Only the summary line changes. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Opus 5.5 --- scripts/weekly.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/weekly.sh b/scripts/weekly.sh index ea861f9a8..468864481 100755 --- a/scripts/weekly.sh +++ b/scripts/weekly.sh @@ -706,7 +706,10 @@ preflight() { say "preflight: no actions.runner.* units on this host: systemd has nothing to inspect" else printf '%s\n' "$health_out" | sed 's/^/preflight: /' - echo "preflight: a runner unit is not accepting jobs (scripts/ci-runner-health.sh): $health_rc" + # Two kinds of FAIL reach here (engine#1504): a unit that is down NOW, and since #2082 a unit that is up + # but whose restart policy would leave it down after the next OOM kill. The FAIL lines above say which; + # this line must not claim "not accepting jobs" for the second, where every runner is active/running. + echo "preflight: a runner unit is down, or its restart policy would leave it down after the next OOM kill (scripts/ci-runner-health.sh exit $health_rc; the FAIL lines above say which, and scripts/install-ci-runner.sh fixes a policy line)" bad=1 fi fi From 75a89242fc9fb56ce616cf1526af7ca16cafb7c9 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:30:57 -0400 Subject: [PATCH 64/98] npu: the text side of a MiniCPM-V 4.7 image: its tokens and the canvas positions, matching the reference (engine#1788) (#2089) vision::image_tokens(plan, markers) is what replaces one <|image_pad|> in the prompt; vision::prompt_positions(ids, markers, grids) is the model's mrope_mode canvas rule: the (t, h, w) planes for a prompt with images, and where generation resumes. Arithmetic on token ids; nothing is fed to the text forward here. Test: npu_vision_prompt, against transformers 5.22.2 (replace_image_token, get_rope_index) for eight prompts, hashed over every id and position, plus torch's rounded linspace for 25,600 cases. Eight mutants caught; two are equivalent for image-only prompts and the header says so. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 5 + docs/npu.md | 22 +++ npu/vision_prompt.h | 252 +++++++++++++++++++++++++++++++ tests/npu_vision_prompt_test.cpp | 177 ++++++++++++++++++++++ 4 files changed, 456 insertions(+) create mode 100644 npu/vision_prompt.h create mode 100644 tests/npu_vision_prompt_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f60faae3..eeed01a8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1238,6 +1238,11 @@ add_executable(npu_vision_image_test tests/npu_vision_image_test.cpp) target_link_libraries(npu_vision_image_test PRIVATE onebit_npu) add_test(NAME npu_vision_image COMMAND npu_vision_image_test) add_dependencies(onebit npu_vision_image_test) +# The text side of an image: its tokens and the canvas positions, against the reference model code's own answers. +add_executable(npu_vision_prompt_test tests/npu_vision_prompt_test.cpp) +target_include_directories(npu_vision_prompt_test PRIVATE npu) +add_test(NAME npu_vision_prompt COMMAND npu_vision_prompt_test) +add_dependencies(onebit npu_vision_prompt_test) add_executable(npu_q4nx_test tests/npu_q4nx_test.cpp) target_include_directories(npu_q4nx_test PRIVATE app) target_link_libraries(npu_q4nx_test PRIVATE onebit_npu) diff --git a/docs/npu.md b/docs/npu.md index 71ac34ff1..92b38eef2 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -1638,6 +1638,28 @@ images on a CPU build with AVX2 (torchvision is not on this box, so that link is no real photograph or image decoder has been through it. The slice separators and placeholder rows on the text side are not written. +**The text side of an image (`npu/vision_prompt.h`).** Two more pieces of arithmetic, on token ids only: + +- `vision::image_tokens(plan, markers)`: what replaces one `<|image_pad|>` in the rendered prompt. + ``, one placeholder per row of the whole view, ``, then each row of slices as + `` placeholders `` per slice, with one newline token between rows. The tower's rows go + into the placeholders in order. (`k` in front is ordinary text, left to the + caller.) +- `vision::prompt_positions(ids, markers, grids)`: the three position planes (t, h, w) for a prompt that + contains images, the model's `mrope_mode: canvas`. Text counts 1, 2, 3 on all three planes. An image + is laid on a canvas whose origin is the next free position: the slices tile it, the whole view is + stretched over it, `` sits one step before it and `` on its far corner, and the text + after it resumes at `origin + max(canvas height, canvas width) + 1`. So an image moves the position + counter by its **side**, not by its row count: a 480 × 640 picture is 203 placeholder rows and + advances the position by 15. + +Both follow transformers' `MiniCPMV4_7Processor.replace_image_token` and +`MiniCPMV4_7Model.get_rope_index`, and `tests/npu_vision_prompt_test.cpp` holds that code's answers for +eight prompts (one to two images, with and without slices, touching, newline-separated, with id +prefixes, and none) as hashes over every token id and position. Images only: video frames are not +handled. Nothing feeds these planes or the tower's rows into the text forward yet; that is the route's +work. + ## Serving The 1bit NPU engine runs the model on the XDNA 2 NPU. With a release package that carries the NPU diff --git a/npu/vision_prompt.h b/npu/vision_prompt.h new file mode 100644 index 000000000..9544cbac2 --- /dev/null +++ b/npu/vision_prompt.h @@ -0,0 +1,252 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The text side of a MiniCPM-V 4.7 image (engine#1788): which tokens stand for it in the prompt, and which position +// every token of a prompt with images gets. Arithmetic on token ids; no tokenizer, no tensor, no file. +// +// TOKENS. One `<|image_pad|>` in the rendered prompt is replaced by +// pad x R0 the whole view, R0 = its rows after the tower +// pad x R ... (one per slice of the row) then each row of slices, +// "\n" rows separated by one newline token +// (transformers' MiniCPMV4_7Processor.replace_image_token). The processor also puts `k` in front; +// that is ordinary text and is the caller's to tokenise. +// +// POSITIONS. The model's `mrope_mode: canvas`: text counts 1, 2, 3 on all three planes; an image is laid on a square +// canvas whose origin is the next free position, the whole view stretched over the canvas the slices tile, every slice +// at its own place, and the text after it resumes at origin + max(canvas height, canvas width) + 1. So an image costs +// positions by its side, not by its row count. This follows MiniCPMV4_7Model.get_rope_index line for line, including: +// - `` sits one step before the canvas (the "halo"), `` on its far corner; +// - the whole view's coordinates are torch.linspace(0, side - 1, n) in float32, rounded half to even; +// - marker and newline tokens directly after an image belong to it (they take the canvas origin). +// The reference also groups frames that only markers separate and steps once more after text between two frames of a +// group. Both are kept below so the code can be read against it, but for prompts that hold only images neither can +// change a position (an image's trailing markers already belong to it, so no such gap exists): removing either leaves +// all eight tested prompts unchanged. They matter for video frames, which this does not handle. +// tests/npu_vision_prompt_test.cpp holds the reference's positions for eight prompts and its linspace for 25,600 cases. +#pragma once + +#include "vision_plan.h" + +#include +#include +#include +#include +#include + +namespace onebit::npu::vision { + +// The model's ids (config.json: image_start_id, image_end_id, slice_start_id, slice_end_id, image_token_id, newline_id). +struct ImageMarkers { + int image_start = 0, image_end = 0, slice_start = 0, slice_end = 0, image_pad = 0, newline = 0; + bool is_marker(int id) const { + return id == slice_start || id == slice_end || id == image_start || id == image_end || id == newline; + } +}; + +// The tokens for one image. Its pieces receive the tower's rows in order: the first run of pads is the whole view, +// then one run per slice, row by row. +inline std::vector image_tokens(const ImagePlan& plan, const ImageMarkers& m) { + std::vector ids; + ids.push_back(m.image_start); + ids.insert(ids.end(), size_t(plan.source.rows), m.image_pad); + ids.push_back(m.image_end); + for (int r = 0; r < plan.slice_rows; ++r) { + if (r) ids.push_back(m.newline); + for (int c = 0; c < plan.slice_cols; ++c) { + ids.push_back(m.slice_start); + ids.insert(ids.end(), size_t(plan.slice.rows), m.image_pad); + ids.push_back(m.slice_end); + } + } + return ids; +} + +// round(torch.linspace(0, last, n))[i] in float32: the first half counts up from 0, the second half down from `last`. +inline int linspace_round(int last, int n, int i) { + if (n <= 1) return 0; + const float step = float(last) / float(n - 1); + const float v = i < n / 2 ? step * float(i) : float(last) - step * float(n - i - 1); + return int(std::nearbyintf(v)); +} + +struct PromptPositions { + std::vector t, h, w; // one per token + int next = 0; // the position of the first token generated after the prompt +}; + +// `grids` is the patch grid (height, width) of every piece in the order its pads appear: each image's whole view, then +// its slices. Throws when the pads and the grids do not describe the same pieces. +inline PromptPositions prompt_positions(const std::vector& ids, const ImageMarkers& m, + const std::vector>& grids) { + const int n = int(ids.size()); + const int merge = 4; // "16x": two 2 x 2 merges + struct Frame { + int thumb_start, thumb_end, end; + std::vector> slices; + }; + struct Group { + int start, end; + std::vector frames; + }; + std::vector groups; + bool open = false; + int previous_end = 0; + for (int i = 0; i < n;) { + if (ids[size_t(i)] != m.image_pad) { + ++i; + continue; + } + int j = i; + while (j < n && ids[size_t(j)] == m.image_pad) ++j; + if (i == 0) throw std::runtime_error("an image placeholder opens the prompt with no in front of it"); + if (open && ids[size_t(i - 1)] == m.slice_start) { + groups.back().frames.back().slices.push_back({i, j}); + } else { + bool adjacent = !groups.empty(); + for (int k = previous_end; adjacent && k < i - 1; ++k) adjacent = m.is_marker(ids[size_t(k)]); + if (!adjacent) groups.push_back({i - 1, 0, {}}); + groups.back().frames.push_back({i, j, 0, {}}); + open = true; + } + previous_end = j; + i = j; + } + for (Group& g : groups) { + for (Frame& f : g.frames) { + int e = f.slices.empty() ? f.thumb_end : f.slices.back().second; + while (e < n && m.is_marker(ids[size_t(e)])) ++e; + f.end = e; + } + g.end = g.frames.back().end; + } + + PromptPositions p; + p.t.resize(size_t(n)); + p.h.resize(size_t(n)); + p.w.resize(size_t(n)); + for (int i = 0; i < n; ++i) p.t[size_t(i)] = p.h[size_t(i)] = p.w[size_t(i)] = i; + const auto text = [&](int from, int to, int pos) { + for (int i = from; i < to; ++i) p.t[size_t(i)] = p.h[size_t(i)] = p.w[size_t(i)] = pos + (i - from); + }; + size_t next_grid = 0; + const auto grid = [&]() { + if (next_grid >= grids.size()) throw std::runtime_error("the prompt holds more image pieces than grids were given"); + const std::pair g = grids[next_grid++]; + if (g.first <= 0 || g.second <= 0 || g.first % merge || g.second % merge) + throw std::runtime_error("a piece's patch grid is not a positive multiple of 4 in both extents"); + return std::pair{g.first / merge, g.second / merge}; + }; + // a piece's tokens: `count` of them, row-major on a (gh, gw) grid + const auto piece = [&](int from, int to, std::pair g, int canvas_h, int canvas_w, int h_off, int w_off, int origin) { + if (to - from != g.first * g.second) + throw std::runtime_error("a piece has " + std::to_string(to - from) + " placeholders but its grid gives " + + std::to_string(g.first * g.second) + " rows"); + for (int y = 0; y < g.first; ++y) + for (int x = 0; x < g.second; ++x) { + const size_t at = size_t(from + y * g.second + x); + p.t[at] = origin; + p.h[at] = origin + h_off + linspace_round((canvas_h ? canvas_h : g.first) - 1, g.first, y); + p.w[at] = origin + w_off + linspace_round((canvas_w ? canvas_w : g.second) - 1, g.second, x); + } + }; + + int current_pos = 0, current_idx = 0; + for (const Group& g : groups) { + if (g.start > current_idx) { + text(current_idx, g.start, current_pos); + current_pos += g.start - current_idx; + } + int cursor = g.start; + for (const Frame& f0 : g.frames) { + const std::pair thumb = grid(); + const int frame_start = f0.thumb_start - 1; + if (frame_start > cursor) { + text(cursor, frame_start, current_pos); + current_pos += frame_start - cursor + 1; + } + const int frame_end = f0.end < g.end ? f0.end : g.end; + const int origin = current_pos, halo = origin > 0 ? origin - 1 : 0; + int canvas_h = thumb.first, canvas_w = thumb.second, slice_h = 0, slice_w = 0, rows = 0, cols = 0; + std::pair first_slice{0, 0}; + const int ns = int(f0.slices.size()); + if (ns) { + cols = ns; + for (int k = 0; k + 1 < ns; ++k) + if (f0.slices[size_t(k + 1)].first - f0.slices[size_t(k)].second > 2) { + cols = k + 1; + break; + } + rows = cols > 0 ? ns / cols : 1; + if (rows * cols != ns) { + rows = 1; + cols = ns; + } + first_slice = grid(); + slice_h = first_slice.first; + slice_w = first_slice.second; + canvas_h = rows * slice_h; + canvas_w = cols * slice_w; + } + for (int i = frame_start; i < frame_end; ++i) p.t[size_t(i)] = p.h[size_t(i)] = p.w[size_t(i)] = origin; + p.h[size_t(frame_start)] = p.w[size_t(frame_start)] = halo; + if (f0.thumb_end < frame_end) { + p.h[size_t(f0.thumb_end)] = origin + canvas_h; + p.w[size_t(f0.thumb_end)] = origin + canvas_w; + } + piece(f0.thumb_start, f0.thumb_end, thumb, canvas_h, canvas_w, 0, 0, origin); + for (int k = 0; k < ns; ++k) { + const int h_off = (k / cols) * slice_h, w_off = (k % cols) * slice_w; + const std::pair sg = k == 0 ? first_slice : grid(); + const int s0 = f0.slices[size_t(k)].first, s1 = f0.slices[size_t(k)].second; + if (s0 - 1 >= frame_start) { + p.h[size_t(s0 - 1)] = origin + h_off; + p.w[size_t(s0 - 1)] = origin + w_off; + } + if (s1 < frame_end) { + p.h[size_t(s1)] = origin + h_off + sg.first - 1; + p.w[size_t(s1)] = origin + w_off + sg.second - 1; + } + piece(s0, s1, sg, 0, 0, h_off, w_off, origin); + } + for (int k = 0; k + 1 < ns; ++k) { + const int gap0 = f0.slices[size_t(k)].second, gap1 = f0.slices[size_t(k + 1)].first; + if (gap1 - gap0 <= 2) continue; + for (int i = gap0 + 1; i < gap1 - 1; ++i) { + p.h[size_t(i)] = origin + (k / cols + 1) * slice_h - 1; + p.w[size_t(i)] = origin + cols * slice_w; + } + } + current_pos = origin + (canvas_h > canvas_w ? canvas_h : canvas_w) + 1; + cursor = frame_end; + } + if (cursor < g.end) { + text(cursor, g.end, current_pos); + current_pos += g.end - cursor; + } + current_idx = g.end; + } + if (current_idx < n) text(current_idx, n, current_pos); + if (next_grid != grids.size()) throw std::runtime_error("more grids were given than the prompt holds image pieces"); + int mx = -1; + for (int i = 0; i < n; ++i) { + if (p.t[size_t(i)] > mx) mx = p.t[size_t(i)]; + if (p.h[size_t(i)] > mx) mx = p.h[size_t(i)]; + if (p.w[size_t(i)] > mx) mx = p.w[size_t(i)]; + } + p.next = mx + 1; + return p; +} + +} // namespace onebit::npu::vision diff --git a/tests/npu_vision_prompt_test.cpp b/tests/npu_vision_prompt_test.cpp new file mode 100644 index 000000000..bc53f91e0 --- /dev/null +++ b/tests/npu_vision_prompt_test.cpp @@ -0,0 +1,177 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// npu_vision_prompt_test +// +// The text side of an image (npu/vision_prompt.h, engine#1788) against transformers 5.22.2's MiniCPM-V 4.7 code: +// MiniCPMV4_7Processor.replace_image_token for the tokens and MiniCPMV4_7Model.get_rope_index for the positions. +// Eight prompts are built here from plan_image() and image_tokens(); the reference built the same prompts from its own +// processor arithmetic, and for each one this file holds its token count, piece count, next position and one FNV-1a +// hash over (ids, t, h, w). A wrong token, a wrong order or one wrong position changes the hash. +// Text tokens are 1000 + k; the marker ids are the published packs'. +#include "../npu/vision_prompt.h" + +#include +#include +#include +#include + +namespace { + +namespace vis = onebit::npu::vision; + +int g_fail = 0, g_arms = 0; +void check(bool ok, const std::string& what) { + ++g_arms; + if (!ok) { + ++g_fail; + std::printf("FAIL %s\n", what.c_str()); + } +} + +const vis::ImageMarkers kM = {248078, 248079, 248088, 248089, 248056, 198}; + +struct Prompt { + std::vector ids; + std::vector> grids; + void text(int n, int base) { + for (int i = 0; i < n; ++i) ids.push_back(1000 + base + i); + } + void raw(std::initializer_list v) { ids.insert(ids.end(), v); } + void image(int h, int w) { + const vis::ImagePlan p = vis::plan_image(h, w); + const std::vector t = vis::image_tokens(p, kM); + ids.insert(ids.end(), t.begin(), t.end()); + grids.push_back({p.source.grid_h, p.source.grid_w}); + for (int k = 0; k < p.slice_rows * p.slice_cols; ++k) grids.push_back({p.slice.grid_h, p.slice.grid_w}); + } +}; + +Prompt scenario(int k) { + Prompt p; + const int NL = kM.newline; + switch (k) { + case 0: p.text(7, 0); p.image(448, 448); p.text(5, 50); break; // one view, no slices + case 1: p.text(3, 0); p.image(480, 640); p.raw({NL}); p.text(4, 50); break; // 1 x 2 slices, a newline after + case 2: p.text(2, 0); p.image(1080, 1920); p.image(100, 100); p.text(3, 50); break; // two images touching + case 3: p.text(1, 0); p.image(720, 1280); p.text(6, 50); p.image(3024, 4032); p.text(2, 80); break; + case 4: p.text(4, 0); p.image(5000, 300); p.text(1, 50); break; // 9 x 1 slices + case 5: // k prefixes + p.text(3, 0); p.raw({248090, 15, 248091}); p.image(600, 800); p.raw({NL, NL}); p.text(2, 50); + p.raw({248090, 16, 248091}); p.image(224, 2000); p.raw({NL}); + break; + case 6: p.text(9, 0); break; // no image at all + case 7: p.text(1, 0); p.image(449, 448); p.raw({NL}); p.image(448, 449); break; // two images, a newline between + } + return p; +} + +// {tokens, pieces, next position, hash(ids, t, h, w)}, from the reference +struct Golden { + int tokens, pieces, next; + uint64_t hash; +}; +const Golden kGolden[8] = { + {78, 1, 21, 0xccb3904ded7cd15bULL}, {217, 3, 22, 0xece6c1beaeb65928ULL}, {668, 10, 47, 0xcd6d6c517226c2afULL}, + {1120, 17, 65, 0x9e0d9e3759a6b555ULL}, {693, 10, 105, 0xbfd3bd95f7383645ULL}, {662, 10, 79, 0xdd390fcf63fc8236ULL}, + {9, 0, 9, 0x2d9fde739c2c5f7cULL}, {407, 6, 27, 0xbe7321611eaa8facULL}, +}; +const uint64_t kLinspaceHash = 0x2f0e4ce031c6cf8eULL; // round(linspace(0, c - 1, n)) for c, n in 1..160, by torch + +uint64_t fnv(const std::vector& v, uint64_t h) { + for (int x : v) + for (int b = 0; b < 4; ++b) h = (h ^ uint64_t((uint32_t(x) >> (8 * b)) & 0xff)) * 0x100000001b3ULL; + return h; +} + +} // namespace + +int main() { + for (int k = 0; k < 8; ++k) { + const Prompt p = scenario(k); + const std::string name = "prompt " + std::to_string(k); + try { + const vis::PromptPositions pos = vis::prompt_positions(p.ids, kM, p.grids); + uint64_t h = fnv(p.ids, 0xcbf29ce484222325ULL); + h = fnv(pos.t, h); + h = fnv(pos.h, h); + h = fnv(pos.w, h); + check(int(p.ids.size()) == kGolden[k].tokens && int(p.grids.size()) == kGolden[k].pieces, + name + ": " + std::to_string(p.ids.size()) + " tokens and " + std::to_string(p.grids.size()) + " pieces, as the reference lays it out"); + check(pos.next == kGolden[k].next, name + ": generation resumes at position " + std::to_string(pos.next)); + check(h == kGolden[k].hash, name + ": every token id and every (t, h, w) is the reference's"); + } catch (const std::exception& e) { + check(false, name + " is laid out (threw: " + e.what() + ")"); + } + } + + // prompt 1, spelled out: 3 text tokens, then a 480 x 640 image (a 7 x 9 view over 1 x 2 slices of 10 x 7) + { + const Prompt p = scenario(1); + const vis::PromptPositions pos = vis::prompt_positions(p.ids, kM, p.grids); + check(p.ids[3] == kM.image_start && pos.t[3] == 3 && pos.h[3] == 2 && pos.w[3] == 2, + " keeps the canvas origin on t and sits one step before it on h and w"); + check(pos.t[4] == 3 && pos.h[4] == 3 && pos.w[4] == 3 && pos.w[5] == 5 && pos.w[6] == 6 && pos.w[7] == 8, + "the whole view is stretched over the slices' canvas: columns 0, 2, 3, 5 ... of 14 for 9 tokens"); + check(p.ids[67] == kM.image_end && pos.t[67] == 3 && pos.h[67] == 13 && pos.w[67] == 17, + " sits on the canvas's far corner (10 down, 14 across)"); + const size_t n = p.ids.size(); + check(pos.t[n - 4] == 18 && pos.t[n - 1] == 21 && pos.next == 22, + "text resumes at origin + max(10, 14) + 1 = 18: the image costs its side, not its 203 rows"); + check(p.ids[n - 5] == kM.newline && pos.t[n - 5] == 3 && pos.h[n - 5] == 3, + "a newline directly after the image belongs to it and takes the canvas origin"); + } + + { + uint64_t h = 0xcbf29ce484222325ULL; + for (int c = 1; c <= 160; ++c) + for (int n = 1; n <= 160; ++n) { + std::vector v(static_cast(n)); + for (int i = 0; i < n; ++i) v[size_t(i)] = vis::linspace_round(c - 1, n, i); + h = fnv(v, h); + } + check(h == kLinspaceHash, "round(linspace(0, c - 1, n)) is torch's for every canvas and count up to 160"); + } + + // pads and grids that do not describe the same pieces + { + Prompt p = scenario(1); + std::string why; + auto g = p.grids; + g.pop_back(); + try { vis::prompt_positions(p.ids, kM, g); } catch (const std::exception& e) { why = e.what(); } + check(why.find("more image pieces than grids") != std::string::npos, "a missing grid is refused"); + why.clear(); + g = p.grids; + g.push_back({32, 32}); + try { vis::prompt_positions(p.ids, kM, g); } catch (const std::exception& e) { why = e.what(); } + check(why.find("more grids were given") != std::string::npos, "an extra grid is refused"); + why.clear(); + g = p.grids; + g[0] = {32, 32}; + try { vis::prompt_positions(p.ids, kM, g); } catch (const std::exception& e) { why = e.what(); } + check(why.find("placeholders but its grid gives") != std::string::npos, "a grid that is not the piece's row count is refused"); + why.clear(); + g = p.grids; + g[1] = {39, 28}; + try { vis::prompt_positions(p.ids, kM, g); } catch (const std::exception& e) { why = e.what(); } + check(why.find("multiple of 4") != std::string::npos, "a grid the tower cannot merge is refused"); + why.clear(); + try { vis::prompt_positions({kM.image_pad, 1000}, kM, {{4, 4}}); } catch (const std::exception& e) { why = e.what(); } + check(why.find("opens the prompt") != std::string::npos, "a placeholder with nothing in front of it is refused"); + } + std::printf("vision prompt: %d checks, %d failed\n", g_arms, g_fail); + return g_fail ? 1 : 0; +} From d11306472768dbe670e125609305aabe37c619cd Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:31:05 -0400 Subject: [PATCH 65/98] tests: the sweep covers tests/*_test.py too, as a set baseline (#2018, correcting the issue's count) (#2088) * tests: the sweep covers tests/*_test.py too, as a set baseline (engine#2018) #1824's row covers tests/*.sh and #1911's covers tools/*.py that declare a self-test; tests/*_test.py was outside both, so a python test whose runner was never added reads as green for ever -- the same defect, in the file kind most new tests are written in. Landing shape follows the repo's own precedent for existing debt (tools/ci/add_test_gaps.txt, engine#1595): a SET baseline (tools/ci/unreferenced_py_tests.txt) rather than 35 allow-list entries with invented reasons. The check FAILS on a name that is not in the baseline and REPORTS a baseline line whose file is registered now, so the list can only shrink. The corpus for this rule is RUNNERS (CMakeLists.txt, cmake/, .github/workflows), not "any mention": a test importing another test is a citation, which is the distinction #1911 established for tools. A file another test imports is therefore not counted -- that is what separates a test from a helper without guessing. CORRECTION to #2018's headline, found while building this: the issue says "6 of 20 sampled tests/*_test.py have no runner" and the true figure is **2 of 42**. The first version of this rule scanned RUNNERS, which holds one FILE (CMakeLists.txt) and two directories, and tested is_dir() on all three -- so every add_python_test(...) registration in CMakeLists.txt was invisible and 35 of 42 looked unregistered. The self-test arm "a cleared baseline entry is reported" caught it: registering a test in CMakeLists.txt did not clear its line. Two independent hand checks confirm the remaining two (tests/hyperloom_benchmark_test.py, tests/moe_expert_chunks_accum_test.py: zero mentions in CMakeLists.txt, cmake/*.cmake and the workflows). Verified on a git-archive extract of main in ~/.cache/scratch-pkg (not /tmp; it is tmpfs): clean tree + 2-entry baseline -> rc=0, "40 python test(s) are registered" a NEW unregistered tests/*_test.py -> rc=1 naming it a test imported by a sibling test -> not counted a baseline line whose test is registered -> reported as no longer needed, rc=0 a baseline line dropped for a real gap -> rc=1 --self-test -> 17 arms, 0 failures (two new) * tests: exempt the new baseline from the copyright check, which needs the full notice (engine#2018) CI went red on "Copyright notice on every file", and my PR body had claimed it would not: a `.txt` has no comment style of its own, so being in EXEMPT is what gives it one, and the notice it then wants is the FULL Apache block rather than the two SPDX lines I wrote. Two corrections, both mine: tools/copyright.py "unreferenced_py_tests.txt" beside add_test_gaps.txt and unreferenced_tests.txt, same reason -- a `#`-comment data file tools/ci/unreferenced_py_tests.txt the full licence header, then the prose that explains the baseline Verified by running the checkers against a git-archive extract: `tools/copyright.py --check` rc=0 (it failed with "no comment style known" before the exemption and "missing copyright notice" with only the SPDX lines), and `tools/ci/tests_referenced_check.py --check` still rc=0 with the two-entry baseline. Second time tonight I asserted a copyright.py behaviour without checking it -- the first was `unreferenced_tests.txt` in #1911, where the same two-line-header assumption cost a red build. The rule to keep: run the repo's own check on the extract before pushing, not after. --- tools/ci/tests_referenced_check.py | 97 +++++++++++++++++++++++++++++- tools/ci/unreferenced_py_tests.txt | 36 +++++++++++ tools/copyright.py | 2 + 3 files changed, 132 insertions(+), 3 deletions(-) mode change 100755 => 100644 tools/ci/tests_referenced_check.py create mode 100644 tools/ci/unreferenced_py_tests.txt diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py old mode 100755 new mode 100644 index 2df91ec28..8dd2f3818 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -22,6 +22,7 @@ reads as green for ever -- the same defect as #1595, one file kind over. need = every tests/*.sh + AND every tests/*_test.py that no RUNNER names and that no other test imports (engine#2018) AND every tools/**/*.py that declares a --self-test (engine#1901): a tool that ships one is claiming it can be verified without its real inputs, so an unregistered one is an arm nobody runs. For those the corpus is RUNNERS only (CMakeLists.txt, cmake/, .github/workflows/) -- a mention in another tool's source is a citation, @@ -51,6 +52,11 @@ import sys ALLOW = pathlib.Path("tools/ci/unreferenced_tests.txt") +# engine#2018: the existing debt, as a SET baseline rather than 35 allow-list entries with invented reasons -- the same +# shape as tools/ci/add_test_gaps.txt (#1595): the check fails when a NEW unregistered test appears, and reports an entry +# that is no longer needed so the list shrinks. A per-file allow-list of "not a row yet" would be 35 lines nobody can +# tell apart from "forgotten". +PYBASE = pathlib.Path("tools/ci/unreferenced_py_tests.txt") SCAN = ("", "cmake", "scripts", "tools", "tests", ".github/workflows") SUFFIXES = {".txt", ".cmake", ".yml", ".yaml", ".sh", ".py"} @@ -73,6 +79,19 @@ def allow_list(root): return out +def py_baseline(root): + """{basename} from the python-test baseline; comments and blanks skipped.""" + path = root / PYBASE + out = set() + if not path.is_file(): + return out + for line in path.read_text(errors="replace").splitlines(): + line = line.split("#")[0].strip() + if line: + out.add(pathlib.Path(line).name) + return out + + def scan_files(root): """Every file that can reference a test, as (path, text) -- docs and the allow-list excluded.""" for sub in SCAN: @@ -135,6 +154,55 @@ def unregistered_selftest_tools(root): return drift +def unregistered_py_tests(root): + """tests/*_test.py that no RUNNER names, excluding files another test imports (engine#2018). + + The corpus is RUNNERS, not "any mention": a test importing another test is a citation, the same distinction the + tools rule makes. The `*_test.py` name plus "not imported by a sibling" is what separates a test from a helper + without guessing -- a helper a test imports is called, and a test nothing calls is dead. + """ + tests = root / "tests" + if not tests.is_dir(): + return [] + cand = sorted(tests.glob("*_test.py")) + if not cand: + return [] + imported = set() + for p in list(tests.glob("*.py")) + sorted((root / "tools").glob("*.py")) if (root / "tools").is_dir() else list(tests.glob("*.py")): + try: + text = p.read_text(errors="replace") + except OSError: + continue + for q in cand: + if p == q: + continue + stem = q.stem + if re.search(r"^\s*(?:import\s+%s\b|from\s+%s\s+import)" % (re.escape(stem), re.escape(stem)), + text, re.M): + imported.add(q.name) + runner_text = [] + for sub in RUNNERS: + base = root / sub + # RUNNERS holds one FILE (CMakeLists.txt) and two directories; testing is_dir() on all three made every + # cmake registration invisible to this rule, which the self-test arm "a cleared baseline entry is reported" + # caught -- registering a python test in CMakeLists.txt did not clear its baseline line. + paths = [base] if base.is_file() else ( + [p for p in sorted(base.rglob("*")) if p.is_file() and p.suffix in {".txt", ".cmake", ".yml", ".yaml", ".py", ".sh"}] + if base.is_dir() else []) + for path in paths: + try: + runner_text.append(path.read_text(errors="replace")) + except OSError: + continue + out = [] + for q in cand: + if q.name in imported: + continue + if not any(q.name in t for t in runner_text): + out.append(str(q.relative_to(root))) + return out + + def check(root): tests = root / "tests" if not tests.is_dir(): @@ -154,15 +222,25 @@ def check(root): for name in idle: print("tests_referenced: %s is in %s but it IS referenced now -- drop the line, it suppresses nothing" % (name, ALLOW)) + # engine#2018: a SET baseline, not an allow-list -- fail on a NEW one, report one that cleared. + py_now = {pathlib.Path(n).name: n for n in unregistered_py_tests(root)} + py_base = py_baseline(root) + py_drift = sorted(rel for name, rel in py_now.items() if name not in py_base) + py_cleared = sorted(n for n in py_base if n not in py_now) + for rel in py_drift: + print("tests_referenced: %s is a test that no runner names -- add a ctest row (add_python_test in " + "CMakeLists.txt or cmake/*.cmake), or add it to %s if it cannot be one yet" % (rel, PYBASE)) + for name in py_cleared: + print("tests_referenced: %s is in %s and no longer needs to be -- drop the line" % (name, PYBASE)) tools_drift = unregistered_selftest_tools(root) for rel in tools_drift: print("tests_referenced: %s declares a self-test and no runner names it -- add a ctest row (add_test in " "CMakeLists.txt or cmake/*.cmake) or a workflow step, or the arm is dead code" % rel) - if drift or gone or idle or tools_drift: + if drift or gone or idle or py_drift or tools_drift: return 1 - print("tests_referenced: all %d test script(s) are referenced%s; %d tool self-test(s) are registered" % ( + print("tests_referenced: all %d test script(s) are referenced%s; %d tool self-test(s) and %d python test(s) are registered" % ( len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))), - len(_selftest_tools(root)))) + len(_selftest_tools(root)), len(list((root / "tests").glob("*_test.py"))) - len(unregistered_py_tests(root)))) return 0 @@ -244,6 +322,19 @@ def root_with(files): str(ALLOW): "tests/keeper2.sh # path form is accepted\n"}) cases.append(("a path-form allow entry is accepted", check(r), 0, None)) + # 9a. engine#2018: a NEW unregistered python test fails, and a test another test IMPORTS does not count + r = root_with({"tests/one.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME one COMMAND bash tests/one.sh)\n", + "tests/fresh_test.py": "def f(): pass\n"}) + cases.append(("an unregistered tests/*_test.py is drift", check(r), 1, "fresh_test.py")) + cases_root["an unregistered tests/*_test.py is drift"] = r + + # 9b. the helper discriminator: imported by a sibling test -> not a test that needs a runner + r = root_with({"tests/one.sh": "#!/bin/sh\n", "CMakeLists.txt": "add_test(NAME one COMMAND bash tests/one.sh)\n", + "tests/helper_test.py": "def h(): pass\n", + "tests/caller_test.py": "from helper_test import h\nh()\n", + "tools/ci/unreferenced_py_tests.txt": "tests/caller_test.py\n"}) + cases.append(("a test another test imports is not drift", check(r), 0, None)) + # 8. no tests/ at all -> cannot tell, not "clean" r = root_with({"CMakeLists.txt": ""}) cases.append(("no tests/ directory is 'cannot tell'", check(r), 2, None)) diff --git a/tools/ci/unreferenced_py_tests.txt b/tools/ci/unreferenced_py_tests.txt new file mode 100644 index 000000000..9cef48019 --- /dev/null +++ b/tools/ci/unreferenced_py_tests.txt @@ -0,0 +1,36 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tests/*.sh that are deliberately NOT a ctest row, one basename per line, with the reason it must stay out. +# `tools/ci/tests_referenced_check.py` fails when a test is referenced by nothing that can run it, and reports a +# line here that is no longer needed so this list can shrink. The bar for an entry is the same one #1806 used: +# a ctest row has to be ABLE TO FAIL. A script that always exits 0 is an operator measurement, not a test. +# +# +# tests/*_test.py that no ctest row and no workflow step names, measured on main 2026-11-10 (engine#2018). +# +# A SET baseline, not an allow-list: `tools/ci/tests_referenced_check.py` FAILS when a name appears here that is not in +# the tree's current set, and REPORTS a line whose file is registered now, so this can only shrink. +# +# The number is 2, and getting it right took a correction worth recording: the first version of the rule scanned +# `RUNNERS`, which contains one FILE (CMakeLists.txt) and two directories, and tested `is_dir()` on all three -- so every +# `add_python_test(...)` registration in CMakeLists.txt was invisible and 35 of 42 tests looked unregistered. The arm +# "a cleared baseline entry is reported" caught it: registering a test in CMakeLists.txt did not clear its line. The true +# figure is 2 of 42, which is a healthy tree; the rule is here to keep it that way rather than to record a backlog. +# +# Each line is a real test whose runner was never added; the fix per file is one `add_python_test(...)`. +# +tests/hyperloom_benchmark_test.py +tests/moe_expert_chunks_accum_test.py diff --git a/tools/copyright.py b/tools/copyright.py index d8db59bd5..840a92fa9 100644 --- a/tools/copyright.py +++ b/tools/copyright.py @@ -44,6 +44,8 @@ # engine#1820: the other sweep's allow-list, same shape and the same reason -- `#`-comment data, `.txt` has no # style, and the header it wants is already at the top of the file. "unreferenced_tests.txt", + # engine#2018: the third test-registration baseline, same shape and the same reason -- data, `.txt` has no style. + "unreferenced_py_tests.txt", "21serial-console", "90onboard-limits", "Dockerfile", "*.Dockerfile", # cycle 17: Puppy's switch that skips its first-run Quick Setup is an empty marker file whose only content # is this notice - Puppy tests for its existence, so a comment is free From 6e28f104c44ca5c1a53b99068499294f64f58fe3 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:42:36 -0400 Subject: [PATCH 66/98] bump llama.cpp to 7a1f2b81c: the prefill kernels stay finite above the f16 range, and hrx-backend-test builds again (fixes #2068) (#2092) Pin 2293f95ea -> 7a1f2b81c (seven commits ahead, none behind): llama.cpp #292 and #298 (engine #1962), #293/#294/#295 (engine #2068), #300 (engine #2069), #299 (engine #2079). registry/architectures.json records the new pin; docs/hrx.md describes the prefill fixes and what is still open under #1962. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/hrx.md | 28 +++++++++++++++++++++++++--- registry/architectures.json | 2 +- third_party/llama.cpp | 2 +- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/hrx.md b/docs/hrx.md index cf6ab2ac7..0b1ebecee 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -300,9 +300,31 @@ Measured on gfx1151 (2026-11-10): the dense inputs of Qwen3-0.6B, Qwen3-4B, ZAYA1-8B and flint. Cases at those two values are in the suite, so the shift has a test on each side. -Still open under #1962: two prefill kernels that re-quantise the activations into 8-bit blocks -(Q5_K and Q6_K weights at 512 tokens) keep a per-block term in half precision and still return NaN -above 65504. And the limit is moved, not removed: an activation above 4.19e6 would overflow again. +The prefill kernels followed at pin `7a1f2b81c`. The two small-batch Q6_K kernels were already done +(llama.cpp #288). The Q4_K and Q6_K prefill kernels read a half-precision copy of the activations; +that copy is now stored at 2^-6 and the sums are scaled back (llama.cpp #298). The Q5_K/IQ4_XS +prefill kernel reads the activations as 8-bit blocks, each with a scale and a scaled sum in half +precision. The scale fits up to 8.3e6; the scaled sum is about as large as the largest activation in +the block, so it was the term that overflowed. The kernel now rebuilds a sum that is not finite from +the 8-bit values it already holds, and uses a finite one as stored (llama.cpp #292). + +Measured on gfx1151 (2026-11-11, fork `7a1f2b81c`, full build): + +- `test-backend-ops -o MUL_MAT` with a resident weight (2048 x 4096), Q4_K, Q5_K, Q6_K, IQ4_XS, Q8_0 + and Q4_0 at 1 to 512 tokens, with activation spikes of 3.0, 9.3e4 and 2e6: 180 of 180 pass (run at + llama.cpp #298's head). Before #292 and #298, Q5_K at 512 tokens and Q4_K and Q6_K at 256 and 512 + tokens returned NaN at 9.3e4. The whole op is 617 of 617; `MUL_MAT_ID` is 845 of 845. +- Perplexity on wikitext-2 (`-c 512 -b 512`, 16 chunks): Qwen3-4B-Q4_K_M 17.1836 (17.1864 before + #298), Qwen3-4B-IQ4_XS 17.6304 (the same before and after #292). +- Prefill rate on Qwen3-4B-Q4_K_M across #298, eight alternating runs on a loaded box: 2683 against + 2735 tok/s (ratio 0.98, pairs between 0.88 and 1.14). No regression shown; a cost of 2 to 3 % is + not excluded. + +Still open under #1962: the final projection of the draft head (a shortlist kernel) narrows its +activations too and has no test that reaches it. The other kernels that read the 8-bit blocks (the +fused SwiGLU, the routed kernels and the low-token Q4 paths) use the stored sum as it is and are not +measured above 65504. And the limits are moved, not removed: 4.19e6 for the half-precision copies, +8.3e6 for the 8-bit blocks. ### Fixed: Qwen3-VL on `HRX0`: it decodes again, and its rotation runs on the device ([engine#1467](https://github.com/1bit-MONSTER/engine/issues/1467)) diff --git a/registry/architectures.json b/registry/architectures.json index ef1ebe7c4..cbe8229e0 100644 --- a/registry/architectures.json +++ b/registry/architectures.json @@ -1,7 +1,7 @@ { "about": "HF architecture -> GGUF architecture and the backends whose code accepts it. Generated by tools/registry_build.py from the pinned sources; do not edit.", "sources": { - "llama.cpp (hrx)": "2293f95ea49dafec71a2460739d95ea31cd27bfc" + "llama.cpp (hrx)": "7a1f2b81ce17aaa84a40b766d1b06e5fd4601196" }, "counts": { "hrx": 298, diff --git a/third_party/llama.cpp b/third_party/llama.cpp index 2293f95ea..7a1f2b81c 160000 --- a/third_party/llama.cpp +++ b/third_party/llama.cpp @@ -1 +1 @@ -Subproject commit 2293f95ea49dafec71a2460739d95ea31cd27bfc +Subproject commit 7a1f2b81ce17aaa84a40b766d1b06e5fd4601196 From fcd8412898cc59d08c540a7d0b5b5247e9b41b7e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:03:04 -0400 Subject: [PATCH 67/98] npu: a sealed set can carry a narrow decode family: 3.26-3.33 -> 3.69-4.21 tok/s decode with a 4-row set (Fixes #2073) (#2074) A decode step on the served 27B route is 416 launches and 70 % device wait; every launch runs a 16-row kernel for one row. A pack (config q4nx__sealed_narrow_elf, q4nx_sealed_narrow_rows) or NPU_NARROW_SEALED_ELF names a sealed ELF built for fewer rows; launches of 1..rows rows take it, anything wider keeps the 16-row family. Same rules as the wide family (#2042), and one loader now serves both. narrow_family_complete / narrow_takes in batch_width.h, 10 arms in tests/npu_pack_test.cpp. Re-applied on main after #2042 merged (the branch was stacked on it); the change itself is the one reviewed at 59f8f4b / 2429b20. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- docs/npu.md | 26 ++++++++++ npu/forward/include/batch_width.h | 26 +++++++++- npu/forward/include/q4nx_forward.h | 2 + npu/forward/src/q4nx_forward.cpp | 80 +++++++++++++++++++----------- npu/target/xdna/xdna_target.cpp | 35 ++++++++++++- npu/target/xdna/xdna_target.h | 9 ++++ tests/npu_pack_test.cpp | 19 +++++++ 7 files changed, 166 insertions(+), 31 deletions(-) diff --git a/docs/npu.md b/docs/npu.md index 92b38eef2..85857e81f 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -573,6 +573,32 @@ Two 16-row blocks against one 32-row block is 30-31 rows/s against 35-37. The 8- identical with the family off and on, and `NPU_BATCH_CHECK=1` on the wide path reports MATCH on every chunk it checks (32-, 4- and 1-row) in both tickets. Only 32 rows has been measured. +**...and a narrow family for decode (#2073).** A decode step on `bonsai2-27b-t2s2` is 416 launches, and +device wait is 211 of its 302 ms: every launch runs a kernel built for 16 rows and carries one, and a +launch's device time does not depend on how many of its rows are filled. So a pack may also name a sealed +ELF built for FEWER rows: `q4nx_t2s_sealed_narrow_elf` (or `q4nx_pb16i_sealed_narrow_elf`) and +`q4nx_sealed_narrow_rows` (default 4, allowed 1..15) in `config.json`; `NPU_NARROW_SEALED_ELF=` +overrides the file, `=0` switches it off, `NPU_NARROW_ROWS` overrides the rows. A launch of 1..rows rows +takes it (a decode step, or a verify pass of a few rows); anything wider keeps the 16-row family. The rules +are the wide family's: all roles or none, one hw_context, shared weight images, and a file that is missing +or does not load is logged (`[forward] narrow family NOT adopted (...)`) while decode stays on the 16-row +kernels. `narrow_family_complete` and `narrow_takes` in `batch_width.h` are pinned by +`tests/npu_pack_test.cpp`. No pack ships a narrow ELF yet. + +Measured with a 4-row set, one binary, arms 16-row / 4-row / 16-row / 4-row, three prompts generating 49 +tokens each (`(tokens - 1) / (wall - wall of a 1-token request)`), two `npu-hold` tickets: + +| ticket, pair | decode, 16-row family | decode, 4-row family | device wait per step | +|---|---|---|---| +| first, pair 1 | 3.26-3.33 tok/s | **3.69-4.21 tok/s** | 210.1 -> 158.1 ms | +| first, pair 2 (another lane loading the box) | 1.88-2.59 tok/s | 2.33-3.14 tok/s | 230.7 -> 208.1 ms | +| second, pair 1 | 2.85-3.29 tok/s | **3.90-4.16 tok/s** | 234.7 -> 151.7 ms | +| second, pair 2 | 2.15-3.32 tok/s | **4.05-4.19 tok/s** | 210.9 -> 150.9 ms | + +The step wall follows the wait: 286.0 -> 233.7 ms and 296.2 -> 225.9 ms in the quiet pairs. The 49-token +greedy text of each prompt is identical in all eight arms, and `NPU_BATCH_CHECK=1` with the narrow family +reports MATCH on every chunk it checks (1-, 4- and 6-row; the 6-row one runs on the 16-row family). + **A t2s container wants a sealed set for the same reason a pb16i one does.** The layout is not what makes a phase change expensive: the **hw_context** is. Measured on this box in-loop on 2026-11-02, a t2s decode family shipped as eight per-role `_t2s2_m16` ELFs paid **+2.0-2.3 ms on every launch, in every role** — more diff --git a/npu/forward/include/batch_width.h b/npu/forward/include/batch_width.h index 3365c7ff0..ebc9c1920 100644 --- a/npu/forward/include/batch_width.h +++ b/npu/forward/include/batch_width.h @@ -75,7 +75,31 @@ inline int chunk_rows(int narrow, int wide) { return wide > narrow ? wide : narr // launch past `wide` rows is never routed here (the caller refuses it by name). inline bool wide_takes(int rows, int narrow, int wide) { return wide > narrow && rows > narrow && rows <= wide; } -// Which file is the wide sealed ELF, if any? The environment wins over the pack so a set can be tried, or switched +// --- a narrow family for decode (engine: decode device wait) ----------------------------------------------------- +// +// The mirror of the wide family: the same roles built for FEWER rows than the 16-row family, for the launches +// that carry only a few rows (a decode step carries one). Complete means every role with a 16-row kernel has a +// narrow kernel of exactly `narrow` rows, and `narrow` is below every 16-row width; a role without one would +// silently stay on its 16-row kernel, which is safe but is not what the log would claim, so it is all or none. +inline bool narrow_family_complete(const int* base_widths, const int* narrow_widths, int n, int narrow) { + if (!base_widths || !narrow_widths || n <= 0 || narrow <= 0) return false; + int roles = 0; + for (int i = 0; i < n; ++i) { + if (base_widths[i] <= 0) { + if (narrow_widths[i] > 0) return false; + continue; + } + if (narrow_widths[i] != narrow || narrow >= base_widths[i]) return false; + ++roles; + } + return roles > 0; +} + +// Does a launch of `rows` rows go to the narrow family? Only when it fits: 1..narrow rows. One row more and the +// launch would write past the narrow kernel's activation buffer, so the bound is the kernel's own row count. +inline bool narrow_takes(int rows, int narrow) { return narrow > 0 && rows >= 1 && rows <= narrow; } + +// Which file is the wide sealed ELF, if any? (The narrow family's file is chosen by the same rule.) The environment wins over the pack so a set can be tried, or switched // off, without editing a pack: `env` "0" means no wide family even when the pack names one, any other non-empty // value is the path as given. Otherwise the pack's name is used: as given when absolute, else inside `kernels_dir` // (where the 16-row sealed ELF lives). Empty answer = no wide family. `env` may be null. diff --git a/npu/forward/include/q4nx_forward.h b/npu/forward/include/q4nx_forward.h index aac96f583..aeb771585 100644 --- a/npu/forward/include/q4nx_forward.h +++ b/npu/forward/include/q4nx_forward.h @@ -371,6 +371,8 @@ class Q4nxNpuForward { std::string pb16i_sealed_; std::string wide_sealed_; // q4nx__sealed_wide_elf: a wide prefill family (engine#1990) int wide_sealed_rows_ = 32; // q4nx_sealed_wide_rows + std::string narrow_sealed_; // q4nx__sealed_narrow_elf: a narrow decode family + int narrow_sealed_rows_ = 4; // q4nx_sealed_narrow_rows std::string sealed_dir_; // the kernels dir the 16-row sealed ELF was found in // ---- the NPU target (engine#799 P4) ---- // Everything that talks to a device lives behind this: the ELF contexts (M=128 prefill and the M=16 diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 61480b018..e3bebd1b2 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -604,44 +604,64 @@ bool Q4nxNpuForward::open_decode_family_local(bool want_m16, xrt::device* dev, c // All roles or none -- max_batch_rows() is the chunk width, so one role without a wide kernel would be // launched past its buffers. A named file that is missing or does not load is logged and the pass stays at // 16 rows: a pack must never stop serving over an optional speed-up. - const std::string wide_path = - onebit::npu::wide_elf_path(getenv("NPU_WIDE_SEALED_ELF"), wide_sealed_, sealed_dir_); - if (const char* wide_elf = wide_path.c_str(); - !wide_path.empty() && pb16i_ && !pb16i_sealed_.empty() && made > 0) { - const char* wr = getenv("NPU_WIDE_ROWS"); - const int rows = wr && *wr ? atoi(wr) : wide_sealed_rows_; - int wide = 0; + // The narrow decode family is the mirror image: the same instances built for FEWER rows (default 4), taken + // by launches of 1..rows rows -- a decode step carries one. Config `q4nx__sealed_narrow_elf` and + // `q4nx_sealed_narrow_rows`; NPU_NARROW_SEALED_ELF / NPU_NARROW_ROWS override, "0" = off. + // One loader for both: `wide` picks the row bound, the target's make/adopt/drop trio and the log text. + auto load_family = [&](bool wide, const std::string& path, int rows) { + if (path.empty() || !pb16i_ || pb16i_sealed_.empty() || made <= 0) return; + const char* tag = wide ? "wide" : "narrow"; + int n = 0; std::string why; - if (rows <= Q4NX_DESIGN_M_DECODE || rows > Q4NX_DESIGN_M_PREFILL) why = "NPU_WIDE_ROWS must be 17..128"; - else if (!file_exists(wide_elf)) why = std::string("no such file: ") + wide_elf; + if (wide && (rows <= Q4NX_DESIGN_M_DECODE || rows > Q4NX_DESIGN_M_PREFILL)) why = "NPU_WIDE_ROWS must be 17..128"; + else if (!wide && (rows < 1 || rows >= Q4NX_DESIGN_M_DECODE)) why = "NPU_NARROW_ROWS must be 1..15"; + else if (!file_exists(path)) why = "no such file: " + path; for (int i = 0; why.empty() && i < ndesigns_; i++) { I8Ctx* c16 = xd_->ctx16(i); if (!c16) continue; const char* inst = pb16i_instance(designs16_[i].role); - I8Ctx* cw = inst ? xd_->make_ctx_wide(i) : nullptr; - if (!cw) { why = std::string("no wide context for role ") + role_tag(designs16_[i].role); break; } - cw->MD = rows; - cw->KD = c16->KD; - cw->ND = c16->ND; - cw->blocked_b = c16->blocked_b; - cw->pb16i = c16->pb16i; - cw->t2s = c16->t2s; - cw->pb16i_planes = c16->pb16i_planes; - if (!cw->init_elf(*dev, wide_elf, rows, c16->KD, c16->ND, 1, inst)) { + I8Ctx* c = !inst ? nullptr : (wide ? xd_->make_ctx_wide(i) : xd_->make_ctx_narrow(i)); + if (!c) { why = std::string("no ") + tag + " context for role " + role_tag(designs16_[i].role); break; } + c->MD = rows; + c->KD = c16->KD; + c->ND = c16->ND; + c->blocked_b = c16->blocked_b; + c->pb16i = c16->pb16i; + c->t2s = c16->t2s; + c->pb16i_planes = c16->pb16i_planes; + if (!c->init_elf(*dev, path.c_str(), rows, c16->KD, c16->ND, 1, inst)) { why = std::string("init_elf failed for instance ") + inst; break; } - wide++; - } - if (why.empty() && wide > 0 && !xd_->adopt_wide(rows)) why = "a role with a decode context has no wide one"; - if (why.empty() && wide > 0) { - fprintf(stderr, " [forward] wide family: %d role(s) at M=%d from %s (chunks of 17..%d rows)\n", - wide, rows, wide_elf, rows); + n++; + } + if (why.empty() && n > 0 && !(wide ? xd_->adopt_wide(rows) : xd_->adopt_narrow(rows))) + why = std::string("a role with a decode context has no ") + tag + " one"; + if (why.empty() && n > 0) { + if (wide) + fprintf(stderr, " [forward] wide family: %d role(s) at M=%d from %s (chunks of 17..%d rows)\n", + n, rows, path.c_str(), rows); + else + fprintf(stderr, " [forward] narrow family: %d role(s) at M=%d from %s (launches of 1..%d rows)\n", + n, rows, path.c_str(), rows); } else { - xd_->drop_wide(); - fprintf(stderr, " [forward] wide family NOT adopted (%s); chunks stay at %d rows\n", - why.empty() ? "no role has a decode context" : why.c_str(), Q4NX_DESIGN_M_DECODE); + if (wide) xd_->drop_wide(); else xd_->drop_narrow(); + if (wide) + fprintf(stderr, " [forward] wide family NOT adopted (%s); chunks stay at %d rows\n", + why.empty() ? "no role has a decode context" : why.c_str(), Q4NX_DESIGN_M_DECODE); + else + fprintf(stderr, " [forward] narrow family NOT adopted (%s); decode stays on the %d-row kernels\n", + why.empty() ? "no role has a decode context" : why.c_str(), Q4NX_DESIGN_M_DECODE); } + }; + { + const char* wr = getenv("NPU_WIDE_ROWS"); + load_family(true, onebit::npu::wide_elf_path(getenv("NPU_WIDE_SEALED_ELF"), wide_sealed_, sealed_dir_), + wr && *wr ? atoi(wr) : wide_sealed_rows_); + const char* nr = getenv("NPU_NARROW_ROWS"); + load_family(false, + onebit::npu::wide_elf_path(getenv("NPU_NARROW_SEALED_ELF"), narrow_sealed_, sealed_dir_), + nr && *nr ? atoi(nr) : narrow_sealed_rows_); } return true; } @@ -738,6 +758,10 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, wide_sealed_ = read_config_string( cfg_path, t2s_ ? "\"q4nx_t2s_sealed_wide_elf\"" : "\"q4nx_pb16i_sealed_wide_elf\"", ""); wide_sealed_rows_ = q4nx_config_int(cfg_path.c_str(), "q4nx_sealed_wide_rows", 32); + // ... and an optional third one built for fewer rows, for decode steps. + narrow_sealed_ = read_config_string( + cfg_path, t2s_ ? "\"q4nx_t2s_sealed_narrow_elf\"" : "\"q4nx_pb16i_sealed_narrow_elf\"", ""); + narrow_sealed_rows_ = q4nx_config_int(cfg_path.c_str(), "q4nx_sealed_narrow_rows", 4); // A folded container also declares the fold: `q4nx_hadamard` names the sign table and // `q4nx_hadamard_block` the Hadamard block. Without it the sidecar images would be // read in the stored (rotated) basis against un-rotated activations. diff --git a/npu/target/xdna/xdna_target.cpp b/npu/target/xdna/xdna_target.cpp index 9ca41c244..43e6ef982 100644 --- a/npu/target/xdna/xdna_target.cpp +++ b/npu/target/xdna/xdna_target.cpp @@ -366,6 +366,28 @@ bool XdnaTarget::adopt_wide(int rows) { wide_rows_ = rows; return true; } +I8Ctx* XdnaTarget::make_ctx_narrow(int role) { + if (role < 0 || role >= Q4NX_MAX_DESIGNS) return nullptr; + ctxn_[role] = std::make_unique(); + return ctxn_[role].get(); +} +bool XdnaTarget::adopt_narrow(int rows) { + int base[Q4NX_MAX_DESIGNS] = {0}, narrow[Q4NX_MAX_DESIGNS] = {0}; + for (int i = 0; i < Q4NX_MAX_DESIGNS; i++) { + if (ctx16_[i]) base[i] = ctx16_[i]->MD; + if (ctxn_[i]) narrow[i] = ctxn_[i]->MD; + } + if (!pb16i_ || !onebit::npu::narrow_family_complete(base, narrow, Q4NX_MAX_DESIGNS, rows)) { + drop_narrow(); + return false; + } + narrow_rows_ = rows; + return true; +} +void XdnaTarget::drop_narrow() { + for (auto& c : ctxn_) c.reset(); + narrow_rows_ = 0; +} void XdnaTarget::drop_wide() { for (auto& c : ctxw_) c.reset(); wide_rows_ = 0; @@ -394,7 +416,8 @@ void XdnaTarget::note_head(int M, double dpk, double dql, double dwt, double drb int XdnaTarget::design_index(I8Ctx* c) const { if (!c) return -1; for (int i = 0; i < ndesigns_ && i < Q4NX_MAX_DESIGNS; i++) - if (ctx_[i].get() == c || (ctx16_[i] && ctx16_[i].get() == c) || (ctxw_[i] && ctxw_[i].get() == c)) + if (ctx_[i].get() == c || (ctx16_[i] && ctx16_[i].get() == c) || (ctxw_[i] && ctxw_[i].get() == c) || + (ctxn_[i] && ctxn_[i].get() == c)) return i; return -1; } @@ -769,7 +792,12 @@ void XdnaTarget::npu_gemm(const GemvArgs& ar, std::vector& out, std::stri const float as = amax > 0.0f ? amax / 127.0f : 1.0f; // Phase-selected activation tile: a one-row call is decode, so it runs this // role's M=16 twin (one core row instead of 128 rows / 8 live tiles). - I8Ctx& kctx = *ctx_for(&ctx, 1, ar.decode_twin); + I8Ctx* kctx1 = ctx_for(&ctx, 1, ar.decode_twin); + // A decode step carries one row: with a narrow family it runs the kernel built for a few rows instead of 16. + if (pb16i_ && ar.role >= 0 && ar.role < Q4NX_MAX_DESIGNS && ctxn_[ar.role] && + onebit::npu::narrow_takes(1, narrow_rows_)) + kctx1 = ctxn_[ar.role].get(); + I8Ctx& kctx = *kctx1; const bool prof = prof_; const auto tp = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; @@ -1050,6 +1078,9 @@ void XdnaTarget::npu_gemm_rows(const GemvArgs& ar, std::vector& out, std: if (pb16i_ && ar.role >= 0 && ar.role < Q4NX_MAX_DESIGNS && ctxw_[ar.role] && onebit::npu::wide_takes(M, Q4NX_DESIGN_M_DECODE, wide_rows_)) kctxp = ctxw_[ar.role].get(); + else if (pb16i_ && ar.role >= 0 && ar.role < Q4NX_MAX_DESIGNS && ctxn_[ar.role] && + onebit::npu::narrow_takes(M, narrow_rows_)) + kctxp = ctxn_[ar.role].get(); // a short chunk (a verify pass of a few rows) fits the narrow kernel I8Ctx& kctx = *kctxp; const bool prof = prof_; const auto tp = prof ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; diff --git a/npu/target/xdna/xdna_target.h b/npu/target/xdna/xdna_target.h index 687289d98..261f6c197 100644 --- a/npu/target/xdna/xdna_target.h +++ b/npu/target/xdna/xdna_target.h @@ -189,6 +189,13 @@ class XdnaTarget final : public NpuTarget { void drop_wide(); bool adopt_wide(int rows); int wide_rows() const { return wide_rows_; } + // The NARROW family: the same roles built for fewer than 16 rows per launch, for decode steps and other + // launches of 1..narrow_rows() rows. Same rules as the wide family (all roles or none, one sealed hw_context); + // a launch of more rows keeps the 16-row family. + I8Ctx* make_ctx_narrow(int role); + bool adopt_narrow(int rows); + void drop_narrow(); + int narrow_rows() const { return narrow_rows_; } // The context a launch of `rows` rows on `role` would use: the M=16 twin when one was adopted and // the row count fits, otherwise the M=128 one. The forward needs this where it launches a GEMM // itself (the LM head) instead of going through gemv(). @@ -232,6 +239,8 @@ class XdnaTarget final : public NpuTarget { std::unique_ptr ctx16_[Q4NX_MAX_DESIGNS]; std::unique_ptr ctxw_[Q4NX_MAX_DESIGNS]; // the wide pb16i/t2s family (engine#1990) int wide_rows_ = 0; // 0 = no wide family + std::unique_ptr ctxn_[Q4NX_MAX_DESIGNS]; // the narrow pb16i/t2s family (decode) + int narrow_rows_ = 0; // 0 = no narrow family std::unique_ptr ctx_head_; bool m16_ = false; bool m16_check_ = false; diff --git a/tests/npu_pack_test.cpp b/tests/npu_pack_test.cpp index 879e43ec6..5fdaec024 100644 --- a/tests/npu_pack_test.cpp +++ b/tests/npu_pack_test.cpp @@ -151,6 +151,25 @@ static void wide_family_arms() { check(wide_elf_path("0", "W.elf", "/k").empty(), "wide elf: NPU_WIDE_SEALED_ELF=0 switches a pack's wide family off"); check(wide_elf_path("0x.elf", "", "/k") == "0x.elf", "wide elf: only the exact value 0 means off"); std::printf(" wide family: 21 arms (complete or not, chunk width, which launch takes it, which file)\n"); + // The narrow (decode) family: the mirror rules. + using onebit::npu::narrow_family_complete; + using onebit::npu::narrow_takes; + const int n4[] = {4, 4, 0, 4}; + check(narrow_family_complete(n16, n4, 4, 4), "narrow family: every decode role has a 4-row kernel -> complete"); + const int nhole[] = {4, 0, 0, 4}; + check(!narrow_family_complete(n16, nhole, 4, 4), "narrow family: one role without a narrow kernel -> not complete"); + const int nmixed[] = {4, 8, 0, 4}; + check(!narrow_family_complete(n16, nmixed, 4, 4), "narrow family: a role built for another row count -> not complete"); + const int nextra[] = {4, 4, 4, 4}; + check(!narrow_family_complete(n16, nextra, 4, 4), "narrow family: a kernel for a role with no decode one -> refused"); + check(!narrow_family_complete(n16, n16, 4, 16), "narrow family: 'narrow' must be narrower than the decode family"); + check(!narrow_family_complete(n16, w32, 4, 32), "narrow family: a wide set is not a narrow one"); + check(!narrow_family_complete(nullptr, n4, 4, 4) && !narrow_family_complete(n16, n4, 0, 4) && + !narrow_family_complete(n16, n4, 4, 0), "narrow family: null, empty and zero-row inputs are not complete"); + check(narrow_takes(1, 4) && narrow_takes(4, 4), "narrow takes: 1 and 4 rows go to a 4-row kernel"); + check(!narrow_takes(5, 4) && !narrow_takes(16, 4), "narrow takes: 5 rows never go to a 4-row kernel"); + check(!narrow_takes(1, 0) && !narrow_takes(0, 4), "narrow takes: no family takes nothing, and 0 rows is not a launch"); + std::printf(" narrow family: 10 arms (complete or not, which launch takes it)\n"); } int main(int argc, char** argv) { From 61ed559158cad9717fe21c68997298fb54f74f66 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:24:58 -0400 Subject: [PATCH 68/98] tests: register moe_expert_chunks_accum_test and shrink the unreferenced baseline to 1 (Refs #2018) (#2105) The baseline file said it itself: 'Each line is a real test whose runner was never added; the fix per file is one add_python_test(...)'. For this one that baseline entry was the wrong outcome -- the test is portable and passes on a plain interpreter (4 arms, 0 failures; worst-case |sum| 11.18e6 stays inside int32 at K=11008, headroom x192), it just had no ctest row. Registering it is the shrink the file's own comment asks for, and it removes a 'never run' record from a test that in fact works. Verified: ctest moe_expert_chunks_accum, tests_referenced, tests_referenced_self, ci_targets* and python_test_order* all pass (7/7); the reference sweep now reports 41 registered python tests and 1 deliberate baseline entry. --- CMakeLists.txt | 5 +++++ tools/ci/unreferenced_py_tests.txt | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eeed01a8f..610142905 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -306,6 +306,11 @@ add_python_test(python_test_order_check ${CMAKE_SOURCE_DIR}/tests/python_test_or add_python_test(python_test_order_check_self ${CMAKE_SOURCE_DIR}/tests/python_test_order_check.py --self-test) add_python_test(ci_targets ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --check --root ${CMAKE_SOURCE_DIR}) add_python_test(ci_targets_self ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --self-test) +# engine#2018: this one was in tools/ci/unreferenced_py_tests.txt as a baseline entry, and it should not have been: the +# file is portable and passes (worst-case |sum| 11.18e6 inside int32 at K=11008, 4 arms), it just never had a runner. A +# baseline records a test as never-run, which for a passing test is the wrong outcome -- so it is registered and the line +# is removed, which is the shrink the baseline's own comment asks for. +add_python_test(moe_expert_chunks_accum_test ${CMAKE_SOURCE_DIR}/tests/moe_expert_chunks_accum_test.py) # engine#1820: the other half of ci_targets_check -- a standalone shell test that nothing references cannot fail. # `need` is every tests/*.sh, `have` is every file that mentions its basename somewhere that can run it, and # tools/ci/unreferenced_tests.txt carries the deliberate exclusions WITH their reasons, so the list can shrink. diff --git a/tools/ci/unreferenced_py_tests.txt b/tools/ci/unreferenced_py_tests.txt index 9cef48019..e27a98d88 100644 --- a/tools/ci/unreferenced_py_tests.txt +++ b/tools/ci/unreferenced_py_tests.txt @@ -24,13 +24,13 @@ # A SET baseline, not an allow-list: `tools/ci/tests_referenced_check.py` FAILS when a name appears here that is not in # the tree's current set, and REPORTS a line whose file is registered now, so this can only shrink. # -# The number is 2, and getting it right took a correction worth recording: the first version of the rule scanned +# The number is 1 (was 2: moe_expert_chunks_accum_test.py is portable and passing, so it is registered rather than +# baselined -- engine#2018), and getting it right took a correction worth recording: the first version of the rule scanned # `RUNNERS`, which contains one FILE (CMakeLists.txt) and two directories, and tested `is_dir()` on all three -- so every # `add_python_test(...)` registration in CMakeLists.txt was invisible and 35 of 42 tests looked unregistered. The arm # "a cleared baseline entry is reported" caught it: registering a test in CMakeLists.txt did not clear its line. The true -# figure is 2 of 42, which is a healthy tree; the rule is here to keep it that way rather than to record a backlog. +# figure was 2 of 42, which is a healthy tree; the rule is here to keep it that way rather than to record a backlog. # # Each line is a real test whose runner was never added; the fix per file is one `add_python_test(...)`. # tests/hyperloom_benchmark_test.py -tests/moe_expert_chunks_accum_test.py From 24479ae97d3053225c42734df0b84a4e30f48c48 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:25:04 -0400 Subject: [PATCH 69/98] npu-pack test: the pack leg's head tile count follows the stored rows, so a padded tail head passes (#2104) A head read from the .fconst tail keeps the pack's padding to whole 64-row tiles (Hunyuan: 120,818 rows are stored as 120,832). The pack leg compared the container's tile count with the vocabulary's, so it failed one check on every such pack although the import was right (Hunyuan-1.8B: 15,104 tiles written, 15,100 expected). head_stored_rows() states the rule once; the import uses it for its tile count and the test for its expectation. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- npu/pack_import.cpp | 7 ++++++- npu/pack_import.h | 6 ++++++ tests/npu_pack_import_test.cpp | 25 ++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index d1e844bc7..f2ad9c895 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3363,6 +3363,11 @@ ContainerGeometry container_geometry(const json& meta, const json& config) { return g; } +int64_t head_stored_rows(int64_t vocab, bool from_fconst_tail) { + if (!from_fconst_tail || vocab <= 0) return vocab; + return (vocab + kTileRows - 1) / kTileRows * kTileRows; +} + std::string qk_norm_refusal(bool declared, bool found) { if (declared && !found) return "this pack declares use_qk_norm and its per-head q/k gains are not where the block layout says " @@ -5350,7 +5355,7 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // (Hunyuan-0.5B: 120,818 rows = 1,887.8 tiles, so the tail holds 1,888 and the last one is partly // padding). The rows past V are dropped before the tensor is written, so the container's head has // exactly V rows. - const int64_t rows = (V + kTileRows - 1) / kTileRows; + const int64_t rows = head_stored_rows(V, true) / kTileRows; // The envelope is the same `64 + 2*hidden` the layer blocks open with (eps, padding, the norm), not a // constant: 1,856 is only its qwen2.5-0.5B value. Qwen3-1.7B's tail is 4,160 + 2,374 x 73,984 to the // byte, which the constant would have missed by 2,304. diff --git a/npu/pack_import.h b/npu/pack_import.h index 513025dbf..5fc37dc35 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -96,6 +96,12 @@ struct QkGainOffsets { QkGainOffsets qk_gain_offsets(int64_t gap_start, int64_t head_dim, int64_t gap_bytes); std::string qk_norm_refusal(bool declared, bool found); +// How many rows a head read from the .fconst tail occupies in the container. The pack stores the head in whole 64-row +// tiles, so a vocabulary that is not a multiple of 64 is carried with the pack's own padding rows (Hunyuan: 120,818 -> +// 120,832) while `vocab_size` and the plan's tensor shape stay the vocabulary. A head that is a graph initializer does +// not go through this (its own padding, to the 32-row tile quantum, is engine#1992's). This is the number a container's lm_head tile count follows from. +int64_t head_stored_rows(int64_t vocab, bool from_fconst_tail); + // Why a pack's two layout witnesses cannot be reconciled, or "" when they can (engine#1985). // // `vote_hdrs` is the number of in-block headers the epsilon-anchor vote recorded (0 when it could not win); `declared_qk` diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 57e3e8862..0dcad1ff0 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -3007,10 +3007,13 @@ void test_pack(const std::string& pack, const std::string& golden_dir) { "pack: 64 columns per gate|up unit and per down_proj group"); int norms = 0, attn_planned = 0, mlp_planned = 0, lm_tiles_planned = 0; + bool head_in_tail = false; for (const auto& t : p.tensors) { if (t.name.find("layernorm") != std::string::npos || t.name == "model.norm.weight") ++norms; - if (t.dtype == "q4_1" && t.refusal.empty() && t.name.compare(0, 9, "lm_head.w") == 0) + if (t.dtype == "q4_1" && t.refusal.empty() && t.name.compare(0, 9, "lm_head.w") == 0) { lm_tiles_planned = int((t.shape[0] / 32) * ((t.shape[1] + 255) / 256)); + head_in_tail = t.source == "fconst:tail"; + } if (t.dtype == "q4_1" && t.refusal.empty() && t.name.find("self_attn") != std::string::npos) ++attn_planned; if (t.dtype == "q4_1" && t.refusal.empty() && t.name.find(".mlp.") != std::string::npos) ++mlp_planned; } @@ -3143,8 +3146,12 @@ void test_pack(const std::string& pack, const std::string& golden_dir) { // the container must open with the engine's own loader onebit::npu::Model m(out); const auto& head = m.tensor("lm_head.weight"); - check(head.shape.at(0) == int64_t((head_rows / 32) * ((p.tensors.at(1).shape.at(1) + 255) / 256)), - "pack: lm_head tile count matches the container"); + // A head read from the .fconst tail keeps the pack's padding to whole 64-row tiles (Hunyuan: 120,818 rows are stored + // as 120,832), so the container's count follows the stored rows; the plan's shape stays the vocabulary. + const int64_t stored_rows = pi::head_stored_rows(head_rows, head_in_tail); + check(head.shape.at(0) == (stored_rows / 32) * ((p.tensors.at(1).shape.at(1) + 255) / 256), + "pack: lm_head tile count matches the container (" + std::to_string(head.shape.at(0)) + " tiles for " + + std::to_string(stored_rows) + " stored rows)"); check(m.tensor("model.layers.0.input_layernorm.weight").shape.at(0) == p.dims.hidden, "pack: a layer norm is `hidden` bf16 values, whatever the hidden size"); check(m.tensor("model.norm.weight").shape.at(0) == int64_t(p.dims.hidden), @@ -5616,6 +5623,17 @@ void test_q4_writer_code_range() { // engine#1889: the arithmetic that claims the per-head QK gains out of the 640 B Hunyuan-7B carries between its last qkv // tile and o_proj. A wrong offset there reads tile bytes as bf16 and mis-rotates every pack of the family, so both the // right shape (2 * (64 + head_dim*2) = 640 at head_dim 128) and the neighbours that must NOT be taken for it are driven. +// The rows a tail head occupies in the container: the pack's own padding to whole 64-row tiles stays. The pack leg's +// tile-count check used the vocabulary, so it failed on every pack whose vocabulary is not a multiple of 64 although +// the import was right (Hunyuan-1.8B: 15,104 tiles written, 15,100 expected). +void test_head_stored_rows() { + check(pi::head_stored_rows(120818, true) == 120832, "head rows: Hunyuan's 120,818 are stored as 120,832"); + check(pi::head_stored_rows(151936, true) == 151936, "head rows: a multiple of 64 is stored as it is"); + check(pi::head_stored_rows(49152 + 32, true) == 49152 + 64, "head rows: a multiple of 32 that is not one of 64 is padded"); + check(pi::head_stored_rows(120818, false) == 120818, "head rows: a graph head is not padded"); + check(pi::head_stored_rows(0, true) == 0, "head rows: no vocabulary, no rows"); +} + void test_qk_gain_offsets() { const pi::QkGainOffsets o = pi::qk_gain_offsets(1000, 128, 640); // engine#1934: block 0 is Q. Hunyuan-7B's own model.onnx carries both gains as FP16 initializers, and q_norm is the @@ -5678,6 +5696,7 @@ int main(int argc, char** argv) { test_encode_tile_code_range(); test_q4_writer_code_range(); test_qk_norm_refusal(); + test_head_stored_rows(); test_qk_gain_offsets(); test_tail_head_placement(); const std::string golden_dir = argc > 1 ? argv[1] : ""; From 50550b2f72c9a26825d38121cb81f00f2ab84548 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:25:11 -0400 Subject: [PATCH 70/98] forward-serve: an answer that starts inside a reasoning block the prompt opened is split as reasoning (Refs #1786) (#2098) Qwen3.5's template ends its generation prompt with , so the model never writes the opening tag. The splitter only entered reasoning on a tag it saw, so the whole reasoning was returned as content and the closing tag dropped. npu::prompt_opens_reasoning() tells AnswerStream, parse_answer and content_mask to start inside the block. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- app/forward_serve.cpp | 9 ++++-- app/npu_answer_parser.h | 32 ++++++++++++++++---- docs/npu-chat-template.md | 9 ++++++ tests/npu_answer_parser_test.cpp | 52 ++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 9 deletions(-) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 0ead20865..1b4249bec 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -625,6 +625,9 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // markup whatever the container is (app/npu_tool_grammar.h), so that reply is split as qwen3. Every // other reply is in the container's own markup. const npu::AnswerFamily family = opts.tool_grammar ? npu::AnswerFamily::qwen3 : e.answer_family; + // A template whose generation prompt already opens the reasoning block leaves the answer inside it + // (npu::prompt_opens_reasoning). Only a rendered chat prompt can: an ids prompt and /v1/completions are raw. + const bool starts_in_reasoning = chat && !prompt_is_ids && npu::prompt_opens_reasoning(prompt, family); auto wrap = [&](const std::string& content, const std::string& finish, const std::vector& lps) { json out; out["id"] = id; @@ -635,7 +638,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // (issue #652 item 3) one shared parser splits what the model wrote into OpenAI's three // parts; a reply that is only tool calls says content: null. const npu::ParsedAnswer ans = - npu::parse_answer(content, family, opts.tools_sent); + npu::parse_answer(content, family, opts.tools_sent, starts_in_reasoning); json message{{"role", "assistant"}}; message["content"] = ans.content_null ? json(nullptr) : json(ans.content); if (!ans.reasoning.empty()) message["reasoning_content"] = ans.reasoning; @@ -656,7 +659,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib std::vector texts; texts.reserve(lps.size()); for (const auto& x : lps) texts.push_back(x.text); - const std::vector keep = npu::content_mask(texts, family, opts.tools_sent); + const std::vector keep = npu::content_mask(texts, family, opts.tools_sent, starts_in_reasoning); std::vector content_lps; for (size_t i = 0; i < lps.size(); i++) if (i < keep.size() && keep[i]) content_lps.push_back(lps[i]); @@ -703,7 +706,7 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib }; // The same splitter as the non-streamed path, fed piece by piece, so a streamed reply // and a non-streamed one cannot disagree about what is reasoning and what is content. - npu::AnswerStream stream(family, opts.tools_sent); + npu::AnswerStream stream(family, opts.tools_sent, starts_in_reasoning); auto emit_pieces = [&](const std::string& t, const std::vector& lp) { std::vector> pieces; stream.feed(t, pieces); diff --git a/app/npu_answer_parser.h b/app/npu_answer_parser.h index 305a6a2c1..ab1c95ee2 100644 --- a/app/npu_answer_parser.h +++ b/app/npu_answer_parser.h @@ -142,8 +142,11 @@ inline AnswerFamily answer_family_of_template(const std::string& template_text) // (is_reasoning, text) pieces; finish() flushes what the holdback still held and reports the calls. class AnswerStream { public: - AnswerStream(AnswerFamily family, bool tools_sent) - : fam_(family), tags_(answer_detail::tags_of(family)), tools_sent_(tools_sent) {} + // `starts_in_reasoning`: the PROMPT already opened the reasoning block (prompt_opens_reasoning below), so the + // answer begins inside it and the first thing to look for is the closing tag. + AnswerStream(AnswerFamily family, bool tools_sent, bool starts_in_reasoning = false) + : fam_(family), tags_(answer_detail::tags_of(family)), tools_sent_(tools_sent), + mode_(starts_in_reasoning ? Mode::reasoning : Mode::content) {} void feed(const std::string& chunk, std::vector>& out) { buf_ += chunk; @@ -376,12 +379,28 @@ class AnswerStream { bool saw_text_ = false; }; +// Does the rendered prompt leave the model INSIDE a reasoning block? Some templates end their generation prompt +// with the opening tag itself -- Qwen3.5's ends `<|im_start|>assistant\n\n` when thinking is on -- so the +// model never writes that tag: its answer starts with the reasoning and the first tag in it is the closing one. +// A splitter that waits to see the block opened then files the whole reasoning under `content` and drops the +// closing tag it never saw opened, so nothing in the reply marks where the answer starts (measured on FastFlowLM's +// Qwen3.5-2B-NPU2, engine#1786). True when the prompt, less trailing whitespace, +// ends with the family's opening tag. A prompt that opens and closes the block (thinking off: +// `\n\n\n\n`) does not end with it, and neither does one that never mentions it. +inline bool prompt_opens_reasoning(const std::string& prompt, AnswerFamily family) { + const std::string open = answer_detail::tags_of(family).think_open; + size_t end = prompt.size(); + while (end > 0 && (prompt[end - 1] == ' ' || prompt[end - 1] == '\n' || prompt[end - 1] == '\r' || prompt[end - 1] == '\t')) --end; + return !open.empty() && end >= open.size() && prompt.compare(end - open.size(), open.size(), open) == 0; +} + // Which of `pieces` (one generated token's text each, in order) belong to `content` -- not to // reasoning and not to call markup. Callers that report a per-token value use this so reasoning and // call markup get no entry (issue #651 item 3). A token that straddles a boundary ("The") // counts as content: its text did reach the caller. -inline std::vector content_mask(const std::vector& pieces, AnswerFamily family, bool tools_sent) { - AnswerStream s(family, tools_sent); +inline std::vector content_mask(const std::vector& pieces, AnswerFamily family, bool tools_sent, + bool starts_in_reasoning = false) { + AnswerStream s(family, tools_sent, starts_in_reasoning); std::vector mask; mask.reserve(pieces.size()); std::vector> out; @@ -406,8 +425,9 @@ inline std::vector content_mask(const std::vector& pieces, An } // The whole answer at once: the same splitter, so the two paths cannot disagree. -inline ParsedAnswer parse_answer(const std::string& text, AnswerFamily family, bool tools_sent) { - AnswerStream s(family, tools_sent); +inline ParsedAnswer parse_answer(const std::string& text, AnswerFamily family, bool tools_sent, + bool starts_in_reasoning = false) { + AnswerStream s(family, tools_sent, starts_in_reasoning); std::vector> pieces; s.feed(text, pieces); s.finish(pieces); diff --git a/docs/npu-chat-template.md b/docs/npu-chat-template.md index 6baf7dac5..b8581feef 100644 --- a/docs/npu-chat-template.md +++ b/docs/npu-chat-template.md @@ -198,6 +198,15 @@ Both give the client the same OpenAI shape: `reasoning_content`, `content`, and is a JSON string. A Gemma call body that is not well formed (a string that never closes, a key with no value, an unclosed object or array, an empty value) is not a call: the markup is returned in `content` unchanged. +**A reasoning block the prompt opened.** Some templates end their generation prompt with the opening tag itself: +Qwen3.5's ends `<|im_start|>assistant\n\n` when thinking is on. The model then never writes that tag; its +answer starts with the reasoning and the first tag in it is the closing one. `forward-serve` checks whether the +rendered prompt, less trailing whitespace, ends with the family's opening tag (`npu::prompt_opens_reasoning`), and +if so splits the answer as starting inside the block. Before engine#1786's row on FastFlowLM's `Qwen3.5-2B-NPU2` +it did not: the whole reasoning came back in `content`, `reasoning_content` was empty, and the closing tag was +dropped, so nothing in the reply marked where the answer began. With thinking off the same template opens and +closes the block in the prompt, and the answer is content from its first token. + One exception: `tool_choice: "required"` or a named function installs a grammar that constrains the reply to the `qwen3` call markup for every container (`app/npu_tool_grammar.h`), so that reply is split as `qwen3`. A grammar in Gemma's own call syntax is not written yet. diff --git a/tests/npu_answer_parser_test.cpp b/tests/npu_answer_parser_test.cpp index f79a067dc..e66c93951 100644 --- a/tests/npu_answer_parser_test.cpp +++ b/tests/npu_answer_parser_test.cpp @@ -129,6 +129,58 @@ int main() { "the qwen3 family's reasoning tags are the ones this test feeds"); } + // A reasoning block the PROMPT opened (engine#1786, FastFlowLM's Qwen3.5-2B-NPU2): the template's generation prompt + // ends with ``, so the answer starts inside the block and the only tag in it is the closing one. + { + const AnswerFamily Q = AnswerFamily::qwen3; + const std::string on = "<|im_start|>user\nWhat is the capital of France?<|im_end|>\n<|im_start|>assistant\n\n"; + const std::string off = "<|im_start|>user\nq<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"; + const std::string plain = "<|im_start|>user\nq<|im_end|>\n<|im_start|>assistant\n"; + check(prompt_opens_reasoning(on, Q), "a prompt that ends with leaves the model inside the block"); + check(!prompt_opens_reasoning(off, Q), "one that opens AND closes it (thinking off) does not"); + check(!prompt_opens_reasoning(plain, Q) && !prompt_opens_reasoning("", Q) && !prompt_opens_reasoning("\n \n", Q), + "nor does one that never mentions it, or an empty one"); + check(!prompt_opens_reasoning("tell me what means<|im_end|>\n<|im_start|>assistant\n", Q), + "the tag earlier in the prompt is not the prompt's end"); + check(!prompt_opens_reasoning(on, AnswerFamily::gemma4), "the tag is the family's own: opens nothing for gemma4"); + + const std::string reply = "Thinking Process:\n1. The capital of France is Paris.\n\n\nParis"; + const auto before = parse_answer(reply, Q, false); // what the route did: the fault, as an assertion + check(before.reasoning.empty() && contains(before.content, "Thinking Process") && !contains(before.content, "") && + contains(before.content, "Paris"), + "without the flag the reasoning is filed under content, with the closing tag dropped, so nothing marks where the answer starts (the measured fault)"); + const auto a = parse_answer(reply, Q, false, /*starts_in_reasoning=*/true); + check(a.reasoning == "Thinking Process:\n1. The capital of France is Paris.\n", "with it the reasoning is the reasoning"); + check(a.content == "\n\nParis" && !contains(a.content, "think"), "and the content is the answer, with no tag in it"); + + // an answer cut off inside the block: all reasoning, nothing invented as content + const auto cut = parse_answer("Thinking Process:\n1. Analyze", Q, false, true); + check(cut.reasoning == "Thinking Process:\n1. Analyze" && cut.content.empty(), "an answer cut inside the block is all reasoning"); + // a call after the prompt-opened block still parses + const auto call = parse_answer("need the tool{\"name\":\"ping\",\"arguments\":{}}", Q, true, true); + check(call.reasoning == "need the tool" && call.calls.size() == 1 && call.calls[0].name == "ping" && call.content_null, + "a tool call after the prompt-opened block is still a call"); + // the model re-opening the block itself is harmless + const auto twice = parse_answer("againok", Q, false, true); + check(twice.content == "ok" && contains(twice.reasoning, "again"), "a model that writes the opening tag anyway still ends up split"); + + // streamed, split at awkward places, the pieces add up to the same thing + AnswerStream st(Q, false, true); + std::vector> out; + for (const std::string& piece : {std::string("Thinking Pro"), std::string("cess:\n1. The capital of France is Paris.\n\n\nPar"), std::string("is")}) + st.feed(piece, out); + st.finish(out); + check(st.reasoning() == a.reasoning && st.content() == a.content, "the streamed split equals the whole-answer split"); + check(!out.empty() && out.front().first, "and its first piece is reasoning, so a client sees it as such from the first token"); + + // per-token mask: reasoning tokens get no logprob entry, content tokens do + const std::vector toks = {"Thinking", " Process", "", "\n\n", "Paris"}; + const std::vector m = content_mask(toks, Q, false, true); + check(m == std::vector({0, 0, 0, 1, 1}), "the content mask starts after the closing tag"); + check(content_mask(toks, Q, false) == std::vector({1, 1, 0, 1, 1}), "where without the flag the reasoning tokens counted as content"); + } + std::printf("%s\n", failures ? "FAILED" : "all cases ok"); return failures ? 1 : 0; } From 5acb6e7a1f26eb73fb7dd492d2360348b4d754d8 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:58:23 -0400 Subject: [PATCH 71/98] bump-hrx: a mirror hold, so a rewrite of AMD's line that needs a hand merge is reported, not failed daily (Refs #1945) (#2110) config/llama-mirror-hold.config: while it names the mirror's commit, the sync reports the hold, moves nothing and succeeds. The run fails, naming the file, when the hold is stale (the mirror moved) or unneeded (AMD's pin contains the held commit again). The shipped file names no commit: it holds nothing until the owner adds the line. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- config/llama-mirror-hold.config | 33 +++++++++++++++++++++ docs/hrx.md | 12 ++++++++ scripts/bump-hrx-sync.sh | 51 +++++++++++++++++++++++++++++++++ tests/bump_hrx_sync_test.sh | 31 ++++++++++++++++++++ 4 files changed, 127 insertions(+) create mode 100644 config/llama-mirror-hold.config diff --git a/config/llama-mirror-hold.config b/config/llama-mirror-hold.config new file mode 100644 index 000000000..e53755909 --- /dev/null +++ b/config/llama-mirror-hold.config @@ -0,0 +1,33 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The commit of 1bit-MONSTER/llama.cpp `1bit/hrx-vulkan` (our mirror of AMD's HRX line) that the daily +# bump-hrx run must not move away from. While a commit is named here, bump-hrx does not move the mirror, +# does not merge AMD's llama.cpp pin into `1bit/hrx-vulkan-patched`, reports the hold in the run summary and +# succeeds. Delete the commit line to lift the hold. +# +# The hold is for one situation only: AMD rewrote its line, so its pin no longer descends from the mirror, +# and taking it is a hand merge that has not been done yet. The run FAILS, naming this file, when the file +# is out of date (the mirror is no longer the named commit) or no longer needed (AMD's pin contains the +# named commit again), so a hold cannot outlive the state it describes. +# +# No commit is named below, so nothing is held. Naming one is the owner's decision. The state it would +# record, as of 2026-11-11 (engine#1945): +# mirror 1bit/hrx-vulkan 44ee5710d19f7f3af79b99bbcbe5fd13979d1f67 (AMD's line before its rebase) +# AMD pin 9bf5871310a8d7de74f4cb0c57009835fbf91c27 (the rebased line, 1777 upstream commits newer) +# merge into 1bit/hrx-vulkan-patched, against the last AMD commit our line took (43b0ec2c41cc): +# 149 conflicted files, 538 hunks (65 files / 83 hunks in the HRX backend) +# To hold, put the mirror's commit on a line of its own: +# 44ee5710d19f7f3af79b99bbcbe5fd13979d1f67 diff --git a/docs/hrx.md b/docs/hrx.md index 0b1ebecee..2faf2db4d 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -98,6 +98,18 @@ fails a pin that leaves a gate model's node off HRX. `ggml/src/ggml-hrx` keep our side; a conflict anywhere else fails the run and names the files; - merges AMD's hrx-system pin into our `1bit/main` the same way; - opens a PR here moving the submodules, listing the llama.cpp commits new to the engine. +- **The mirror hold.** A rewrite of AMD's line that the rules above cannot merge leaves the daily run with + nothing to do but fail, every day, until somebody does the merge by hand. While + [`config/llama-mirror-hold.config`](../config/llama-mirror-hold.config) names the mirror's commit, the run + instead reports the hold in its summary (`mirror 1bit/hrx-vulkan HELD at …; AMD's pin is …, on a line that + does not contain it`), moves nothing and succeeds; the engine's pin still follows our own line. The hold + describes one state and fails the run when that state ends, so it cannot be forgotten: **stale** when the + mirror is no longer the named commit, **unneeded** when AMD's pin contains the named commit again (an ordinary + sync would work). `scripts/bump-hrx-sync.sh llama-hold ` prints the decision alone + (`none`, `held`, `stale`, `unneeded`). The file ships naming no commit, so it holds nothing; naming one is + the owner's decision. It was added for AMD's 2026-11-09 rebase (engine#1945): taking that pin is a merge of + 149 conflicted files (538 hunks) into `1bit/hrx-vulkan-patched`, measured against the last AMD commit our + line took. - **The hold.** While [`config/hrx-system-hold.config`](../config/hrx-system-hold.config) names a commit, the run does not merge AMD's hrx-system, keeps `third_party/hrx-system` where it is, and takes only AMD's llama.cpp changes outside the HRX backend paths diff --git a/scripts/bump-hrx-sync.sh b/scripts/bump-hrx-sync.sh index c8beae0a9..3e9e1e80f 100755 --- a/scripts/bump-hrx-sync.sh +++ b/scripts/bump-hrx-sync.sh @@ -34,6 +34,8 @@ # A mirror commit that is on no AMD branch fails the run and is named, as one of two # things (off_branch_commits): AMD's own commit from a line AMD rebased and dropped, to be # merged by hand, or a commit that is not AMD's, to be moved to our line. +# - the mirror is held (config/llama-mirror-hold.config names its commit): the run reports the hold and +# moves nothing. For a rewrite that cannot be merged by rule (engine#1945); see llama_hold_decision. # - hrx-system is held (config/hrx-system-hold.config). AMD's ggml-hrx is written against # AMD's hrx-system (e.g. #139 moved the kernel corpus to the `multiple_of` predicate that # our held hrx-system cannot parse), so only AMD's changes OUTSIDE the HRX backend paths @@ -48,6 +50,7 @@ set -euo pipefail cmd=${1:-} DRY_RUN=${DRY_RUN:-0} HOLD_FILE=${HOLD_FILE:-config/hrx-system-hold.config} +LLAMA_HOLD_FILE=${LLAMA_HOLD_FILE:-config/llama-mirror-hold.config} ENGINE_DIR=$(pwd) # The HRX backend paths of the llama.cpp tree (what AMD's hrx-graph line adds on top of ggml). # tests/CMakeLists.txt belongs here: it is where the HRX test targets are registered, so it moves @@ -85,6 +88,34 @@ hold_sha() { grep -v '^[[:space:]]*#' "$ENGINE_DIR/$HOLD_FILE" | grep -oE '^[0-9a-f]{40}' | head -n1 || true } +# The commit 1bit/hrx-vulkan (our mirror of AMD's line) is held at, or empty. Same file shape as the hrx-system hold. +llama_hold_sha() { + local f=$LLAMA_HOLD_FILE + case "$f" in /*) ;; *) f=$ENGINE_DIR/$f ;; esac # relative to the engine checkout unless given whole + [ -f "$f" ] || return 0 + grep -v '^[[:space:]]*#' "$f" | grep -oE '^[0-9a-f]{40}' | head -n1 || true +} + +# What a mirror hold means for this run. Reads repo $1; $2 is the held commit (may be empty), $3 the mirror tip, +# $4 AMD's pin. One word: +# none no commit is held: the sync runs as usual +# held the mirror is the held commit and AMD's pin does not descend from it: report it, move nothing +# stale a commit is held but the mirror is somewhere else: somebody moved the mirror, the file is out of date +# unneeded the mirror is the held commit and AMD's pin contains it: an ordinary fast-forward would work +# 2026-11-10 (engine#1945): AMD re-committed its HRX series on a base 1777 upstream commits newer. Taking that +# pin is a merge of about 150 conflicted files into our line, which is days of work for the lanes that own +# those files, and until it is done the daily run can only fail. A failure that is known, sized and assigned +# is not news; a hold says so in the run's summary and lets the run stay green for everything else it checks. +# The hold is deliberately narrow: `stale` and `unneeded` both FAIL the run, so the file cannot outlive the +# state it describes -- the day AMD's pin is reachable again, or the mirror moves, the run says to delete it. +llama_hold_decision() { + local repo=$1 hold=$2 mirror=$3 pin=$4 + if [ -z "$hold" ]; then echo none; return; fi + if [ "$hold" != "$mirror" ]; then echo stale; return; fi + if [ "$mirror" = "$pin" ] || git -C "$repo" merge-base --is-ancestor "$mirror" "$pin" 2>/dev/null; then echo unneeded; return; fi + echo held +} + # Why a mirror commit is on no AMD branch. Reads repo $1 (commits only are enough), whose refs/amd/* # are AMD's branches; $2 is the mirror tip and $3 AMD's new pin. One line per such commit: # rebased the new pin holds a commit with the same author, author date and @@ -128,6 +159,24 @@ sync_llama() { patched=$(git rev-parse refs/remotes/fork/patched) echo "AMD pin $pin, mirror 1bit/hrx-vulkan $mirror, our line 1bit/hrx-vulkan-patched $patched" + # 0. A mirror hold (config/llama-mirror-hold.config): the run reports it and moves nothing. + local lhold; lhold=$(llama_hold_sha) + case "$(llama_hold_decision . "$lhold" "$mirror" "$pin")" in + stale) + die "$LLAMA_HOLD_FILE holds 1bit/hrx-vulkan at ${lhold:0:12}, but the mirror is ${mirror:0:12}: the hold is out of date. Name the mirror's commit in it, or delete the commit line to let the sync run." ;; + unneeded) + die "$LLAMA_HOLD_FILE holds 1bit/hrx-vulkan at ${lhold:0:12}, and AMD's pin ${pin:0:12} contains that commit: an ordinary sync would work. Delete the commit line." ;; + held) + note "1bit/hrx-vulkan is held at ${mirror:0:12} by $LLAMA_HOLD_FILE: AMD's pin ${pin:0:12} does not descend from it (AMD rewrote its line). Nothing was moved; the merge is tracked where the hold file says." + summary "- llama.cpp: mirror \`1bit/hrx-vulkan\` HELD at \`${mirror:0:12}\` ($LLAMA_HOLD_FILE); AMD's pin is \`${pin:0:12}\`, on a line that does not contain it. Not merged." + output "rebased=$patched" + output "llama_held=1bit/hrx-vulkan HELD at \`${mirror:0:12}\` by \`$LLAMA_HOLD_FILE\` (AMD's pin is \`${pin:0:12}\`)." + output "patches</dev/null || true) +EOF" + return 0 ;; + esac + # 1. The mirror. It holds AMD's commits only; ours live on 1bit/hrx-vulkan-patched. local rewritten=0 if [ "$mirror" != "$pin" ] && ! git merge-base --is-ancestor "$mirror" "$pin"; then @@ -293,5 +342,7 @@ case "$cmd" in hrx) sync_hrx ;; # For tests/bump_hrx_sync_test.sh: the classification alone, on a local repository, no network. off-branch) off_branch_commits "${2:?repo}" "${3:?mirror}" "${4:?pin}" ;; + # The mirror-hold decision alone, on a local repository: llama-hold , hold file from LLAMA_HOLD_FILE. + llama-hold) llama_hold_decision "${2:?repo}" "$(llama_hold_sha)" "${3:?mirror}" "${4:?pin}" ;; *) echo "usage: $0 llama|hrx" >&2; exit 2 ;; esac diff --git a/tests/bump_hrx_sync_test.sh b/tests/bump_hrx_sync_test.sh index 4108db8ce..4191ec487 100755 --- a/tests/bump_hrx_sync_test.sh +++ b/tests/bump_hrx_sync_test.sh @@ -102,5 +102,36 @@ other=$(g rev-parse HEAD) expect "the same subject and author at another author date is foreign" "$(run "$other" "$pin" | head -n1)" \ "foreign $(g rev-parse --short "$other") hrx: something new" +# ---- the mirror hold (config/llama-mirror-hold.config): what a named commit means for a run ------------------- +holdf=$work/hold.config +hold() { LLAMA_HOLD_FILE=$holdf bash "$script" llama-hold "$repo" "$1" "$2"; } +# a pin that descends from the mirror: build one +g checkout -q --detach "$mirror" +commit "$amd" "$amd_mail" 2026-11-10T00:00:00Z 2026-11-10T00:00:00Z "hrx: one more on the old line" +desc=$(g rev-parse HEAD) + +rm -f "$holdf" +expect "no hold file: the sync runs as usual" "$(hold "$mirror" "$pin")" "none" +printf '# a comment\n# %s\n' "$mirror" > "$holdf" +expect "a commit that is only mentioned in a comment holds nothing" "$(hold "$mirror" "$pin")" "none" +printf '# held\n%s\n' "$mirror" > "$holdf" +expect "the mirror is the held commit and AMD's pin is on a rewritten line: held" "$(hold "$mirror" "$pin")" "held" +expect "the mirror has moved away from the held commit: the file is stale" "$(hold "$ours" "$pin")" "stale" +expect "AMD's pin contains the held commit again: the hold is not needed" "$(hold "$mirror" "$desc")" "unneeded" +expect "the pin IS the held commit: not needed either" "$(hold "$mirror" "$mirror")" "unneeded" +printf 'not-a-sha\n%s extra words\n' "$mirror" > "$holdf" +expect "the commit line may carry a trailing note; a line that is not a commit is ignored" "$(hold "$mirror" "$pin")" "held" +printf '%s\n' "${mirror:0:12}" > "$holdf" +expect "a short hash is not a hold: forty hex digits or nothing" "$(hold "$mirror" "$pin")" "none" + +# The file shipped in the tree names no commit, so merging it holds nothing. +shipped=$(cd "$(dirname "$script")/.." && pwd)/config/llama-mirror-hold.config +if [ -f "$shipped" ]; then + expect "the shipped hold file holds nothing until a commit line is added" \ + "$(LLAMA_HOLD_FILE=$shipped bash "$script" llama-hold "$repo" "$mirror" "$pin")" "none" +else + fail "config/llama-mirror-hold.config is in the tree" +fi + if [ "$fails" -ne 0 ]; then echo "$fails check(s) failed"; exit 1; fi echo "bump_hrx_sync: ok" From 0f68f1ef1d11386107628f0a65539bfd40151452 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:58:29 -0400 Subject: [PATCH 72/98] weekly: the runner preflight warns on a short restart policy and blocks only on a dead runner (Refs #1504) (#2109) ci-runner-health.sh exits 3 (not 1) when every runner unit is accepting jobs but a restart policy would leave one down after the next OOM kill, and prints those lines as WARN. weekly.sh's preflight blocks on exit 1 and warns on exit 3 (owner decision on #1504, 2026-11-11). A missing ceiling alone no longer reads 'stops for good'. docs/ci-runners.md carries the two drop-in files and the three owner commands (scripts/ci-runner-restart.conf), and says that exit 143 in a build step is the memory guard. tests/weekly_runner_preflight_test.sh runs the real health script and the real preflight function against a stub systemctl. Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 4 ++ docs/ci-runners.md | 75 ++++++++++++++++++++++- docs/hw-gate.md | 4 +- scripts/ci-runner-health.sh | 28 +++++++-- scripts/ci-runner-restart.conf | 29 +++++++++ scripts/weekly.sh | 42 +++++++------ tests/weekly_runner_preflight_test.sh | 88 +++++++++++++++++++++++++++ 7 files changed, 245 insertions(+), 25 deletions(-) create mode 100644 scripts/ci-runner-restart.conf create mode 100755 tests/weekly_runner_preflight_test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 610142905..ce9aeeed1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -374,6 +374,10 @@ add_test(NAME pack_version COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_version_test.s # arm asserts the row's message, and a clean tree is the control. No package build, no GPU, runs in CI. add_test(NAME pack_check COMMAND ${CMAKE_SOURCE_DIR}/tests/pack_check_test.sh) add_test(NAME check_ci_runners COMMAND ${CMAKE_SOURCE_DIR}/tests/check_ci_runners_test.sh) +# engine #1504: the release preflight blocks on a runner unit that is down and only warns on a short restart policy. +# The real scripts/ci-runner-health.sh and weekly.sh's preflight_runner_units() against a stub systemctl; no systemd. +add_test(NAME weekly_runner_preflight COMMAND ${CMAKE_SOURCE_DIR}/tests/weekly_runner_preflight_test.sh) +add_test(NAME ci_runner_health_selftest COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/ci-runner-health.sh --self-test) # engine #1750: the carried amdxdna series applies in order to the pinned driver, and the context-restart code it # produces fails a context's jobs (and logs it) when the restart fails, instead of leaving them parked or running # them at close on a half-built firmware context. The driver's own functions in a user-space harness: no NPU, no diff --git a/docs/ci-runners.md b/docs/ci-runners.md index f3195c82a..107760817 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -306,7 +306,9 @@ here: a new workflow changes what the repository's token may do, which is an RFC ## Two rows, one question: are the runners healthy? Since #1682 and #1546 both landed there are two checks, and neither alone is enough — each is blind to what the -other sees. Both run in `scripts/weekly.sh`'s preflight, so a release refuses to start on a runner it cannot trust. +other sees. Both run in `scripts/weekly.sh`'s preflight. A runner that is **not accepting jobs** stops the release +before it starts. A runner that is up but whose **restart policy is short** is a warning, and the release goes on +(owner decision on engine#1504, 2026-11-11): see [the next section](#a-short-restart-policy-warns-it-does-not-block). | | `scripts/check-ci-runners.sh` | `scripts/ci-runner-health.sh` | |---|---|---| @@ -315,7 +317,7 @@ other sees. Both run in `scripts/weekly.sh`'s preflight, so a release refuses to | notices | a registration that was revoked, expired or never made | `Result=oom-kill`, a restart window, a unit that stopped | | cannot see | *why* it died — no `oom-kill`, no restart count | a unit that is running fine while its registration is gone | | speed | after the heartbeat times out | the moment the unit dies | -| exit | 0 online · 1 an offender, named · **2 cannot tell** (unreadable or truncated list) | 0 all accepting · 1 one is not · 2 no such units on this host | +| exit | 0 online · 1 an offender, named · **2 cannot tell** (unreadable or truncated list) | 0 all accepting, policy right · 1 one is not accepting jobs · 2 no such units on this host · **3 all accepting, a restart policy is short** | Run either by hand — both are read-only and neither restarts anything: @@ -335,3 +337,72 @@ rather than report green. One case neither row covers: a runner that is `online` in the API *and* active locally, but wedged so it takes no jobs. That is a hung process, not a registration or unit state; the practical signal for it is a queue that stops moving while both rows stay green. + +## A short restart policy warns, it does not block + +`scripts/ci-runner-health.sh` prints two lines per unit: its state (`PASS`/`FAIL`) and its restart policy +(`PASS`/`WARN`). The exit code keeps them apart, because they call for different things: + +| exit | meaning | `scripts/weekly.sh` preflight | +|---|---|---| +| 0 | every unit is accepting jobs and would come back after an OOM kill | passes | +| 1 | a unit is not accepting jobs now (dead, or in its restart window) | **blocks**: the checks after the release would queue for ever | +| 2 | no `actions.runner.*` units on this host | a note | +| 3 | every unit is accepting jobs, but a `WARN` line names a policy setting | **warns** and goes on | + +A unit that is down outranks a policy line, so 1 wins over 3. The warning in the release log reads +`preflight: WARNING, not blocking: every runner unit is accepting jobs, but a restart policy would leave one down +after the next OOM kill`, with the `WARN` lines above it naming the unit and the setting. + +Why a warning is enough: the policy is a box configuration change, which the owner applies (below), and a release +that cannot start does not make the unit any safer. Meanwhile the heavy builds run under +`scripts/mem-guard.sh --term`, which ends the build before the kernel's OOM killer reaches the runner. **A build step +that ends with exit 143 was stopped by that guard**: the log has a `GUARD: MemAvailable N GB < floor GB` line, the +runner is still alive, and the job is re-run once memory frees. The guard covers only the steps that use it. A server +or a bench started without it can still take the unit down, which is why the drop-ins below are the real fix. + +`tests/weekly_runner_preflight_test.sh` (ctest `weekly_runner_preflight`) runs the real health script and the real +`preflight_runner_units()` from `scripts/weekly.sh` against a stub `systemctl`: the four exit codes, block on a dead +unit, warn on a short policy. + +### The two drop-ins (owner step) + +On strixhalo, `strixhalo-bench` and `strixhalo-org` still run the units that predate `scripts/install-ci-runner.sh`: +`Restart=no`, `OOMPolicy=stop`. One OOM kill of any process in the unit stops it for good. `strixhalo-engine` got +the same drop-in on 2026-11-10. This needs `sudo`, so it is the owner's to run. It does not re-register a runner, +change its user, or stop a job in flight. + +Both files have the same content, [`scripts/ci-runner-restart.conf`](../scripts/ci-runner-restart.conf): + +```ini +# /etc/systemd/system/actions.runner.1bit-MONSTER-engine.strixhalo-bench.service.d/restart.conf +# /etc/systemd/system/actions.runner.1bit-MONSTER.strixhalo-org.service.d/restart.conf +[Unit] +StartLimitIntervalSec=0 + +[Service] +Restart=always +RestartSec=15 +OOMPolicy=continue +``` + +The three commands, from the root of an engine checkout: + +```sh +sudo install -Dm644 scripts/ci-runner-restart.conf /etc/systemd/system/actions.runner.1bit-MONSTER-engine.strixhalo-bench.service.d/restart.conf +sudo install -Dm644 scripts/ci-runner-restart.conf /etc/systemd/system/actions.runner.1bit-MONSTER.strixhalo-org.service.d/restart.conf +sudo systemctl daemon-reload +``` + +Check it took (read-only, no `sudo`): + +```sh +systemctl show actions.runner.1bit-MONSTER-engine.strixhalo-bench.service actions.runner.1bit-MONSTER.strixhalo-org.service -p Id -p Restart -p OOMPolicy +scripts/ci-runner-health.sh; echo "exit $?" +``` + +Want `Restart=always` and `OOMPolicy=continue` on both. The health script still exits 3 afterwards, with one `WARN` +per unit naming only `MemoryMax=infinity(want a ceiling)`: the drop-in sets no memory ceiling. The ceiling this page +documents (`MemoryHigh=43G`, `MemoryMax=48G`) comes with a full `scripts/install-ci-runner.sh` install, or with +`sudo systemctl set-property MemoryHigh=43G MemoryMax=48G` per unit. To undo a drop-in, delete the +`restart.conf` and run `sudo systemctl daemon-reload`. diff --git a/docs/hw-gate.md b/docs/hw-gate.md index 1d7afdeef..67402e213 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -72,7 +72,9 @@ hrx-system) through `cmake/hrx.cmake`, then links `onebit` with its HRX wiring. - **Memory.** The box is shared, and on Strix Halo GTT is host RAM. Before it compiles, the job waits for 24 GB of `MemAvailable` (`scripts/mem-wait.sh`, polled once a minute for up to 15 minutes). If memory doesn't free up, the job fails with the reading, and a re-run is the fix. The compile itself runs under `scripts/mem-guard.sh --term 12`, - which stops the whole `thermal-run` scope if `MemAvailable` falls below 12 GB. On 2026-11-08 both self-hosted + which stops the whole `thermal-run` scope if `MemAvailable` falls below 12 GB. **Exit 143 in a build step is + that guard, not a compiler failure**: the step log has a `GUARD: MemAvailable N GB < 12 GB` line just above it, the + runner is still alive, and a re-run once memory frees is the fix. On 2026-11-08 both self-hosted runners were OOM-killed, the second time 76 s after a restart handed this job a compile (engine#1504). - **What it is not.** It is a compile and link check of the public plumbing only. It does not build the private HIP add-on (the gate builds that) and does not dispatch anything. Green means the pinned pair builds into the engine. It diff --git a/scripts/ci-runner-health.sh b/scripts/ci-runner-health.sh index 20f47c874..0cd0fa2cf 100755 --- a/scripts/ci-runner-health.sh +++ b/scripts/ci-runner-health.sh @@ -30,8 +30,12 @@ # scripts/ci-runner-health.sh [...] # named units, e.g. for a specific runner # scripts/ci-runner-health.sh --self-test # the verdict logic, with systemctl stubbed # -# Exit: 0 when every runner is active/running with no failure result; 1 when any is not; 2 when there is nothing to inspect -# (no such units on this host, so a silent pass would be the defect). +# Exit: 0 when every runner is active/running with a restart policy that survives an OOM kill; 1 when a runner is not +# accepting jobs NOW; 2 when there is nothing to inspect (no such units on this host, so a silent pass would be the defect); +# 3 when every runner is accepting jobs but a restart policy would leave one down after the next OOM kill. 1 and 3 are +# separate because they call for different things (engine#1504, owner 2026-11-11): a dead runner blocks a release, since +# its checks would queue for ever, while a policy line is a box configuration change the owner applies +# (docs/ci-runners.md "The two drop-ins"), so scripts/weekly.sh warns on 3 and goes on. The policy lines print as WARN. set -uo pipefail # -> one verdict line. The state checks above answer "is it alive", which is what the @@ -53,6 +57,13 @@ policy_verdict() { # an infinite ceiling means the cgroup never hits its own limit, so the box's OOM killer chooses - and the bench runner # is the unit whose jobs are memory-heavy by definition. [ -n "${_mmax:-}" ] && [ "$_mmax" != infinity ] || _bad="$_bad MemoryMax=${_mmax:-?}(want a ceiling);" + case "$_bad" in + "") ;; + *Restart=*|*OOMPolicy=*) ;; + # Restart and OOMPolicy are right and only the ceiling is missing: the unit does come back, so do not say it will not. + *) printf 'FAIL the runner would come back after an OOM kill, but its cgroup has no ceiling:%s the box-wide OOM killer picks the victim, not the limit of this unit' "$_bad" + return 0 ;; + esac if [ -n "$_bad" ]; then printf 'FAIL the restart policy cannot restart this runner after an OOM kill:%s one OOM kill and the unit stops for good, while jobs read as pending' "$_bad" return 0 @@ -109,7 +120,12 @@ self_test() { chk FAIL "$(policy_verdict always stop 51539607552)" # an OOM kill of a JOB stops the RUNNER chk FAIL "$(policy_verdict always continue infinity)" # no cgroup ceiling: the box's OOM killer chooses chk FAIL "$(policy_verdict '' '' '')" # nothing readable is not a pass - [ "$rc" = 0 ] && echo "ci-runner-health self-test: ok (15 cases)" || echo "ci-runner-health self-test: FAIL" + # a missing ceiling alone must not be reported as "the unit stops for good" (the live strixhalo-engine unit, 2026-11-11) + case "$(policy_verdict always continue infinity)" in + *'would come back'*) printf 'ok a missing ceiling alone says the runner comes back\n' ;; + *) printf 'FAIL a missing ceiling alone should not say the unit stops for good\n'; rc=1 ;; + esac + [ "$rc" = 0 ] && echo "ci-runner-health self-test: ok (16 cases)" || echo "ci-runner-health self-test: FAIL" return $rc } @@ -125,7 +141,7 @@ if [ ${#units[@]} -eq 0 ]; then exit 2 fi -rc=0 +rc=0 policy=0 for u in "${units[@]}"; do line=$("$SC" show "$u" -p ActiveState -p SubState -p Result -p NRestarts -p Restart -p OOMPolicy -p MemoryMax 2>/dev/null) st=$(printf '%s\n' "$line" | sed -n 's/^ActiveState=//p') @@ -144,7 +160,9 @@ for u in "${units[@]}"; do p=$(policy_verdict "$rst" "$oom" "$mmax") case "$p" in PASS*) printf 'PASS %-52s %s\n' "" "${p#PASS }" ;; - *) printf 'FAIL %-52s %s\n' "" "${p#FAIL }"; rc=1 ;; + *) printf 'WARN %-52s %s\n' "" "${p#FAIL }"; policy=1 ;; esac done +# a runner that is down outranks a policy line: 1 if any unit is not accepting jobs, else 3 if only a policy is short. +[ "$rc" = 0 ] && [ "$policy" = 1 ] && rc=3 exit "$rc" diff --git a/scripts/ci-runner-restart.conf b/scripts/ci-runner-restart.conf new file mode 100644 index 000000000..ee193a792 --- /dev/null +++ b/scripts/ci-runner-restart.conf @@ -0,0 +1,29 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# systemd drop-in for a self-hosted runner unit that predates scripts/install-ci-runner.sh (engine#1504). Installed as +# /etc/systemd/system/.d/restart.conf by the owner; docs/ci-runners.md "The two drop-ins" has the commands. +# It gives the unit the crash-recovery lines of the documented unit and changes nothing else (same user, same paths). +[Unit] +# never give up restarting: a runner that hit the start limit is as dead as one that was never restarted +StartLimitIntervalSec=0 + +[Service] +# always, not on-failure: runsvc.sh exits 0 when it is killed, so on-failure never fires +Restart=always +RestartSec=15 +# the kernel's OOM victim is one process; with the default (stop) that ends the unit. continue lets the job die and +# the runner keep running, so the job fails with a reason instead of the runner vanishing +OOMPolicy=continue diff --git a/scripts/weekly.sh b/scripts/weekly.sh index 468864481..d8ce116fc 100755 --- a/scripts/weekly.sh +++ b/scripts/weekly.sh @@ -635,6 +635,29 @@ stage_release() { say "released $tag" } +# The systemd row of the runner preflight (engine#1504): $1 is scripts/ci-runner-health.sh. Returns 1 only when a runner +# unit is not accepting jobs NOW (its exit 1): the checks after the release would queue for ever. Its exit 3 - every +# runner up, but a restart policy that would leave one down after the next OOM kill - is a WARNING and the release goes +# on (owner 2026-11-11): the policy is a box configuration change the owner applies (docs/ci-runners.md "The two +# drop-ins"), every heavy build already runs under scripts/mem-guard.sh --term, which ends the build (exit 143) before +# the kernel's OOM killer reaches the runner, and a release that cannot start does not make the unit any safer. +# A function of its own so tests/weekly_runner_preflight_test.sh can run it against a stub. +preflight_runner_units() { + local health=$1 health_out health_rc=0 + [ -x "$health" ] || return 0 + health_out=$("$health" 2>&1) || health_rc=$? + case "$health_rc" in + 0) printf '%s\n' "$health_out" | grep -c '^PASS' | xargs -r -I{} echo "preflight: runner units: {} PASS in systemd's view" ;; + 2) say "preflight: no actions.runner.* units on this host: systemd has nothing to inspect" ;; + 3) printf '%s\n' "$health_out" | sed 's/^/preflight: /' + echo "preflight: WARNING, not blocking: every runner unit is accepting jobs, but a restart policy would leave one down after the next OOM kill (scripts/ci-runner-health.sh exit 3; the WARN lines above name the unit and the setting, docs/ci-runners.md \"The two drop-ins\" has the fix)" ;; + *) printf '%s\n' "$health_out" | sed 's/^/preflight: /' + echo "preflight: a runner unit is not accepting jobs, so the checks after this release would queue for ever (scripts/ci-runner-health.sh exit $health_rc; the FAIL lines above say which; WEEKLY_SKIP_RUNNER_CHECK=1 to release anyway)" + return 1 ;; + esac + return 0 +} + # --- preflight ---------------------------------------------------------------------------- # What the stages need from the machine, checked before the first one runs, so that a missing # tool stops the run at the start rather than after an hour of build (2026-10-19 rehearsal). @@ -686,6 +709,7 @@ preflight() { # scripts/ci-runner-health.sh - systemd's view (#1546): Result=oom-kill, a restart window, a unit that is not # active. Says WHY, and fires the moment a unit dies. Exit 2 = no such units on # this host, i.e. a fresh box, which is a note rather than a failure. + # Exit 3 = all up but a restart policy is short: a warning (preflight_runner_units). # WEEKLY_SKIP_RUNNER_CHECK=1 is the release-during-an-outage escape, and it skips BOTH: a flag that skipped only # one of them would be useless in exactly the outage it exists for, because both fail for the same dead runner. if [ "${WEEKLY_SKIP_RUNNER_CHECK:-0}" != 1 ]; then @@ -697,23 +721,7 @@ preflight() { echo "preflight: a self-hosted runner is not online, so the checks after this release would queue for ever (WEEKLY_SKIP_RUNNER_CHECK=1 to release anyway)" bad=1 fi - if [ -x "$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" ]; then - if health_out=$("$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" 2>&1); then - printf '%s\n' "$health_out" | grep -c '^PASS' | xargs -r -I{} echo "preflight: runner units: {} PASS in systemd's view" - else - health_rc=$? - if [ "$health_rc" = 2 ]; then - say "preflight: no actions.runner.* units on this host: systemd has nothing to inspect" - else - printf '%s\n' "$health_out" | sed 's/^/preflight: /' - # Two kinds of FAIL reach here (engine#1504): a unit that is down NOW, and since #2082 a unit that is up - # but whose restart policy would leave it down after the next OOM kill. The FAIL lines above say which; - # this line must not claim "not accepting jobs" for the second, where every runner is active/running. - echo "preflight: a runner unit is down, or its restart policy would leave it down after the next OOM kill (scripts/ci-runner-health.sh exit $health_rc; the FAIL lines above say which, and scripts/install-ci-runner.sh fixes a policy line)" - bad=1 - fi - fi - fi + preflight_runner_units "$(cd "$(dirname "$0")" && pwd)/ci-runner-health.sh" || bad=1 else # a skip that says nothing is the failure mode these rows exist for: a release log where the runner checks # silently did not run reads exactly like one where they passed. diff --git a/tests/weekly_runner_preflight_test.sh b/tests/weekly_runner_preflight_test.sh new file mode 100755 index 000000000..381f590a5 --- /dev/null +++ b/tests/weekly_runner_preflight_test.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# weekly_runner_preflight_test.sh: the release preflight BLOCKS on a runner unit that is down and only WARNS on a restart +# policy that is short (engine#1504, owner 2026-11-11). Two halves, both the real code: +# 1. scripts/ci-runner-health.sh against a stub systemctl: exit 0 / 1 / 2 / 3, and a down unit outranks a policy line. +# 2. scripts/weekly.sh's preflight_runner_units(), cut out of the script and run against the real health script on the +# same stub: returns 0 with a WARNING line on exit 3, returns 1 on exit 1. +# No systemd, no network, no build. +set -u +root=$(cd "$(dirname "$0")/.." && pwd) +tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT +fails=0 +ok() { printf 'ok %s\n' "$1"; } +bad() { printf 'FAIL %s\n' "$1"; fails=$((fails + 1)); } + +# stub systemctl: STUB_UNITS = lines "unit state sub result restart oompolicy memorymax" +cat > "$tmp/systemctl" <<'STUB' +#!/usr/bin/env bash +case "$1" in + list-units) awk '{print $1, "loaded", $2, $3}' "$STUB_UNITS" ;; + show) awk -v u="$2" '$1 == u { + printf "ActiveState=%s\nSubState=%s\nResult=%s\nNRestarts=0\nRestart=%s\nOOMPolicy=%s\nMemoryMax=%s\n", $2, $3, $4, $5, $6, $7 }' "$STUB_UNITS" ;; +esac +STUB +chmod +x "$tmp/systemctl" +export SYSTEMCTL="$tmp/systemctl" STUB_UNITS="$tmp/units" +health="$root/scripts/ci-runner-health.sh" + +good="actions.runner.a.service active running success always continue 51539607552" +policy="actions.runner.b.service active running success no stop infinity" # the live bench unit, 2026-11-11 +dead="actions.runner.c.service failed failed oom-kill no stop infinity" # the 11-08 shape + +expect_rc() { # + local name=$1 want=$2; shift 2 + printf '%s\n' "$@" > "$STUB_UNITS"; [ $# -eq 0 ] && : > "$STUB_UNITS" + out=$(bash "$health" 2>&1); rc=$? + if [ "$rc" = "$want" ]; then ok "health: $name -> exit $rc"; else bad "health: $name: want exit $want, got $rc: $out"; fi +} +expect_rc "all healthy" 0 "$good" +expect_rc "a unit is down" 1 "$good" "$dead" +expect_rc "no units" 2 +expect_rc "all up, one policy short" 3 "$good" "$policy" +expect_rc "down outranks policy" 1 "$policy" "$dead" +printf '%s\n' "$good" "$policy" > "$STUB_UNITS" +out=$(bash "$health" 2>&1) +if printf '%s\n' "$out" | grep -q '^WARN .*Restart=no(want always)' && ! printf '%s\n' "$out" | grep -q '^FAIL'; then + ok "health: a short policy prints WARN with the setting, and no FAIL line" +else bad "health: a short policy should print WARN and no FAIL: $out"; fi + +# half 2: the function weekly.sh really runs +sed -n '/^preflight_runner_units() {$/,/^}$/p' "$root/scripts/weekly.sh" > "$tmp/fn.sh" +grep -q 'health_rc' "$tmp/fn.sh" || { bad "could not cut preflight_runner_units() out of scripts/weekly.sh"; echo "weekly_runner_preflight: FAIL"; exit 1; } +grep -q 'preflight_runner_units "' "$root/scripts/weekly.sh" || bad "scripts/weekly.sh's preflight no longer calls preflight_runner_units" +say() { echo "$*"; } +# shellcheck disable=SC1091 +. "$tmp/fn.sh" +row() { # + local name=$1 want=$2 text=$3; shift 3 + printf '%s\n' "$@" > "$STUB_UNITS"; [ $# -eq 0 ] && : > "$STUB_UNITS" + out=$(preflight_runner_units "$health" 2>&1); rc=$? + if [ "$rc" = "$want" ] && printf '%s\n' "$out" | grep -q -- "$text"; then ok "weekly: $name -> return $rc" + else bad "weekly: $name: want return $want with '$text', got $rc: $out"; fi +} +row "all healthy passes" 0 "PASS in systemd's view" "$good" +row "a short policy warns and does not block" 0 "WARNING, not blocking" "$good" "$policy" +row "the warning names the setting" 0 "OOMPolicy=stop(want continue)" "$good" "$policy" +row "a dead unit blocks" 1 "not accepting jobs" "$good" "$dead" +row "a dead unit blocks even beside a short policy" 1 "after an OOM kill" "$policy" "$dead" +row "no units is a note" 0 "nothing to inspect" +out=$(preflight_runner_units "$tmp/absent.sh" 2>&1); rc=$? +[ "$rc" = 0 ] && [ -z "$out" ] && ok "weekly: no health script, no row" || bad "weekly: a missing health script should be silent: $rc $out" + +[ "$fails" = 0 ] && { echo "weekly_runner_preflight: ok"; exit 0; } +echo "weekly_runner_preflight: FAIL ($fails)"; exit 1 From 697db7dc205e08251da98c23381c3ad7304c6354 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:58:36 -0400 Subject: [PATCH 73/98] npu-pack: say what main does with a graph head (stored as is; an unaligned one is refused), review of #2104 (#2107) Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- npu/pack_import.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/npu/pack_import.h b/npu/pack_import.h index 5fc37dc35..8977766f2 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -98,8 +98,10 @@ std::string qk_norm_refusal(bool declared, bool found); // How many rows a head read from the .fconst tail occupies in the container. The pack stores the head in whole 64-row // tiles, so a vocabulary that is not a multiple of 64 is carried with the pack's own padding rows (Hunyuan: 120,818 -> -// 120,832) while `vocab_size` and the plan's tensor shape stay the vocabulary. A head that is a graph initializer does -// not go through this (its own padding, to the 32-row tile quantum, is engine#1992's). This is the number a container's lm_head tile count follows from. +// 120,832) while `vocab_size` and the plan's tensor shape stay the vocabulary. A head that is a graph initializer is +// stored as it is: on main nothing pads it, and one whose row count is not a multiple of 32 is refused by the tile +// writer ("q4_1 tiles need N % 32 == 0"). If graph-head padding lands (engine#1992 was closed unmerged), its quantum +// belongs here. This is the number a container's lm_head tile count follows from. int64_t head_stored_rows(int64_t vocab, bool from_fconst_tail); // Why a pack's two layout witnesses cannot be reconciled, or "" when they can (engine#1985). From aa11d5127fbb9c333a0eac519af91a3b1877dd6d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:09:22 -0400 Subject: [PATCH 74/98] ci: a skipped test is a test that did not run, and now something says so (refs #1946) (#2095) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: a skipped test is a test that did not run, and now something says so (refs #1946) Five of the defects this session kept finding are one defect: **green because nothing ran.** * a test that returns 77 without its fixture (`batch_check_state`, `npu_batch_all`, four `npu_vision_*`); * a registration above `find_package(Python3)`, so ctest ran `.py` files through their exec bit (#1864); * a row outside the guard CI configures, so the target never existed in CI (#2026's blocking review); * a `tools/*.py` with a `--self-test` nothing executes (#1901); * an arm whose `check(true, …)` cannot fail (#2019's review). `ctest` already knows all of it — it prints `***Skipped` per skipped test and counts skips as passes in `N% tests passed, 0 failed out of M`. Nothing read it. `tools/ci/ctest_skips.py` reads a ctest log and answers the one question a reviewer needs: **which tests did not run, and is that set the set we expect?** * `***Skipped` (the test ran and returned 77) → against `tools/ci/ctest_skips.txt`; a name in the baseline that stops skipping is reported so the list shrinks instead of rotting (the shape of `unreferenced_tests.txt`, #1824); * **`(Not Run)` — the binary is missing — is never baselineable.** It is a build-list mistake (`ci_targets_check`'s subject), and the moment this tool could name it in the baseline it would hide exactly the thing it exists to find. `--write` prints those separately and refuses to put them in the file. ## Measured on the CI-shaped tree (`ONEBIT_NPU=OFF`, 192 tests) ``` $ ctest --output-on-failure --timeout 300 > log $ tools/ci/ctest_skips.py --log log 7 tests ***Skipped -> baselined, one line each, with the reason 53 tests （Not Run) -> reported as missing binaries, not baselined 56 failed -> the summary line, unchanged ``` The 53 are this developer tree's unbuilt targets, not CI's — in CI the Build step lists what it builds and `ci_targets_check` keeps that list honest. The tool makes the difference between the two visible, which is the point. ## Self-test Eight arms on synthetic logs: no skips, both skip shapes, a baselined skip accepted, an unbaselined skip refused, a baseline entry that stopped skipping reported, a missing binary refused with no way to baseline it, the two shapes yielding different verdicts, and a log without a summary being "cannot tell" (rc 2). `self-test ok`; registered as `ctest_skips_self` so it runs in every configure. ## What this PR does not do It does not wire itself into `ci.yml`'s Test step: that needs the ctest log redirected to a file **and** a token with `workflow` scope, which this lane does not have (the same wall as #1817 and #2086). The wiring is one line — `ctest … | tee build/ctest.log` then `tools/ci/ctest_skips.py --log build/ctest.log` — and it belongs to whoever can push a workflow change. Refs #1946, #1864, #2026, #1901, #2019, #1824. * ci: the skip baseline carries its notice, and the copyright gate now knows the name (CI found this on #2095) CI was right and the check did its job: ``` build Copyright notice on every file ##[error]Process completed with exit code 1 no comment style known (add one to LINE_STYLES/BLOCK_STYLES or EXEMPT): tools/ci/ctest_skips.txt ``` `.txt` has no comment style of its own, so `tools/copyright.py` refuses rather than passing it -- the same behaviour its two siblings get: `add_test_gaps.txt` (#1595) and `unreferenced_tests.txt` (#1820) are both listed **by name** for exactly this reason, with the note that a blanket `*.txt` would demand a notice in files that are data rather than code. So this does what those two do, and nothing more: * `ctest_skips.txt` joins the by-name list in `tools/copyright.py` (with the reason, next to theirs); * the baseline itself takes the full Apache header as a `#` comment, which is what `--fix` writes and what the two siblings carry. Verified: `tools/copyright.py --check` rc 0; `ctest_skips.py --self-test` ok; `ctest -R ctest_skips` 1/1 passed. Refs #2095 (this is the fix for its failing build), #1946, #1595, #1820. --------- Co-authored-by: t --- CMakeLists.txt | 5 + tools/ci/ctest_skips.py | 195 +++++++++++++++++++++++++++++++++++++++ tools/ci/ctest_skips.txt | 26 ++++++ tools/copyright.py | 4 + 4 files changed, 230 insertions(+) create mode 100755 tools/ci/ctest_skips.py create mode 100644 tools/ci/ctest_skips.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index ce9aeeed1..b6b578068 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -358,6 +358,11 @@ add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test- # two false conclusions about a pack in one night. `--self-test` drives the rule on a synthetic shard (no network): # the same offset from the data section returns the tensor, from byte 8 it does not. add_python_test(hf_tensor_self ${CMAKE_SOURCE_DIR}/tools/hf_tensor.py --self-test) +# engine#1946: a test that returns 77 is green, and green-because-it-did-not-run is the failure this fleet keeps paying +# for. `tools/ci/ctest_skips.py` reads a ctest log and reports every test that did not execute -- `***Skipped` (77) against +# `tools/ci/ctest_skips.txt`, and `(Not Run)` (a missing binary) as a build-list mistake that can never be baselined. +# Self-test only here: wiring it into ci.yml's Test step needs the log plus a `workflow`-scoped token (engine#1817). +add_python_test(ctest_skips_self ${CMAKE_SOURCE_DIR}/tools/ci/ctest_skips.py --self-test) add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) # engine#856: one HRX process's GTT split into the loader's copy and HRX's packed second copy, from its own -v log. add_python_test(hrx_gtt_split_self ${CMAKE_SOURCE_DIR}/tools/hrx_gtt_split.py --self-test) diff --git a/tools/ci/ctest_skips.py b/tools/ci/ctest_skips.py new file mode 100755 index 000000000..c5f088b99 --- /dev/null +++ b/tools/ci/ctest_skips.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""ctest_skips.py -- make "green because it did not run" visible in a ctest log. + +Two nights of reviewing turned up the same failure five times: a test that returns 77 without its fixture, a +registration above `find_package`, a row outside the guard CI configures, a `.py` nothing runs, an arm whose +`check(true, ...)` cannot fail. Each one is *green*, and each one is green because nothing executed. + +`ctest` already knows: it prints `***Skipped` per skipped test and a `N% tests passed, 0 failed out of M` line that +counts skips as passes. This tool reads a ctest log and answers the one question a reviewer needs -- **which tests did +not run, and is that set the set we expect?** -- against a checked-in baseline. + + tools/ci/ctest_skips.py --log build/ctest.log # report; exit 1 on a NEW skip + tools/ci/ctest_skips.py --log build/ctest.log --write # refresh the baseline + tools/ci/ctest_skips.py --self-test + +Exit: 0 every skip is in the baseline, 1 a test skipped that the baseline does not name (named), 2 cannot tell +(no log, or no summary line). + +The baseline is `tools/ci/ctest_skips.txt`: one test name per line with an optional `# reason`. An entry that no longer +skips is reported so the list shrinks instead of rotting -- the same shape as `unreferenced_tests.txt` (#1824). +""" + +import argparse +import pathlib +import re +import sys + +BASELINE = pathlib.Path("tools/ci/ctest_skips.txt") +# ctest writes "did not run" in two shapes, and the second is the worse one: +# " 16/192 Test #16: npu_repack_path .......***Skipped 0.72 sec" the test ran and returned 77 +# "\t182 - npu_private_route (Not Run)" the binary was not built at all +# Only the first is a fixture deciding it cannot work here; the second is a build-list mistake (ci_targets_check's +# subject) and has no legitimate baseline entry. +SKIP_LINE = re.compile(r"^\s*\d+/\d+ Test\s+#\d+:\s+(\S+)\s+\.*\*\*\*Skipped", re.MULTILINE) +NOT_RUN_LINE = re.compile(r"^\s*\d+ - (\S+) \(Not Run\)", re.MULTILINE) +SUMMARY = re.compile(r"^\s*(\d+)% tests passed, (\d+) tests failed out of (\d+)") + + +def skipped(log_text: str): + """{test name: 'skipped' or 'not-run'} -- every test that did not execute.""" + found = {name: "skipped" for name in SKIP_LINE.findall(log_text)} + for name in NOT_RUN_LINE.findall(log_text): + found[name] = "not-run" + return found + + +def summary(log_text: str): + """(failed, total) from ctest's summary line, or None when the log has no summary.""" + for line in log_text.splitlines(): + m = SUMMARY.match(line) + if m: + return int(m.group(2)), int(m.group(3)) + return None + + +def baseline(root: pathlib.Path): + """{test name: reason} from the baseline file; comments and blanks skipped.""" + path = root / BASELINE + out = {} + if not path.is_file(): + return out + for i, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + line = line.strip() + if not line or line.startswith("#"): + continue + name, _, reason = line.partition("#") + name = name.strip() + if name: + out[name] = reason.strip() + return out + + +def check(root: pathlib.Path, log_text: str) -> int: + total = summary(log_text) + if total is None: + print("ctest_skips: cannot tell: the log has no 'N% tests passed' summary line", file=sys.stderr) + return 2 + failed, count = total + did_not_run = skipped(log_text) + allow = baseline(root) + new = sorted(n for n in did_not_run if n not in allow and did_not_run[n] == "skipped") + not_run = sorted(n for n in did_not_run if did_not_run[n] == "not-run") + gone = sorted(n for n in allow if n not in did_not_run) + for name in not_run: + print(f"ctest_skips: {name} did NOT RUN because its binary is missing -- that is a build-list mistake, not a " + f"fixture decision: add the target to ci.yml's Build step (tools/ci/ci_targets_check.py is the same " + f"class), and it needs no baseline entry") + for name in new: + print(f"ctest_skips: {name} SKIPPED and the baseline does not name it -- it did not run, and a test that does " + f"not run cannot fail. Give it its fixture, move it where CI builds it, or add it to {BASELINE} with " + f"the reason it cannot run here") + for name in gone: + print(f"ctest_skips: {name} is in {BASELINE} but it no longer skips -- drop the line, the list should shrink") + if new or gone or not_run: + return 1 + if did_not_run: + print(f"ctest_skips: {len(did_not_run)} of {count} test(s) did not run, all named in the baseline " + f"({failed} failed)") + else: + print(f"ctest_skips: all {count} test(s) ran ({failed} failed)") + return 0 + + +def write_baseline(root: pathlib.Path, log_text: str) -> int: + did_not_run = skipped(log_text) + # Only a test that RAN and returned 77 can be baselined. "Not Run" means the binary is missing, which is a build-list + # mistake (ci_targets_check's subject) and would become invisible the moment this file named it. + skipped_only = sorted(n for n, kind in did_not_run.items() if kind == "skipped") + not_run = sorted(n for n, kind in did_not_run.items() if kind == "not-run") + path = root / BASELINE + lines = ["# Tests that legitimately do not run in the CI configuration, one name per line with the reason.", + "# Written by tools/ci/ctest_skips.py --write. A name here that stops skipping is reported, so the list", + "# shrinks rather than rots (the shape of tools/ci/unreferenced_tests.txt, engine#1824).", + "# A test whose BINARY is missing is never baselined here: that is a build-list mistake, not a fixture.", + ""] + for name in skipped_only: + lines.append(f"{name} # TODO: why this cannot run here") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"ctest_skips: wrote {path} with {len(skipped_only)} skipped test(s)") + if not_run: + print(f"ctest_skips: {len(not_run)} test(s) did NOT RUN (binary missing) and are deliberately not baselined: " + f"{', '.join(not_run[:6])}{' …' if len(not_run) > 6 else ''}", file=sys.stderr) + return 0 + + +def self_test() -> int: + """A check that only ever sees a clean log proves nothing: drive all four verdicts.""" + import tempfile + + fails = 0 + + def ok(cond, what): + nonlocal fails + print(("ok " if cond else "FAIL ") + what) + fails += 0 if cond else 1 + + clean = "100% tests passed, 0 tests failed out of 3\n" + with_skip = (" 12/13 Test #12: npu_pack_import ..................***Skipped 0.01 sec\n" + "100% tests passed, 0 tests failed out of 13\n") + with_notrun = ("\t182 - npu_private_route (Not Run)\n" + "100% tests passed, 0 tests failed out of 13\n") + root = pathlib.Path(tempfile.mkdtemp()) + (root / BASELINE).parent.mkdir(parents=True, exist_ok=True) + (root / BASELINE).write_text("# reasons\nnpu_pack_import # needs a container\n") + + ok(not skipped(clean), "a log with no skip lines reports none") + ok(list(skipped(with_skip)) == ["npu_pack_import"], "a '***Skipped'-style line is read") + ok(check(root, clean) == 1, "a baseline entry that stopped skipping is drift (rc 1)") + ok(check(root, with_skip) == 0, "a skip the baseline names is accepted (rc 0)") + ok(summary(with_skip) == (0, 13), "the summary counts skips as passes, which is the point") + (root / BASELINE).write_text("# empty\n") + ok(check(root, with_skip) == 1, "a skip the baseline does NOT name is refused (rc 1)") + ok(check(root, "no summary here\n") == 2, "a log without a summary is 'cannot tell' (rc 2)") + ok(list(skipped(with_notrun)) == ["npu_private_route"], "a '(Not Run)' line is read") + ok(check(root, with_notrun) == 1, "a missing binary is refused with no way to baseline it (rc 1)") + ok(skipped(with_skip) != skipped(with_notrun), "... and the two shapes are not the same verdict") + print("self-test ok" if not fails else f"self-test FAILED ({fails})") + return 1 if fails else 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--log", default="build/ctest.log", help="a ctest log (default: build/ctest.log)") + ap.add_argument("--root", default=".") + ap.add_argument("--write", action="store_true") + ap.add_argument("--self-test", action="store_true") + a = ap.parse_args() + if a.self_test: + return self_test() + path = pathlib.Path(a.log) + if not path.is_file(): + print(f"ctest_skips: cannot tell: no log at {path}", file=sys.stderr) + return 2 + text = path.read_text(encoding="utf-8", errors="replace") + root = pathlib.Path(a.root).resolve() + return write_baseline(root, text) if a.write else check(root, text) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci/ctest_skips.txt b/tools/ci/ctest_skips.txt new file mode 100644 index 000000000..b41bd8a6d --- /dev/null +++ b/tools/ci/ctest_skips.txt @@ -0,0 +1,26 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Tests that legitimately do not run in the CI configuration, one name per line with the reason. +# Written by tools/ci/ctest_skips.py --write. A name here that stops skipping is reported, so the list +# shrinks rather than rots (the shape of tools/ci/unreferenced_tests.txt, engine#1824). +# A test whose BINARY is missing is never baselined here: that is a build-list mistake, not a fixture. + +amdxdna_restart # returns 77: needs a fixture this configuration does not have (see the test's own skip message) +npu_import_contract # returns 77: needs a fixture this configuration does not have (see the test's own skip message) +npu_repack_path # returns 77: needs a fixture this configuration does not have (see the test's own skip message) +patch_sfs_atomic # returns 77: needs a fixture this configuration does not have (see the test's own skip message) +serve_client_hangup_cpu # returns 77: needs a fixture this configuration does not have (see the test's own skip message) +serve_context_limit_cpu # returns 77: needs a fixture this configuration does not have (see the test's own skip message) +serve_embed_vectors_cpu # returns 77: needs a fixture this configuration does not have (see the test's own skip message) diff --git a/tools/copyright.py b/tools/copyright.py index 840a92fa9..580a41350 100644 --- a/tools/copyright.py +++ b/tools/copyright.py @@ -44,6 +44,10 @@ # engine#1820: the other sweep's allow-list, same shape and the same reason -- `#`-comment data, `.txt` has no # style, and the header it wants is already at the top of the file. "unreferenced_tests.txt", + # engine#1946: the ctest skip baseline, the same shape and the same reason -- a '#'-comment data file whose header + # is the notice it would otherwise be asked for. (CI caught this on the PR: the tool refuses an unknown style rather + # than passing it, which is the behaviour worth having.) + "ctest_skips.txt", # engine#2018: the third test-registration baseline, same shape and the same reason -- data, `.txt` has no style. "unreferenced_py_tests.txt", "21serial-console", "90onboard-limits", "Dockerfile", "*.Dockerfile", From cad81fd2ce8cd74f7f483f2bbf0cbcccfc6b0865 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:31:23 -0400 Subject: [PATCH 75/98] tools: fit the routing head over --dump-encoder output, so before/after accuracy has an after (engine#1313 item 3) (#2112) Co-authored-by: t --- CMakeLists.txt | 3 + docs/laya-models.md | 13 ++ tools/laya_head_finetune.py | 335 ++++++++++++++++++++++++++++++++++++ 3 files changed, 351 insertions(+) create mode 100644 tools/laya_head_finetune.py diff --git a/CMakeLists.txt b/CMakeLists.txt index b6b578068..45a7daa0a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -364,6 +364,9 @@ add_python_test(hf_tensor_self ${CMAKE_SOURCE_DIR}/tools/hf_tensor.py --self-tes # Self-test only here: wiring it into ci.yml's Test step needs the log plus a `workflow`-scoped token (engine#1817). add_python_test(ctest_skips_self ${CMAKE_SOURCE_DIR}/tools/ci/ctest_skips.py --self-test) add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) +# engine#1313 item 3: the "after" of "before/after routing accuracy" -- a ridge head over --dump-encoder output, with the +# table's own logits as the before arm. Self-test only, so it needs no dump and no model. +add_python_test(laya_head_finetune ${CMAKE_SOURCE_DIR}/tools/laya_head_finetune.py --self-test) # engine#856: one HRX process's GTT split into the loader's copy and HRX's packed second copy, from its own -v log. add_python_test(hrx_gtt_split_self ${CMAKE_SOURCE_DIR}/tools/hrx_gtt_split.py --self-test) # engine#679: time per HRX dispatch from the backend's serial trace (the id hash and the per-kernel sums). diff --git a/docs/laya-models.md b/docs/laya-models.md index 6da158d8b..ffdb8d913 100644 --- a/docs/laya-models.md +++ b/docs/laya-models.md @@ -145,6 +145,19 @@ A line whose decision was cached still carries the state it was cached under, so same decisions serve made. A decision whose `state_len` exceeds `len(state)` must be dropped or re-sent with a shorter prompt: `--eval` would otherwise classify a prefix the demo never sent. +**The head itself: `tools/laya_head_finetune.py` (engine#1313 item 3).** `--dump-encoder` writes the encoder state and +`--eval` scores a classifier; this takes the dump and fits the classifier, so "before/after routing accuracy" has both +halves on the same cases, encoder and class order: + +```sh +1bit route --laya-models table.json --laya-model DIR --dump-encoder cases.jsonl --out dump/ +tools/laya_head_finetune.py dump/ --holdout 0.5 # before (the table's logits) vs after (a ridge head), held out +``` + +The split is by **case and stratified by label**, not by index: a decision log is written in arrival order and a demo's +arrival order is class-contiguous, so an index split holds out a class rather than a sample of each. `--holdout 0` fits +and scores on everything and reports `in-sample`, because a number is not a gate until you know which it is. + `--eval` prints accuracy, per-class support, recall and precision, the confusion matrix (rows true, columns chosen), serve's mean confidence, the mean top probability, ECE (10 equal bins over the top probability), log loss and the decision time p50 and p90. Size-gated cases are counted (`gated`), since serve would gate them too. diff --git a/tools/laya_head_finetune.py b/tools/laya_head_finetune.py new file mode 100644 index 000000000..d0bf6b6e2 --- /dev/null +++ b/tools/laya_head_finetune.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Fine-tune the routing head over `1bit route --dump-encoder` output, and report before/after accuracy +(engine#1313, item 3). + +The other two thirds of item 3 already exist: `--eval` prints accuracy, per-class recall/precision, the confusion +matrix, ECE and log loss over labelled cases, and `--dump-encoder` writes the head's own input. What was missing is +the middle: nothing took the dump and turned it into a head, so "before/after routing accuracy" had no "after". + +`before` is the table's decision, read from each case's own `logits` in the dump (argmax) -- so the two arms are +scored on the same cases, the same encoder and the same class order, and the only thing that changes is the head. +`after` is a ridge regression from the pooled encoder state to the class one-hot, which is closed-form, deterministic +and needs no framework: on a few hundred cases with a few thousand input rows the normal equations are exact. + +The split is by CASE, never by row: a case's encoder rows are all in one half, or a head could be scored on rows of a +case it was fitted on and the number would be in-sample while looking held out. `--holdout 0` fits and scores on +everything and says so (`in-sample`), because a gate must not read a number without knowing which it is. + +Usage: + tools/laya_head_finetune.py DUMP_DIR [--holdout F] [--ridge L] [--pool mean|last] [--json] + tools/laya_head_finetune.py --self-test + +The dump is what `1bit route --laya-models T --laya-model DIR --dump-encoder cases.jsonl --out DUMP` writes: + DUMP/h.f32 float32 encoder rows, concatenated + DUMP/cases.jsonl one line per case: {"src", "label", "options", "offset", "len", "logits", ...} + DUMP/dump.json {"d": encoder width, "rows", "cases", ...} + +A line whose `label` is -1 was written without a label; it is skipped, exactly as `--eval` skips it. +""" +import argparse +import json +import math +import os +import struct +import sys + + +def read_dump(root): + with open(os.path.join(root, "dump.json")) as f: + info = json.load(f) + d = int(info["d"]) + cases = [] + with open(os.path.join(root, "cases.jsonl")) as f: + for line in f: + line = line.strip() + if line: + cases.append(json.loads(line)) + raw = open(os.path.join(root, "h.f32"), "rb").read() + rows = len(raw) // (4 * d) if d else 0 + encoded = struct.unpack("<%df" % (rows * d), raw[: rows * d * 4]) + return d, encoded, cases + + +def pool_case(encoded, d, off, length, how): + """One feature vector per case: the mean (or the last) encoder row.""" + if length <= 0: + return None + if how == "last": + start = (off + length - 1) * d + return list(encoded[start : start + d]) + acc = [0.0] * d + for r in range(length): + base = (off + r) * d + for i in range(d): + acc[i] += encoded[base + i] + inv = 1.0 / length + return [v * inv for v in acc] + + +def argmax(v): + best = 0 + for i in range(1, len(v)): + if v[i] > v[best]: + best = i + return best + + +def fit_ridge(xs, ys, k, lam): + """Closed-form W (d x k) mapping features to one-hot, with a bias column. Solves (X'X + L)^-1 X'Y.""" + n = len(xs) + d = len(xs[0]) + D = d + 1 + # A = X'X + lam*I (bias not regularised), B = X'Y + A = [[0.0] * D for _ in range(D)] + B = [[0.0] * k for _ in range(D)] + for x, y in zip(xs, ys): + xb = list(x) + [1.0] + for i in range(D): + xi = xb[i] + if xi == 0.0: + continue + row = A[i] + for j in range(D): + row[j] += xi * xb[j] + B[i][y] += xi + for i in range(d): + A[i][i] += lam + # Gaussian elimination with partial pivoting on [A|B] + M = [A[i] + B[i] for i in range(D)] + for col in range(D): + piv = max(range(col, D), key=lambda r: abs(M[r][col])) + if abs(M[piv][col]) < 1e-12: + raise SystemExit("laya_head_finetune: the design matrix is singular at column %d (raise --ridge)" % col) + if piv != col: + M[col], M[piv] = M[piv], M[col] + pv = M[col][col] + M[col] = [v / pv for v in M[col]] + for r in range(D): + if r != col and M[r][col] != 0.0: + f = M[r][col] + Mr, Mc = M[r], M[col] + for c in range(col, D + k): + Mr[c] -= f * Mc[c] + # After elimination the top D rows are [I | A^-1 B], so the weights live in columns D..D+k-1. + # Transposed to one row per class (each D long, bias last) so predict() is a plain dot product. + return [[M[i][D + c] for i in range(D)] for c in range(k)] + + +def predict(W, x): + xb = list(x) + [1.0] + return [sum(w * v for w, v in zip(row, xb)) for row in W] + + +def evaluate(feats, cases, W, idxs, k): + """(correct, total, confusion) for one arm; W is None for the table's own decision.""" + conf = [[0] * k for _ in range(k)] + correct = 0 + for i in idxs: + want = cases[i]["label"] + got = argmax(cases[i]["logits"]) if W is None else argmax(predict(W, feats[i])) + if 0 <= want < k and 0 <= got < k: + conf[want][got] += 1 + correct += 1 if want == got else 0 + return correct, len(idxs), conf + + +def run(args): + d, encoded, cases = read_dump(args.dump) + usable, feats, labels = [], [], [] + for c in cases: + if c.get("label", -1) < 0: + continue + f = pool_case(encoded, d, int(c["offset"]), int(c["len"]), args.pool) + if f is None: + continue + usable.append(c) + feats.append(f) + labels.append(int(c["label"])) + if not usable: + raise SystemExit("laya_head_finetune: no labelled cases in the dump") + k = max(len(usable[0]["options"]), max(labels) + 1) + for c, y in zip(usable, labels): + if not (0 <= y < len(c["options"])): + raise SystemExit("laya_head_finetune: %s: label %d is outside its options" % (c.get("src", "?"), y)) + + # Split by CASE and STRATIFIED BY LABEL, deterministically (no RNG: a re-run on one dump reports one number). + # + # Not by index. A decision log is written in arrival order, and a demo's arrival order is class-contiguous -- + # a session of reflex steps, then a planning stretch. An index split then holds out a whole CLASS rather than a + # sample of each, and the "after" number is measured on classes the head never saw while the "before" arm (which + # needs no training) scores them fine. My own self-test hit exactly this: 20+20+20 cases, holdout 0.5, train = + # classes 0 and 1 only, so the ceiling for any head was 1/3. Stratifying by label is what makes "held out" mean + # held out from the same population. With --holdout 0 nothing is held out and the report says in-sample. + n = len(usable) + by_label = {} + for i, y in enumerate(labels): + by_label.setdefault(y, []).append(i) + train, test = [], [] + if args.holdout == 0: + train = list(range(n)) + test = list(range(n)) # in-sample: fit and score on everything, and say so + for y in (sorted(by_label) if args.holdout > 0 else []): + idxs = by_label[y] + hold = int(round(len(idxs) * args.holdout)) + if args.holdout > 0 and hold == 0: + raise SystemExit("laya_head_finetune: class %d has %d case(s), too few for a held-out split; " + "add cases or use --holdout 0 (in-sample)" % (y, len(idxs))) + test.extend(idxs[len(idxs) - hold:] if hold else []) + train.extend(idxs[: len(idxs) - hold] if hold else idxs) + if not train: + raise SystemExit("laya_head_finetune: the holdout left no training cases (use a smaller --holdout)") + + W = fit_ridge([feats[i] for i in train], [labels[i] for i in train], k, args.ridge) + b_correct, b_total, b_conf = evaluate(feats, usable, None, test, k) + a_correct, a_total, a_conf = evaluate(feats, usable, W, test, k) + b_acc = b_correct / b_total if b_total else 0.0 + a_acc = a_correct / a_total if a_total else 0.0 + out = { + "cases": n, + "train": len(train), + "test": len(test), + "holdout": args.holdout, + "arm": "in-sample" if args.holdout == 0 else "held-out", + "pool": args.pool, + "classes": [c["options"] for c in usable[:1]][0], + "before": {"correct": b_correct, "total": b_total, "accuracy": b_acc}, + "after": {"correct": a_correct, "total": a_total, "accuracy": a_acc}, + "delta": a_acc - b_acc, + "confusion_before": b_conf, + "confusion_after": a_conf, + } + return out + + +def self_test(): + """A dump can be synthesised exactly: rows in class-conditional clusters with a known boundary. The check is + that the fine-tune LEARNS it (the table's logits, made deliberately uninformative, cannot), and that the split + is by case -- a head fitted on a case's rows must not be scored on the same case.""" + import tempfile + + fails = 0 + + def check(ok, what): + nonlocal fails + print(("ok " if ok else "FAIL ") + what) + if not ok: + fails += 1 + + rng = 12345 + + def rnd(): + nonlocal rng + rng = (1103515245 * rng + 12345) % (1 << 31) + return rng / float(1 << 31) + + d, k, per_class, rows_per_case = 8, 3, 20, 3 + centres = [[0.0] * d for _ in range(k)] + for c in range(k): + for i in range(d): + centres[c][i] = 4.0 if (i % k) == c else -4.0 + with tempfile.TemporaryDirectory() as tmp: + h = open(os.path.join(tmp, "h.f32"), "wb") + meta = open(os.path.join(tmp, "cases.jsonl"), "w") + off = 0 + made = [] + for c in range(k): + for _ in range(per_class): + vals = [] + for _r in range(rows_per_case): + for i in range(d): + vals.append(centres[c][i] + (rnd() - 0.5)) + h.write(struct.pack("<%df" % len(vals), *vals)) + # Uninformative table logits: identical for every class, so "before" can only be class 0. + meta.write(json.dumps({"src": "s%d" % len(made), "label": c, "options": ["a", "b", "c"][:k], + "offset": off, "len": rows_per_case, "logits": [0.0] * k}) + "\n") + off += rows_per_case + made.append(c) + h.close() + meta.close() + json.dump({"d": d, "rows": off, "cases": len(made)}, open(os.path.join(tmp, "dump.json"), "w")) + + class A: + dump = tmp + holdout = 0.5 + ridge = 1e-3 + pool = "mean" + json = False + + out = run(A) + check(out["arm"] == "held-out", "the default run is scored held-out, and says so") + check(out["after"]["accuracy"] > out["before"]["accuracy"], + "the fine-tuned head beats the table on the same held-out cases (%.2f vs %.2f)" + % (out["after"]["accuracy"], out["before"]["accuracy"])) + check(out["after"]["accuracy"] >= 0.9, + "and it actually learns a separated boundary (%.2f)" % out["after"]["accuracy"]) + check(out["test"] + out["train"] == out["cases"], "every case lands in exactly one side of the split") + + # In-sample is reported as in-sample: same dump, holdout 0. + A.holdout = 0.0 + ins = run(A) + check(ins["arm"] == "in-sample", "holdout 0 is labelled in-sample rather than passing as a held-out number") + check(ins["before"]["total"] == ins["cases"] and ins["test"] == ins["cases"], + "in-sample scores every case (%d of %d)" % (ins["test"], ins["cases"])) + + # An unlabelled line is skipped, as `--eval` skips it. + with open(os.path.join(tmp, "cases.jsonl"), "a") as f: + f.write(json.dumps({"src": "unlabelled", "label": -1, "options": ["a", "b", "c"][:k], + "offset": 0, "len": rows_per_case, "logits": [0.0] * k}) + "\n") + A.holdout = 0.5 + out2 = run(A) + check(out2["cases"] == out["cases"], "a label -1 line is skipped, not counted as a class") + + if fails: + print("laya_head_finetune: %d FAILED" % fails) + return 1 + print("laya_head_finetune: all self-test arms passed") + return 0 + + +def main(argv): + ap = argparse.ArgumentParser(add_help=True) + ap.add_argument("dump", nargs="?") + ap.add_argument("--holdout", type=float, default=0.5, help="fraction of CASES held out (0 = in-sample)") + ap.add_argument("--ridge", type=float, default=1e-3, help="ridge strength on the feature weights") + ap.add_argument("--pool", choices=["mean", "last"], default="mean", help="how a case's encoder rows become one vector") + ap.add_argument("--json", action="store_true") + ap.add_argument("--self-test", action="store_true") + args = ap.parse_args(argv) + if args.self_test: + return self_test() + if not args.dump: + ap.error("a DUMP_DIR or --self-test is required") + if not (0.0 <= args.holdout < 1.0): + ap.error("--holdout must be in [0, 1)") + out = run(args) + if args.json: + print(json.dumps(out, indent=1)) + return 0 + print("cases %d (%d train, %d held out) arm=%s pool=%s" % + (out["cases"], out["train"], out["test"], out["arm"], out["pool"])) + print(" before (the table): %3d/%3d %.3f" % (out["before"]["correct"], out["before"]["total"], out["before"]["accuracy"])) + print(" after (fine-tune): %3d/%3d %.3f delta %+0.3f" % + (out["after"]["correct"], out["after"]["total"], out["after"]["accuracy"], out["delta"])) + for name, conf in (("before", out["confusion_before"]), ("after", out["confusion_after"])): + print(" %s confusion (rows true, cols chosen):" % name) + for r, row in enumerate(conf): + print(" %d: %s" % (r, " ".join("%3d" % v for v in row))) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 875272bc7a83b6e3d518cb3c460a126fdd125d5f Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:31:32 -0400 Subject: [PATCH 76/98] npu-pack: a tied head must correlate AND not be mis-scaled, or it imports cleanly and answers nonsense (engine#1503) (#2101) * npu-pack: a tied head must correlate AND not be mis-scaled, or it imports cleanly and answers nonsense (engine#1503) Two gemma-2-2b hybrid packs measured on this box import with every check green and serve garbage: pack corr rel_err serves Qwen-2.5_1.5B_Instruct_rai_1.9.0_npu 0.996901 0.000000 'The capital of France is Paris.' x3 gemma-2-2b-onnx-ryzenai-hybrid 0.991539 0.136140 'myself myself ...' x3 gemma-2-2b-awq-...-g32-fp16-onnx-hybrid 0.996053 0.094874 'myselfGetBytes...' x3 A head that is a SCALED copy of its own embedding still correlates, so the correlation-only check passes it -- and the import printed the number that distinguishes them (rel_err) as an advisory field next to the correlation it gated on. Both packs cost a device ticket to discover. - one predicate, selfcheck_head_ok(corr, rel), so the four sites that set selfcheck_tied cannot drift; - kSelfcheckRelMax = 0.02, the measured gap: 0.000000 on the pack that serves, 0.095/0.136 on the two that do not; - Report::selfcheck_scaled, set at all four sites next to selfcheck_tied; - main() refuses on it and NAMES both numbers and the threshold. --allow-incomplete does not override: that flag means 'write the tensors I could not represent', and this is 'the tensors I wrote do not decode'. Verified: both translation units pass g++ -fsyntax-only (the json header from a neighbour build), and the predicate's six arms pass against the three measured triples plus a control, a no-correlation case and the inclusive boundary (~ /wt/relcheck.cpp, run). NOT verified here: the full npu_pack_import_test binary and any container import -- that needs a configured build, which the disk floor did not allow at the time; the two test arms are written for CI. * npu-pack: name the partial output in the mis-scaled-head refusal (engine#1503) Demonstrated on the real pack, not argued from the fixture: importing ~/scratch-packs/gemma22b (gemma-2-2b-onnx-ryzenai-hybrid, the pack that served 'myself myself ...' for 16 tokens) now stops at refusing to import ...: the head correlates with the pack's own embedding (0.991539) but its relative error is 0.136140, which is a scaled or partially decoded head rather than a rounding difference (the threshold is 0.020). A pack in this state imports cleanly and answers garbage on the device; --allow-incomplete does not override this. That run also showed the check happens after import_pack() has already copied the text artifacts and written model.q4nx, so a refusal left five files behind. An importer must not delete a directory the caller named, so the message now says the partial output exists and is not servable -- the operator deletes it before retrying. Verified by rebuilding onebit and re-running the same import, and the test target still passes its six arms. --------- Co-authored-by: pi-mosaic --- npu/pack_import.cpp | 31 +++++++++++++++++++++++++++++++ npu/pack_import.h | 11 +++++++++++ tests/npu_pack_import_test.cpp | 16 ++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index f2ad9c895..463c49a36 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -498,6 +498,11 @@ const uint8_t* at(const Fconst& f, int64_t off, int64_t len) { // field, the readout and the note all have to agree on it: three copies of 0.99 is how they drifted apart before. constexpr double kTieCorr = 0.99; +// engine#1503: tied AND not mis-scaled. Correlation alone passes a head that is a scaled copy of the embedding -- two +// packs on this box report corr 0.9915/0.9961 with rel_err 0.095/0.136 and serve garbage, while the pack that serves +// correctly reports rel_err 0.000000. One predicate, so the four sites that set selfcheck_tied cannot drift apart. + + // Every entry was measured by diffing the 1.9 pack against the 1.8 pack of the SAME model: the bundle's tile shape and // its rotation are not derivable from the pack, and neither is `prefix`. It was tempting to read the extra bytes as // Qwen3's per-head QK norm gains, but Llama-3.2-1B -- which has no QK norms at all -- carries the same +4,352 at hidden @@ -5016,6 +5021,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons // `tied` is false there while the head plainly reproduces its embedding, and the readout used to call those // packs untied while the container it wrote said tied. Same threshold the container's own decision uses. r.selfcheck_tied = r.lmhead_corr > kTieCorr; + + r.selfcheck_scaled = r.selfcheck_tied && !selfcheck_head_ok(r.lmhead_corr, r.lmhead_rel); const Q4 swap = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), zp, int(mm.N), int(mm.K), mm.block, /*swap_nibbles*/ true, false); const Q4 nozp = decode_matmulnbits(bytes_of(wt, qw_need), scales.data(), nullptr, int(mm.N), int(mm.K), @@ -5103,6 +5110,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons } r.selfcheck_reference = true; r.selfcheck_tied = c32 > kTieCorr; + + r.selfcheck_scaled = r.selfcheck_tied && !selfcheck_head_ok(r.lmhead_corr, r.lmhead_rel); double sxy = 0, sx = 0, sy = 0, sxx = 0, syy = 0; const int nsh = std::min(2048, int(V) - 1); for (int rr = 0; rr < nsh; ++rr) @@ -5316,6 +5325,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons r.lmhead_corr = corr(q, embed_tbl, q.N); r.selfcheck_reference = true; r.selfcheck_tied = r.lmhead_corr > kTieCorr; + + r.selfcheck_scaled = r.selfcheck_tied && !selfcheck_head_ok(r.lmhead_corr, r.lmhead_rel); double sxy = 0, sx = 0, sy = 0, sxx = 0, syy = 0; const int nsh = std::min(2048, q.N - 1); for (int rr = 0; rr < nsh; ++rr) @@ -5428,6 +5439,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons r.lmhead_corr = corr(q, embed_tbl, q.N); r.selfcheck_reference = true; r.selfcheck_tied = r.lmhead_corr > kTieCorr; + + r.selfcheck_scaled = r.selfcheck_tied && !selfcheck_head_ok(r.lmhead_corr, r.lmhead_rel); // The control this path can actually run: the same tiles and the same decoder, with each // head row matched against the NEXT embedding row. If a misaligned head still correlated, // the number above would be measuring the vocabulary's mean brightness rather than the @@ -6025,6 +6038,20 @@ int run(int argc, char** argv) { } const Report r = import_pack(dir, out, Options{allow, layout}); + // engine#1503: a head that correlates but is mis-scaled imports cleanly and answers nonsense on the device (two + // packs measured: rel_err 0.095 and 0.136 against 0.000000 on the pack that serves). `--allow-incomplete` is not an + // override here: that flag means "write the tensors I could not represent", and this is "the tensors I wrote do not + // decode", which no amount of writing fixes. + if (r.selfcheck_scaled) { + std::fprintf(stderr, + "1bit npu-pack: refusing to import %s: the head correlates with the pack's own embedding (%.6f) but " + "its relative error is %.6f, which is a scaled or partially decoded head rather than a rounding " + "difference (the threshold is %.3f). A pack in this state imports cleanly and answers garbage on " + "the device; --allow-incomplete does not override this. The partial output this run already\n" + "wrote under %s is not servable -- delete it before retrying.\n", + dir.c_str(), r.lmhead_corr, r.lmhead_rel, kSelfcheckRelMax, out.c_str()); + return 3; + } if (!r.refused.empty() && !allow) { std::fprintf(stderr, "1bit npu-pack: refusing to import %s: %zu of the pack's tensors cannot be " @@ -6278,5 +6305,9 @@ void sweep_fconst_scratch_dirs(const std::vector& dirs, const std:: } } +// engine#1503: the tied-head rule, defined here so it is the namespace-scope function the header declares (a file-local +// copy is ambiguous with it at every call site). Correlated AND not a scaled copy of the embedding -- the two states this +// box measures are rel_err 0.000000 (serves) and 0.095/0.136 (imports cleanly, answers garbage). +bool selfcheck_head_ok(double corr, double rel) { return corr > kTieCorr && rel <= kSelfcheckRelMax; } } // namespace onebit::npu::pack_import diff --git a/npu/pack_import.h b/npu/pack_import.h index 8977766f2..812903775 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -450,6 +450,11 @@ struct Report { // check; an untied pack's head and embedding are separate matrices by construction, and the same number is // meaningless -- reported as such rather than left looking like a failed import. bool selfcheck_tied = false; + // engine#1503: tied, correlated -- but MIS-SCALED. Correlation alone passes a head that is a scaled copy of the + // embedding: two packs measured on this box report corr 0.9915/0.9961 with rel_err 0.095/0.136 and serve garbage + // ("myself" forever), while the pack that serves correctly reports rel_err 0.000000. A pack in this state imports + // cleanly and answers nonsense, so it is refused rather than reported. + bool selfcheck_scaled = false; double lmhead_corr = 0, lmhead_rel = 0, ctrl_corr_nibble = 0, ctrl_corr_nozp = 0; // The tail (.fconst) path has no MatMulNBits bytes to mis-decode, so its control is an ALIGNMENT one: the same // tiles, the same decoder, rows matched against the neighbouring embedding row. A correct decode correlates ~1 @@ -780,6 +785,12 @@ bool fconst_scratch_ready(const std::string& dir); const Fconst19Rule* select_fconst19_rule_from(const Fconst19Rule* rules, int count, int hidden, int heads, int layers, std::string* why); +// engine#1503: the tied-head rule, as ONE predicate so the four call sites cannot drift: the head must reproduce the +// embedding (corr) AND not be a scaled copy of it (rel_err). The constant is the measured gap between the two states -- +// 0.000000 for the pack that serves, 0.095/0.136 for the two that do not -- so 0.02 sits far from both. +inline constexpr double kSelfcheckRelMax = 0.02; +bool selfcheck_head_ok(double corr, double rel); + int64_t attn_tile_bytes(int64_t k); // Pearson correlation between a quantised head and a bf16 embedding table (engine#1690; the self-check's own statistic, // exported so a test can pin the buffer width it decodes). diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 0dcad1ff0..dfceaa69a 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -5683,7 +5683,23 @@ void test_header_witness_refusal() { std::printf(" header witness: 8 arms (one-header vote vs declaration refuses; 3/1-without-declaration/0 allowed)\n"); } +// engine#1503: the tied-head rule as ONE predicate -- correlated AND not mis-scaled. Two packs on this box correlate +// (0.9915 / 0.9961) while being scaled copies of their own embedding (rel_err 0.1361 / 0.0949): they import cleanly and +// serve garbage, where the pack that serves correctly reports rel_err 0.000000. Correlation alone cannot tell them apart, +// which is why the correlation-only check passed both and cost two device tickets. +static void selfcheck_head_rule_arms() { + check(pi::selfcheck_head_ok(0.996901, 0.000000), "head rule: the pack that serves correctly passes (corr .9969, rel 0)"); + check(!pi::selfcheck_head_ok(0.991539, 0.136140), "head rule: a correlated but scaled head is refused (corr .9915, rel .1361)"); + check(!pi::selfcheck_head_ok(0.996053, 0.094874), "head rule: the second measured pack is refused too (corr .9961, rel .0949)"); + check(pi::selfcheck_head_ok(0.996053, 0.000000), + "head rule: correlated and unscaled passes -- the rule is not 'refuse everything'"); + check(!pi::selfcheck_head_ok(0.50, 0.000000), "head rule: an uncorrelated head is still refused on correlation"); + check(pi::selfcheck_head_ok(0.996053, pi::kSelfcheckRelMax), "head rule: exactly at the threshold passes (inclusive bound)"); + std::printf(" head rule: 6 arms (two measured bad packs, the serving one, the unscaled control, no-corr, boundary)\n"); +} + int main(int argc, char** argv) { + selfcheck_head_rule_arms(); // engine#1907: every crafted 1.9 import writes a canonical copy into the scratch cache. Without this the suite left 0-byte // copies in the user's real ~/.cache/1bit/fconst19 and in /tmp on every run. Give the run its own directory, removed at exit. const fs::path run_scratch = ::onebit::test::tmp_root() / ("1bit-pack-test-scratch-" + std::to_string(::getpid())); From afb46403be6066c85327db44537fa00e53db7fb6 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:03:12 -0400 Subject: [PATCH 77/98] forward-serve: a text request to a decision-only pack is a 400 that names the decision route (Refs #1786) (#2121) A pack with non-causal attention has no decode to run. /v1/chat/completions and /v1/completions tried anyway and failed inside the forward with HTTP 500 and its internal wording (measured on FastFlowLM's D3-Nano-NPU2). Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- app/forward_serve.cpp | 6 ++++++ app/npu_systemone.h | 16 ++++++++++++++++ docs/npu.md | 5 +++++ tests/npu_chatml_test.cpp | 15 +++++++++++++++ 4 files changed, 42 insertions(+) diff --git a/app/forward_serve.cpp b/app/forward_serve.cpp index 1b4249bec..b5da1fa66 100644 --- a/app/forward_serve.cpp +++ b/app/forward_serve.cpp @@ -807,6 +807,12 @@ void handle_generate_impl(ForwardEngine& e, const httplib::Request& req, httplib // this thread waits for the first frame or the end, and only then decides between an ordinary response -- the 400s // and 500s a request can earn before it has produced anything keep their status codes -- and a stream. void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat) { + // A non-causal decision pack has no decode to run (npu::decision_only_refusal): say so before any work. + if (e.fwd && e.fwd->noncausal()) { + res.status = 400; + res.set_content(npu::decision_only_refusal(e.id, e.head != nullptr).dump(), "application/json"); + return; + } bool wants_stream = false; try { const json body = json::parse(req.body); diff --git a/app/npu_systemone.h b/app/npu_systemone.h index 22623f7cc..2a6c0bfd4 100644 --- a/app/npu_systemone.h +++ b/app/npu_systemone.h @@ -124,6 +124,22 @@ inline nlohmann::json systemone_question_error(const std::string& kind, const st {"message", message}}; } +// What a text route answers on a pack that cannot generate text. A decision pack that declares +// `attention_mode: noncausal_full_attention` encodes a whole prompt in one pass and reads an answer off the last +// position; there is no token-by-token decode to run, so `/v1/chat/completions` and `/v1/completions` can never +// succeed on it. They used to try, and failed inside the forward with HTTP 500 and the forward's own wording +// ("forward step failed: attention.mode noncausal_full_attention encodes a prompt in one batched pass, but this +// call starts at position 23 ..."; measured on FastFlowLM's D3-Nano-NPU2, engine#1786). That is the client's +// request against the wrong route, so it is a 400 that names the right one. `has_head` says whether the decision +// route is actually up on this serve, because pointing a client at a route that answers 503 is no help. +inline nlohmann::json decision_only_refusal(const std::string& model, bool has_head) { + const std::string where = has_head ? "send decisions to POST /v1/systemone" + : "and this serve has no decision head loaded, so it has no route for it"; + return {{"error", + {{"message", model + " is a decision model (non-causal attention): it does not generate text; " + where}, + {"type", "decision_model_error"}}}}; +} + inline std::string systemone_length_message(size_t tokens, size_t limit) { return "the question prompt has " + std::to_string(tokens) + " tokens, over the maximum context length of " + std::to_string(limit) + " tokens; nothing was truncated"; diff --git a/docs/npu.md b/docs/npu.md index 85857e81f..922997b93 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2107,6 +2107,11 @@ forward-serve` serves that too, from the same process and the same weights: | `POST /v1/systemone` (alias `POST /v1/decisions`) | `{"state": …, "questions": {id: question}}` → `{"model", "answers": {id: answer}, "usage"}` | | `GET /v1/systemone/models` | the decision model this build serves: name, description, `max_input_tokens`, `modalities` | +A pack that declares `attention_mode: noncausal_full_attention` (below) is a decision model and nothing else: +it encodes a prompt in one pass and has no token-by-token decode. On such a pack `/v1/chat/completions` and +`/v1/completions` answer **400** with `type: decision_model_error` and a message that names `/v1/systemone` +(or says there is no decision head, when the serve loaded none), before any work is done. + `questions` is an **object**, and its ids come back as the keys of `answers` in request order. Question types are `choice` (1–255 options), `noul` and `score` (2–10 ordered levels); a decision runs one backbone prefill per question, pools the **last position's hidden state after the final norm** (`pooling: last`), diff --git a/tests/npu_chatml_test.cpp b/tests/npu_chatml_test.cpp index d43909570..dd407c1d1 100644 --- a/tests/npu_chatml_test.cpp +++ b/tests/npu_chatml_test.cpp @@ -154,6 +154,21 @@ int main(int argc, char** argv) { check(pin_probe_messages(oreq).size() == 2, "the same for the ordered JSON the template path uses"); } + // A text route on a decision-only pack (engine#1786, D3-Nano-NPU2): a 400 that names the right route + { + const nlohmann::json up = onebit::npu::decision_only_refusal("D3-Nano-NPU2", true); + const std::string m = up["error"]["message"].get(); + check(m.find("D3-Nano-NPU2") == 0 && m.find("does not generate text") != std::string::npos && + m.find("POST /v1/systemone") != std::string::npos, + "the refusal names the model, says it does not generate text, and names the decision route"); + check(up["error"]["type"] == "decision_model_error", "and carries a type a client can match on"); + const std::string none = onebit::npu::decision_only_refusal("x", false)["error"]["message"].get(); + check(none.find("POST /v1/systemone") == std::string::npos && none.find("no decision head") != std::string::npos, + "with no head loaded it does not send the client to a route that would answer 503"); + check(m.find("forward step failed") == std::string::npos && m.find("position") == std::string::npos, + "none of the forward's internal wording reaches the client"); + } + if (failures == 0) std::printf("npu_chatml: ok\n"); return failures == 0 ? 0 : 1; } From a97e39855190d0250dd4ff4187b4dbb53c36d101 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:03:18 -0400 Subject: [PATCH 78/98] tools: npu_serve_smoke, the release gate's check that npu serve is on the fast route at speed (refs #2106, #2102) (#2119) * tools: npu_serve_smoke, the release gate's check that npu serve is on the fast route at speed (refs #2106, #2102) A serve that falls back to the per-op forward still answers, so a check that asks 'did it answer' passes on the wrong route. This starts 1bit serve --device npu, sends three prompts with thinking off, and asserts the route's own log line, a decode floor (60 tok/s), the first token (600 ms) and non-empty, non-degenerate content. Measured, Qwen3-0.6B, one npu-hold ticket each: the 2026-11-07 release candidate PASSes (median 84.4 tok/s, 277 ms); a main build without the fast route FAILs route, rate (1.9 tok/s) and first-token while its answers are fine. --self-test drives the judging on 15 shapes (the silent fallback, thinking left on, a degenerate reply, the median, missing timings); registered in ctest as npu_serve_smoke_self. Co-Authored-By: Claude Fable 5.2 * tools: npu_serve_smoke's self-test pins the median (a mean mutant passed all 15 cases) Co-Authored-By: Claude Fable 5.2 * tools: npu_serve_smoke takes the two route texts as options, so it can follow the serve's wording Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: bong-water-water-bong Co-authored-by: Claude Fable 5.2 --- CMakeLists.txt | 4 + docs/hw-gate.md | 40 ++++++ tools/npu_serve_smoke.py | 280 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 324 insertions(+) create mode 100755 tools/npu_serve_smoke.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 45a7daa0a..0a8719d47 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -363,6 +363,10 @@ add_python_test(hf_tensor_self ${CMAKE_SOURCE_DIR}/tools/hf_tensor.py --self-tes # `tools/ci/ctest_skips.txt`, and `(Not Run)` (a missing binary) as a build-list mistake that can never be baselined. # Self-test only here: wiring it into ci.yml's Test step needs the log plus a `workflow`-scoped token (engine#1817). add_python_test(ctest_skips_self ${CMAKE_SOURCE_DIR}/tools/ci/ctest_skips.py --self-test) +# engine#2106 / #2102: the release gate's NPU serve check asserts the fast route, a decode floor, the first token and a +# non-empty answer, because a serve on the slow per-op route still answers. --self-test drives the judging on the shape +# of each failure (the silent fallback at 1.9 tok/s, thinking left on, a degenerate reply); no device, no binary. +add_python_test(npu_serve_smoke_self ${CMAKE_SOURCE_DIR}/tools/npu_serve_smoke.py --self-test) add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) # engine#1313 item 3: the "after" of "before/after routing accuracy" -- a ridge head over --dump-encoder output, with the # table's own logits as the before arm. Self-test only, so it needs no dump and no model. diff --git a/docs/hw-gate.md b/docs/hw-gate.md index 67402e213..c8cb13604 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -252,6 +252,46 @@ The settings are environment variables at the top of the script: `HW_GATE_STORE` `$HW_GATE_OUT` (default `~/.cache/hw-gate/runs/