diff --git a/bench/npu-leaderboard/lfm2-12b-perop-decode-20261111.json b/bench/npu-leaderboard/lfm2-12b-perop-decode-20261111.json new file mode 100644 index 000000000..e9e137e94 --- /dev/null +++ b/bench/npu-leaderboard/lfm2-12b-perop-decode-20261111.json @@ -0,0 +1,28 @@ +{ + "id": "lfm2-12b-perop-decode-20261111", + "host": "strixhalo", + "model": "LFM2-1.2B-NPU2", + "route": "npu", + "quant": "Q4NX symmetric 4-bit tiles (FastFlowLM's published container, listed as Q4_0), per-op int8 GEMM, blocked family", + "metric": { + "name": "decode", + "value": 4.28, + "unit": "tok/s" + }, + "settings": { + "device": "accel0", + "engine": "1bit built at the code of main 1131acf0 (engine#2170)", + "container": "FastFlowLM/LFM2-1.2B-NPU2 model.q4nx, sha256 2755ab40cd927e3c…, served as published; the only edit is \"q4nx_weight_layout\": \"b8\" in a copy of config.json", + "kernels": "full_i8 blocked: QKV K2048 N3072, QKV K2048 N6144 (the conv layers' in_proj), O K2048 N2048, GU K2048 N16384, D K8192 N2048, each with its M=16 twin", + "ticket": "npu-hold -l cl-flm-lfm2-rows, both LFM2 packs in one 107 s ticket", + "sampling": "temperature 0, streamed, max_tokens 64", + "prompt_tokens": 20, + "load1": "17.5 at start, 14.7 at end", + "known_answer": "3 requests of 'What is the capital of France? Answer with one word.' returned 'Paris' three times, identical" + }, + "instrument": "streamed chat completion of 64 tokens, usage read back; decode rate from the server's own timings; two requests, 4.25 and 4.30 tok/s, same text both times", + "repeats": 2, + "timestamp": "2026-11-11T13:32:47Z", + "source": "~/.cache/scratch-cl-flm/rows.log and out-rows-LFM2-1.2B-NPU2/ on strixhalo (row-serve.sh + row-probe.py, lane cl-flm; engine#1786)", + "notes": "A hybrid: 10 of 16 layers are gated short convolutions, run as two device GEMMs with the gate and a 3-tap causal conv on the host. This is the per-op route's speed, not what the array does on a geometry with a fused set: FastFlowLM 1.0.12's runtime decoded the same file at 42 to 83 tok/s in the paired tickets of the same day. What would make it wrong: two requests, box load 15 to 19 during the ticket (the same pack measured 3.7 tok/s in a paired ticket earlier that day)." +} diff --git a/bench/npu-leaderboard/lfm2-12b-perop-first-token-20261111.json b/bench/npu-leaderboard/lfm2-12b-perop-first-token-20261111.json new file mode 100644 index 000000000..15babf812 --- /dev/null +++ b/bench/npu-leaderboard/lfm2-12b-perop-first-token-20261111.json @@ -0,0 +1,28 @@ +{ + "id": "lfm2-12b-perop-first-token-20261111", + "host": "strixhalo", + "model": "LFM2-1.2B-NPU2", + "route": "npu", + "quant": "Q4NX symmetric 4-bit tiles (FastFlowLM's published container, listed as Q4_0), per-op int8 GEMM, blocked family", + "metric": { + "name": "first token (warm)", + "value": 0.756, + "unit": "s" + }, + "settings": { + "device": "accel0", + "engine": "1bit built at the code of main 1131acf0 (engine#2170)", + "container": "FastFlowLM/LFM2-1.2B-NPU2 model.q4nx, sha256 2755ab40cd927e3c…, served as published; the only edit is \"q4nx_weight_layout\": \"b8\" in a copy of config.json", + "kernels": "full_i8 blocked: QKV K2048 N3072, QKV K2048 N6144 (the conv layers' in_proj), O K2048 N2048, GU K2048 N16384, D K8192 N2048, each with its M=16 twin", + "ticket": "npu-hold -l cl-flm-lfm2-rows, both LFM2 packs in one 107 s ticket", + "sampling": "temperature 0, streamed, max_tokens 64", + "prompt_tokens": 20, + "load1": "17.5 at start, 14.7 at end", + "known_answer": "3 requests of 'What is the capital of France? Answer with one word.' returned 'Paris' three times, identical" + }, + "instrument": "time from request to the first streamed chunk on a 20-token prompt; two requests, 0.747 and 0.764 s", + "repeats": 2, + "timestamp": "2026-11-11T13:32:47Z", + "source": "~/.cache/scratch-cl-flm/rows.log and out-rows-LFM2-1.2B-NPU2/ on strixhalo (row-serve.sh + row-probe.py, lane cl-flm; engine#1786)", + "notes": "Same ticket as lfm2-12b-perop-decode-20261111. Batched prefill, no prefix cache hit (a different prompt had been served before it). The server's own prompt_ms was 745 and 762 ms." +} diff --git a/bench/npu-leaderboard/lfm25-12b-perop-decode-20261111.json b/bench/npu-leaderboard/lfm25-12b-perop-decode-20261111.json new file mode 100644 index 000000000..7c80b37f1 --- /dev/null +++ b/bench/npu-leaderboard/lfm25-12b-perop-decode-20261111.json @@ -0,0 +1,28 @@ +{ + "id": "lfm25-12b-perop-decode-20261111", + "host": "strixhalo", + "model": "LFM2.5-1.2B-NPU2", + "route": "npu", + "quant": "Q4NX symmetric 4-bit tiles (FastFlowLM's published container, listed as Q4_0), per-op int8 GEMM, blocked family", + "metric": { + "name": "decode", + "value": 4.2, + "unit": "tok/s" + }, + "settings": { + "device": "accel0", + "engine": "1bit built at the code of main 1131acf0 (engine#2170)", + "container": "FastFlowLM/LFM2.5-1.2B-NPU2 model.q4nx, sha256 b341d42021e617a5…, served as published; the only edit is \"q4nx_weight_layout\": \"b8\" in a copy of config.json", + "kernels": "full_i8 blocked: QKV K2048 N3072, QKV K2048 N6144 (the conv layers' in_proj), O K2048 N2048, GU K2048 N16384, D K8192 N2048, each with its M=16 twin", + "ticket": "npu-hold -l cl-flm-lfm2-rows, both LFM2 packs in one 107 s ticket", + "sampling": "temperature 0, streamed, max_tokens 64", + "prompt_tokens": 20, + "load1": "18.6 at start, 17.5 at end", + "known_answer": "3 requests of 'What is the capital of France? Answer with one word.' returned 'Paris' three times, identical" + }, + "instrument": "streamed chat completion of 64 tokens, usage read back; decode rate from the server's own timings; two requests, 4.20 and 4.19 tok/s, same text both times", + "repeats": 2, + "timestamp": "2026-11-11T13:31:50Z", + "source": "~/.cache/scratch-cl-flm/rows.log and out-rows-LFM2.5-1.2B-NPU2/ on strixhalo (row-serve.sh + row-probe.py, lane cl-flm; engine#1786)", + "notes": "A hybrid: 10 of 16 layers are gated short convolutions, run as two device GEMMs with the gate and a 3-tap causal conv on the host. This is the per-op route's speed, not what the array does on a geometry with a fused set: FastFlowLM 1.0.12's runtime decoded the same file at 42 to 83 tok/s in the paired tickets of the same day. What would make it wrong: two requests, box load 15 to 19 during the ticket (the same pack measured 4.6 tok/s at load 3 to 4 and 3.5 to 3.7 at load 8 to 10 earlier that day, so load is not the only term)." +} diff --git a/bench/npu-leaderboard/lfm25-12b-perop-first-token-20261111.json b/bench/npu-leaderboard/lfm25-12b-perop-first-token-20261111.json new file mode 100644 index 000000000..b3dba39a6 --- /dev/null +++ b/bench/npu-leaderboard/lfm25-12b-perop-first-token-20261111.json @@ -0,0 +1,28 @@ +{ + "id": "lfm25-12b-perop-first-token-20261111", + "host": "strixhalo", + "model": "LFM2.5-1.2B-NPU2", + "route": "npu", + "quant": "Q4NX symmetric 4-bit tiles (FastFlowLM's published container, listed as Q4_0), per-op int8 GEMM, blocked family", + "metric": { + "name": "first token (warm)", + "value": 0.929, + "unit": "s" + }, + "settings": { + "device": "accel0", + "engine": "1bit built at the code of main 1131acf0 (engine#2170)", + "container": "FastFlowLM/LFM2.5-1.2B-NPU2 model.q4nx, sha256 b341d42021e617a5…, served as published; the only edit is \"q4nx_weight_layout\": \"b8\" in a copy of config.json", + "kernels": "full_i8 blocked: QKV K2048 N3072, QKV K2048 N6144 (the conv layers' in_proj), O K2048 N2048, GU K2048 N16384, D K8192 N2048, each with its M=16 twin", + "ticket": "npu-hold -l cl-flm-lfm2-rows, both LFM2 packs in one 107 s ticket", + "sampling": "temperature 0, streamed, max_tokens 64", + "prompt_tokens": 20, + "load1": "18.6 at start, 17.5 at end", + "known_answer": "3 requests of 'What is the capital of France? Answer with one word.' returned 'Paris' three times, identical" + }, + "instrument": "time from request to the first streamed chunk on a 20-token prompt; two requests, 1.094 and 0.763 s", + "repeats": 2, + "timestamp": "2026-11-11T13:31:50Z", + "source": "~/.cache/scratch-cl-flm/rows.log and out-rows-LFM2.5-1.2B-NPU2/ on strixhalo (row-serve.sh + row-probe.py, lane cl-flm; engine#1786)", + "notes": "Same ticket as lfm25-12b-perop-decode-20261111. Batched prefill, no prefix cache hit (a different prompt had been served before it). The server's own prompt_ms was 766 and 760 ms, so the 1.094 s of the first request is client-side time before the first chunk, not prefill." +} diff --git a/bench/npu-leaderboard/lfm25-12b-thinking-perop-decode-20261111.json b/bench/npu-leaderboard/lfm25-12b-thinking-perop-decode-20261111.json new file mode 100644 index 000000000..969b05342 --- /dev/null +++ b/bench/npu-leaderboard/lfm25-12b-thinking-perop-decode-20261111.json @@ -0,0 +1,28 @@ +{ + "id": "lfm25-12b-thinking-perop-decode-20261111", + "host": "strixhalo", + "model": "LFM2.5-1.2B-Thinking-NPU2", + "route": "npu", + "quant": "Q4NX symmetric 4-bit tiles (FastFlowLM's published container, listed as Q4_0), per-op int8 GEMM, blocked family", + "metric": { + "name": "decode", + "value": 4.3, + "unit": "tok/s" + }, + "settings": { + "device": "accel0", + "engine": "1bit built at the code of main 1131acf0 (engine#2170)", + "container": "FastFlowLM/LFM2.5-1.2B-Thinking-NPU2 model.q4nx, sha256 b9c4bacba8ed7251…, served as published; the only edit is \"q4nx_weight_layout\": \"b8\" in a copy of config.json", + "kernels": "full_i8 blocked: QKV K2048 N3072, QKV K2048 N6144 (the conv layers' in_proj), O K2048 N2048, GU K2048 N16384, D K8192 N2048, each with its M=16 twin", + "ticket": "npu-hold -l cl-flm-lfm25tk-row, 234 s", + "sampling": "temperature 0, max_tokens 320, not streamed", + "prompt_tokens": "the pack's own template (chat_template.jinja) on 'What is the capital of France? Answer with one word.'", + "load1": "10.1 at start, 13.0 at end", + "known_answer": "3 requests returned the same message three times: 287 reasoning tokens split into reasoning_content, then content that begins 'Paris' and runs to the token limit (finish_reason length)" + }, + "instrument": "usage.completion_tokens over the server's own predicted_ms, 320 tokens per request; three requests, 4.30, 4.25 and 4.37 tok/s, the median", + "repeats": 3, + "timestamp": "2026-11-11T14:14:56Z", + "source": "~/.cache/scratch-cl-flm/rows-25tk.log and out-rows-lfm25tk/ on strixhalo (row-serve.sh + think-row.py, lane cl-flm; engine#1786)", + "notes": "The thinking variant of the LFM2.5 hybrid. The reply does not stop after its answer inside 320 tokens. FastFlowLM 1.0.12's own runtime, same file, same prompt, same budget, one ticket later (two requests, identical): content empty, finish length, and 830 characters of reasoning that begin 'Okay derivative of the function f(x) = x^2 + 3x - 4. Find its vertex.' -- a different problem from the one asked -- at 65.5 tok/s. So on this pack the two runtimes do not produce comparable text, and this row is not a parity row: it says this route reasons about the question and reaches 'Paris'. What would make it wrong: one prompt." +} diff --git a/bench/npu-leaderboard/qwen3-17b-fused-decode-flm-pair-20261111.json b/bench/npu-leaderboard/qwen3-17b-fused-decode-flm-pair-20261111.json new file mode 100644 index 000000000..a1a0a64e4 --- /dev/null +++ b/bench/npu-leaderboard/qwen3-17b-fused-decode-flm-pair-20261111.json @@ -0,0 +1,26 @@ +{ + "id": "qwen3-17b-fused-decode-flm-pair-20261111", + "host": "strixhalo", + "model": "Qwen3-1.7B-NPU2", + "route": "npu", + "quant": "Q4NX q4_1 tiles (FastFlowLM's published container, unmodified), sealed fused set 2048x6144x16x8", + "metric": { + "name": "decode", + "value": 39.25, + "unit": "tok/s" + }, + "settings": { + "device": "accel0", + "engine": "1bit 80aceca152a909d5 (lemonade-onebit-rc-2026-11-07-c32)", + "container": "FastFlowLM/Qwen3-1.7B-NPU2 model.q4nx, sha256 b1c1fe9d6c4a6d0c…, the same file both runtimes loaded", + "ticket": "one npu-hold ticket, 38 s, arms in the order theirs, ours, ours, theirs", + "sampling": "greedy, streamed", + "reply": "63 streamed chunks per request", + "load1": "4.34 at start, 3.30 at end" + }, + "instrument": "streamed chat completion, decode rate from the stream's own chunk times; 3 repeats in each of two arms, median of the six (range 39.20 to 39.32)", + "repeats": 6, + "timestamp": "2026-11-11T05:42:39Z", + "source": "~/lb/aval/clperf-f9-abba-20261111T054201Z/run.log on strixhalo (lane cl-perf's f9-npu-abba.sh, unmodified; engine#1787)", + "notes": "Side by side with FastFlowLM 1.0.12's own runtime on the same array and the same model.q4nx, same ticket, A B B A: theirs decoded at 45.57 tok/s (median of its six, range 45.08 to 45.93), ours at 39.25, so theirs is 1.16 times ours on this pack. This row is ours; theirs is quoted here because a row is one run of this engine. What would make it wrong: it is one ticket at load 3 to 4, one prompt, and the shipped c32 build rather than main." +} diff --git a/bench/npu-leaderboard/qwen3-17b-fused-first-token-flm-pair-20261111.json b/bench/npu-leaderboard/qwen3-17b-fused-first-token-flm-pair-20261111.json new file mode 100644 index 000000000..02d8ca330 --- /dev/null +++ b/bench/npu-leaderboard/qwen3-17b-fused-first-token-flm-pair-20261111.json @@ -0,0 +1,26 @@ +{ + "id": "qwen3-17b-fused-first-token-flm-pair-20261111", + "host": "strixhalo", + "model": "Qwen3-1.7B-NPU2", + "route": "npu", + "quant": "Q4NX q4_1 tiles (FastFlowLM's published container, unmodified), sealed fused set 2048x6144x16x8", + "metric": { + "name": "first token (warm)", + "value": 0.568, + "unit": "s" + }, + "settings": { + "device": "accel0", + "engine": "1bit 80aceca152a909d5 (lemonade-onebit-rc-2026-11-07-c32)", + "container": "FastFlowLM/Qwen3-1.7B-NPU2 model.q4nx, sha256 b1c1fe9d6c4a6d0c…, the same file both runtimes loaded", + "ticket": "one npu-hold ticket, 38 s, arms in the order theirs, ours, ours, theirs", + "sampling": "greedy, streamed", + "reply": "63 streamed chunks per request", + "load1": "4.34 at start, 3.30 at end" + }, + "instrument": "time from request to the first streamed chunk; 3 repeats in each of two arms, median of the six (range 0.566 to 0.578)", + "repeats": 6, + "timestamp": "2026-11-11T05:42:39Z", + "source": "~/lb/aval/clperf-f9-abba-20261111T054201Z/run.log on strixhalo (lane cl-perf's f9-npu-abba.sh, unmodified; engine#1787)", + "notes": "Same ticket as qwen3-17b-fused-decode-flm-pair-20261111. FastFlowLM 1.0.12's runtime on the same model.q4nx: 0.591 s (median of its six). The two are level within the ticket's own spread; the first request of each of their arms was slower (0.658 and 0.653 s) and is inside their median." +} diff --git a/docs/arch-gaps.md b/docs/arch-gaps.md index 40bda945e..b0322bb06 100644 --- a/docs/arch-gaps.md +++ b/docs/arch-gaps.md @@ -1041,7 +1041,10 @@ takes the reviewed set: such a pack prints `. FLM ... registry: none (reviewed, Still open, and alerting on purpose: `MiniCPMV4_7ForConditionalGeneration` (needs one converter registration, not an architecture), `MiniCPMOcrForConditionalGeneration` (own `minicpm-ocr` library), `smolvla-NPU2` (no config at all), and the `!! NPU FLM` families whose registry mapping is HRX-only (`hy_v3`, `dflash`, `gemma4`, `hunyuan-dense`, `gemma3`) — -that last group is engine #588 (the family must be served on the NPU route), not a registry edit. +that last group is **not a registry edit**: the family has to be served on the NPU route. It used to be recorded here as +engine #588, which is **closed** (2026-10-27) and was about *watching FLM's releases*, not about serving a family on the +route. The issue that does own this class is engine#1786 (every `FastFlowLM/*-NPU2` source serves on `--device npu`, +opened 2026-11-10). See the last section of this page for the arrivals that are sitting in it. ## Census run 2026-11-09 (`engine #1667`) @@ -1072,9 +1075,11 @@ Left as they are, on purpose, under the policy above: the six `. REMOTE-CODE` cl (`HajeriOperatorForCausalLM`, `MM200ForCausalLM`, `K2HorizonForCausalLM`, `KLMForCausalLM`, `KimiK31SuperExpertForCausalLM`, `ThalamusReasoningQwen3_5ForConditionalGeneration`); the `? UNVERIFIABLE` gated repos, which are retried on every run; and the `!! FLM` lines. Those keep the 2026-11-08 reasons: `MiniCPMV4_7`, `MiniCPMOcr` -and `smolvla` as above, and `!! NPU FLM` is engine #588. This run's issue window also includes the `gpt-oss` +and `smolvla` as above, and `!! NPU FLM` was recorded as engine #588 — which is **closed** (2026-10-27) and was +about watching FLM's releases, not about serving a family on the NPU route; the class is owned by engine#1786 +(corrected 2026-11-11; see the section at the end of this page). This run's issue window also includes the `gpt-oss` (`GPT-OSS-20B`, `GPT-OSS-Safeguard-20b`) and `lfm2` (`LFM2-1.2B`, `LFM2-2.6B`) Q4NX packs. Their mapping is HRX-only -too, so they belong to #588, not to a registry edit. +too, so they are the same class — a route to serve, not a registry edit. ## 2026-11-10: `DeepseekV4FlashSuperExpertForCausalLM` — reviewed, and recorded here @@ -1098,3 +1103,45 @@ the single undocumented entry out of 37. The other spelling, `OpenSciForCausalLM` (7 models, same `model_type` and the same `auto_map`), is a separate exact key: `census_watch.py` matches architecture classes exactly, so that spelling needs its own entry and is not covered by this one. + +## FLM 1.0.12's build on the box: one family root with nothing, three that are already somewhere (engine#1787) + +F9 of `TNPU-ALL` §9 asks for a row for *every family FastFlowLM's build serves and we do not*. The build is on the box, +so the checklist was enumerated from it rather than from the release notes: + +``` +$ ls ~/flm-f9/xclbins | wc -l -> 54 (per-family xclbin directories) +$ ls ~/flm-f9/lib/*.so | wc -l -> 34 (per-family shared objects) +``` + +A word on the name: #1787 calls this "FLM 1.0.12's HRX build", and the section used to. What is counted here is a +directory of **xclbins, every one named `*-NPU2`**: they are NPU artifacts. A reader looking for GPU families will not +find them in this list. + +Mapping each of the 54 by **family root** (not by model name: `Gemma3-1B` normalises to `gemma31b`, which appears nowhere, +and matching that way reported 53 of 54 as unknown) against `registry/*.json`, `docs/registry.md` and this page gave 48 +with a trace and 6 xclbins over 4 roots without one. **That second number was wrong** (review of engine#2065): the root +was compared as a literal against class names and `model_type`s that do not contain it (`nanbeige4.1` against +`NanbeigeForCausalLM`, `medgemma` against `gemma3`). Each of the four, searched on `main` case-insensitively and +restated 2026-11-11: + +| xclbin directory | family root | what `main` has | what is actually owed | +|---|---|---|---| +| `Translategemma-4B-Instruct-NPU2` | `translategemma` | nothing: `git grep -il translategemma` finds 0 files | everything. This is the one root with no trace. By its name it would be a `gemma3` arrival like `medgemma`; nobody has opened its config, so that is a guess and not a mapping | +| `LFM2.5-1.2B-NPU2`, `LFM2.5-1.2B-Thinking-NPU2` | `lfm2.5` | the packs' own `config.json` says `Lfm2ForCausalLM` / `model_type: lfm2`, which the registry maps, so there is no "2.5 class" to add; `models/pins.json` pins the `LFM2.5-2.6B` and `LFM2.5-8B-A1B` GGUFs; and since engine#2170 both packs **serve on the NPU route** with measured rows in [npu.md](npu.md) ("FastFlowLM's own containers") | nothing in the registry. On the route: speed (per-op, about 4 tok/s against their 48 to 83), not coverage | +| `Medgemma-4B-NPU2`, `Medgemma-1.5-4B-NPU2` | `medgemma` | a `gemma3` arrival, already routed: [npu.md](npu.md) has the row `gemma3` / `medgemma-1.5-4b-it-NPU2` / hrx / not served, and `tests/census_watch_test.py` pins that pack to `Gemma3ForCausalLM` | an NPU row. `Medgemma-4B-NPU2` is on the box. From its `config.json` (H 2560, 8 heads, 4 KV heads, head_dim 256, IM 10240) its four shapes are `QKV K2560 N4096`, `O K2048 N2560`, `GU K2560 N20480` and `D K10240 N2560`. In the unblocked kernel family the last three are built (for example `~/models/npu/qwen3-coder-kernels/`); in the blocked (`_b8`) family only `O` is. `QKV K2560 N4096` is built in neither, so it is one kernel shape away, not a model-type item (counted 2026-11-11 with `find ~ -name 'full_i8_*.elf'`, FastFlowLM's own files excluded) | +| `Nanbeige4.1-3B-NPU2` | `nanbeige4.1` | mapped: `registry/architectures.json` has `NanbeigeForCausalLM` (gguf `nanbeige`), `registry/census.json` counts it, `registry/support_pins.json` lists its converter config, and `q4nx_semantics.h` and [tokenizers.md](tokenizers.md) name `Nanbeige4.1-3B` itself (11 files) | an NPU row. Nothing on [npu.md](npu.md) says whether it serves there, and it has not been tried | + +So: **one** family root with nothing (`translategemma`), and three that are mapped or served and owe a measured NPU row +or nothing at all. The count of 48 "with a trace" was spot-checked, not re-derived, and after this correction it should +be read as "at least 51". + +**The rows that are still owed are un-inspected, and say so on purpose.** Nothing above describes what `translategemma` +or `nanbeige4.1` computes: an xclbin name is evidence that *their* build carries a compiled family for it, and that is +all. §1's five conditions are what converts an entry from *listed* to *settled* — imports with zero gaps and serving on +the device, a known-answer prompt under the model's own template with three identical answers, decode tok/s and TTFT +under a ticket, and the kernel set and engine commit named. The two `lfm2.5` packs are the example of an entry that +went from listed to settled that way. + +The `!! NPU FLM` class above is where the owed rows belong. Its owning issue is engine#1786 (every `*-NPU2` source +serves), which is open. diff --git a/docs/npu-leaderboard.md b/docs/npu-leaderboard.md index 182931b12..0a5168f12 100644 --- a/docs/npu-leaderboard.md +++ b/docs/npu-leaderboard.md @@ -29,6 +29,13 @@ One row is one measured run of this engine. A record is only ranked if it names | Flash-Next (IQ4_XS 4.25 bpw) | hrx | IQ4_XS | prefill pp512 | 286.63 tok/s | ngl=99, fa=1, ub=512, engine=0d0c846 | 3 | llama-bench -p 512, 3 repeats, s.d. 0.94 tok/s | | Flash-Next (IQ4_XS 4.25 bpw) | hrx | IQ4_XS | prefill p1024 ub576 | 269.34 tok/s | ngl=99, fa=1, ub=576, engine=0d0c846 | 1 | llama-bench -p 1024, 1 repeat | | Flash-Next (IQ4_XS 4.25 bpw) | hrx | IQ4_XS | prefill pp4096 | 260.59 tok/s | ngl=99, fa=1, ub=512, engine=0d0c846 | 3 | llama-bench -p 4096, 3 repeats, s.d. 4.51 tok/s | +| LFM2-1.2B-NPU2 | npu | Q4NX symmetric 4-bit tiles (FastFlowLM's published container, listed as Q4_0), per-op int8 GEMM, blocked family | decode | 4.28 tok/s | device=accel0, engine=1bit built at the code of main 1131acf0 (engine#2170), container=FastFlowLM/LFM2-1.2B-NPU2 model.q4nx, sha256 2755ab40cd927e3c…, served as published; the only edit is "q4nx_weight_layout": "b8" in a copy of config.json, kernels=full_i8 blocked: QKV K2048 N3072, QKV K2048 N6144 (the conv layers' in_proj), O K2048 N2048, GU K2048 N16384, D K8192 N2048, each with its M=16 twin, ticket=npu-hold -l cl-flm-lfm2-rows, both LFM2 packs in one 107 s ticket, sampling=temperature 0, streamed, max_tokens 64, prompt_tokens=20, load1=17.5 at start, 14.7 at end, known_answer=3 requests of 'What is the capital of France? Answer with one word.' returned 'Paris' three times, identical | 2 | streamed chat completion of 64 tokens, usage read back; decode rate from the server's own timings; two requests, 4.25 and 4.30 tok/s, same text both times | +| LFM2-1.2B-NPU2 | npu | Q4NX symmetric 4-bit tiles (FastFlowLM's published container, listed as Q4_0), per-op int8 GEMM, blocked family | first token (warm) | 0.756 s | device=accel0, engine=1bit built at the code of main 1131acf0 (engine#2170), container=FastFlowLM/LFM2-1.2B-NPU2 model.q4nx, sha256 2755ab40cd927e3c…, served as published; the only edit is "q4nx_weight_layout": "b8" in a copy of config.json, kernels=full_i8 blocked: QKV K2048 N3072, QKV K2048 N6144 (the conv layers' in_proj), O K2048 N2048, GU K2048 N16384, D K8192 N2048, each with its M=16 twin, ticket=npu-hold -l cl-flm-lfm2-rows, both LFM2 packs in one 107 s ticket, sampling=temperature 0, streamed, max_tokens 64, prompt_tokens=20, load1=17.5 at start, 14.7 at end, known_answer=3 requests of 'What is the capital of France? Answer with one word.' returned 'Paris' three times, identical | 2 | time from request to the first streamed chunk on a 20-token prompt; two requests, 0.747 and 0.764 s | +| LFM2.5-1.2B-NPU2 | npu | Q4NX symmetric 4-bit tiles (FastFlowLM's published container, listed as Q4_0), per-op int8 GEMM, blocked family | decode | 4.2 tok/s | device=accel0, engine=1bit built at the code of main 1131acf0 (engine#2170), container=FastFlowLM/LFM2.5-1.2B-NPU2 model.q4nx, sha256 b341d42021e617a5…, served as published; the only edit is "q4nx_weight_layout": "b8" in a copy of config.json, kernels=full_i8 blocked: QKV K2048 N3072, QKV K2048 N6144 (the conv layers' in_proj), O K2048 N2048, GU K2048 N16384, D K8192 N2048, each with its M=16 twin, ticket=npu-hold -l cl-flm-lfm2-rows, both LFM2 packs in one 107 s ticket, sampling=temperature 0, streamed, max_tokens 64, prompt_tokens=20, load1=18.6 at start, 17.5 at end, known_answer=3 requests of 'What is the capital of France? Answer with one word.' returned 'Paris' three times, identical | 2 | streamed chat completion of 64 tokens, usage read back; decode rate from the server's own timings; two requests, 4.20 and 4.19 tok/s, same text both times | +| LFM2.5-1.2B-NPU2 | npu | Q4NX symmetric 4-bit tiles (FastFlowLM's published container, listed as Q4_0), per-op int8 GEMM, blocked family | first token (warm) | 0.929 s | device=accel0, engine=1bit built at the code of main 1131acf0 (engine#2170), container=FastFlowLM/LFM2.5-1.2B-NPU2 model.q4nx, sha256 b341d42021e617a5…, served as published; the only edit is "q4nx_weight_layout": "b8" in a copy of config.json, kernels=full_i8 blocked: QKV K2048 N3072, QKV K2048 N6144 (the conv layers' in_proj), O K2048 N2048, GU K2048 N16384, D K8192 N2048, each with its M=16 twin, ticket=npu-hold -l cl-flm-lfm2-rows, both LFM2 packs in one 107 s ticket, sampling=temperature 0, streamed, max_tokens 64, prompt_tokens=20, load1=18.6 at start, 17.5 at end, known_answer=3 requests of 'What is the capital of France? Answer with one word.' returned 'Paris' three times, identical | 2 | time from request to the first streamed chunk on a 20-token prompt; two requests, 1.094 and 0.763 s | +| LFM2.5-1.2B-Thinking-NPU2 | npu | Q4NX symmetric 4-bit tiles (FastFlowLM's published container, listed as Q4_0), per-op int8 GEMM, blocked family | decode | 4.3 tok/s | device=accel0, engine=1bit built at the code of main 1131acf0 (engine#2170), container=FastFlowLM/LFM2.5-1.2B-Thinking-NPU2 model.q4nx, sha256 b9c4bacba8ed7251…, served as published; the only edit is "q4nx_weight_layout": "b8" in a copy of config.json, kernels=full_i8 blocked: QKV K2048 N3072, QKV K2048 N6144 (the conv layers' in_proj), O K2048 N2048, GU K2048 N16384, D K8192 N2048, each with its M=16 twin, ticket=npu-hold -l cl-flm-lfm25tk-row, 234 s, sampling=temperature 0, max_tokens 320, not streamed, prompt_tokens=the pack's own template (chat_template.jinja) on 'What is the capital of France? Answer with one word.', load1=10.1 at start, 13.0 at end, known_answer=3 requests returned the same message three times: 287 reasoning tokens split into reasoning_content, then content that begins 'Paris' and runs to the token limit (finish_reason length) | 3 | usage.completion_tokens over the server's own predicted_ms, 320 tokens per request; three requests, 4.30, 4.25 and 4.37 tok/s, the median | +| Qwen3-1.7B-NPU2 | npu | Q4NX q4_1 tiles (FastFlowLM's published container, unmodified), sealed fused set 2048x6144x16x8 | decode | 39.25 tok/s | device=accel0, engine=1bit 80aceca152a909d5 (lemonade-onebit-rc-2026-11-07-c32), container=FastFlowLM/Qwen3-1.7B-NPU2 model.q4nx, sha256 b1c1fe9d6c4a6d0c…, the same file both runtimes loaded, ticket=one npu-hold ticket, 38 s, arms in the order theirs, ours, ours, theirs, sampling=greedy, streamed, reply=63 streamed chunks per request, load1=4.34 at start, 3.30 at end | 6 | streamed chat completion, decode rate from the stream's own chunk times; 3 repeats in each of two arms, median of the six (range 39.20 to 39.32) | +| Qwen3-1.7B-NPU2 | npu | Q4NX q4_1 tiles (FastFlowLM's published container, unmodified), sealed fused set 2048x6144x16x8 | first token (warm) | 0.568 s | device=accel0, engine=1bit 80aceca152a909d5 (lemonade-onebit-rc-2026-11-07-c32), container=FastFlowLM/Qwen3-1.7B-NPU2 model.q4nx, sha256 b1c1fe9d6c4a6d0c…, the same file both runtimes loaded, ticket=one npu-hold ticket, 38 s, arms in the order theirs, ours, ours, theirs, sampling=greedy, streamed, reply=63 streamed chunks per request, load1=4.34 at start, 3.30 at end | 6 | time from request to the first streamed chunk; 3 repeats in each of two arms, median of the six (range 0.566 to 0.578) | | Qwen3-4B | hrx | as shipped (1bit-hrx-3092596) | decode (eval) | 41.48 tok/s | parallel=4, kv_unified=true, requests=60 | 1 | llama-server timings in the response body | | Qwen3-Coder-30B-A3B-Instruct | hrx | Q4_K_M | decode aggregate | 87.73 tok/s | parallel=4, requests_in_flight=4, kv_unified=True, n_ctx=8192, per_stream_tok_s=21.94, solo_same_server_tok_s=84.22, engine=38e29a2 + i612-npu-streams (b78868c), llama_cpp=40c4ccb76, model_file=Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf | 5 | wall clock per request + usage.completion_tokens, n_predict=128, temperature 0, greedy, distinct prompts | | Qwen3-Coder-30B-A3B-Instruct | hrx | Q4_K_M | decode | 84.33 tok/s | parallel=1, requests_in_flight=1, n_ctx=8192, engine=38e29a2 + i612-npu-streams (b78868c), llama_cpp=40c4ccb76, model_file=Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf | 5 | wall clock per request + usage.completion_tokens, n_predict=128, temperature 0, greedy, distinct prompts | @@ -62,6 +69,13 @@ One row is one measured run of this engine. A record is only ranked if it names - `flashnext-pp512-20261027` — 2026-10-27T02:38:52Z; source: ~/lb/572/run-20261027T015408Z/a1-timing.out (strixhalo); gufo reference on the same box is 1679 tok/s; the analysis is in docs/hrx.md - `flashnext-p1024-ub576-20261027` — 2026-10-27T02:50:00Z; source: ~/lb/572/run-20261027T015408Z/a4-ub576.out (strixhalo); shows the ubatch bound above 512 no longer refuses the run; single repeat, do not read as a tuned number - `flashnext-pp4096-20261027` — 2026-10-27T02:41:00Z; source: ~/lb/572/run-20261027T015408Z/a1-timing.out (strixhalo) +- `lfm2-12b-perop-decode-20261111` — 2026-11-11T13:32:47Z; source: ~/.cache/scratch-cl-flm/rows.log and out-rows-LFM2-1.2B-NPU2/ on strixhalo (row-serve.sh + row-probe.py, lane cl-flm; engine#1786); A hybrid: 10 of 16 layers are gated short convolutions, run as two device GEMMs with the gate and a 3-tap causal conv on the host. This is the per-op route's speed, not what the array does on a geometry with a fused set: FastFlowLM 1.0.12's runtime decoded the same file at 42 to 83 tok/s in the paired tickets of the same day. What would make it wrong: two requests, box load 15 to 19 during the ticket (the same pack measured 3.7 tok/s in a paired ticket earlier that day). +- `lfm2-12b-perop-first-token-20261111` — 2026-11-11T13:32:47Z; source: ~/.cache/scratch-cl-flm/rows.log and out-rows-LFM2-1.2B-NPU2/ on strixhalo (row-serve.sh + row-probe.py, lane cl-flm; engine#1786); Same ticket as lfm2-12b-perop-decode-20261111. Batched prefill, no prefix cache hit (a different prompt had been served before it). The server's own prompt_ms was 745 and 762 ms. +- `lfm25-12b-perop-decode-20261111` — 2026-11-11T13:31:50Z; source: ~/.cache/scratch-cl-flm/rows.log and out-rows-LFM2.5-1.2B-NPU2/ on strixhalo (row-serve.sh + row-probe.py, lane cl-flm; engine#1786); A hybrid: 10 of 16 layers are gated short convolutions, run as two device GEMMs with the gate and a 3-tap causal conv on the host. This is the per-op route's speed, not what the array does on a geometry with a fused set: FastFlowLM 1.0.12's runtime decoded the same file at 42 to 83 tok/s in the paired tickets of the same day. What would make it wrong: two requests, box load 15 to 19 during the ticket (the same pack measured 4.6 tok/s at load 3 to 4 and 3.5 to 3.7 at load 8 to 10 earlier that day, so load is not the only term). +- `lfm25-12b-perop-first-token-20261111` — 2026-11-11T13:31:50Z; source: ~/.cache/scratch-cl-flm/rows.log and out-rows-LFM2.5-1.2B-NPU2/ on strixhalo (row-serve.sh + row-probe.py, lane cl-flm; engine#1786); Same ticket as lfm25-12b-perop-decode-20261111. Batched prefill, no prefix cache hit (a different prompt had been served before it). The server's own prompt_ms was 766 and 760 ms, so the 1.094 s of the first request is client-side time before the first chunk, not prefill. +- `lfm25-12b-thinking-perop-decode-20261111` — 2026-11-11T14:14:56Z; source: ~/.cache/scratch-cl-flm/rows-25tk.log and out-rows-lfm25tk/ on strixhalo (row-serve.sh + think-row.py, lane cl-flm; engine#1786); The thinking variant of the LFM2.5 hybrid. The reply does not stop after its answer inside 320 tokens. FastFlowLM 1.0.12's own runtime, same file, same prompt, same budget, one ticket later (two requests, identical): content empty, finish length, and 830 characters of reasoning that begin 'Okay derivative of the function f(x) = x^2 + 3x - 4. Find its vertex.' -- a different problem from the one asked -- at 65.5 tok/s. So on this pack the two runtimes do not produce comparable text, and this row is not a parity row: it says this route reasons about the question and reaches 'Paris'. What would make it wrong: one prompt. +- `qwen3-17b-fused-decode-flm-pair-20261111` — 2026-11-11T05:42:39Z; source: ~/lb/aval/clperf-f9-abba-20261111T054201Z/run.log on strixhalo (lane cl-perf's f9-npu-abba.sh, unmodified; engine#1787); Side by side with FastFlowLM 1.0.12's own runtime on the same array and the same model.q4nx, same ticket, A B B A: theirs decoded at 45.57 tok/s (median of its six, range 45.08 to 45.93), ours at 39.25, so theirs is 1.16 times ours on this pack. This row is ours; theirs is quoted here because a row is one run of this engine. What would make it wrong: it is one ticket at load 3 to 4, one prompt, and the shipped c32 build rather than main. +- `qwen3-17b-fused-first-token-flm-pair-20261111` — 2026-11-11T05:42:39Z; source: ~/lb/aval/clperf-f9-abba-20261111T054201Z/run.log on strixhalo (lane cl-perf's f9-npu-abba.sh, unmodified; engine#1787); Same ticket as qwen3-17b-fused-decode-flm-pair-20261111. FastFlowLM 1.0.12's runtime on the same model.q4nx: 0.591 s (median of its six). The two are level within the ticket's own spread; the first request of each of their arms was slower (0.658 and 0.653 s) and is inside their median. - `qwen3-4b-hrx-parallel4-20261027` — 2026-10-27T12:36:00Z; source: ~/lb/528/run-20261027T123606Z-none (strixhalo); #528 repro arm: only one slot was used, so this is a single-slot decode rate and the NaN report stayed unreproduced - `coder30b-hrx-parallel4-20261030` — 2026-10-30T21:16:28Z; source: ~/lb/612-hrx-20261030T211252Z (tools/showcase-612-hrx.sh 2 4 8 on an idle strixhalo, under gpu-hold label 'pi i612'); reproduces ~/lb/612-hrx-20261030T064957Z; 1.04x: four streams only recover the single-stream baseline (91.11 / 1.08x at 06:45Z). llama-server reports all four slots decoding in one batch at 24.88 t/s each, so the batching is real; the aggregate is flat because the model is bandwidth-bound. No --parallel 8 row is published: at 8 streams the rate is non-stationary inside a run (llama-server's own tg per slot goes 14.99 15.12 14.34 12.10 7.09 6.78 5.70 5.64 as the answers grow, then 14.17 for a fresh batch, with die temp 61-71 C and iGPU sclk 694-2127 MHz swinging normally), so an 8-stream number is a function of where in that decay it is sampled: 112.89 / 1.34x at 06:45Z, 78.87 / 0.97x and 68.29 / 0.92x later the same day. - `coder30b-hrx-solo-20261030` — 2026-10-30T21:16:28Z; source: ~/lb/612-hrx-20261030T211252Z (tools/showcase-612-hrx.sh 2 4 8 on an idle strixhalo, under gpu-hold label 'pi i612'); reproduces ~/lb/612-hrx-20261030T064957Z; One request at a time, the baseline the batched rows are read against. Measured as the solo arm of the --parallel 2/4 servers (84.33 / 84.22, ten requests) and 81.39 on the --parallel 8 server; across both runs today the same load gave 74.03-88.34, i.e. +-8 %, so compare ratios, not absolute tok/s. diff --git a/docs/npu.md b/docs/npu.md index f50585a33..e96ddc3d7 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2062,7 +2062,8 @@ does on a geometry that has a sealed set: `Qwen3-1.7B-NPU2` is 3.0 tok/s here an | `Qwen3-VL-4B-Instruct-NPU2`, text tower only | `qwen3` | identical, `Paris` | 1.4 to 1.5 tok/s | | `Hy-MT2-1.8B-NPU2` | `hunyuan_v1_dense` | identical, `Paris`; 4 of 5 translations byte-identical to FastFlowLM's runtime (below) | 2.3 to 2.8 tok/s | | `LFM2.5-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | identical, `Paris`; against FastFlowLM's runtime below | 3.5 to 4.6 tok/s | -| `LFM2-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | `Paris`; one request per prompt, against FastFlowLM's runtime below | 3.7 tok/s | +| `LFM2-1.2B-NPU2` | `lfm2`, hybrid: 10 conv layers of 16 | identical, `Paris`; against FastFlowLM's runtime below | 3.7 to 4.3 tok/s | +| `LFM2.5-1.2B-Thinking-NPU2` | `lfm2`, hybrid, thinking | identical; `Paris` after 287 reasoning tokens, no clean stop (below) | 4.3 tok/s | `D3-Nano-NPU2` is a decision pack: it loads on the `Qwen3.5-2B` kernel set, its readout converts with `tools/safetensors_to_q4nx.py decision`, and `/v1/systemone` returned three identical answers to the documented @@ -2117,13 +2118,19 @@ the gap "Per-op route" above describes. at the merged head of that change: two of five answers byte-identical (`Paris`; the count from one to ten), three that share 21, 58 and 53 characters with FastFlowLM's and then continue differently, each a fluent answer to the question (a Fibonacci function that returns `0` for `n <= 0` where theirs returns a message). Decode 3.7 tok/s -against 42 to 47. Its logprobs at the three points were not taken, and it was asked each prompt once, so this row -does not carry the three-identical-answers condition. - -`LFM2.5-1.2B-Thinking-NPU2` loads on the same directory too: its template is read from the pack's -`chat_template.jinja`, the reasoning is split out, and at 96 tokens every one of the five replies was still -reasoning, so there is no answer text to compare and no row; the reasoning differs from FastFlowLM's from its -first words. +against 42 to 47. Its logprobs at the three points were not taken. A later ticket at the merged code (main `1131acf0`) asked both +packs the known-answer prompt three times each: `Paris`, identical, with decode at 4.2 to 4.3 tok/s; those four +figures are records in `bench/npu-leaderboard/`. + +`LFM2.5-1.2B-Thinking-NPU2` serves on the same directory too: its template is read from the pack's +`chat_template.jinja` and the reasoning is split out. Asked the known-answer prompt three times with 320 tokens +of room, it reasoned for 287 tokens and then answered, the three replies identical in reasoning and content, at +4.3 tok/s. The content begins `Paris` and does not end there: it goes on (`\boxed{Paris}`, then a restatement) +until the token limit, so the row has the answer and not a clean stop. FastFlowLM's own runtime, given the same +file, prompt and budget, returned no content at all: 320 tokens of reasoning that begin `Okay derivative of the +function f(x) = x^2 + 3x - 4. Find its vertex.`, a problem nobody asked, the same on both of two requests. So +on this pack the two runtimes' texts are not comparable and the row is not a parity row; it records that this +route reasons about the question that was asked. **`Hy-MT2-1.8B-NPU2`: a third tile.** This pack's projections are stored 4,608 bytes to a tile where every other pack on the box uses 5,120, and no reader knew the size, so it was refused at the first dequant. The tile is