npu: the Whisper host reference (#1931) with its blockers cleared, onto main — supersedes #1931 and #2035 (Refs #1783)

CMakeLists.txt

@@ -804,7 +804,7 @@ if(ONEBIT_NPU_HOST)
 find_package(PkgConfig REQUIRED)
 pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8)
 add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp
-                            npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp
+                            npu/audio_exec.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp
                             npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp)
 target_include_directories(onebit_npu PUBLIC npu)
 target_link_libraries(onebit_npu PUBLIC nlohmann_json::nlohmann_json PRIVATE PkgConfig::PCRE2)
@@ -1378,6 +1378,28 @@ endif()
 # q4nx_pack.c is listed here rather than reached through onebit_npu: the integer host arm's checks
 # (engine#1808) call q4nx_host_gemm_i8*() directly, and those live in that file.
 add_executable(npu_pack_test tests/npu_pack_test.cpp npu/forward/src/q4nx_pack.c npu/forward/src/q4nx_dequant.c)
+# The Whisper encoder front-end's four f32 host kernels: no container, no device, so it runs wherever this file is built
+# (the same reason #1830/#1863 put their arms in a binary CI already compiles). Values come from the encoder reference.
+add_executable(audio_ops_test tests/audio_ops_test.cpp)
+target_include_directories(audio_ops_test PRIVATE npu/forward/include)
+# The audio pack's graph, read through the engine: pins the topological-order assumption the executor needs and the op
+# census the host reference was written against. Skips (77) without a path.
+add_executable(onnx_graph_test tests/onnx_graph_test.cpp)
+target_link_libraries(onnx_graph_test PRIVATE onebit_npu)
+add_executable(whisper_transcribe tools/whisper_transcribe.cpp)
+target_include_directories(whisper_transcribe PRIVATE npu/forward/include)
+target_link_libraries(whisper_transcribe PRIVATE onebit_npu)
+add_executable(audio_mel_test tests/audio_mel_test.cpp)
+target_include_directories(audio_mel_test PRIVATE npu/forward/include)
+add_executable(audio_exec_test tests/audio_exec_test.cpp)
+target_link_libraries(audio_exec_test PRIVATE onebit_npu)
+# The executor's fork-join (npu/audio_parallel.h): MatMul bit-identical across ONEBIT_AUDIO_THREADS, and a throwing unit
+# is a caught exception rather than std::terminate. The graph is built in memory, so: no pack, no device.
+add_executable(audio_parallel_test tests/audio_parallel_test.cpp)
+target_link_libraries(audio_parallel_test PRIVATE onebit_npu Threads::Threads)
+# ci.yml's build list is not ours to edit without `workflow` scope, so these reach CI the way ops_dump_test does: as build
+# dependencies of a target it already builds (tools/ci/ci_targets_check.py follows this edge).
+add_dependencies(onebit audio_ops_test audio_mel_test onnx_graph_test audio_exec_test audio_parallel_test)
 # A DFlash draft package as a model source (engine#1780): config + container -> a checked model whose tensors decode
 # one at a time. The loader is compiled here and not into onebit_npu because it needs the q4_1 tile decoder, which
 # lives with the forward (npu/forward/src/q4nx_dequant.c); it moves there with the draft's forward. Synthetic package,
@@ -1729,6 +1751,14 @@ endif()
 if(ONEBIT_NPU_MODEL AND ONEBIT_NPU_BO_DUMP)
     add_test(NAME npu_pack_model COMMAND npu_pack_test ${ONEBIT_NPU_MODEL} ${ONEBIT_NPU_BO_DUMP})
 endif()
+add_test(NAME audio_ops COMMAND audio_ops_test)
+add_test(NAME audio_mel COMMAND audio_mel_test)
+set_tests_properties(audio_ops PROPERTIES SKIP_RETURN_CODE 77)
+add_test(NAME onnx_graph COMMAND onnx_graph_test ${ONEBIT_AUDIO_ONNX})
+add_test(NAME audio_exec COMMAND audio_exec_test ${ONEBIT_AUDIO_ONNX})
+add_test(NAME audio_parallel COMMAND audio_parallel_test)
+set_tests_properties(audio_exec PROPERTIES SKIP_RETURN_CODE 77)
+set_tests_properties(onnx_graph PROPERTIES SKIP_RETURN_CODE 77)
 # The private NPU route registry (npu/private_route.h), without an add-on.
 add_executable(npu_private_route_test tests/npu_private_route_test.cpp)
 target_link_libraries(npu_private_route_test PRIVATE onebit_npu)

npu/audio_exec.h

@@ -0,0 +1,55 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// audio_exec.h -- run an audio pack's graph on the host in f32. Not a general ONNX runtime: it covers exactly the ops a
+// Whisper encoder uses (measured on both exports -- 447 nodes / 11 op types for whisper-small, 1031 / 10 for large-v3), and
+// it refuses an op it does not implement *by name* rather than producing a plausible activation from a silently skipped node.
+//
+// The four f32 kernels this stands on are in npu/forward/include/audio_ops.h, which is device-free and known-answer tested;
+// this file adds the shape bookkeeping and the wiring. The caller compares its output against the reference, and against
+// onnxruntime, before anything here is used to argue a device number.
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "onnx_graph.h"
+
+namespace onebit::npu::pack_import {
+
+struct AudioRunResult {
+    std::vector<int64_t> dims;      // dims of the requested output
+    std::vector<float> data;        // row-major f32 payload
+    std::vector<int64_t> i64;       // int64 payload, when the requested tensor is an index/token tensor
+    std::vector<uint8_t> b;         // bool payload, when it is a mask
+};
+
+// Two entry points, because a Whisper pack has two graphs: the encoder consumes the mel window as f32, the decoder consumes
+// a token window as int64 plus the encoder's output. Everything else -- node order, weights, refusals -- is shared.
+//
+// mel: (mel_bins, frames) row-major f32, the encoder's graph input in the layout its own first Conv consumes.
+// out_name: the graph output to return (empty = the graph's declared output).
+// Returns true and fills `out`; on failure returns false and sets `why` to the node and the reason.
+bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames,
+                     const std::string& out_name, AudioRunResult* out, std::string* why);
+
+// The decoder: `tokens` is the int64 token window for the graph's first input; `xa` (the encoder output, row-major f32) is
+// bound to the graph's second input when the graph has one. `xa_dims` may be empty when there is no cross-attention input.
+bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_tokens, const float* xa,
+                       const std::vector<int64_t>& xa_dims, const std::string& out_name, AudioRunResult* out,
+                       std::string* why);
+
+}  // namespace onebit::npu::pack_import

npu/audio_parallel.h

@@ -0,0 +1,81 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// audio_parallel.h -- the fork-join the host audio executor splits independent work over (engine#1931).
+//
+// Deterministic by construction: unit u always runs fn(u) exactly once, and a unit writes only its own slice of the
+// output, so every element's arithmetic is unchanged by the thread count and the result is bit-identical to the serial
+// loop. That is the property a reference implementation has to keep, and it is asserted, not assumed:
+// tests/audio_parallel_test.cpp compares the executor's MatMul byte for byte across thread counts.
+//
+// A unit that throws does not take the process down. A std::thread whose function lets an exception escape calls
+// std::terminate, so each worker catches, every worker is joined, and the first exception (lowest worker index, which
+// makes the choice repeatable) is rethrown on the calling thread, where the executor turns it into a refusal with a
+// reason like every other failure.
+//
+// Header-only and free of the executor's types, so the test needs no pack and no device.
+
+#pragma once
+
+#include <cstddef>
+#include <cstdlib>
+#include <exception>
+#include <thread>
+#include <vector>
+
+namespace onebit::npu::pack_import {
+
+// How many threads the audio executor may use. ONEBIT_AUDIO_THREADS=N sets it; anything that is not a positive number
+// means 1 (serial, the control for measuring the split), so a typo cannot turn into "as many as the parse produced".
+// Unset: the hardware's count, at most 8.
+inline size_t audio_thread_count() {
+    if (const char* env = std::getenv("ONEBIT_AUDIO_THREADS")) {
+        const long n = std::strtol(env, nullptr, 10);
+        return n > 0 ? size_t(n) : 1;
+    }
+    const size_t hw = std::thread::hardware_concurrency();
+    return hw == 0 ? 1 : (hw > 8 ? 8 : hw);
+}
+
+// Run fn(0) .. fn(units-1), each exactly once, on at most `want` threads. Worker t takes units t, t+want, t+2*want, ...
+template <typename Fn>
+void parallel_units(size_t units, size_t want, Fn&& fn) {
+    if (want > units) want = units;
+    if (want <= 1) {
+        for (size_t u = 0; u < units; ++u) fn(u);
+        return;
+    }
+    std::vector<std::exception_ptr> failed(want);
+    std::vector<std::thread> pool;
+    pool.reserve(want);
+    for (size_t t = 0; t < want; ++t)
+        pool.emplace_back([&, t]() {
+            try {
+                for (size_t u = t; u < units; u += want) fn(u);
+            } catch (...) {
+                failed[t] = std::current_exception();
+            }
+        });
+    for (auto& th : pool) th.join();
+    for (const auto& ep : failed)
+        if (ep) std::rethrow_exception(ep);
+}
+
+template <typename Fn>
+void parallel_units(size_t units, Fn&& fn) {
+    parallel_units(units, audio_thread_count(), static_cast<Fn&&>(fn));
+}
+
+}  // namespace onebit::npu::pack_import

npu/forward/include/audio_mel.h

@@ -0,0 +1,138 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// audio_mel.h -- the log-mel front-end: hann-windowed STFT (n_fft 400, hop 160, reflect padding), power spectrum, a
+// slaney-normalised mel filterbank, then log10 and the (max - 8, +4)/4 scaling a Whisper encoder's input expects.
+//
+// This is the one part of the pipeline that is *not* in the pack: the pack's graph starts at the mel window, so the numbers
+// here have to match the reference extractor's, and tests/audio_mel_test.cpp checks them against it.
+//
+// The filterbank is built, not shipped: librosa's slaney mel scale (f_sp = 200/3 Hz, linear below 1 kHz, log above,
+// logstep = ln(6.4)/27) with slaney normalisation, which is what Whisper's feature extractor uses.
+
+#pragma once
+
+#include <cmath>
+#include <cstddef>
+#include <vector>
+
+namespace onebit::audio {
+
+inline double hz_to_mel(double hz) {
+    const double f_sp = 200.0 / 3.0;
+    const double min_log_hz = 1000.0, min_log_mel = min_log_hz / f_sp, logstep = std::log(6.4) / 27.0;
+    if (hz < min_log_hz) return hz / f_sp;
+    return min_log_mel + std::log(hz / min_log_hz) / logstep;
+}
+
+inline double mel_to_hz(double mel) {
+    const double f_sp = 200.0 / 3.0;
+    const double min_log_hz = 1000.0, min_log_mel = min_log_hz / f_sp, logstep = std::log(6.4) / 27.0;
+    if (mel < min_log_mel) return mel * f_sp;
+    return min_log_hz * std::exp(logstep * (mel - min_log_mel));
+}
+
+// (bins, 1 + n_fft/2): triangular filters, slaney-normalised, spanning 0..nyquist.
+inline std::vector<float> mel_filterbank(int bins, int n_fft, int sample_rate) {
+    const int nfreqs = n_fft / 2 + 1;
+    const double nyq = sample_rate / 2.0;
+    const double mlo = hz_to_mel(0.0), mhi = hz_to_mel(nyq);
+    std::vector<double> f(bins + 2);
+    for (int i = 0; i < bins + 2; ++i) f[i] = mel_to_hz(mlo + (mhi - mlo) * double(i) / double(bins + 1));
+    // FFT bin centres in Hz, and the per-filter bandwidth normalisation.
+    std::vector<double> hz(nfreqs);
+    for (int k = 0; k < nfreqs; ++k) hz[k] = double(k) * nyq / double(nfreqs - 1);
+    std::vector<double> enorm(bins);
+    for (int i = 0; i < bins; ++i) {
+        const double w = f[i + 2] - f[i];
+        enorm[i] = w > 0 ? 2.0 / w : 0.0;
+    }
+    std::vector<float> m(size_t(bins) * size_t(nfreqs), 0.0f);
+    for (int i = 0; i < bins; ++i) {
+        const double lo = f[i], ctr = f[i + 1], hi = f[i + 2];
+        for (int k = 0; k < nfreqs; ++k) {
+            const double x = hz[k];
+            double v = 0.0;
+            if (x >= lo && x <= ctr && ctr > lo) v = (x - lo) / (ctr - lo);
+            else if (x > ctr && x <= hi && hi > ctr) v = (hi - x) / (hi - ctr);
+            m[size_t(i) * nfreqs + k] = float(v * enorm[i]);
+        }
+    }
+    return m;
+}
+
+struct MelConfig {
+    int sample_rate = 16000;
+    int n_fft = 400;
+    int hop = 160;
+    int bins = 80;
+    int frames = 3000;   // the encoder's window: 30 s
+};
+
+// audio: `samples` mono f32 in [-1,1]. out: (bins, frames) row-major. Returns false only on a size the config cannot fill.
+inline bool log_mel_spectrogram(const float* audio, size_t samples, const MelConfig& cfg, std::vector<float>* out) {
+    const int pad = cfg.n_fft / 2;
+    const int nfreqs = cfg.n_fft / 2 + 1;
+    if (samples + 2 * size_t(pad) < size_t(cfg.n_fft)) return false;
+
+    // reflect padding, both sides (torch's center=True default for stft)
+    std::vector<double> x(samples + 2 * size_t(pad));
+    for (int i = 0; i < pad; ++i) x[size_t(i)] = audio[pad - i];
+    for (size_t i = 0; i < samples; ++i) x[size_t(pad) + i] = audio[i];
+    for (int i = 0; i < pad; ++i) x[size_t(pad) + samples + size_t(i)] = audio[samples - 2 - size_t(i)];
+
+    std::vector<double> win(cfg.n_fft);
+    for (int i = 0; i < cfg.n_fft; ++i)   // periodic hann, as torch.hann_window(w) is by default
+        win[size_t(i)] = 0.5 - 0.5 * std::cos(2.0 * M_PI * double(i) / double(cfg.n_fft));
+
+    const std::vector<float> fb = mel_filterbank(cfg.bins, cfg.n_fft, cfg.sample_rate);
+    out->assign(size_t(cfg.bins) * size_t(cfg.frames), 0.0f);
+    std::vector<double> re(nfreqs), im(nfreqs), power(nfreqs);
+    for (int t = 0; t < cfg.frames; ++t) {
+        const size_t base = size_t(t) * size_t(cfg.hop);
+        for (int k = 0; k < nfreqs; ++k) {
+            double sr = 0.0, si = 0.0;
+            for (int n = 0; n < cfg.n_fft; ++n) {
+                const double v = x[base + size_t(n)] * win[size_t(n)];
+                const double ang = -2.0 * M_PI * double(k) * double(n) / double(cfg.n_fft);
+                sr += v * std::cos(ang);
+                si += v * std::sin(ang);
+            }
+            re[size_t(k)] = sr;
+            im[size_t(k)] = si;
+            power[size_t(k)] = sr * sr + si * si;
+        }
+        for (int b = 0; b < cfg.bins; ++b) {
+            const float* row = fb.data() + size_t(b) * nfreqs;
+            double acc = 0.0;
+            for (int k = 0; k < nfreqs; ++k) acc += double(row[k]) * power[size_t(k)];
+            out->at(size_t(b) * size_t(cfg.frames) + size_t(t)) = float(acc);
+        }
+    }
+    // log10 with a floor, then Whisper's dynamic-range clamp and scale.
+    double mx = -1e30;
+    for (float& v : *out) {
+        v = float(std::log10(std::max(double(v), 1e-10)));
+        if (v > mx) mx = v;
+    }
+    const double floor_db = mx - 8.0;
+    for (float& v : *out) {
+        const double c = std::max(double(v), floor_db);
+        v = float((c + 4.0) / 4.0);
+    }
+    return true;
+}
+
+}  // namespace onebit::audio