npu: read an audio pack's graph whole and run its encoder and decoder on the host (refs #1783)

CMakeLists.txt

@@ -212,6 +212,16 @@ add_test(NAME bump_hrx_report COMMAND bash ${CMAKE_SOURCE_DIR}/tests/bump_hrx_re
 add_executable(audio_ops_test tests/audio_ops_test.cpp)
 target_include_directories(audio_ops_test PRIVATE npu/forward/include)
 add_test(NAME audio_ops COMMAND audio_ops_test)
+# engine#1783 (F8): the engine reads an audio pack's graph and runs the whole encoder + decoder on the host -- 447 + 786 nodes,
+# matched against onnxruntime. The reader keeps every node in graph order (the census walk keeps only the MatMul family).
+add_executable(onnx_graph_test tests/onnx_graph_test.cpp)
+target_link_libraries(onnx_graph_test PRIVATE onebit_npu)
+add_test(NAME onnx_graph COMMAND onnx_graph_test ${ONEBIT_AUDIO_ONNX})
+set_tests_properties(onnx_graph PROPERTIES SKIP_RETURN_CODE 77)
+add_executable(audio_exec_test tests/audio_exec_test.cpp)
+target_link_libraries(audio_exec_test PRIVATE onebit_npu)
+add_test(NAME audio_exec COMMAND audio_exec_test ${ONEBIT_AUDIO_ONNX})
+set_tests_properties(audio_exec PROPERTIES SKIP_RETURN_CODE 77)
 add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $<TARGET_FILE:onebit>)
 # serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU.
 add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $<TARGET_FILE:onebit>)
@@ -754,7 +764,7 @@ if(ONEBIT_NPU_HOST)
 find_package(PkgConfig REQUIRED)
 pkg_check_modules(PCRE2 REQUIRED IMPORTED_TARGET libpcre2-8)
 add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_elf.cpp npu/model.cpp npu/vision_pack.cpp npu/vision_tower.cpp npu/vision_image.cpp
-                            npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp
+                            npu/audio_exec.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp
                             npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp)
 target_include_directories(onebit_npu PUBLIC npu)
 target_link_libraries(onebit_npu PUBLIC nlohmann_json::nlohmann_json PRIVATE PkgConfig::PCRE2)

npu/audio_exec.h

@@ -0,0 +1,43 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// audio_exec.h -- run an audio pack's graph on the host in f32. Not a general ONNX runtime: it covers exactly the ops a
+// Whisper encoder uses (measured on both exports -- 447 nodes / 11 op types for whisper-small, 1031 / 10 for large-v3), and
+// it refuses an op it does not implement *by name* rather than producing a plausible activation from a silently skipped node.
+//
+// The four f32 kernels this stands on are in npu/forward/include/audio_ops.h, which is device-free and known-answer tested;
+// this file adds the shape bookkeeping and the wiring. The caller compares its output against the reference, and against
+// onnxruntime, before anything here is used to argue a device number.
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "onnx_graph.h"
+
+namespace onebit::npu::pack_import {
+
+struct AudioRunResult {
+    std::vector<int64_t> dims;      // dims of the requested output
+    std::vector<float> data;        // row-major f32 payload
+    std::vector<int64_t> i64;       // int64 payload, when the requested tensor is an index/token tensor
+    std::vector<uint8_t> b;         // bool payload, when it is a mask
+};
+
+// Two entry points, because a Whisper pack has two graphs: the encoder consumes the mel window as f32, the decoder consumes
+// a token window as int64 plus the encoder's output. Everything else -- node order, weights, refusals -- is shared.
+//
+// mel: (mel_bins, frames) row-major f32, the encoder's graph input in the layout its own first Conv consumes.
+// out_name: the graph output to return (empty = the graph's declared output).
+// Returns true and fills `out`; on failure returns false and sets `why` to the node and the reason.
+bool run_audio_graph(const OnnxGraphInfo& g, const float* mel, int64_t mel_bins, int64_t frames,
+                     const std::string& out_name, AudioRunResult* out, std::string* why);
+
+// The decoder: `tokens` is the int64 token window for the graph's first input; `xa` (the encoder output, row-major f32) is
+// bound to the graph's second input when the graph has one. `xa_dims` may be empty when there is no cross-attention input.
+bool run_decoder_graph(const OnnxGraphInfo& g, const int64_t* tokens, int64_t n_tokens, const float* xa,
+                       const std::vector<int64_t>& xa_dims, const std::string& out_name, AudioRunResult* out,
+                       std::string* why);
+
+}  // namespace onebit::npu::pack_import

npu/forward/include/audio_mel.h

@@ -0,0 +1,126 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// audio_mel.h -- the log-mel front-end: hann-windowed STFT (n_fft 400, hop 160, reflect padding), power spectrum, a
+// slaney-normalised mel filterbank, then log10 and the (max - 8, +4)/4 scaling a Whisper encoder's input expects.
+//
+// This is the one part of the pipeline that is *not* in the pack: the pack's graph starts at the mel window, so the numbers
+// here have to match the reference extractor's, and tests/audio_mel_test.cpp checks them against it.
+//
+// The filterbank is built, not shipped: librosa's slaney mel scale (f_sp = 200/3 Hz, linear below 1 kHz, log above,
+// logstep = ln(6.4)/27) with slaney normalisation, which is what Whisper's feature extractor uses.
+
+#pragma once
+
+#include <cmath>
+#include <cstddef>
+#include <vector>
+
+namespace onebit::audio {
+
+inline double hz_to_mel(double hz) {
+    const double f_sp = 200.0 / 3.0;
+    const double min_log_hz = 1000.0, min_log_mel = min_log_hz / f_sp, logstep = std::log(6.4) / 27.0;
+    if (hz < min_log_hz) return hz / f_sp;
+    return min_log_mel + std::log(hz / min_log_hz) / logstep;
+}
+
+inline double mel_to_hz(double mel) {
+    const double f_sp = 200.0 / 3.0;
+    const double min_log_hz = 1000.0, min_log_mel = min_log_hz / f_sp, logstep = std::log(6.4) / 27.0;
+    if (mel < min_log_mel) return mel * f_sp;
+    return min_log_hz * std::exp(logstep * (mel - min_log_mel));
+}
+
+// (bins, 1 + n_fft/2): triangular filters, slaney-normalised, spanning 0..nyquist.
+inline std::vector<float> mel_filterbank(int bins, int n_fft, int sample_rate) {
+    const int nfreqs = n_fft / 2 + 1;
+    const double nyq = sample_rate / 2.0;
+    const double mlo = hz_to_mel(0.0), mhi = hz_to_mel(nyq);
+    std::vector<double> f(bins + 2);
+    for (int i = 0; i < bins + 2; ++i) f[i] = mel_to_hz(mlo + (mhi - mlo) * double(i) / double(bins + 1));
+    // FFT bin centres in Hz, and the per-filter bandwidth normalisation.
+    std::vector<double> hz(nfreqs);
+    for (int k = 0; k < nfreqs; ++k) hz[k] = double(k) * nyq / double(nfreqs - 1);
+    std::vector<double> enorm(bins);
+    for (int i = 0; i < bins; ++i) {
+        const double w = f[i + 2] - f[i];
+        enorm[i] = w > 0 ? 2.0 / w : 0.0;
+    }
+    std::vector<float> m(size_t(bins) * size_t(nfreqs), 0.0f);
+    for (int i = 0; i < bins; ++i) {
+        const double lo = f[i], ctr = f[i + 1], hi = f[i + 2];
+        for (int k = 0; k < nfreqs; ++k) {
+            const double x = hz[k];
+            double v = 0.0;
+            if (x >= lo && x <= ctr && ctr > lo) v = (x - lo) / (ctr - lo);
+            else if (x > ctr && x <= hi && hi > ctr) v = (hi - x) / (hi - ctr);
+            m[size_t(i) * nfreqs + k] = float(v * enorm[i]);
+        }
+    }
+    return m;
+}
+
+struct MelConfig {
+    int sample_rate = 16000;
+    int n_fft = 400;
+    int hop = 160;
+    int bins = 80;
+    int frames = 3000;   // the encoder's window: 30 s
+};
+
+// audio: `samples` mono f32 in [-1,1]. out: (bins, frames) row-major. Returns false only on a size the config cannot fill.
+inline bool log_mel_spectrogram(const float* audio, size_t samples, const MelConfig& cfg, std::vector<float>* out) {
+    const int pad = cfg.n_fft / 2;
+    const int nfreqs = cfg.n_fft / 2 + 1;
+    if (samples + 2 * size_t(pad) < size_t(cfg.n_fft)) return false;
+
+    // reflect padding, both sides (torch's center=True default for stft)
+    std::vector<double> x(samples + 2 * size_t(pad));
+    for (int i = 0; i < pad; ++i) x[size_t(i)] = audio[pad - i];
+    for (size_t i = 0; i < samples; ++i) x[size_t(pad) + i] = audio[i];
+    for (int i = 0; i < pad; ++i) x[size_t(pad) + samples + size_t(i)] = audio[samples - 2 - size_t(i)];
+
+    std::vector<double> win(cfg.n_fft);
+    for (int i = 0; i < cfg.n_fft; ++i)   // periodic hann, as torch.hann_window(w) is by default
+        win[size_t(i)] = 0.5 - 0.5 * std::cos(2.0 * M_PI * double(i) / double(cfg.n_fft));
+
+    const std::vector<float> fb = mel_filterbank(cfg.bins, cfg.n_fft, cfg.sample_rate);
+    out->assign(size_t(cfg.bins) * size_t(cfg.frames), 0.0f);
+    std::vector<double> re(nfreqs), im(nfreqs), power(nfreqs);
+    for (int t = 0; t < cfg.frames; ++t) {
+        const size_t base = size_t(t) * size_t(cfg.hop);
+        for (int k = 0; k < nfreqs; ++k) {
+            double sr = 0.0, si = 0.0;
+            for (int n = 0; n < cfg.n_fft; ++n) {
+                const double v = x[base + size_t(n)] * win[size_t(n)];
+                const double ang = -2.0 * M_PI * double(k) * double(n) / double(cfg.n_fft);
+                sr += v * std::cos(ang);
+                si += v * std::sin(ang);
+            }
+            re[size_t(k)] = sr;
+            im[size_t(k)] = si;
+            power[size_t(k)] = sr * sr + si * si;
+        }
+        for (int b = 0; b < cfg.bins; ++b) {
+            const float* row = fb.data() + size_t(b) * nfreqs;
+            double acc = 0.0;
+            for (int k = 0; k < nfreqs; ++k) acc += double(row[k]) * power[size_t(k)];
+            out->at(size_t(b) * size_t(cfg.frames) + size_t(t)) = float(acc);
+        }
+    }
+    // log10 with a floor, then Whisper's dynamic-range clamp and scale.
+    double mx = -1e30;
+    for (float& v : *out) {
+        v = float(std::log10(std::max(double(v), 1e-10)));
+        if (v > mx) mx = v;
+    }
+    const double floor_db = mx - 8.0;
+    for (float& v : *out) {
+        const double c = std::max(double(v), floor_db);
+        v = float((c + 4.0) / 4.0);
+    }
+    return true;
+}
+
+}  // namespace onebit::audio

npu/onnx_graph.h

@@ -0,0 +1,57 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// onnx_graph.h -- a *reader* of an ONNX graph, as opposed to the census walk in npu/pack_import.cpp, which keeps only
+// the MatMul-family nodes because that is all a quantised LLM pack needs. An audio pack (the Whisper encoder) is a plain
+// f32 graph: 32 transformer blocks of Conv/MatMul/Add/LayerNormalization/Gelu/Softmax/Transpose/Mul, and to execute it the
+// engine needs every node in order, with its inputs, outputs and integer attributes, and the set of initializer names.
+//
+// Nothing here executes anything. It exists so the executor's ordering assumption -- "a node's inputs are graph inputs,
+// initializers, or outputs of an earlier node" -- is a *checked* fact about a real pack rather than a hope, and so a pack
+// that breaks it is refused by name.
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <string>
+#include <vector>
+
+namespace onebit::npu::pack_import {
+
+struct OnnxNodeInfo {
+    std::string op;
+    std::string name;
+    std::vector<std::string> ins;
+    std::vector<std::string> outs;
+    std::map<std::string, int64_t> attrs;   // integer attributes only (axis, perm as a list is not an int attr)
+    std::map<std::string, std::vector<int64_t>> attrs_list;   // `perm`, `pads`, `strides`, `dilations`
+    std::map<std::string, float> attrs_float;                 // `alpha`, `epsilon` (as stored)
+};
+
+// One initializer: its shape, dtype and where its bytes are (inline `raw_data`, or a side file via external_data).
+struct OnnxInit {
+    std::string name;
+    int dtype = 0;                     // TensorProto.DataType: 1 = FLOAT, 7 = INT64, 10 = FLOAT16, ...
+    std::vector<int64_t> dims;
+    const uint8_t* raw = nullptr;      // valid only while the mapped file is open; the caller copies what it needs
+    size_t raw_len = 0;
+    std::string location;              // external_data side file, relative to the graph's directory
+    uint64_t offset = 0;
+    uint64_t length = 0;
+};
+
+struct OnnxGraphInfo {
+    std::string dir;                   // directory of the .onnx file, for resolving external_data
+    std::vector<OnnxInit> init_tensors;
+    std::vector<std::string> inputs;    // graph inputs, in declaration order
+    std::vector<std::string> outputs;   // graph outputs
+    std::vector<OnnxNodeInfo> nodes;    // graph order, which ONNX guarantees is topological
+    std::vector<std::string> initializers;   // weight/bias names
+    std::vector<std::string> external;       // initializers whose data is in a side file (location set)
+};
+
+// Throws std::runtime_error with the path and the protobuf complaint on a malformed file.
+OnnxGraphInfo parse_onnx_graph(const std::string& path);
+
+}  // namespace onebit::npu::pack_import

npu/pack_import.cpp

@@ -15,6 +15,8 @@
 
 #include "pack_import.h"
 
+#include "onnx_graph.h"
+
 #include "file_map.h"
 #include "pack.h"
 #include "q4nx.h"
@@ -85,6 +87,9 @@ std::vector<Field> pb(const uint8_t* p, size_t n) {
             }
             case 5:
                 if (size_t(end - p) < 4) throw std::runtime_error("protobuf: truncated fixed32");
+                // engine#1783: keep the four bytes in v. A float attribute (LayerNormalization's epsilon) is wire 5, and
+                // without this every such attribute reads as 0.0 -- indistinguishable from an absent one.
+                f.v = uint32_t(p[0]) | (uint32_t(p[1]) << 8) | (uint32_t(p[2]) << 16) | (uint32_t(p[3]) << 24);
                 p += 4;
                 break;
             default: throw std::runtime_error("protobuf: unsupported wire type " + std::to_string(f.wire));
@@ -6310,4 +6315,86 @@ void sweep_fconst_scratch_dirs(const std::vector<std::string>& dirs, const std::
 // box measures are rel_err 0.000000 (serves) and 0.095/0.136 (imports cleanly, answers garbage).
 bool selfcheck_head_ok(double corr, double rel) { return corr > kTieCorr && rel <= kSelfcheckRelMax; }
 
+// engine#1783: a full graph reader, for packs that are plain f32 graphs (the Whisper encoder) rather than quantised LLM
+// packs. The census walk above keeps only the MatMul family by design; this one keeps every node in graph order, which ONNX
+// guarantees is topological, plus the initializer names -- so an executor can tell "a weight" from "a graph input", and a
+// pack that breaks the ordering is refused by name instead of producing plausible activations.
+OnnxGraphInfo parse_onnx_graph(const std::string& path) {
+    size_t size = 0;
+    std::string why;
+    const uint8_t* base = map_file(path, size, &why);
+    if (base == nullptr) throw std::runtime_error(path + ": " + why);
+    const auto model = pb(base, size);              // ModelProto
+    const auto* gfield = field1(model, 7);          // ModelProto.graph
+    if (gfield == nullptr) throw std::runtime_error(path + ": no ModelProto.graph (field 7)");
+    const auto g = pb(gfield->p, gfield->n);
+
+    OnnxGraphInfo out;
+    const size_t slash = path.find_last_of('/');
+    out.dir = (slash == std::string::npos) ? std::string(".") : path.substr(0, slash);
+    for (const auto* f : fields(g, 11)) {           // graph.input
+        const auto vi = pb(f->p, f->n);
+        if (const auto* nm = field1(vi, 1)) out.inputs.push_back(pb_str(*nm));
+    }
+    for (const auto* f : fields(g, 12)) {           // graph.output
+        const auto vi = pb(f->p, f->n);
+        if (const auto* nm = field1(vi, 1)) out.outputs.push_back(pb_str(*nm));
+    }
+    for (const auto* f : fields(g, 5)) {            // graph.initializer
+        const auto t = pb(f->p, f->n);
+        const auto* nm = field1(t, 8);
+        if (nm == nullptr) continue;
+        out.initializers.push_back(pb_str(*nm));
+        OnnxInit ti;
+        ti.name = pb_str(*nm);
+        if (const auto* dt = field1(t, 2)) ti.dtype = int(dt->v);
+        for (const auto* d : fields(t, 1)) ti.dims.push_back(int64_t(d->v));
+        if (const auto* rd = field1(t, 9)) {
+            ti.raw = rd->p;
+            ti.raw_len = rd->n;
+        }
+        for (const auto* d : fields(t, 13)) {       // TensorProto.external_data: key/value string pairs
+            const auto kv = pb(d->p, d->n);
+            const auto* k = field1(kv, 1);
+            const auto* v = field1(kv, 2);
+            if (k == nullptr || v == nullptr) continue;
+            const std::string key = pb_str(*k), val = pb_str(*v);
+            if (key == "location") ti.location = val;
+            else if (key == "offset") ti.offset = std::stoull(val);
+            else if (key == "length") ti.length = std::stoull(val);
+        }
+        if (!ti.location.empty()) out.external.push_back(ti.name);
+        out.init_tensors.push_back(std::move(ti));
+    }
+    for (const auto* f : fields(g, 1)) {            // graph.node
+        const auto node = pb(f->p, f->n);
+        OnnxNodeInfo ni;
+        if (const auto* o = field1(node, 4)) ni.op = pb_str(*o);
+        if (const auto* nm = field1(node, 3)) ni.name = pb_str(*nm);
+        for (const auto* i : fields(node, 1)) ni.ins.push_back(pb_str(*i));
+        for (const auto* o : fields(node, 2)) ni.outs.push_back(pb_str(*o));
+        for (const auto* a : fields(node, 5)) {     // AttributeProto
+            const auto at = pb(a->p, a->n);
+            const auto* nm = field1(at, 1);
+            if (nm == nullptr) continue;
+            const std::string key = pb_str(*nm);
+            if (const auto* iv = field1(at, 3)) ni.attrs[key] = int64_t(iv->v);   // INT
+            if (const auto* fv = field1(at, 2)) {                                 // FLOAT (wire 5, see fields())
+                const uint32_t bits = uint32_t(fv->v);
+                float f32 = 0.0f;
+                std::memcpy(&f32, &bits, sizeof f32);
+                ni.attrs_float[key] = f32;
+            }
+            std::vector<int64_t> list;
+            for (const auto* iv : fields(at, 8)) list.push_back(int64_t(iv->v));  // INTS
+            if (!list.empty()) ni.attrs_list[key] = std::move(list);
+        }
+        out.nodes.push_back(std::move(ni));
+    }
+    // NOT unmapped: init_tensors[].raw points into this mapping, and the executor copies what it needs while it is alive.
+    // shortcut: the mapping is released when the process exits; a long-lived caller that parses many graphs should own the
+    // mapping itself, which is the refactor to make when a second consumer appears.
+    return out;
+}
+
 }  // namespace onebit::npu::pack_import