From 855163de61e03b94789a6bface7952c0c29ba0a0 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:53:06 -0400 Subject: [PATCH] tests: the plans checked in under configs/plans/ are loaded by a gate (engine#799) `docs/npu-plan.md` says four plans are checked in "one per model directory on the build box", and that `tools/npu_plan.py --check` proves a directory's copy still matches. Both true - and the second needs the model directory, which CI has not. So the shipped plans had **no gate at all**: `git grep configs/plans` found them mentioned in two docs and read by nothing, and a hand edit or a generator change could leave one refused by `Plan::load()` with the first evidence being a lane loading it on the box. $ (driving the four through the same loader the forward uses) configs/plans/bonsai2-27b-NPU-b8.plan.json LOADS (layers=64) configs/plans/bonsai2-27b-NPU.plan.json LOADS (layers=64) configs/plans/bonsai2-27b-pb16i.plan.json LOADS (layers=64) configs/plans/Qwen3-0.6B.plan.json LOADS (layers=28) They are all valid today - which is the reason to pin them, not a reason to skip it. The test loads each shipped plan through `Plan::load()`, so the refusal vocabulary is the engine's own rather than a JSON schema written twice: a format bump is refused by name plan: configs/plans/Qwen3-0.6B.plan.json format is "1bit-npu-plan/2", this engine reads "1bit-npu-plan/1" and it also asserts `plan.format == kFormat`, so a future format bump has to touch this test instead of silently accepting yesterday's plans. It needs no model, no device and no pack. An empty or wrong directory FAILS rather than passing over zero plans - the vacuous green this test exists to prevent. Verified both arms: a bumped format fails with the sentence above (rc 1), an empty directory fails (rc 1), the real directory passes (rc 0), and through ctest in a real configure (`Test #239 npu_plan_shipped`, 100% passed). Registered through cmake/tests-*.cmake (#1979's glob), so no CMakeLists.txt hunk. What it does NOT prove: that a plan still matches its model directory (that is `--check`, and it needs the pack), or that a plan is right - only that it is one the engine can read. --- cmake/tests-plan-shipped.cmake | 27 +++++++++++ tests/npu_plan_shipped_test.cpp | 79 +++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 cmake/tests-plan-shipped.cmake create mode 100644 tests/npu_plan_shipped_test.cpp diff --git a/cmake/tests-plan-shipped.cmake b/cmake/tests-plan-shipped.cmake new file mode 100644 index 000000000..f16e41634 --- /dev/null +++ b/cmake/tests-plan-shipped.cmake @@ -0,0 +1,27 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# engine#799: the plans checked in under configs/plans/ were loaded by nothing. docs/npu-plan.md says +# `tools/npu_plan.py --check` proves a directory's copy still matches - true, and it needs the model +# directory, which CI has not. So the shipped plans had no gate: a hand edit or a generator change could leave one +# refused by Plan::load() and the first evidence would be a lane loading it on the box. +# +# This loads each one through the same loader the forward uses. No model, no device, no pack. +add_executable(npu_plan_shipped_test tests/npu_plan_shipped_test.cpp) +target_include_directories(npu_plan_shipped_test PRIVATE npu) +target_link_libraries(npu_plan_shipped_test PRIVATE nlohmann_json::nlohmann_json) +add_test(NAME npu_plan_shipped + COMMAND npu_plan_shipped_test ${CMAKE_SOURCE_DIR}/configs/plans) +add_dependencies(onebit npu_plan_shipped_test) diff --git a/tests/npu_plan_shipped_test.cpp b/tests/npu_plan_shipped_test.cpp new file mode 100644 index 000000000..4f963b66a --- /dev/null +++ b/tests/npu_plan_shipped_test.cpp @@ -0,0 +1,79 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#799: the plans checked in under configs/plans/ are reference artefacts, and nothing loaded them. +// +// `docs/npu-plan.md` says four plans are checked in "one per model directory on the build box", and that +// `tools/npu_plan.py --check` proves a directory's copy still matches - which is true, and which needs the +// model directory. CI has no packs, so the shipped plans had no gate at all: a hand edit, or a generator change, +// could leave one refused by `Plan::load()` and the first evidence would be a lane loading it on the box. +// +// This loads each shipped plan through the SAME loader the forward uses, so the refusal vocabulary is the real +// one (PlanError, "1bit-npu-plan/1"), not a JSON schema written twice. It needs no model, no device and no pack. +// +// What it does NOT prove: that a plan still matches its model directory (that is `--check`, and it needs the +// pack), or that a plan is *right* - only that it is one the engine can read. + +#include "plan/plan.h" + +#include +#include +#include +#include + +namespace { + +int bad = 0; + +void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++bad; +} + +} // namespace + +int main(int argc, char** argv) { + namespace fs = std::filesystem; + const fs::path dir = argc > 1 ? fs::path(argv[1]) : fs::path("configs/plans"); + std::vector plans; + std::error_code ec; + for (const auto& e : fs::directory_iterator(dir, ec)) { + if (ec) break; + if (e.is_regular_file() && e.path().extension() == ".json") plans.push_back(e.path()); + } + if (!plans.empty()) std::sort(plans.begin(), plans.end()); + + // An empty directory is not a pass: it means the path is wrong (a test run from a build dir), and a suite that + // passed over zero plans would be the vacuous green this whole test exists to prevent. + check(!ec && !plans.empty(), + "the shipped plans directory holds at least one .json (" + dir.string() + (ec ? ": " + ec.message() : "") + ")"); + if (plans.empty()) { std::printf("npu_plan_shipped: FAIL\n"); return 1; } + + for (const auto& p : plans) { + const std::string name = p.filename().string(); + try { + const auto plan = onebit::npu::plan::Plan::load(p.string()); + check(plan.layers > 0, name + ": loads through Plan::load() with " + std::to_string(plan.layers) + " layer(s)"); + // The loader sets `format` to kFormat when it accepts the file; asserting it here means a future + // format bump has to touch this test as well, rather than silently accepting the old plans. + check(plan.format == onebit::npu::plan::kFormat, name + ": carries the format this build reads"); + } catch (const std::exception& e) { + check(false, name + ": REFUSED by Plan::load(): " + e.what()); + } + } + + std::printf("npu_plan_shipped: %s\n", bad ? "FAIL" : "ok"); + return bad ? 1 : 0; +}