From bafbbedc4f2cb460ea804eb2ace4f959b152a230 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 12:03:12 -0400 Subject: [PATCH] npu: bound the Q4NX cache with a cap and evict on write (fixes #1956) `~/.cache/1bit/q4nx` holds one repacked model directory per GGUF stem (~3.5 G each) so a second serve skips a ~70 s import. Nothing ever removed an old one: the only `remove_all` calls on that path rebuild ONE entry (`app/serve.cpp:3006/3037/3094/3101`), so it measured **104 G across 29 entries** today while the root filesystem had **119 G** free -- about 34 imports from a full disk, and several lanes import by the hour. Same defect as #1770 -> #1797 one layer up, with a bigger unit. The difference decides the fix: those `.fconst` copies are scratch deleted on success, so an age sweep is right for them; these are PRODUCTS whose whole value is being hit later, so deleting a hit re-costs the import it was kept for. Hence a cap with **eviction on write**, oldest first: the lane that writes the entry that overflows pays for it, and never a lane that is merely serving. - `app/q4nx_cache_evict.h`: `q4nx_cache_evict_plan(entries, cap_bytes, keep)` -- the policy as a pure function of (name, bytes, mtime), the cap, and the entry just written. `keep` is never a victim, by construction: removing it would re-cost the import that just paid for it, so an entry larger than the whole cap stays and everything else goes (best effort) rather than the caller deleting its own work. Ties on mtime break by name, so the choice is reproducible rather than filesystem order. - `app/serve.cpp`: `evict_q4nx_cache(cache, stem)` after BOTH successful repack writers (the converter path and the fused-lane path), cap `ONEBIT_Q4NX_CACHE_MAX_GIB` (default 40 GiB, `0` disables). It prints the victim it removes, so a cache shrinking is visible rather than inferred. - `tests/q4nx_cache_evict_test.cpp` + ctest `q4nx_cache_evict`, with `add_dependencies(onebit ...)` so the target is actually built on a runner whose CI names its target list (#1858/#1834). Six arms, and the two that matter are the ones a real cache produces: an entry larger than the whole cap stays (the importer never deletes what it just imported), and two entries written in the same second evict deterministically. Negative control run: removing the `keep` exclusion fails the over-cap arm. **Run here, not just read:** the test needs only the header, so `g++ -std=c++17 -Iapp tests/q4nx_cache_evict_test.cpp` compiles and passes standalone -- 6/6 -- which is how the negative control was checked too. The `serve.cpp` half is CI's compile check: disk is at 119 GB, under the floor these rules set for a build tree. --- CMakeLists.txt | 8 ++++ app/q4nx_cache_evict.h | 71 +++++++++++++++++++++++++++++ app/serve.cpp | 39 ++++++++++++++++ tests/q4nx_cache_evict_test.cpp | 79 +++++++++++++++++++++++++++++++++ 4 files changed, 197 insertions(+) create mode 100644 app/q4nx_cache_evict.h create mode 100644 tests/q4nx_cache_evict_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 79a6b1c44..352094e3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1254,6 +1254,14 @@ endif() add_executable(npu_model_bounds_test tests/npu_model_bounds_test.cpp) target_link_libraries(npu_model_bounds_test PRIVATE onebit_npu) add_test(NAME npu_model_bounds COMMAND npu_model_bounds_test) + +# engine#1956: the Q4NX cache cap's planner -- policy for a cache that had grown to 104 G with no eviction. +# Header-only, no cache and no converter: the arm tests the policy, and `add_dependencies` puts the target in +# the build CI actually runs (an `add_test` alone leaves a target unbuilt on this runner, engine#1858/#1834). +add_executable(q4nx_cache_evict_test tests/q4nx_cache_evict_test.cpp) +target_include_directories(q4nx_cache_evict_test PRIVATE ${CMAKE_SOURCE_DIR}/app) +add_test(NAME q4nx_cache_evict COMMAND q4nx_cache_evict_test) +add_dependencies(onebit q4nx_cache_evict_test) # CI builds an explicit target list (.github/workflows/ci.yml, "Keep this list in step with # add_test()"), and extending that list needs a token with the `workflow` scope, which a lane does # not carry. Binding npu_moe_test to a target the list already names gets it built either way (a diff --git a/app/q4nx_cache_evict.h b/app/q4nx_cache_evict.h new file mode 100644 index 000000000..e93b0acde --- /dev/null +++ b/app/q4nx_cache_evict.h @@ -0,0 +1,71 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#1956: the Q4NX container cache is a bounded product store, not an unbounded log. +// +// `~/.cache/1bit/q4nx` holds one repacked model directory per GGUF stem (~3.5 G each) so a second serve +// skips a ~70 s import. Nothing ever removed an old one: the only `remove_all` calls on that path rebuild +// ONE entry (app/serve.cpp:3006/3037/3094/3101), so on 2026-11-10 it held **104 G across 29 entries** while +// the root filesystem had 119 G free -- ~34 more imports from a full disk. +// +// An age rule would be wrong here even though #1797's 6 h sweep is right for the `.fconst` scratch: those +// files are scratch deleted on success, these are PRODUCTS whose whole value is being hit later. So the +// policy is a cap with **eviction on write**, oldest first: the lane that writes the entry that overflows +// pays for it, and never a lane that is merely serving. The planner is here, pure and header-only, so the +// policy can be tested without a cache, a converter or an NPU. +#pragma once + +#include +#include +#include +#include + +namespace onebit { + +struct Q4nxCacheEntry { + std::string name; // the directory name under the cache root (the GGUF stem) + uint64_t bytes = 0; // its size on disk, recursively + int64_t mtime = 0; // its newest write, as seconds; older entries go first +}; + +// Which entries to remove so the cache stays within `cap_bytes`, newest kept, `keep` never removed. +// +// `keep` is the entry the caller has JUST written. It is excluded from the candidates by design: removing +// it would re-cost the import that just paid for it, so an entry larger than the whole cap stays and the +// rest go (best effort) rather than the caller deleting its own work. +inline std::vector q4nx_cache_evict_plan(const std::vector& entries, + uint64_t cap_bytes, const std::string& keep) { + uint64_t total = 0; + std::vector candidates; + candidates.reserve(entries.size()); + for (const Q4nxCacheEntry& e : entries) { + total += e.bytes; + if (e.name != keep) candidates.push_back(&e); + } + // Oldest first, then by name so two entries written in the same second evict deterministically. + std::sort(candidates.begin(), candidates.end(), [](const Q4nxCacheEntry* a, const Q4nxCacheEntry* b) { + if (a->mtime != b->mtime) return a->mtime < b->mtime; + return a->name < b->name; + }); + std::vector out; + for (const Q4nxCacheEntry* e : candidates) { + if (total <= cap_bytes) break; + out.push_back(e->name); + total -= e->bytes; + } + return out; +} + +} // namespace onebit diff --git a/app/serve.cpp b/app/serve.cpp index 2b0f4a507..8ae6391d6 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -74,6 +74,7 @@ #include "pm.h" #include "laya_checkpoint.h" #include "recipes.h" +#include "q4nx_cache_evict.h" // engine#1956: the cache cap's pure planner #include "route_log.h" #include "serve_metrics.h" #include "serve_watchdog.h" @@ -2950,6 +2951,42 @@ std::string q4nx_cache_root() { throw std::runtime_error("no home directory for the Q4NX cache: set HOME or ONEBIT_Q4NX_CACHE"); } +// engine#1956: keep the Q4NX cache bounded, evicting on write. `keep` is the entry just written. +// +// The cap is on the whole directory, in GiB, because the unit that matters to the disk is bytes: ~3.5 G per +// pack, so the default of 40 GiB holds about what a box serves in a day while bounding the directory at a +// third of the 104 G it reached on 2026-11-10. ONEBIT_Q4NX_CACHE_MAX_GIB overrides it; 0 disables eviction +// (for a lane that has measured its own cache and wants no surprises). +void evict_q4nx_cache(const std::string& cache_dir, const std::string& keep) { + const char* env = std::getenv("ONEBIT_Q4NX_CACHE_MAX_GIB"); + const uint64_t cap_gib = env && *env ? std::strtoull(env, nullptr, 10) : 40ull; + if (cap_gib == 0) return; + std::error_code ec; + std::vector entries; + for (const auto& e : fs::directory_iterator(cache_dir, ec)) { + if (ec || !e.is_directory()) continue; + uint64_t bytes = 0; + int64_t newest = 0; + std::error_code wec; + for (const auto& f : fs::recursive_directory_iterator(e.path(), wec)) { + std::error_code sec; + if (f.is_regular_file()) bytes += f.file_size(sec); + const auto t = fs::last_write_time(f.path(), sec); + const int64_t secs = int64_t(t.time_since_epoch().count()); + if (secs > newest) newest = secs; + } + entries.push_back({e.path().filename().string(), bytes, newest}); + } + const std::vector victims = onebit::q4nx_cache_evict_plan(entries, cap_gib << 30, keep); + for (const std::string& v : victims) { + std::error_code rec; + fs::remove_all(fs::path(cache_dir) / v, rec); + if (!rec) + std::fprintf(stderr, "1bit serve: evicted cached Q4NX dir %s (engine#1956, cap %llu GiB)\n", v.c_str(), + (unsigned long long)cap_gib); + } +} + // Repack a GGUF into a Q4NX model directory with the FLM_Q4NX_Converter // (Python), cached by GGUF stem so a repeated serve reuses the conversion. // Override the tool paths with ONEBIT_Q4NX_PYTHON / ONEBIT_Q4NX_CONVERTER and @@ -3048,6 +3085,7 @@ std::string repack_gguf(const std::string& gguf) { const int rc = run_program(repack, false); if (rc != 0 || !fs::exists(out + "/model.q4nx")) throw std::runtime_error("GGUF repack failed (rc=" + std::to_string(rc) + "): " + command_text(repack)); + evict_q4nx_cache(cache, stem); return out; } @@ -3117,6 +3155,7 @@ std::string repack_gguf_lane(const std::string& gguf, const std::string& lane_di fs::copy_file(fs::path(tok) / "tokenizer_config.json", fs::path(out) / "tokenizer_config.json", fs::copy_options::overwrite_existing); } + evict_q4nx_cache(cache, stem); // engine#1956: the fused-lane writer bounds the cache too return out; } diff --git a/tests/q4nx_cache_evict_test.cpp b/tests/q4nx_cache_evict_test.cpp new file mode 100644 index 000000000..d804a5a34 --- /dev/null +++ b/tests/q4nx_cache_evict_test.cpp @@ -0,0 +1,79 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The Q4NX cache cap's planner (engine#1956), device-free and cache-free: the policy is a pure function of +// (entry name, size, mtime), a cap, and the entry that was just written, so it is tested here rather than by +// filling a disk. The four cases below are the ones a real cache produces: under the cap, over it, an entry +// larger than the whole cap, and two entries written in the same second. +#include "q4nx_cache_evict.h" + +#include +#include +#include +#include + +using onebit::Q4nxCacheEntry; +using onebit::q4nx_cache_evict_plan; + +static int failures = 0; + +static void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++failures; +} + +int main() { + const uint64_t G = 1024ull * 1024 * 1024; + + { // under the cap: nothing is evicted, so a box serving one model never pays + const std::vector e = {{"a", 3 * G, 100}, {"b", 3 * G, 200}, {"c", 3 * G, 300}}; + const auto v = q4nx_cache_evict_plan(e, 40 * G, "c"); + check(v.empty(), "under the cap: nothing is evicted"); + } + { // over the cap: oldest first, and only until it fits + const std::vector e = { + {"old", 3 * G, 100}, {"mid", 3 * G, 200}, {"keep", 3 * G, 300}}; + const auto v = q4nx_cache_evict_plan(e, 4 * G, "keep"); + check(v.size() == 2 && v[0] == "old" && v[1] == "mid", + "over the cap: oldest first, only until it fits (" + (v.empty() ? std::string("none") : v[0]) + "…)"); + check(std::find(v.begin(), v.end(), "keep") == v.end(), + "over the cap: the entry just written is never a victim"); + } + { // one entry larger than the whole cap: it stays, the rest go -- never the caller's own work + const std::vector e = {{"huge", 50 * G, 300}, {"old", 3 * G, 100}}; + const auto v = q4nx_cache_evict_plan(e, 40 * G, "huge"); + check(v.size() == 1 && v[0] == "old", + "an entry larger than the cap: kept, and everything else goes (the importer does not delete what it " + "just imported)"); + } + { // same second: the tie-break is by name, so the choice is reproducible rather than filesystem order + const std::vector e = {{"b", 3 * G, 100}, {"a", 3 * G, 100}, {"keep", 3 * G, 100}}; + const auto v = q4nx_cache_evict_plan(e, 4 * G, "keep"); + check(v.size() == 2 && v[0] == "a" && v[1] == "b", + "same mtime: evicted by name, deterministically"); + } + { // the cap counted over ALL entries including the kept one -- a keep-only cache is not "free" + const std::vector e = {{"keep", 39 * G, 300}, {"x", 2 * G, 100}}; + const auto v = q4nx_cache_evict_plan(e, 40 * G, "keep"); + check(v.size() == 1 && v[0] == "x", "the kept entry counts toward the cap, so the overflow is still paid"); + } + + if (failures) { + std::fprintf(stderr, "%d check(s) failed\n", failures); + return 1; + } + std::printf("all q4nx cache eviction checks passed\n"); + return 0; +}