diff --git a/CMakeLists.txt b/CMakeLists.txt index 79a6b1c44..352094e3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1254,6 +1254,14 @@ endif() add_executable(npu_model_bounds_test tests/npu_model_bounds_test.cpp) target_link_libraries(npu_model_bounds_test PRIVATE onebit_npu) add_test(NAME npu_model_bounds COMMAND npu_model_bounds_test) + +# engine#1956: the Q4NX cache cap's planner -- policy for a cache that had grown to 104 G with no eviction. +# Header-only, no cache and no converter: the arm tests the policy, and `add_dependencies` puts the target in +# the build CI actually runs (an `add_test` alone leaves a target unbuilt on this runner, engine#1858/#1834). +add_executable(q4nx_cache_evict_test tests/q4nx_cache_evict_test.cpp) +target_include_directories(q4nx_cache_evict_test PRIVATE ${CMAKE_SOURCE_DIR}/app) +add_test(NAME q4nx_cache_evict COMMAND q4nx_cache_evict_test) +add_dependencies(onebit q4nx_cache_evict_test) # CI builds an explicit target list (.github/workflows/ci.yml, "Keep this list in step with # add_test()"), and extending that list needs a token with the `workflow` scope, which a lane does # not carry. Binding npu_moe_test to a target the list already names gets it built either way (a diff --git a/app/q4nx_cache_evict.h b/app/q4nx_cache_evict.h new file mode 100644 index 000000000..e93b0acde --- /dev/null +++ b/app/q4nx_cache_evict.h @@ -0,0 +1,71 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// engine#1956: the Q4NX container cache is a bounded product store, not an unbounded log. +// +// `~/.cache/1bit/q4nx` holds one repacked model directory per GGUF stem (~3.5 G each) so a second serve +// skips a ~70 s import. Nothing ever removed an old one: the only `remove_all` calls on that path rebuild +// ONE entry (app/serve.cpp:3006/3037/3094/3101), so on 2026-11-10 it held **104 G across 29 entries** while +// the root filesystem had 119 G free -- ~34 more imports from a full disk. +// +// An age rule would be wrong here even though #1797's 6 h sweep is right for the `.fconst` scratch: those +// files are scratch deleted on success, these are PRODUCTS whose whole value is being hit later. So the +// policy is a cap with **eviction on write**, oldest first: the lane that writes the entry that overflows +// pays for it, and never a lane that is merely serving. The planner is here, pure and header-only, so the +// policy can be tested without a cache, a converter or an NPU. +#pragma once + +#include +#include +#include +#include + +namespace onebit { + +struct Q4nxCacheEntry { + std::string name; // the directory name under the cache root (the GGUF stem) + uint64_t bytes = 0; // its size on disk, recursively + int64_t mtime = 0; // its newest write, as seconds; older entries go first +}; + +// Which entries to remove so the cache stays within `cap_bytes`, newest kept, `keep` never removed. +// +// `keep` is the entry the caller has JUST written. It is excluded from the candidates by design: removing +// it would re-cost the import that just paid for it, so an entry larger than the whole cap stays and the +// rest go (best effort) rather than the caller deleting its own work. +inline std::vector q4nx_cache_evict_plan(const std::vector& entries, + uint64_t cap_bytes, const std::string& keep) { + uint64_t total = 0; + std::vector candidates; + candidates.reserve(entries.size()); + for (const Q4nxCacheEntry& e : entries) { + total += e.bytes; + if (e.name != keep) candidates.push_back(&e); + } + // Oldest first, then by name so two entries written in the same second evict deterministically. + std::sort(candidates.begin(), candidates.end(), [](const Q4nxCacheEntry* a, const Q4nxCacheEntry* b) { + if (a->mtime != b->mtime) return a->mtime < b->mtime; + return a->name < b->name; + }); + std::vector out; + for (const Q4nxCacheEntry* e : candidates) { + if (total <= cap_bytes) break; + out.push_back(e->name); + total -= e->bytes; + } + return out; +} + +} // namespace onebit diff --git a/app/serve.cpp b/app/serve.cpp index 2b0f4a507..8ae6391d6 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -74,6 +74,7 @@ #include "pm.h" #include "laya_checkpoint.h" #include "recipes.h" +#include "q4nx_cache_evict.h" // engine#1956: the cache cap's pure planner #include "route_log.h" #include "serve_metrics.h" #include "serve_watchdog.h" @@ -2950,6 +2951,42 @@ std::string q4nx_cache_root() { throw std::runtime_error("no home directory for the Q4NX cache: set HOME or ONEBIT_Q4NX_CACHE"); } +// engine#1956: keep the Q4NX cache bounded, evicting on write. `keep` is the entry just written. +// +// The cap is on the whole directory, in GiB, because the unit that matters to the disk is bytes: ~3.5 G per +// pack, so the default of 40 GiB holds about what a box serves in a day while bounding the directory at a +// third of the 104 G it reached on 2026-11-10. ONEBIT_Q4NX_CACHE_MAX_GIB overrides it; 0 disables eviction +// (for a lane that has measured its own cache and wants no surprises). +void evict_q4nx_cache(const std::string& cache_dir, const std::string& keep) { + const char* env = std::getenv("ONEBIT_Q4NX_CACHE_MAX_GIB"); + const uint64_t cap_gib = env && *env ? std::strtoull(env, nullptr, 10) : 40ull; + if (cap_gib == 0) return; + std::error_code ec; + std::vector entries; + for (const auto& e : fs::directory_iterator(cache_dir, ec)) { + if (ec || !e.is_directory()) continue; + uint64_t bytes = 0; + int64_t newest = 0; + std::error_code wec; + for (const auto& f : fs::recursive_directory_iterator(e.path(), wec)) { + std::error_code sec; + if (f.is_regular_file()) bytes += f.file_size(sec); + const auto t = fs::last_write_time(f.path(), sec); + const int64_t secs = int64_t(t.time_since_epoch().count()); + if (secs > newest) newest = secs; + } + entries.push_back({e.path().filename().string(), bytes, newest}); + } + const std::vector victims = onebit::q4nx_cache_evict_plan(entries, cap_gib << 30, keep); + for (const std::string& v : victims) { + std::error_code rec; + fs::remove_all(fs::path(cache_dir) / v, rec); + if (!rec) + std::fprintf(stderr, "1bit serve: evicted cached Q4NX dir %s (engine#1956, cap %llu GiB)\n", v.c_str(), + (unsigned long long)cap_gib); + } +} + // Repack a GGUF into a Q4NX model directory with the FLM_Q4NX_Converter // (Python), cached by GGUF stem so a repeated serve reuses the conversion. // Override the tool paths with ONEBIT_Q4NX_PYTHON / ONEBIT_Q4NX_CONVERTER and @@ -3048,6 +3085,7 @@ std::string repack_gguf(const std::string& gguf) { const int rc = run_program(repack, false); if (rc != 0 || !fs::exists(out + "/model.q4nx")) throw std::runtime_error("GGUF repack failed (rc=" + std::to_string(rc) + "): " + command_text(repack)); + evict_q4nx_cache(cache, stem); return out; } @@ -3117,6 +3155,7 @@ std::string repack_gguf_lane(const std::string& gguf, const std::string& lane_di fs::copy_file(fs::path(tok) / "tokenizer_config.json", fs::path(out) / "tokenizer_config.json", fs::copy_options::overwrite_existing); } + evict_q4nx_cache(cache, stem); // engine#1956: the fused-lane writer bounds the cache too return out; } diff --git a/tests/q4nx_cache_evict_test.cpp b/tests/q4nx_cache_evict_test.cpp new file mode 100644 index 000000000..d804a5a34 --- /dev/null +++ b/tests/q4nx_cache_evict_test.cpp @@ -0,0 +1,79 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The Q4NX cache cap's planner (engine#1956), device-free and cache-free: the policy is a pure function of +// (entry name, size, mtime), a cap, and the entry that was just written, so it is tested here rather than by +// filling a disk. The four cases below are the ones a real cache produces: under the cap, over it, an entry +// larger than the whole cap, and two entries written in the same second. +#include "q4nx_cache_evict.h" + +#include +#include +#include +#include + +using onebit::Q4nxCacheEntry; +using onebit::q4nx_cache_evict_plan; + +static int failures = 0; + +static void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) ++failures; +} + +int main() { + const uint64_t G = 1024ull * 1024 * 1024; + + { // under the cap: nothing is evicted, so a box serving one model never pays + const std::vector e = {{"a", 3 * G, 100}, {"b", 3 * G, 200}, {"c", 3 * G, 300}}; + const auto v = q4nx_cache_evict_plan(e, 40 * G, "c"); + check(v.empty(), "under the cap: nothing is evicted"); + } + { // over the cap: oldest first, and only until it fits + const std::vector e = { + {"old", 3 * G, 100}, {"mid", 3 * G, 200}, {"keep", 3 * G, 300}}; + const auto v = q4nx_cache_evict_plan(e, 4 * G, "keep"); + check(v.size() == 2 && v[0] == "old" && v[1] == "mid", + "over the cap: oldest first, only until it fits (" + (v.empty() ? std::string("none") : v[0]) + "…)"); + check(std::find(v.begin(), v.end(), "keep") == v.end(), + "over the cap: the entry just written is never a victim"); + } + { // one entry larger than the whole cap: it stays, the rest go -- never the caller's own work + const std::vector e = {{"huge", 50 * G, 300}, {"old", 3 * G, 100}}; + const auto v = q4nx_cache_evict_plan(e, 40 * G, "huge"); + check(v.size() == 1 && v[0] == "old", + "an entry larger than the cap: kept, and everything else goes (the importer does not delete what it " + "just imported)"); + } + { // same second: the tie-break is by name, so the choice is reproducible rather than filesystem order + const std::vector e = {{"b", 3 * G, 100}, {"a", 3 * G, 100}, {"keep", 3 * G, 100}}; + const auto v = q4nx_cache_evict_plan(e, 4 * G, "keep"); + check(v.size() == 2 && v[0] == "a" && v[1] == "b", + "same mtime: evicted by name, deterministically"); + } + { // the cap counted over ALL entries including the kept one -- a keep-only cache is not "free" + const std::vector e = {{"keep", 39 * G, 300}, {"x", 2 * G, 100}}; + const auto v = q4nx_cache_evict_plan(e, 40 * G, "keep"); + check(v.size() == 1 && v[0] == "x", "the kept entry counts toward the cap, so the overflow is still paid"); + } + + if (failures) { + std::fprintf(stderr, "%d check(s) failed\n", failures); + return 1; + } + std::printf("all q4nx cache eviction checks passed\n"); + return 0; +}