Skip to content

npu: the device-form conversion sweeps the Q4NX cache it writes (refs #2263) #2271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bong-water-water-bong merged 1 commit into from
Nov 12, 2026
Merged

npu: the device-form conversion sweeps the Q4NX cache it writes (refs #2263) #2271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ file(WRITE ${CMAKE_BINARY_DIR}/generated/laya_first_default.h
"// generated from config/laya-first.json by CMakeLists.txt\n#pragma once\n"
"static const char kDefaultLayaFirstJson[] = R\"ONEBITLAYAFIRST(${ONEBIT_LAYA_FIRST_JSON})ONEBITLAYAFIRST\";\n")
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/config/laya-first.json)
# engine#2263: the Q4NX cache's sweep, below both of its callers: `onebit` (serve start, `1bit q4nx-cache`) and onebit_npu
# (npu/q4nx_device.cpp, the conversion that writes entries). Its own library so each binary links exactly one copy.
add_library(onebit_q4nx_cache STATIC npu/q4nx_cache_sweep.cpp)
target_include_directories(onebit_q4nx_cache PUBLIC ${CMAKE_SOURCE_DIR}/npu)
add_executable(onebit app/main.cpp app/serve.cpp app/recipes.cpp app/pm.cpp app/route_serve.cpp app/route_eval.cpp app/laya_checkpoint.cpp app/laya_first.cpp
app/embed_serve.cpp app/embed_model.cpp app/q4nx_cache.cpp)
# config/recipes.json (tuned llama-server settings per model and route, docs/recipes.md) is
Expand Down Expand Up @@ -149,7 +153,7 @@ if(CMAKE_GENERATOR MATCHES "Makefiles")
COMMAND ${CMAKE_COMMAND} -E false
VERBATIM)
endif()
target_link_libraries(onebit PRIVATE nlohmann_json::nlohmann_json httplib::httplib Threads::Threads)
target_link_libraries(onebit PRIVATE nlohmann_json::nlohmann_json httplib::httplib Threads::Threads onebit_q4nx_cache)

# ── The chat-template renderer (engine #963, docs/npu-chat-template.md) ──────
# The NPU chat route renders the container's own `chat_template` (app/npu_template.h) with the pinned
Expand Down Expand Up @@ -847,7 +851,7 @@ add_library(onebit_npu STATIC npu/file_map.cpp npu/flm_head_import.cpp npu/full_
npu/audio_exec.cpp npu/moe.cpp npu/pack.cpp npu/pack_import.cpp npu/private_route.cpp
npu/q4nx.cpp npu/q4nx_device.cpp npu/tokenizer.cpp)
target_include_directories(onebit_npu PUBLIC npu)
target_link_libraries(onebit_npu PUBLIC nlohmann_json::nlohmann_json PRIVATE PkgConfig::PCRE2)
target_link_libraries(onebit_npu PUBLIC nlohmann_json::nlohmann_json onebit_q4nx_cache PRIVATE PkgConfig::PCRE2)

# ── npu/forward's forward C++ compiles without the NPU runtime (engine #898) ───────────────
# CI configures without ONEBIT_NPU, so npu/forward/src/q4nx_forward.cpp - where every A1 defect so
Expand Down Expand Up @@ -1264,6 +1268,7 @@ add_executable(embed_model_test tests/embed_model_test.cpp app/embed_model.cpp)
# age window, and the disabled case). Listed explicitly like the other app/ units so the test needs no `onebit`.
add_executable(q4nx_cache_test tests/q4nx_cache_test.cpp app/q4nx_cache.cpp)
target_include_directories(q4nx_cache_test PRIVATE app)
target_link_libraries(q4nx_cache_test PRIVATE onebit_q4nx_cache)
add_test(NAME q4nx_cache COMMAND q4nx_cache_test)
# engine#1956 review: an `add_test` whose target ci.yml never builds never runs in CI -- the documented route, no workflow edit.
add_dependencies(onebit q4nx_cache_test)
Expand Down
95 changes: 0 additions & 95 deletions app/q4nx_cache.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,101 +25,6 @@

namespace fs = std::filesystem;

int64_t q4nx_dir_bytes(const std::string& path) {
std::error_code ec;
if (!fs::is_directory(path, ec)) return 0;
int64_t total = 0;
for (auto it = fs::recursive_directory_iterator(path, fs::directory_options::skip_permission_denied, ec);
!ec && it != fs::recursive_directory_iterator(); it.increment(ec)) {
std::error_code fec;
if (it->is_regular_file(fec)) total += int64_t(fs::file_size(it->path(), fec));
if (fec) continue;
}
return total;
}

std::vector<Q4nxCacheCandidate> q4nx_cache_candidates(const std::string& root, const std::string& keep,
int64_t max_bytes, int64_t min_age_seconds) {
std::vector<Q4nxCacheCandidate> out;
std::error_code ec;
if (max_bytes <= 0 || !fs::is_directory(root, ec)) return out;
struct Ent { std::string path; int64_t bytes; fs::file_time_type mtime; };
std::vector<Ent> all;
for (auto it = fs::directory_iterator(root, fs::directory_options::skip_permission_denied, ec);
!ec && it != fs::directory_iterator(); it.increment(ec)) {
Ent e;
e.path = it->path().string();
std::error_code tec;
if (!it->is_directory(tec)) continue;
e.bytes = q4nx_dir_bytes(e.path);
if (!keep.empty() && (e.path == keep || fs::path(e.path).filename() == fs::path(keep).filename())) continue;
e.mtime = fs::last_write_time(it->path(), tec);
if (tec) continue;
all.push_back(std::move(e));
}
if (all.empty()) return out;
const auto now = fs::file_time_type::clock::now();
const auto min_age = std::chrono::seconds(min_age_seconds);
std::vector<Ent*> cands;
for (auto& e : all)
if (now - e.mtime > min_age) cands.push_back(&e);
std::sort(cands.begin(), cands.end(), [](const Ent* a, const Ent* b) { return a->mtime < b->mtime; });
int64_t after = 0;
for (const auto& e : all) after += e.bytes;
for (Ent* e : cands) {
if (after <= max_bytes) break;
out.push_back({fs::path(e->path).filename().string(), e->bytes});
after -= e->bytes;
}
return out;
}

Q4nxCacheSweep q4nx_cache_evict(const std::string& root, const std::string& keep, int64_t max_bytes,
int64_t min_age_seconds) {
Q4nxCacheSweep r;
r.bytes_before = q4nx_dir_bytes(root);
r.bytes_after = r.bytes_before;
// One selection, shared with `--dry-run`: the list is what this function removes, in this order.
const std::vector<Q4nxCacheCandidate> cands = q4nx_cache_candidates(root, keep, max_bytes, min_age_seconds);
for (const auto& cand : cands) {
std::error_code rec;
fs::remove_all(fs::path(root) / cand.name, rec);
if (rec) continue;
r.bytes_after -= cand.bytes;
++r.removed;
}
if (r.removed > 0) {
r.reported = true;
std::fprintf(stderr,
"note: q4nx cache: removed %d entr%s, %.1f GiB -> %.1f GiB (bound %.1f GiB, %s); entries younger than "
"%lld s and the one just written are kept (engine#1956)\n",
r.removed, r.removed == 1 ? "y" : "ies", double(r.bytes_before) / (1 << 30),
double(r.bytes_after) / (1 << 30), double(max_bytes) / (1 << 30), root.c_str(),
(long long)min_age_seconds);
}
return r;
}

int64_t q4nx_cache_bound_bytes(const char* text, int64_t default_gib) {
const int64_t max_gib = INT64_MAX >> 30; // the largest GiB count whose byte count still fits
long long gib = default_gib;
if (text && *text) {
char* end = nullptr;
errno = 0;
const long long v = std::strtoll(text, &end, 10);
if (end != text) gib = v; // a number (ERANGE saturates, which the clamp below handles)
}
if (gib <= 0) return 0;
if (gib > max_gib) gib = max_gib;
return int64_t(gib) << 30;
}

void q4nx_cache_touch(const std::string& entry) {
std::error_code ec;
if (fs::is_directory(entry, ec)) fs::last_write_time(entry, fs::file_time_type::clock::now(), ec);
}


int run_q4nx_cache(int argc, char** argv) {
const auto usage = [] {
std::fprintf(stderr,
Expand Down
56 changes: 1 addition & 55 deletions app/q4nx_cache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,62 +13,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#pragma once

#include <cstdint>
#include <string>
#include <vector>

// The Q4NX container cache (engine#1956): app/serve.cpp repacks a GGUF into `$ONEBIT_Q4NX_CACHE` (default
// ~/.cache/1bit/q4nx) and reuses the result, and until this existed nothing ever removed an old entry. Measured on strixhalo
// 2026-11-10: 104 G in 29 entries, out of a 118 G cache, on a box whose root filesystem was down to 119 G -- which is why two
// lanes' build gates closed the same day. This is the same defect as #1770 -> #1797 one layer down (the .fconst scratch that
// only ever accumulated), with one difference that matters: this cache is SERVED FROM, so an entry may be in use.
struct Q4nxCacheSweep {
int removed = 0;
int64_t bytes_before = 0, bytes_after = 0;
bool reported = false; // true when a line was printed (i.e. something was removed)
};

// Remove least-recently-used entries under `root` until the total is at most `max_bytes`.
//
// Not evicted, ever: `keep` (the entry just written), and any entry whose mtime is within `min_age_seconds` -- a serve may be
// reading it right now, and this sweep cannot see other processes' open files. That is a heuristic, not a guarantee: a lane
// serving an entry older than the age window could still have it removed underneath it, which costs that lane a re-import and
// not a crash (the entry is regenerable from its GGUF). `max_bytes <= 0` disables the sweep entirely, which is what a lane that
// wants the old unbounded behaviour sets.
//
// Prints ONE line when it removes anything (never one per sweep, never one per entry): a lane that loses a 3.5 G container to
// the bound should see why in its log instead of re-importing into the same squeeze.
Q4nxCacheSweep q4nx_cache_evict(const std::string& root, const std::string& keep, int64_t max_bytes,
int64_t min_age_seconds = 600);

// engine#1788 follow-up, from USING the command: `--dry-run` reported totals only, so an operator could see "3 entries, 12 GiB"
// but not WHICH entries -- which is the question before deleting another lane's imports. This is the same selection
// q4nx_cache_evict() makes (LRU first, `keep` and the age window respected, stop at max_bytes), returned instead of applied,
// so the two cannot drift: evict() is now this list plus a remove loop.
struct Q4nxCacheCandidate {
std::string name; // the entry directory's name, as it appears under the root
int64_t bytes = 0;
};
std::vector<Q4nxCacheCandidate> q4nx_cache_candidates(const std::string& root, const std::string& keep, int64_t max_bytes,
int64_t min_age_seconds = 600);

// Directory bytes under `path` (0 when it does not exist).
int64_t q4nx_dir_bytes(const std::string& path);

// The byte bound for ONEBIT_Q4NX_CACHE_MAX_GIB's value (`text` may be null). Unset, empty or not a number gives the default
// (`default_gib`); zero or negative gives 0, which q4nx_cache_evict() reads as "no sweep". The result never exceeds
// INT64_MAX: the plain `gib << 30` this replaces overflowed from 2^33 GiB up, so a very large number written to mean "no
// limit" could come out as a 1 GiB bound, and a typo ("sixty") silently turned the sweep off.
int64_t q4nx_cache_bound_bytes(const char* text, int64_t default_gib = 60);

// Mark `entry` (a directory in the cache) as used now. The sweep orders by the entry directory's mtime, which the filesystem
// only moves when a file is created or removed inside it -- so without this the order is by CREATION, and a container
// imported last week and served every day is evicted before yesterday's one-off. Call it on every cache hit. It also puts a
// just-opened entry inside the sweep's age window, which is what makes "a serve may be reading it" true for a reused entry
// and not only for a new one. Failure is ignored (a read-only cache still serves).
void q4nx_cache_touch(const std::string& entry);
#include "q4nx_cache_sweep.h" // the sweep itself (npu/, engine#2263)

// engine#1788 / disk pressure 11-12: the sweep as a COMMAND, because until now the only trigger was a serve starting
// (`app/serve.cpp` calls q4nx_cache_evict() on its way up). With the NPU off nothing serves, so nothing sweeps and the
Expand Down
7 changes: 6 additions & 1 deletion docs/npu.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,12 @@ the bf16 rounding reappears for the device, never in the container.

**The cache is bounded, and the bound is least-recently-used (engine#1956).** After writing an
entry, `1bit serve` removes the least-recently-used entries until the cache is within
`ONEBIT_Q4NX_CACHE_MAX_GIB` (default 60; `0` or a negative number turns the sweep off). The entry
`ONEBIT_Q4NX_CACHE_MAX_GIB` (default 60; `0` or a negative number turns the sweep off). The
forward's device-form conversion (`npu/q4nx_device.cpp`) sweeps the same way after it writes a new
entry (engine#2263), so a box that converts through the forward and never starts a serve stays
within the bound too. The sweep itself is `npu/q4nx_cache_sweep.h`, shared by both writers and by
`1bit q4nx-cache evict`. A box that writes nothing never sweeps; `1bit q4nx-cache evict` is the
manual trigger for that case. The entry
just written and any entry used in the last ten minutes are never removed. "Used" means reused,
not only written: a cache hit — a repacked GGUF found in the cache, or a device-form conversion
reused at load — marks the entry's directory as used now, so a container imported once and served
Expand Down
119 changes: 119 additions & 0 deletions npu/q4nx_cache_sweep.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "q4nx_cache_sweep.h"

#include <algorithm>
#include <cerrno>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <vector>

namespace fs = std::filesystem;

int64_t q4nx_dir_bytes(const std::string& path) {
std::error_code ec;
if (!fs::is_directory(path, ec)) return 0;
int64_t total = 0;
for (auto it = fs::recursive_directory_iterator(path, fs::directory_options::skip_permission_denied, ec);
!ec && it != fs::recursive_directory_iterator(); it.increment(ec)) {
std::error_code fec;
if (it->is_regular_file(fec)) total += int64_t(fs::file_size(it->path(), fec));
if (fec) continue;
}
return total;
}

std::vector<Q4nxCacheCandidate> q4nx_cache_candidates(const std::string& root, const std::string& keep,
int64_t max_bytes, int64_t min_age_seconds) {
std::vector<Q4nxCacheCandidate> out;
std::error_code ec;
if (max_bytes <= 0 || !fs::is_directory(root, ec)) return out;
struct Ent { std::string path; int64_t bytes; fs::file_time_type mtime; };
std::vector<Ent> all;
for (auto it = fs::directory_iterator(root, fs::directory_options::skip_permission_denied, ec);
!ec && it != fs::directory_iterator(); it.increment(ec)) {
Ent e;
e.path = it->path().string();
std::error_code tec;
if (!it->is_directory(tec)) continue;
e.bytes = q4nx_dir_bytes(e.path);
if (!keep.empty() && (e.path == keep || fs::path(e.path).filename() == fs::path(keep).filename())) continue;
e.mtime = fs::last_write_time(it->path(), tec);
if (tec) continue;
all.push_back(std::move(e));
}
if (all.empty()) return out;
const auto now = fs::file_time_type::clock::now();
const auto min_age = std::chrono::seconds(min_age_seconds);
std::vector<Ent*> cands;
for (auto& e : all)
if (now - e.mtime > min_age) cands.push_back(&e);
std::sort(cands.begin(), cands.end(), [](const Ent* a, const Ent* b) { return a->mtime < b->mtime; });
int64_t after = 0;
for (const auto& e : all) after += e.bytes;
for (Ent* e : cands) {
if (after <= max_bytes) break;
out.push_back({fs::path(e->path).filename().string(), e->bytes});
after -= e->bytes;
}
return out;
}

Q4nxCacheSweep q4nx_cache_evict(const std::string& root, const std::string& keep, int64_t max_bytes,
int64_t min_age_seconds) {
Q4nxCacheSweep r;
r.bytes_before = q4nx_dir_bytes(root);
r.bytes_after = r.bytes_before;
// One selection, shared with `--dry-run`: the list is what this function removes, in this order.
const std::vector<Q4nxCacheCandidate> cands = q4nx_cache_candidates(root, keep, max_bytes, min_age_seconds);
for (const auto& cand : cands) {
std::error_code rec;
fs::remove_all(fs::path(root) / cand.name, rec);
if (rec) continue;
r.bytes_after -= cand.bytes;
++r.removed;
}
if (r.removed > 0) {
r.reported = true;
std::fprintf(stderr,
"note: q4nx cache: removed %d entr%s, %.1f GiB -> %.1f GiB (bound %.1f GiB, %s); entries younger than "
"%lld s and the one just written are kept (engine#1956)\n",
r.removed, r.removed == 1 ? "y" : "ies", double(r.bytes_before) / (1 << 30),
double(r.bytes_after) / (1 << 30), double(max_bytes) / (1 << 30), root.c_str(),
(long long)min_age_seconds);
}
return r;
}

int64_t q4nx_cache_bound_bytes(const char* text, int64_t default_gib) {
const int64_t max_gib = INT64_MAX >> 30; // the largest GiB count whose byte count still fits
long long gib = default_gib;
if (text && *text) {
char* end = nullptr;
errno = 0;
const long long v = std::strtoll(text, &end, 10);
if (end != text) gib = v; // a number (ERANGE saturates, which the clamp below handles)
}
if (gib <= 0) return 0;
if (gib > max_gib) gib = max_gib;
return int64_t(gib) << 30;
}

void q4nx_cache_touch(const std::string& entry) {
std::error_code ec;
if (fs::is_directory(entry, ec)) fs::last_write_time(entry, fs::file_time_type::clock::now(), ec);
}
Loading
Loading