Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions ggml/src/ggml-hrx/ggml-hrx.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1655,12 +1655,13 @@ static std::unique_ptr<ggml_backend_hrx_reg_context> create_registry_context() {
// Nothing fits under the ceiling: refuse the device here, loudly, instead of planning a launch that can
// reach the fault line (#680). Report only, do not try to be clever about it.
const size_t reserve = ggml::hrx::onebit::pool_reserve_bytes();
GGML_LOG_ERROR("ggml_hrx: %s: refusing to use the device: GTT total %.1f GiB with %.1f GiB free leaves "
"nothing under the %.1f GiB ceiling (reserve GGML_HRX_GTT_RESERVE_MB, override "
"GGML_HRX_POOL_LIMIT_MB); free GTT or lower the reserve (engine #680)\n",
device_ctx->name.c_str(), device_total / (1024.0 * 1024.0 * 1024.0),
device_free / (1024.0 * 1024.0 * 1024.0),
(device_total > reserve ? device_total - reserve : 0) / (1024.0 * 1024.0 * 1024.0));
// engine#2069: worded from the terms, not from the ceiling -- `budget == 0` can come from the HOST term, and
// then "free GTT" names the resource that was not binding. The sentence lives in failure-status.cpp so a host
// test can assert it (this file compiles only with GGML_HRX=ON).
const std::string refusal = ggml::hrx::pool_refusal_text(
device_ctx->name.c_str(), device_total, device_free,
device_total > reserve ? device_total - reserve : 0, terms);
GGML_LOG_ERROR("%s", refusal.c_str());
hrx_device_release(hrx_device);
continue;
}
Expand Down
30 changes: 30 additions & 0 deletions ggml/src/ggml-hrx/runtime/failure-status.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
#include "runtime/failure-status.h"
#include "hrx-1bit-safety.h" // engine#2069: the terms the refusal is worded from

#include <cstdio>

#include <cctype>
#include <cstring>
Expand Down Expand Up @@ -62,4 +65,31 @@ enum ggml_status failure_status(const std::vector<std::string> & messages, enum
return failure_looks_like_allocation(messages) ? GGML_STATUS_ALLOC_FAILED : fallback;
}

std::string pool_refusal_text(const char * device_name, size_t device_total, size_t device_free, size_t ceiling_bytes,
const onebit::pool_budget_terms_t & terms) {
constexpr double GiB = 1024.0 * 1024.0 * 1024.0;
const char * name = device_name ? device_name : "(unnamed)";
char buf[1024];
// The host term wins when it is smaller than the GTT term, and then GTT is not what to free. The GTT figures are still
// printed, because they are part of why the budget is what it is, but they are labelled as the non-binding side.
if (terms.host_bound) {
// A measured zero is a symptom of its own (engine#1942: an argument-evaluation-order bug made a readable figure read as
// zero), so it says "measured 0.0 GiB" rather than printing a number that looks like a reading.
snprintf(buf, sizeof buf,
"ggml_hrx: %s: refusing to use the device: host MemAvailable %.1f GiB is below the GTT term "
"(GTT total %.1f GiB with %.1f GiB free, ceiling %.1f GiB), so HOST memory is what is binding here; "
"free RAM%s (engine#680, engine#2069)\n",
name, terms.host_available / GiB, device_total / GiB, device_free / GiB, ceiling_bytes / GiB,
terms.host_available == 0 ? " - and note the host figure measured 0.0 GiB, which is a reported zero rather than a reading"
: "");
return std::string(buf);
}
snprintf(buf, sizeof buf,
"ggml_hrx: %s: refusing to use the device: GTT total %.1f GiB with %.1f GiB free leaves nothing under the "
"%.1f GiB ceiling (reserve GGML_HRX_GTT_RESERVE_MB, override GGML_HRX_POOL_LIMIT_MB); free GTT or lower the "
"reserve (engine#680)\n",
name, device_total / GiB, device_free / GiB, ceiling_bytes / GiB);
return std::string(buf);
}

} // namespace ggml::hrx
21 changes: 21 additions & 0 deletions ggml/src/ggml-hrx/runtime/failure-status.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,25 @@ const std::string * failure_unmatched_allocation(const std::vector<std::string>
// caller with nothing better to say gets, and it is what the test exercises for the preserved-code case.
enum ggml_status failure_status(const std::vector<std::string> & messages, enum ggml_status fallback = GGML_STATUS_FAILED);

// The sentence a device refusal prints when nothing fits under the pool ceiling (engine#2069).
//
// The refusal used to word itself from the ceiling alone: "GTT total X with Y free leaves nothing under the Z ceiling ...
// free GTT or lower the reserve". But `budget == 0` can come from the HOST term -- pool_budget_terms() takes the smaller of
// the GTT term and MemAvailable, and sets `host_bound` when the host one wins -- and then the advice names the wrong
// resource and the numbers printed are the ones that were NOT binding. Measured: a self-built HRX configuration refused with
// "GTT total 120.0 GiB with 78.0 GiB free leaves nothing under the 104.0 GiB ceiling", which cannot explain anything (78 free
// against a 104 ceiling leaves ~62 GiB), while the real cause was the host figure reading as 0 (engine#1942's
// argument-evaluation-order bug, fixed by #1960).
//
// Lives here rather than inline in ggml-hrx.cpp for the same reason kPoolRefusalToken does: ggml-hrx.cpp compiles only with
// GGML_HRX=ON, so a sentence written there cannot be asserted by a host test. Keeping the wording in a header lets the arms below
// run everywhere, and leaves the call site a single line.
namespace onebit {
struct pool_budget_terms_t; // defined in hrx-1bit-safety.h, which ggml-hrx.cpp includes; the wording needs only the type
} // namespace onebit

std::string pool_refusal_text(const char * device_name, size_t device_total, size_t device_free, size_t ceiling_bytes,
const onebit::pool_budget_terms_t & terms);


} // namespace ggml::hrx
7 changes: 7 additions & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,13 @@ if (TARGET ggml-hrx)
target_include_directories(test-hrx-failure-status PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)
add_test(NAME test-hrx-failure-status COMMAND test-hrx-failure-status)

# a device refusal must name the resource that was binding (engine#2069): host MemAvailable vs the GTT ceiling. Pure
# helper, no device and no HRX link, for the same reason kPoolRefusalToken lives in a header the tests can reach.
add_executable(test-hrx-pool-refusal-text test-hrx-pool-refusal-text.cpp
../ggml/src/ggml-hrx/runtime/failure-status.cpp)
target_include_directories(test-hrx-pool-refusal-text PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)
add_test(NAME test-hrx-pool-refusal-text COMMAND test-hrx-pool-refusal-text)

add_executable(test-hrx-pool-refusal-budget test-hrx-pool-refusal-budget.cpp
../ggml/src/ggml-hrx/hrx-1bit-safety.cpp)
target_include_directories(test-hrx-pool-refusal-budget PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)
Expand Down
73 changes: 73 additions & 0 deletions tests/test-hrx-pool-refusal-text.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright 2026 bong-water-water-bong. All rights reserved.
// Proprietary and confidential. Distributed only as compiled binaries under the
// 1bit Binary License (LICENSE-BINARY).
//
// A device refusal must name the resource that was actually binding (engine#2069). `budget == 0` can come from the HOST
// term -- pool_budget_terms() takes the smaller of the GTT term and MemAvailable and sets `host_bound` -- but the refusal
// used to word itself from the ceiling alone, so it told the operator to "free GTT" and printed the three GTT figures, none
// of which was binding. Measured: a self-built HRX configuration refused with "GTT total 120.0 GiB with 78.0 GiB free
// leaves nothing under the 104.0 GiB ceiling", which cannot explain anything (78 free against a 104 ceiling leaves ~62 GiB),
// while the real cause was the host figure reading as 0 (the #1942 argument-evaluation-order bug).
//
// Four arms, and the second half of each is what makes them worth having: the host-bound text must NOT advise freeing GTT,
// and the GTT text must NOT mention RAM -- a wording function can otherwise satisfy every arm by mentioning everything.
#include "runtime/failure-status.h"
#include "hrx-1bit-safety.h"

#include <cstdio>
#include <cstdlib>
#include <cstdarg>

// ggml-base is not in this link: the arms exercise the wording, and ggml_abort only appears on paths they never take.
extern "C" void ggml_abort(const char *, int, const char *, ...) { std::abort(); }
extern "C" void ggml_log_internal(enum ggml_log_level, const char *, ...) {}
#include <string>

using ggml::hrx::onebit::pool_budget_terms_t;
using ggml::hrx::pool_refusal_text;

static int fails = 0;
static void check(bool ok, const char * what, const std::string & text = std::string()) {
if (!ok) { std::printf("FAIL %s\n text: %s", what, text.c_str()); ++fails; }
else std::printf("ok %s\n", what);
}
static bool has(const std::string & h, const char * n) { return h.find(n) != std::string::npos; }

int main() {
constexpr size_t GiB = 1024ull * 1024ull * 1024ull;
const size_t total = 120 * GiB, free_gtt = 78 * GiB, ceiling = 104 * GiB;

// 1. the measured case: the host term was binding (a measured zero from the #1942 argument-order bug)
pool_budget_terms_t host0 = {};
host0.host_available = 0; host0.host_bound = true; host0.budget = 0;
std::string t = pool_refusal_text("HRX0", total, free_gtt, ceiling, host0);
check(has(t, "host MemAvailable"), "host-bound refusal names the HOST term", t);
check(has(t, "free RAM"), "and tells the operator to free RAM", t);
check(!has(t, "free GTT or lower"), "and does NOT tell them to free GTT", t);
check(has(t, "measured 0.0 GiB"), "a measured zero is called out as a reported zero", t);

// 2. a real host figure: named, and still no "free GTT" advice
pool_budget_terms_t host8 = {};
host8.host_available = 8 * GiB; host8.host_bound = true; host8.budget = 8 * GiB;
t = pool_refusal_text("HRX0", total, free_gtt, ceiling, host8);
check(has(t, "8.0 GiB") && has(t, "free RAM") && !has(t, "free GTT or lower"),
"a non-zero host term is printed with its figure and still no GTT advice", t);
check(!has(t, "measured 0.0 GiB"), "and no zero warning when nothing was zero", t);

// 3. GTT binding: the #680 wording survives verbatim
pool_budget_terms_t gtt = {};
gtt.host_available = 64 * GiB; gtt.host_bound = false; gtt.budget = 26 * GiB;
t = pool_refusal_text("HRX0", total, free_gtt, ceiling, gtt);
check(has(t, "free GTT or lower the reserve") && has(t, "GGML_HRX_GTT_RESERVE_MB") && has(t, "engine#680"),
"the GTT wording (and #680's rationale) is unchanged when GTT binds", t);
check(!has(t, "free RAM"), "and it does not mention RAM", t);

// 4. kHostUnknown: the host term decided nothing, so the GTT wording is the right one
pool_budget_terms_t unk = {};
unk.host_available = ggml::hrx::onebit::kHostUnknown; unk.host_bound = false; unk.budget = 0;
t = pool_refusal_text("HRX0", total, free_gtt, ceiling, unk);
check(has(t, "free GTT or lower"), "an unreadable host figure leaves the GTT wording", t);

std::printf(fails ? "%d arm(s) failed\n" : "all arms passed\n", fails);
return fails ? 1 : 0;
}