diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index d70ceece75b8..b200996bdcb2 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -1655,12 +1655,13 @@ static std::unique_ptr create_registry_context() { // Nothing fits under the ceiling: refuse the device here, loudly, instead of planning a launch that can // reach the fault line (#680). Report only, do not try to be clever about it. const size_t reserve = ggml::hrx::onebit::pool_reserve_bytes(); - GGML_LOG_ERROR("ggml_hrx: %s: refusing to use the device: GTT total %.1f GiB with %.1f GiB free leaves " - "nothing under the %.1f GiB ceiling (reserve GGML_HRX_GTT_RESERVE_MB, override " - "GGML_HRX_POOL_LIMIT_MB); free GTT or lower the reserve (engine #680)\n", - device_ctx->name.c_str(), device_total / (1024.0 * 1024.0 * 1024.0), - device_free / (1024.0 * 1024.0 * 1024.0), - (device_total > reserve ? device_total - reserve : 0) / (1024.0 * 1024.0 * 1024.0)); + // engine#2069: worded from the terms, not from the ceiling -- `budget == 0` can come from the HOST term, and + // then "free GTT" names the resource that was not binding. The sentence lives in failure-status.cpp so a host + // test can assert it (this file compiles only with GGML_HRX=ON). + const std::string refusal = ggml::hrx::pool_refusal_text( + device_ctx->name.c_str(), device_total, device_free, + device_total > reserve ? device_total - reserve : 0, terms); + GGML_LOG_ERROR("%s", refusal.c_str()); hrx_device_release(hrx_device); continue; } diff --git a/ggml/src/ggml-hrx/runtime/failure-status.cpp b/ggml/src/ggml-hrx/runtime/failure-status.cpp index b3876f0f6066..298c22b554ae 100644 --- a/ggml/src/ggml-hrx/runtime/failure-status.cpp +++ b/ggml/src/ggml-hrx/runtime/failure-status.cpp @@ -1,4 +1,7 @@ #include "runtime/failure-status.h" +#include "hrx-1bit-safety.h" // engine#2069: the terms the refusal is worded from + +#include #include #include @@ -62,4 +65,31 @@ enum ggml_status failure_status(const std::vector & messages, enum return failure_looks_like_allocation(messages) ? GGML_STATUS_ALLOC_FAILED : fallback; } +std::string pool_refusal_text(const char * device_name, size_t device_total, size_t device_free, size_t ceiling_bytes, + const onebit::pool_budget_terms_t & terms) { + constexpr double GiB = 1024.0 * 1024.0 * 1024.0; + const char * name = device_name ? device_name : "(unnamed)"; + char buf[1024]; + // The host term wins when it is smaller than the GTT term, and then GTT is not what to free. The GTT figures are still + // printed, because they are part of why the budget is what it is, but they are labelled as the non-binding side. + if (terms.host_bound) { + // A measured zero is a symptom of its own (engine#1942: an argument-evaluation-order bug made a readable figure read as + // zero), so it says "measured 0.0 GiB" rather than printing a number that looks like a reading. + snprintf(buf, sizeof buf, + "ggml_hrx: %s: refusing to use the device: host MemAvailable %.1f GiB is below the GTT term " + "(GTT total %.1f GiB with %.1f GiB free, ceiling %.1f GiB), so HOST memory is what is binding here; " + "free RAM%s (engine#680, engine#2069)\n", + name, terms.host_available / GiB, device_total / GiB, device_free / GiB, ceiling_bytes / GiB, + terms.host_available == 0 ? " - and note the host figure measured 0.0 GiB, which is a reported zero rather than a reading" + : ""); + return std::string(buf); + } + snprintf(buf, sizeof buf, + "ggml_hrx: %s: refusing to use the device: GTT total %.1f GiB with %.1f GiB free leaves nothing under the " + "%.1f GiB ceiling (reserve GGML_HRX_GTT_RESERVE_MB, override GGML_HRX_POOL_LIMIT_MB); free GTT or lower the " + "reserve (engine#680)\n", + name, device_total / GiB, device_free / GiB, ceiling_bytes / GiB); + return std::string(buf); +} + } // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/runtime/failure-status.h b/ggml/src/ggml-hrx/runtime/failure-status.h index d50d96be34a6..d8ba6941248e 100644 --- a/ggml/src/ggml-hrx/runtime/failure-status.h +++ b/ggml/src/ggml-hrx/runtime/failure-status.h @@ -41,4 +41,25 @@ const std::string * failure_unmatched_allocation(const std::vector // caller with nothing better to say gets, and it is what the test exercises for the preserved-code case. enum ggml_status failure_status(const std::vector & messages, enum ggml_status fallback = GGML_STATUS_FAILED); +// The sentence a device refusal prints when nothing fits under the pool ceiling (engine#2069). +// +// The refusal used to word itself from the ceiling alone: "GTT total X with Y free leaves nothing under the Z ceiling ... +// free GTT or lower the reserve". But `budget == 0` can come from the HOST term -- pool_budget_terms() takes the smaller of +// the GTT term and MemAvailable, and sets `host_bound` when the host one wins -- and then the advice names the wrong +// resource and the numbers printed are the ones that were NOT binding. Measured: a self-built HRX configuration refused with +// "GTT total 120.0 GiB with 78.0 GiB free leaves nothing under the 104.0 GiB ceiling", which cannot explain anything (78 free +// against a 104 ceiling leaves ~62 GiB), while the real cause was the host figure reading as 0 (engine#1942's +// argument-evaluation-order bug, fixed by #1960). +// +// Lives here rather than inline in ggml-hrx.cpp for the same reason kPoolRefusalToken does: ggml-hrx.cpp compiles only with +// GGML_HRX=ON, so a sentence written there cannot be asserted by a host test. Keeping the wording in a header lets the arms below +// run everywhere, and leaves the call site a single line. +namespace onebit { +struct pool_budget_terms_t; // defined in hrx-1bit-safety.h, which ggml-hrx.cpp includes; the wording needs only the type +} // namespace onebit + +std::string pool_refusal_text(const char * device_name, size_t device_total, size_t device_free, size_t ceiling_bytes, + const onebit::pool_budget_terms_t & terms); + + } // namespace ggml::hrx diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7111758c8825..4a39ea139794 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -369,6 +369,13 @@ if (TARGET ggml-hrx) target_include_directories(test-hrx-failure-status PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) add_test(NAME test-hrx-failure-status COMMAND test-hrx-failure-status) + # a device refusal must name the resource that was binding (engine#2069): host MemAvailable vs the GTT ceiling. Pure + # helper, no device and no HRX link, for the same reason kPoolRefusalToken lives in a header the tests can reach. + add_executable(test-hrx-pool-refusal-text test-hrx-pool-refusal-text.cpp + ../ggml/src/ggml-hrx/runtime/failure-status.cpp) + target_include_directories(test-hrx-pool-refusal-text PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) + add_test(NAME test-hrx-pool-refusal-text COMMAND test-hrx-pool-refusal-text) + add_executable(test-hrx-pool-refusal-budget test-hrx-pool-refusal-budget.cpp ../ggml/src/ggml-hrx/hrx-1bit-safety.cpp) target_include_directories(test-hrx-pool-refusal-budget PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) diff --git a/tests/test-hrx-pool-refusal-text.cpp b/tests/test-hrx-pool-refusal-text.cpp new file mode 100644 index 000000000000..fb545bf945e9 --- /dev/null +++ b/tests/test-hrx-pool-refusal-text.cpp @@ -0,0 +1,73 @@ +// Copyright 2026 bong-water-water-bong. All rights reserved. +// Proprietary and confidential. Distributed only as compiled binaries under the +// 1bit Binary License (LICENSE-BINARY). +// +// A device refusal must name the resource that was actually binding (engine#2069). `budget == 0` can come from the HOST +// term -- pool_budget_terms() takes the smaller of the GTT term and MemAvailable and sets `host_bound` -- but the refusal +// used to word itself from the ceiling alone, so it told the operator to "free GTT" and printed the three GTT figures, none +// of which was binding. Measured: a self-built HRX configuration refused with "GTT total 120.0 GiB with 78.0 GiB free +// leaves nothing under the 104.0 GiB ceiling", which cannot explain anything (78 free against a 104 ceiling leaves ~62 GiB), +// while the real cause was the host figure reading as 0 (the #1942 argument-evaluation-order bug). +// +// Four arms, and the second half of each is what makes them worth having: the host-bound text must NOT advise freeing GTT, +// and the GTT text must NOT mention RAM -- a wording function can otherwise satisfy every arm by mentioning everything. +#include "runtime/failure-status.h" +#include "hrx-1bit-safety.h" + +#include +#include +#include + +// ggml-base is not in this link: the arms exercise the wording, and ggml_abort only appears on paths they never take. +extern "C" void ggml_abort(const char *, int, const char *, ...) { std::abort(); } +extern "C" void ggml_log_internal(enum ggml_log_level, const char *, ...) {} +#include + +using ggml::hrx::onebit::pool_budget_terms_t; +using ggml::hrx::pool_refusal_text; + +static int fails = 0; +static void check(bool ok, const char * what, const std::string & text = std::string()) { + if (!ok) { std::printf("FAIL %s\n text: %s", what, text.c_str()); ++fails; } + else std::printf("ok %s\n", what); +} +static bool has(const std::string & h, const char * n) { return h.find(n) != std::string::npos; } + +int main() { + constexpr size_t GiB = 1024ull * 1024ull * 1024ull; + const size_t total = 120 * GiB, free_gtt = 78 * GiB, ceiling = 104 * GiB; + + // 1. the measured case: the host term was binding (a measured zero from the #1942 argument-order bug) + pool_budget_terms_t host0 = {}; + host0.host_available = 0; host0.host_bound = true; host0.budget = 0; + std::string t = pool_refusal_text("HRX0", total, free_gtt, ceiling, host0); + check(has(t, "host MemAvailable"), "host-bound refusal names the HOST term", t); + check(has(t, "free RAM"), "and tells the operator to free RAM", t); + check(!has(t, "free GTT or lower"), "and does NOT tell them to free GTT", t); + check(has(t, "measured 0.0 GiB"), "a measured zero is called out as a reported zero", t); + + // 2. a real host figure: named, and still no "free GTT" advice + pool_budget_terms_t host8 = {}; + host8.host_available = 8 * GiB; host8.host_bound = true; host8.budget = 8 * GiB; + t = pool_refusal_text("HRX0", total, free_gtt, ceiling, host8); + check(has(t, "8.0 GiB") && has(t, "free RAM") && !has(t, "free GTT or lower"), + "a non-zero host term is printed with its figure and still no GTT advice", t); + check(!has(t, "measured 0.0 GiB"), "and no zero warning when nothing was zero", t); + + // 3. GTT binding: the #680 wording survives verbatim + pool_budget_terms_t gtt = {}; + gtt.host_available = 64 * GiB; gtt.host_bound = false; gtt.budget = 26 * GiB; + t = pool_refusal_text("HRX0", total, free_gtt, ceiling, gtt); + check(has(t, "free GTT or lower the reserve") && has(t, "GGML_HRX_GTT_RESERVE_MB") && has(t, "engine#680"), + "the GTT wording (and #680's rationale) is unchanged when GTT binds", t); + check(!has(t, "free RAM"), "and it does not mention RAM", t); + + // 4. kHostUnknown: the host term decided nothing, so the GTT wording is the right one + pool_budget_terms_t unk = {}; + unk.host_available = ggml::hrx::onebit::kHostUnknown; unk.host_bound = false; unk.budget = 0; + t = pool_refusal_text("HRX0", total, free_gtt, ceiling, unk); + check(has(t, "free GTT or lower"), "an unreadable host figure leaves the GTT wording", t); + + std::printf(fails ? "%d arm(s) failed\n" : "all arms passed\n", fails); + return fails ? 1 : 0; +}