ggml-hrx: a failed host-weight materialize names the size and the pool budget (#1618)

ggml/src/ggml-hrx/hrx-1bit-safety.cpp

@@ -83,6 +83,11 @@ bool gtt_counters(size_t & total, size_t & used) {
     return false;
 }
 
+bool pool_budget_is_fixed_limit() {
+    const char * env = std::getenv("GGML_HRX_POOL_LIMIT_MB");
+    return env != nullptr && std::atoll(env) > 0;
+}
+
 size_t pool_budget(size_t memory_total, size_t memory_free) {
     const char * env = std::getenv("GGML_HRX_POOL_LIMIT_MB");
     if (env != nullptr) {

ggml/src/ggml-hrx/hrx-1bit-safety.h

@@ -29,6 +29,9 @@ namespace ggml::hrx::onebit {
 // already hold (memory_total - memory_free). A GTT plan past this budget is refused at allocation instead of
 // faulting the device: #680 is a gfxhub page fault plus a MODE2 reset at GTT 106.5 of 120 GiB.
 size_t pool_budget(size_t memory_total, size_t memory_free);
+// true when GGML_HRX_POOL_LIMIT_MB is in force: pool_budget() then returns that fixed limit rather than a
+// remainder, so a caller that prints the figure has to word it differently.
+bool   pool_budget_is_fixed_limit();
 
 // The budget after an allocation the allocator refused, given the size it refused (engine #1502).
 //

ggml/src/ggml-hrx/runtime/command-program-executor.cpp

@@ -10,6 +10,7 @@
 #include "runtime/hrx-sleeping-wait.h"
 #include "runtime/kernel-executable-cache.h"
 #include "runtime/transient-arena.h"
+#include "hrx-1bit-safety.h"
 
 #include <algorithm>
 #include <cstdio>
@@ -711,7 +712,29 @@ static CommandProgramBindings materialize_host_bindings(const CommandProgramExec
             HostWeightAcquireResult resident =
                 context.host_weights->acquire(context.device, context.stream, *context.host_transfers, source);
             if (!resident.valid()) {
-                status.log("materialize host weight value %d failed", binding.value.value);
+                // engine #1618: this line and the HSA error behind it carried no pool figure,
+                // so an 89 GB model materialized against a 66.6 GiB budget read as a
+                // flash-attention bug. Name the size and the budget the upload competed for.
+                size_t gtt_total = 0, gtt_used = 0;
+                if (ggml::hrx::onebit::gtt_counters(gtt_total, gtt_used)) {
+                    const bool  fixed    = ggml::hrx::onebit::pool_budget_is_fixed_limit();
+                    const size_t gtt_free = gtt_total > gtt_used ? gtt_total - gtt_used : 0;
+                    const size_t left     = ggml::hrx::onebit::pool_budget(gtt_total, gtt_free);
+                    const double as_gib   = 1.0 / (1024.0 * 1024.0 * 1024.0);
+                    // #1618 review: pool_budget() is the headroom LEFT NOW (ceiling - in use), not the figure the
+                    // open-time guardrail line calls the pool budget, and under GGML_HRX_POOL_LIMIT_MB it is a fixed
+                    // limit instead. Naming it correctly is the whole point of this line.
+                    status.log("materialize host weight value %d failed: %zu bytes requested, "
+                               "GTT %.2f of %.2f GiB in use, %s %.2f GiB%s",
+                               binding.value.value, source.length,
+                               (double)gtt_used * as_gib, (double)gtt_total * as_gib,
+                               fixed ? "fixed limit" : "headroom now", (double)left * as_gib,
+                               fixed ? " (GGML_HRX_POOL_LIMIT_MB)"
+                                     : " (ceiling - in use; the open-time pool budget is the guardrail line)");
+                } else {
+                    status.log("materialize host weight value %d failed: %zu bytes requested",
+                               binding.value.value, source.length);
+                }
                 status.append(resident.status);
                 materialized.push_back(binding);
                 continue;