From dddb47854e62522f1205a4c574e8b049f88b5b28 Mon Sep 17 00:00:00 2001 From: pi-hrx-sched Date: Mon, 9 Nov 2026 04:21:26 -0400 Subject: [PATCH 1/2] ggml-hrx: a failed host-weight materialize names the size and the pool budget engine #1618 was filed as 'flash attention compute-errors on qwen4exp' and spent a day there. The operator-facing line was: E graph_compute: also: materialize host weight value 3 failed and everything behind it came from another repo's file (slab_provider.c:663, RESOURCE_EXHAUSTED from hsa_amd_memory_pool_allocate). Nothing in the engine's own output said that an 89 GB model was materializing weights against a 66.6 GiB pool budget, so the '-fa on vs -fa off' contrast looked like a kernel defect when it was the tipping point of a memory shortfall. The failure branch now logs the requested bytes and, when the counters are readable, the live GTT use and the same pool_budget() the launch guardrail prints at open, so the line and the budget are in one place. gtt_counters()/pool_budget() are the existing hrx-1bit-safety.h helpers; there is no new state and no behaviour change - the branch only runs after the acquire has already failed. Compile-verified: amdclang++ -fsyntax-only with this file's own flags from an existing HRX build's compile_commands.json, 0 diagnostics, in the fork tree (the same command was re-run against the submodule checkout before the patch was moved here). --- .../runtime/command-program-executor.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-hrx/runtime/command-program-executor.cpp b/ggml/src/ggml-hrx/runtime/command-program-executor.cpp index 3485ff955fcf..9c4a98301842 100644 --- a/ggml/src/ggml-hrx/runtime/command-program-executor.cpp +++ b/ggml/src/ggml-hrx/runtime/command-program-executor.cpp @@ -10,6 +10,7 @@ #include "runtime/hrx-sleeping-wait.h" #include "runtime/kernel-executable-cache.h" #include "runtime/transient-arena.h" +#include "hrx-1bit-safety.h" #include #include @@ -711,7 +712,23 @@ static CommandProgramBindings materialize_host_bindings(const CommandProgramExec HostWeightAcquireResult resident = context.host_weights->acquire(context.device, context.stream, *context.host_transfers, source); if (!resident.valid()) { - status.log("materialize host weight value %d failed", binding.value.value); + // engine #1618: this line and the HSA error behind it carried no pool figure, + // so an 89 GB model materialized against a 66.6 GiB budget read as a + // flash-attention bug. Name the size and the budget the upload competed for. + size_t gtt_total = 0, gtt_used = 0; + if (ggml::hrx::onebit::gtt_counters(gtt_total, gtt_used)) { + const size_t gtt_free = gtt_total > gtt_used ? gtt_total - gtt_used : 0; + const size_t budget = ggml::hrx::onebit::pool_budget(gtt_total, gtt_free); + status.log("materialize host weight value %d failed: %zu bytes requested, " + "GTT %.2f of %.2f GiB in use, pool budget %.2f GiB", + binding.value.value, source.length, + (double)gtt_used / (1024.0 * 1024.0 * 1024.0), + (double)gtt_total / (1024.0 * 1024.0 * 1024.0), + (double)budget / (1024.0 * 1024.0 * 1024.0)); + } else { + status.log("materialize host weight value %d failed: %zu bytes requested", + binding.value.value, source.length); + } status.append(resident.status); materialized.push_back(binding); continue; From cf8a8a0b82c8daa78ab3292e1ecb602e3d58ae7c Mon Sep 17 00:00:00 2001 From: pi-hrx-sched Date: Mon, 9 Nov 2026 04:51:04 -0400 Subject: [PATCH 2/2] ggml-hrx: call the materialize failure figure what it is - headroom, not the pool budget Review point on #255, and it was right: pool_budget(total, free) computes ceiling - in_use, i.e. what is LEFT at the moment it is called. At device open that was the 66.6 GiB of #1618; at a failed upload the process's own weights are resident, so the same call returns the remainder (19.73 GiB in the example). Printing both under one name is exactly the confusion this line exists to remove - and under GGML_HRX_POOL_LIMIT_MB the same function returns a fixed limit, a third meaning. The line now says 'headroom now N GiB (ceiling - in use; the open-time pool budget is the guardrail line)', or 'fixed limit N GiB (GGML_HRX_POOL_LIMIT_MB)' when the override is set. The question of which it is lives in the safety module, not in the caller: pool_budget_is_fixed_limit() is new beside pool_budget() so the env var is still read in one place. The open-time budget itself is not reachable from here - CommandProgramExecutionContext's device is an opaque hrx_device_t, while memory_budget lives on the ggml-hrx backend context - so the label carries the pointer to the guardrail line instead of a second number, as the review allowed. amdclang++ -fsyntax-only with each file's own flags from an existing HRX build: 0 diagnostics for both changed TUs. --- ggml/src/ggml-hrx/hrx-1bit-safety.cpp | 5 +++++ ggml/src/ggml-hrx/hrx-1bit-safety.h | 3 +++ .../runtime/command-program-executor.cpp | 18 ++++++++++++------ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-hrx/hrx-1bit-safety.cpp b/ggml/src/ggml-hrx/hrx-1bit-safety.cpp index e188bb154c95..dda9059197ff 100644 --- a/ggml/src/ggml-hrx/hrx-1bit-safety.cpp +++ b/ggml/src/ggml-hrx/hrx-1bit-safety.cpp @@ -83,6 +83,11 @@ bool gtt_counters(size_t & total, size_t & used) { return false; } +bool pool_budget_is_fixed_limit() { + const char * env = std::getenv("GGML_HRX_POOL_LIMIT_MB"); + return env != nullptr && std::atoll(env) > 0; +} + size_t pool_budget(size_t memory_total, size_t memory_free) { const char * env = std::getenv("GGML_HRX_POOL_LIMIT_MB"); if (env != nullptr) { diff --git a/ggml/src/ggml-hrx/hrx-1bit-safety.h b/ggml/src/ggml-hrx/hrx-1bit-safety.h index 45d4e2139fb3..d1e4faec6f4b 100644 --- a/ggml/src/ggml-hrx/hrx-1bit-safety.h +++ b/ggml/src/ggml-hrx/hrx-1bit-safety.h @@ -29,6 +29,9 @@ namespace ggml::hrx::onebit { // already hold (memory_total - memory_free). A GTT plan past this budget is refused at allocation instead of // faulting the device: #680 is a gfxhub page fault plus a MODE2 reset at GTT 106.5 of 120 GiB. size_t pool_budget(size_t memory_total, size_t memory_free); +// true when GGML_HRX_POOL_LIMIT_MB is in force: pool_budget() then returns that fixed limit rather than a +// remainder, so a caller that prints the figure has to word it differently. +bool pool_budget_is_fixed_limit(); // The budget after an allocation the allocator refused, given the size it refused (engine #1502). // diff --git a/ggml/src/ggml-hrx/runtime/command-program-executor.cpp b/ggml/src/ggml-hrx/runtime/command-program-executor.cpp index 9c4a98301842..5a72243e4388 100644 --- a/ggml/src/ggml-hrx/runtime/command-program-executor.cpp +++ b/ggml/src/ggml-hrx/runtime/command-program-executor.cpp @@ -717,14 +717,20 @@ static CommandProgramBindings materialize_host_bindings(const CommandProgramExec // flash-attention bug. Name the size and the budget the upload competed for. size_t gtt_total = 0, gtt_used = 0; if (ggml::hrx::onebit::gtt_counters(gtt_total, gtt_used)) { - const size_t gtt_free = gtt_total > gtt_used ? gtt_total - gtt_used : 0; - const size_t budget = ggml::hrx::onebit::pool_budget(gtt_total, gtt_free); + const bool fixed = ggml::hrx::onebit::pool_budget_is_fixed_limit(); + const size_t gtt_free = gtt_total > gtt_used ? gtt_total - gtt_used : 0; + const size_t left = ggml::hrx::onebit::pool_budget(gtt_total, gtt_free); + const double as_gib = 1.0 / (1024.0 * 1024.0 * 1024.0); + // #1618 review: pool_budget() is the headroom LEFT NOW (ceiling - in use), not the figure the + // open-time guardrail line calls the pool budget, and under GGML_HRX_POOL_LIMIT_MB it is a fixed + // limit instead. Naming it correctly is the whole point of this line. status.log("materialize host weight value %d failed: %zu bytes requested, " - "GTT %.2f of %.2f GiB in use, pool budget %.2f GiB", + "GTT %.2f of %.2f GiB in use, %s %.2f GiB%s", binding.value.value, source.length, - (double)gtt_used / (1024.0 * 1024.0 * 1024.0), - (double)gtt_total / (1024.0 * 1024.0 * 1024.0), - (double)budget / (1024.0 * 1024.0 * 1024.0)); + (double)gtt_used * as_gib, (double)gtt_total * as_gib, + fixed ? "fixed limit" : "headroom now", (double)left * as_gib, + fixed ? " (GGML_HRX_POOL_LIMIT_MB)" + : " (ceiling - in use; the open-time pool budget is the guardrail line)"); } else { status.log("materialize host weight value %d failed: %zu bytes requested", binding.value.value, source.length);