diff --git a/ggml/src/ggml-hrx/CMakeLists.txt b/ggml/src/ggml-hrx/CMakeLists.txt index 30228641d990..68b538b3805e 100644 --- a/ggml/src/ggml-hrx/CMakeLists.txt +++ b/ggml/src/ggml-hrx/CMakeLists.txt @@ -367,6 +367,7 @@ ggml_add_backend_library(ggml-hrx runtime/host-memory.cpp runtime/host-memory.h runtime/host-memory-available.cpp + runtime/failure-status.cpp runtime/host-memory-available.h runtime/host-memory-ternary.cpp runtime/host-memory-ternary.h diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index 1a1bf0614859..5f01aa3002e6 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -2,6 +2,7 @@ #include "dispatch_registration/common/moe-placement-guard.h" #include "ggml-hrx.h" #include "hrx-1bit-safety.h" +#include "runtime/failure-status.h" #include "runtime/host-memory-available.h" #include "ggml-hrx-dmabuf.h" @@ -826,12 +827,36 @@ static enum ggml_status graph_compute(ggml_backend_t backend, ggml_cgraph * grap auto * context = static_cast(backend->context); const ggml::hrx::GraphExecutor executor = ggml::hrx::GraphExecutor(*context); const ggml::hrx::GraphExecutionResult result = executor.execute(*graph); + // engine #1618: a resource failure must not leave this backend as a generic failure. llama.cpp maps ALLOC_FAILED to + // its own -2 and FAILED to -3, and the server prints one sentence for "out of memory" and another for "Compute error" + // - so without this classification a host-memory allocation the pool could not serve reads to the client as a kernel + // fault, which is what made #1618 look like a flash-attention defect for a day. + enum ggml_status code = result.success() ? GGML_STATUS_SUCCESS + : ggml::hrx::failure_status(result.status.errors(), result.code); if (!result.success()) { // Print the LAST reason: a matcher that declined also logs why, and those lines come first, so // errors().front() named a matcher while the graph failed on the node left without a dispatch // (2026-10-20 census: a q8_0 KV cache printed the qwen postprocess refusal for a graph that broke // on another node). The earlier reasons follow, bounded, for context. const std::vector & reasons = result.status.errors(); + // The classification above reads the WHOLE set, while the headline below prints the last reason. When the two differ, + // a resource failure would return ALLOC_FAILED while the log's first sentence points at a dispatch decision - the + // same "the log knew, the client did not" split in miniature (engine #1618 review), so print the classifying message + // as the headline and keep the rest as context. + if (const std::string * suspicious = ggml::hrx::failure_unmatched_allocation(reasons)) { + GGML_LOG_WARN("%s: an allocation-looking failure did not match the resource needles, so it is reported as a " + "compute failure: %s (extend kResourceNeedles if this is an out-of-memory status)\n", __func__, + suspicious->c_str()); + } + const std::string * classifying = nullptr; + for (const std::string & reason : reasons) { + if (ggml::hrx::failure_looks_like_allocation({ reason })) { + classifying = &reason; + } + } + if (classifying != nullptr && !reasons.empty() && classifying != &reasons.back()) { + GGML_LOG_ERROR("%s: %s\n", __func__, classifying->c_str()); + } if (reasons.empty()) { GGML_LOG_ERROR("%s: %s\n", __func__, status_first_error(result.status)); } else { @@ -845,7 +870,7 @@ static enum ggml_status graph_compute(ggml_backend_t backend, ggml_cgraph * grap } } } - return result.code; + return code; } static const ggml_backend_i backend_i = { diff --git a/ggml/src/ggml-hrx/runtime/failure-status.cpp b/ggml/src/ggml-hrx/runtime/failure-status.cpp new file mode 100644 index 000000000000..427d61883b45 --- /dev/null +++ b/ggml/src/ggml-hrx/runtime/failure-status.cpp @@ -0,0 +1,63 @@ +#include "runtime/failure-status.h" + +#include +#include + +namespace ggml::hrx { + +namespace { + +// iree's status name for a pool that cannot serve the request, and HSA's for the same condition one layer down. Both appear +// verbatim in the failures this exists for: +// al...: RESOURCE_EXHAUSTED; [hsa_amd_memory_pool_allocate] HSA_STATUS_ERROR_OUT_OF_RESOURCES +constexpr const char * kResourceNeedles[] = { "RESOURCE_EXHAUSTED", "OUT_OF_RESOURCES" }; + +// Case-insensitive on purpose: the *classification* needles are exact because they are identifiers as iree and HSA render +// them, but the *hint* exists to catch spellings we have not seen -- `hipErrorOutOfMemory` is real hip and mixed case, and a +// case-sensitive match would have missed exactly the case this probe was added for (the test caught that). +constexpr const char * kAllocationHints[] = { "memory", "alloc" }; + +bool contains_any_of_ci(const std::string & message, const char * const * needles, size_t count) { + std::string lowered; + lowered.reserve(message.size()); + for (char c : message) { + lowered.push_back(static_cast(std::tolower(static_cast(c)))); + } + for (size_t i = 0; i < count; ++i) { + if (lowered.find(needles[i]) != std::string::npos) { + return true; + } + } + return false; +} + +} // namespace + +const std::string * failure_unmatched_allocation(const std::vector & messages) { + if (failure_looks_like_allocation(messages)) { + return nullptr; // it matched: nothing to warn about + } + for (const std::string & message : messages) { + if (contains_any_of_ci(message, kAllocationHints, sizeof(kAllocationHints) / sizeof(kAllocationHints[0]))) { + return &message; + } + } + return nullptr; +} + +bool failure_looks_like_allocation(const std::vector & messages) { + for (const std::string & message : messages) { + for (const char * needle : kResourceNeedles) { + if (std::strstr(message.c_str(), needle) != nullptr) { + return true; + } + } + } + return false; +} + +enum ggml_status failure_status(const std::vector & messages, enum ggml_status fallback) { + return failure_looks_like_allocation(messages) ? GGML_STATUS_ALLOC_FAILED : fallback; +} + +} // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/runtime/failure-status.h b/ggml/src/ggml-hrx/runtime/failure-status.h new file mode 100644 index 000000000000..1b1f8c41abb3 --- /dev/null +++ b/ggml/src/ggml-hrx/runtime/failure-status.h @@ -0,0 +1,35 @@ +#pragma once + +#include "ggml.h" + +#include +#include + +namespace ggml::hrx { + +// Whether a backend failure is a *resource* failure rather than a compute one (engine #1618). +// +// The distinction matters outside this backend: `src/llama-context.cpp` maps GGML_STATUS_ALLOC_FAILED to llama_decode's -2 +// and GGML_STATUS_FAILED to -3, and the server prints a different sentence for each. Before this, every HRX failure left +// `GraphExecutionResult`'s default GGML_STATUS_FAILED, so a host-memory allocation the pool could not serve reached the client +// as "Compute error." - i.e. a resource failure read as a kernel fault, which is how #1618 was filed and how it stayed +// misdiagnosed for a day (the log said HSA_STATUS_ERROR_OUT_OF_RESOURCES while the client said "Compute error"). +// +// Text-based on purpose: the code lives in libhrx's `iree_status_t`, and `status.h` keeps only the rendered messages by the +// time a failure reaches this backend. The two strings matched below are iree's own status name and HSA's error name, so they +// survive rewording of the prose around them. +bool failure_looks_like_allocation(const std::vector & messages); + +// A message that *looks* like an allocation failure (it mentions memory or allocation) but matched neither needle, so it is +// about to be classified as a compute failure (engine #1618 review). Returns nullptr when nothing looks suspicious. This exists +// so the next spelling of an out-of-memory status -- `hipErrorOutOfMemory`, `HSA_STATUS_ERROR_INVALID_ALLOCATION`, whatever the +// runtime renders tomorrow -- costs a log line rather than another day of misdiagnosis: the fallback stays the safe direction +// (a false OOM is worse), but it stops being silent. +const std::string * failure_unmatched_allocation(const std::vector & messages); + +// The ggml status to return for these failure messages: GGML_STATUS_ALLOC_FAILED when they describe a resource failure, +// |fallback| otherwise. graph_compute() passes its own code (so a caller that already has one keeps it); the default is what a +// caller with nothing better to say gets, and it is what the test exercises for the preserved-code case. +enum ggml_status failure_status(const std::vector & messages, enum ggml_status fallback = GGML_STATUS_FAILED); + +} // namespace ggml::hrx diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b314c97e2f76..f5b4c7c8cfbf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -363,6 +363,12 @@ if (TARGET ggml-hrx) add_test(NAME test-hrx-graph-program-budget COMMAND test-hrx-graph-program-budget) # the host-weight arena refusal's wording: pure helper, no device and no HRX link (engine #900) + # the backend's failure classification: resource failure vs compute failure, pure (engine #1618) + add_executable(test-hrx-failure-status test-hrx-failure-status.cpp + ../ggml/src/ggml-hrx/runtime/failure-status.cpp) + target_include_directories(test-hrx-failure-status PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) + add_test(NAME test-hrx-failure-status COMMAND test-hrx-failure-status) + add_executable(test-hrx-pool-refusal-budget test-hrx-pool-refusal-budget.cpp ../ggml/src/ggml-hrx/hrx-1bit-safety.cpp) target_include_directories(test-hrx-pool-refusal-budget PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) diff --git a/tests/test-hrx-failure-status.cpp b/tests/test-hrx-failure-status.cpp new file mode 100644 index 000000000000..4e019cff68ee --- /dev/null +++ b/tests/test-hrx-failure-status.cpp @@ -0,0 +1,89 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Host test for failure-status.h: whether a backend failure is a resource failure (engine #1618). Pure - no device, no +// runtime library - because the rule is a function of the messages libhrx renders. +// +// The case this exists for: a Flash-Next decode with flash attention on failed with +// `al...: RESOURCE_EXHAUSTED; [hsa_amd_memory_pool_allocate] HSA_STATUS_ERROR_OUT_OF_RESOURCES` while materializing a host +// weight, and the client was told "Compute error." - a resource failure served as a kernel fault. + +#include "runtime/failure-status.h" + +#include +#include +#include +#include + +#define REQUIRE(condition) \ + do { \ + if (!(condition)) { \ + std::fprintf(stderr, "%s:%d: requirement failed: %s\n", __FILE__, __LINE__, #condition); \ + std::abort(); \ + } \ + } while (false) + +using ggml::hrx::failure_looks_like_allocation; +using ggml::hrx::failure_status; + +int main() { + // The issue's own line, verbatim from the log that was misread. + const std::vector pool_exhausted = { + "allocate host memory device buffer: hrx-system/runtime/src/iree/hal/drivers/amdgpu/slab_provider.c:663: " + "RESOURCE_EXHAUSTED; [hsa_amd_memory_pool_allocate] HSA_STATUS_ERROR_OUT_OF_RESOURCES", + }; + REQUIRE(failure_looks_like_allocation(pool_exhausted)); + REQUIRE(failure_status(pool_exhausted) == GGML_STATUS_ALLOC_FAILED); + + // HSA's name alone is enough: libhrx renders both, and a caller may see only one of them. + REQUIRE(failure_looks_like_allocation({ "materialize host weight value 3 failed: HSA_STATUS_ERROR_OUT_OF_RESOURCES" })); + REQUIRE(failure_status({ "x: RESOURCE_EXHAUSTED" }) == GGML_STATUS_ALLOC_FAILED); + + // A real compute failure must stay a compute failure: this is the direction that would be a regression, because a + // kernel error reported as "out of memory" sends the reader to free memory that was never the problem. + const std::vector kernel_fault = { + "execute HRX command program failed: compute status: -1", + "hip.qsa_attn_select rejected node 20: unsupported layout", + }; + REQUIRE(!failure_looks_like_allocation(kernel_fault)); + REQUIRE(failure_status(kernel_fault) == GGML_STATUS_FAILED); + + // Empty reasons keep the caller's code rather than inventing one. + REQUIRE(!failure_looks_like_allocation({})); + REQUIRE(failure_status({}, GGML_STATUS_FAILED) == GGML_STATUS_FAILED); + REQUIRE(failure_status({}, GGML_STATUS_ABORTED) == GGML_STATUS_ABORTED); + + // Any message in the list counts, not just the last: graph_compute prints the last reason but classifies the whole set. + REQUIRE(failure_looks_like_allocation({ "plan rejected: operand layout", "x: RESOURCE_EXHAUSTED" })); + + // The uncovered set is as visible as the covered one (review of #261): a hip OOM spelling that matches neither needle + // stays a compute failure -- the safe direction -- but it must be NAMED, so the next person gets a log line instead of a + // lost day. `hipErrorOutOfMemory` is a real hip name and matches neither needle on purpose. + using ggml::hrx::failure_unmatched_allocation; + const std::vector unmatched = { "hip.qsa_attn_select: hipErrorOutOfMemory" }; + REQUIRE(!failure_looks_like_allocation(unmatched)); + REQUIRE(failure_status(unmatched) == GGML_STATUS_FAILED); + const std::string * hint = failure_unmatched_allocation(unmatched); + REQUIRE(hint != nullptr && hint->find("hipErrorOutOfMemory") != std::string::npos); + + // ...and a matched failure produces no warning, so the signal means "uncovered", not "allocation-ish". + REQUIRE(failure_unmatched_allocation(pool_exhausted) == nullptr); + // A plain kernel fault is not allocation-looking and stays quiet. + REQUIRE(failure_unmatched_allocation(kernel_fault) == nullptr); + REQUIRE(failure_unmatched_allocation({}) == nullptr); + + std::printf("test-hrx-failure-status: all requirements passed\n"); + return 0; +} diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index fa74ed5c1b28..3a19ac65fc72 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3856,7 +3856,15 @@ struct server_context_impl { err = "Invalid input batch."; } - if (ret < -1) { + if (ret == -2) { + // llama_decode's -2 is GGML_STATUS_ALLOC_FAILED: the backend could not get the memory a node needed + // (engine #1618). Saying "Compute error" here is what made a pool exhaustion read as a kernel fault + // to every client, so name the resource and the way out. + err = "Out of memory: the device could not allocate the memory this step needs. Free the device " + "(another process's resident model, a smaller -c/-ngl, or a smaller -ub) and retry."; + } + + if (ret < -2) { // TODO: update slot state based on llama_memory_seq_pos_min() and llama_memory_seq_pos_max() err = "Compute error."; }