From 585ad55e0c70dd79b45a2e2be5987a49623eea6d Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 9 Nov 2026 08:52:26 -0400 Subject: [PATCH 1/3] ggml-hrx: the forward declaration says bool, matching the definition, so the pin builds again `main` cannot be compiled with `GGML_HRX=ON` at 8f124f906. Two merges each added a forward declaration of the same function - #252's `static void device_free_total(...)` and #240's `static bool ...` - and `void`/`bool` is an ambiguous redeclaration: ggml-hrx.cpp:517:13: error: ambiguating new declaration of 'bool {anonymous}::device_free_total(ggml_backend_hrx_device_context*, size_t&, size_t&)' ggml-hrx.cpp:383:13: note: old declaration 'void {anonymous}::device_free_total(ggml_backend_hrx_device_context*, size_t&, size_t&)' The definition is `bool` (it reports whether the driver counters were available), so the `void` declaration is the wrong one. Both hunks are textually far apart, which is why the merge was clean and the break is only visible when the backend is compiled - and CI never compiles it, because `GGML_HRX=ON` wants the private runtime. One word. `ggml-hrx.cpp` now compiles, and this is the same line that let the #1502 work build llama-cli end to end on 1bit/1502-host-term. scope: the declaration's type. No behaviour change; the function, its callers and its definition are untouched. --- ggml/src/ggml-hrx/ggml-hrx.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index 705dcc54155a..b45dd2abe0d6 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -380,7 +380,11 @@ static const ggml_backend_buffer_i buffer_i = { // engine #1502 (#251) calls this from buffer_alloc, eight hundred lines above its definition, with no declaration in // scope - so the translation unit does not compile ("'device_free_total' was not declared in this scope"). One forward // declaration is enough; the definition and its callers are unchanged. -static void device_free_total(ggml_backend_hrx_device_context * device_ctx, size_t & free_bytes, size_t & total_bytes); +// The declaration must match the definition's type: it is `bool` (it reports whether the driver counters were +// available). Two merges each added a forward declaration - #252's above, #240's `bool` one further down - and the +// `void`/`bool` pair is an ambiguous redeclaration, so the translation unit fails to compile at the `bool` declaration +// and the pin builds for nobody. One word, and every lane that builds this pin is unblocked. +static bool device_free_total(ggml_backend_hrx_device_context * device_ctx, size_t & free_bytes, size_t & total_bytes); static ggml_backend_buffer_t buffer_alloc(ggml_backend_buffer_type_t buft, size_t size) { ggml::hrx::onebit::poison_selftest(); // no-op unless GGML_HRX_POISON_SELFTEST=1 From af49ced0e569fdd68ca21fe802113715922ab0d6 Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 9 Nov 2026 10:48:42 -0400 Subject: [PATCH 2/3] hrx: an allocation failure leaves the backend as ALLOC_FAILED, and the server says out of memory (fixes engine #1618) #1618 was filed as "llama-server reports 'Compute error' when a host-memory device buffer allocation fails on a Flash-Next request, so a resource failure reads as a kernel fault", and it stayed misdiagnosed for a day. The log had the answer the client never saw: al...: RESOURCE_EXHAUSTED; [hsa_amd_memory_pool_allocate] HSA_STATUS_ERROR_OUT_OF_RESOURCES while the client got "Compute error." - because two seams each collapsed the distinction: 1. `ggml-hrx` never returned GGML_STATUS_ALLOC_FAILED (0 hits over ggml/src/ggml-hrx/), so every failure left `GraphExecutionResult`'s default GGML_STATUS_FAILED. llama.cpp already distinguishes them - llama-context.cpp maps ALLOC_FAILED to llama_decode's -2 and FAILED to -3 - and the distinction was simply never produced. 2. `tools/server/server-context.cpp` collapsed everything with `ret < -1` into "Compute error.", so -2 (no memory) and -3 (a real compute failure) reached the client as the same sentence. So: graph_compute() now classifies its failure reasons (runtime/failure-status.*), and the server prints an out-of-memory sentence naming the way out for -2, keeping "Compute error." for everything else. The classification is text-based on purpose. The code lives in libhrx's iree_status_t, and `status.h` keeps only the rendered messages by the time a failure reaches this backend, so the rule matches iree's own status name (RESOURCE_EXHAUSTED) and HSA's error name (OUT_OF_RESOURCES) - strings that survive rewording of the prose around them. Both directions are pinned by the test, because the regression that matters is the other one: a kernel error reported as "out of memory" would send the reader to free memory that was never the problem. Proof: tests/test-hrx-failure-status.cpp is pure (no device, no runtime library) and drives the issue's own log line, each needle alone, a multi-message list, two genuine compute failures, and the empty/fallback cases. All pass. ggml-hrx.cpp and server-context.cpp compile. scope: the status the backend returns and the sentence the server prints. No change to which allocations are attempted, no change to kernels, and it does not make the 89 GB model fit - it makes the failure say what it is. Stacked on #260 (the pin does not compile without it). --- ggml/src/ggml-hrx/CMakeLists.txt | 1 + ggml/src/ggml-hrx/ggml-hrx.cpp | 9 ++- ggml/src/ggml-hrx/runtime/failure-status.cpp | 31 +++++++++ ggml/src/ggml-hrx/runtime/failure-status.h | 27 ++++++++ tests/CMakeLists.txt | 6 ++ tests/test-hrx-failure-status.cpp | 73 ++++++++++++++++++++ tools/server/server-context.cpp | 10 ++- 7 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 ggml/src/ggml-hrx/runtime/failure-status.cpp create mode 100644 ggml/src/ggml-hrx/runtime/failure-status.h create mode 100644 tests/test-hrx-failure-status.cpp diff --git a/ggml/src/ggml-hrx/CMakeLists.txt b/ggml/src/ggml-hrx/CMakeLists.txt index 30228641d990..68b538b3805e 100644 --- a/ggml/src/ggml-hrx/CMakeLists.txt +++ b/ggml/src/ggml-hrx/CMakeLists.txt @@ -367,6 +367,7 @@ ggml_add_backend_library(ggml-hrx runtime/host-memory.cpp runtime/host-memory.h runtime/host-memory-available.cpp + runtime/failure-status.cpp runtime/host-memory-available.h runtime/host-memory-ternary.cpp runtime/host-memory-ternary.h diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index b45dd2abe0d6..3bd4be0af2dd 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -2,6 +2,7 @@ #include "dispatch_registration/common/moe-placement-guard.h" #include "ggml-hrx.h" #include "hrx-1bit-safety.h" +#include "runtime/failure-status.h" #include "ggml-hrx-dmabuf.h" #include "backend-buffer-binding.h" @@ -803,6 +804,12 @@ static enum ggml_status graph_compute(ggml_backend_t backend, ggml_cgraph * grap auto * context = static_cast(backend->context); const ggml::hrx::GraphExecutor executor = ggml::hrx::GraphExecutor(*context); const ggml::hrx::GraphExecutionResult result = executor.execute(*graph); + // engine #1618: a resource failure must not leave this backend as a generic failure. llama.cpp maps ALLOC_FAILED to + // its own -2 and FAILED to -3, and the server prints one sentence for "out of memory" and another for "Compute error" + // - so without this classification a host-memory allocation the pool could not serve reads to the client as a kernel + // fault, which is what made #1618 look like a flash-attention defect for a day. + enum ggml_status code = result.success() ? GGML_STATUS_SUCCESS + : ggml::hrx::failure_status(result.status.errors(), result.code); if (!result.success()) { // Print the LAST reason: a matcher that declined also logs why, and those lines come first, so // errors().front() named a matcher while the graph failed on the node left without a dispatch @@ -822,7 +829,7 @@ static enum ggml_status graph_compute(ggml_backend_t backend, ggml_cgraph * grap } } } - return result.code; + return code; } static const ggml_backend_i backend_i = { diff --git a/ggml/src/ggml-hrx/runtime/failure-status.cpp b/ggml/src/ggml-hrx/runtime/failure-status.cpp new file mode 100644 index 000000000000..c98cc5195027 --- /dev/null +++ b/ggml/src/ggml-hrx/runtime/failure-status.cpp @@ -0,0 +1,31 @@ +#include "runtime/failure-status.h" + +#include + +namespace ggml::hrx { + +namespace { + +// iree's status name for a pool that cannot serve the request, and HSA's for the same condition one layer down. Both appear +// verbatim in the failures this exists for: +// al...: RESOURCE_EXHAUSTED; [hsa_amd_memory_pool_allocate] HSA_STATUS_ERROR_OUT_OF_RESOURCES +constexpr const char * kResourceNeedles[] = { "RESOURCE_EXHAUSTED", "OUT_OF_RESOURCES" }; + +} // namespace + +bool failure_looks_like_allocation(const std::vector & messages) { + for (const std::string & message : messages) { + for (const char * needle : kResourceNeedles) { + if (std::strstr(message.c_str(), needle) != nullptr) { + return true; + } + } + } + return false; +} + +enum ggml_status failure_status(const std::vector & messages, enum ggml_status fallback) { + return failure_looks_like_allocation(messages) ? GGML_STATUS_ALLOC_FAILED : fallback; +} + +} // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/runtime/failure-status.h b/ggml/src/ggml-hrx/runtime/failure-status.h new file mode 100644 index 000000000000..f544a36b9800 --- /dev/null +++ b/ggml/src/ggml-hrx/runtime/failure-status.h @@ -0,0 +1,27 @@ +#pragma once + +#include "ggml.h" + +#include +#include + +namespace ggml::hrx { + +// Whether a backend failure is a *resource* failure rather than a compute one (engine #1618). +// +// The distinction matters outside this backend: `src/llama-context.cpp` maps GGML_STATUS_ALLOC_FAILED to llama_decode's -2 +// and GGML_STATUS_FAILED to -3, and the server prints a different sentence for each. Before this, every HRX failure left +// `GraphExecutionResult`'s default GGML_STATUS_FAILED, so a host-memory allocation the pool could not serve reached the client +// as "Compute error." - i.e. a resource failure read as a kernel fault, which is how #1618 was filed and how it stayed +// misdiagnosed for a day (the log said HSA_STATUS_ERROR_OUT_OF_RESOURCES while the client said "Compute error"). +// +// Text-based on purpose: the code lives in libhrx's `iree_status_t`, and `status.h` keeps only the rendered messages by the +// time a failure reaches this backend. The two strings matched below are iree's own status name and HSA's error name, so they +// survive rewording of the prose around them. +bool failure_looks_like_allocation(const std::vector & messages); + +// The ggml status to return for these failure messages: GGML_STATUS_ALLOC_FAILED when they describe a resource failure, +// |fallback| otherwise (so a caller that already has a code keeps it). +enum ggml_status failure_status(const std::vector & messages, enum ggml_status fallback = GGML_STATUS_FAILED); + +} // namespace ggml::hrx diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c592e6b41034..e16fd93ff651 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -356,6 +356,12 @@ if (TARGET ggml-hrx) add_test(NAME test-hrx-graph-program-budget COMMAND test-hrx-graph-program-budget) # the host-weight arena refusal's wording: pure helper, no device and no HRX link (engine #900) + # the backend's failure classification: resource failure vs compute failure, pure (engine #1618) + add_executable(test-hrx-failure-status test-hrx-failure-status.cpp + ../ggml/src/ggml-hrx/runtime/failure-status.cpp) + target_include_directories(test-hrx-failure-status PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) + add_test(NAME test-hrx-failure-status COMMAND test-hrx-failure-status) + add_executable(test-hrx-pool-refusal-budget test-hrx-pool-refusal-budget.cpp ../ggml/src/ggml-hrx/hrx-1bit-safety.cpp) target_include_directories(test-hrx-pool-refusal-budget PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) diff --git a/tests/test-hrx-failure-status.cpp b/tests/test-hrx-failure-status.cpp new file mode 100644 index 000000000000..6e9640f404d7 --- /dev/null +++ b/tests/test-hrx-failure-status.cpp @@ -0,0 +1,73 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Host test for failure-status.h: whether a backend failure is a resource failure (engine #1618). Pure - no device, no +// runtime library - because the rule is a function of the messages libhrx renders. +// +// The case this exists for: a Flash-Next decode with flash attention on failed with +// `al...: RESOURCE_EXHAUSTED; [hsa_amd_memory_pool_allocate] HSA_STATUS_ERROR_OUT_OF_RESOURCES` while materializing a host +// weight, and the client was told "Compute error." - a resource failure served as a kernel fault. + +#include "runtime/failure-status.h" + +#include +#include +#include +#include + +#define REQUIRE(condition) \ + do { \ + if (!(condition)) { \ + std::fprintf(stderr, "%s:%d: requirement failed: %s\n", __FILE__, __LINE__, #condition); \ + std::abort(); \ + } \ + } while (false) + +using ggml::hrx::failure_looks_like_allocation; +using ggml::hrx::failure_status; + +int main() { + // The issue's own line, verbatim from the log that was misread. + const std::vector pool_exhausted = { + "allocate host memory device buffer: hrx-system/runtime/src/iree/hal/drivers/amdgpu/slab_provider.c:663: " + "RESOURCE_EXHAUSTED; [hsa_amd_memory_pool_allocate] HSA_STATUS_ERROR_OUT_OF_RESOURCES", + }; + REQUIRE(failure_looks_like_allocation(pool_exhausted)); + REQUIRE(failure_status(pool_exhausted) == GGML_STATUS_ALLOC_FAILED); + + // HSA's name alone is enough: libhrx renders both, and a caller may see only one of them. + REQUIRE(failure_looks_like_allocation({ "materialize host weight value 3 failed: HSA_STATUS_ERROR_OUT_OF_RESOURCES" })); + REQUIRE(failure_status({ "x: RESOURCE_EXHAUSTED" }) == GGML_STATUS_ALLOC_FAILED); + + // A real compute failure must stay a compute failure: this is the direction that would be a regression, because a + // kernel error reported as "out of memory" sends the reader to free memory that was never the problem. + const std::vector kernel_fault = { + "execute HRX command program failed: compute status: -1", + "hip.qsa_attn_select rejected node 20: unsupported layout", + }; + REQUIRE(!failure_looks_like_allocation(kernel_fault)); + REQUIRE(failure_status(kernel_fault) == GGML_STATUS_FAILED); + + // Empty reasons keep the caller's code rather than inventing one. + REQUIRE(!failure_looks_like_allocation({})); + REQUIRE(failure_status({}, GGML_STATUS_FAILED) == GGML_STATUS_FAILED); + REQUIRE(failure_status({}, GGML_STATUS_ABORTED) == GGML_STATUS_ABORTED); + + // Any message in the list counts, not just the last: graph_compute prints the last reason but classifies the whole set. + REQUIRE(failure_looks_like_allocation({ "plan rejected: operand layout", "x: RESOURCE_EXHAUSTED" })); + + std::printf("test-hrx-failure-status: all requirements passed\n"); + return 0; +} diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index fa74ed5c1b28..ec32d5898f39 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3856,7 +3856,15 @@ struct server_context_impl { err = "Invalid input batch."; } - if (ret < -1) { + if (ret == -2) { + // llama_decode's -2 is GGML_STATUS_ALLOC_FAILED: the backend could not get the memory a node needed + // (engine #1618). Saying "Compute error" here is what made a pool exhaustion read as a kernel fault + // to every client, so name the resource and the way out. + err = "Out of memory: the device could not allocate the memory this step needs. Free the device " + "(another process's resident model, a smaller -c/-ngl, or a smaller -ub) and retry."; + } + + if (ret < -1 && ret != -2) { // TODO: update slot state based on llama_memory_seq_pos_min() and llama_memory_seq_pos_max() err = "Compute error."; } From d69678a7a73dfd0a22ee675d5e22330e14f94e6d Mon Sep 17 00:00:00 2001 From: agent Date: Mon, 9 Nov 2026 11:26:09 -0400 Subject: [PATCH 3/3] hrx: the log names the reason the status was classified by, and the uncovered spellings are visible (review of #261) Three points from the review, all of them taken. 1. The status and the log disagreed in miniature. graph_compute() classifies over the WHOLE reasons set but printed only the last reason, so a resource-caused failure whose last logged line was a matcher decline returned ALLOC_FAILED while the log's headline pointed at a dispatch decision - the same "the log knew, the client did not" split this PR exists to remove, one layer down. The classifying message is now printed as the headline when it is not the last one, and the rest stays as context. 2. The fallback was silent, so the next unlisted spelling would cost another day. `failure_unmatched_allocation()` names a message that mentions memory or allocation without matching a needle; graph_compute() logs it once, with "extend kResourceNeedles if this is an out-of-memory status". The classification itself is unchanged - a false OOM is worse than a missed one - but a miss is no longer invisible. The hint match is case-insensitive and the classification needles are not, deliberately: the needles are identifiers as iree and HSA render them, while the hint exists for spellings nobody has seen yet. The test caught that: `hipErrorOutOfMemory` is real hip and mixed case, and a case-sensitive hint would have missed exactly the case the probe was added for. 3. `if (ret < -1 && ret != -2)` is now `ret < -2`, and failure_status()'s default fallback is documented as what a caller with nothing better to say gets (graph_compute passes its own code; the test uses the default). Proof: test-hrx-failure-status passes, now including the uncovered case (`hipErrorOutOfMemory` stays FAILED and is named), the matched case producing no warning, a plain kernel fault staying quiet, and the empty set. The ggml-hrx and server translation units compile. --- ggml/src/ggml-hrx/ggml-hrx.cpp | 18 +++++++++++ ggml/src/ggml-hrx/runtime/failure-status.cpp | 32 ++++++++++++++++++++ ggml/src/ggml-hrx/runtime/failure-status.h | 10 +++++- tests/test-hrx-failure-status.cpp | 16 ++++++++++ tools/server/server-context.cpp | 2 +- 5 files changed, 76 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index 3bd4be0af2dd..bdaeda7eaf4a 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -816,6 +816,24 @@ static enum ggml_status graph_compute(ggml_backend_t backend, ggml_cgraph * grap // (2026-10-20 census: a q8_0 KV cache printed the qwen postprocess refusal for a graph that broke // on another node). The earlier reasons follow, bounded, for context. const std::vector & reasons = result.status.errors(); + // The classification above reads the WHOLE set, while the headline below prints the last reason. When the two differ, + // a resource failure would return ALLOC_FAILED while the log's first sentence points at a dispatch decision - the + // same "the log knew, the client did not" split in miniature (engine #1618 review), so print the classifying message + // as the headline and keep the rest as context. + if (const std::string * suspicious = ggml::hrx::failure_unmatched_allocation(reasons)) { + GGML_LOG_WARN("%s: an allocation-looking failure did not match the resource needles, so it is reported as a " + "compute failure: %s (extend kResourceNeedles if this is an out-of-memory status)\n", __func__, + suspicious->c_str()); + } + const std::string * classifying = nullptr; + for (const std::string & reason : reasons) { + if (ggml::hrx::failure_looks_like_allocation({ reason })) { + classifying = &reason; + } + } + if (classifying != nullptr && !reasons.empty() && classifying != &reasons.back()) { + GGML_LOG_ERROR("%s: %s\n", __func__, classifying->c_str()); + } if (reasons.empty()) { GGML_LOG_ERROR("%s: %s\n", __func__, status_first_error(result.status)); } else { diff --git a/ggml/src/ggml-hrx/runtime/failure-status.cpp b/ggml/src/ggml-hrx/runtime/failure-status.cpp index c98cc5195027..427d61883b45 100644 --- a/ggml/src/ggml-hrx/runtime/failure-status.cpp +++ b/ggml/src/ggml-hrx/runtime/failure-status.cpp @@ -1,5 +1,6 @@ #include "runtime/failure-status.h" +#include #include namespace ggml::hrx { @@ -11,8 +12,39 @@ namespace { // al...: RESOURCE_EXHAUSTED; [hsa_amd_memory_pool_allocate] HSA_STATUS_ERROR_OUT_OF_RESOURCES constexpr const char * kResourceNeedles[] = { "RESOURCE_EXHAUSTED", "OUT_OF_RESOURCES" }; +// Case-insensitive on purpose: the *classification* needles are exact because they are identifiers as iree and HSA render +// them, but the *hint* exists to catch spellings we have not seen -- `hipErrorOutOfMemory` is real hip and mixed case, and a +// case-sensitive match would have missed exactly the case this probe was added for (the test caught that). +constexpr const char * kAllocationHints[] = { "memory", "alloc" }; + +bool contains_any_of_ci(const std::string & message, const char * const * needles, size_t count) { + std::string lowered; + lowered.reserve(message.size()); + for (char c : message) { + lowered.push_back(static_cast(std::tolower(static_cast(c)))); + } + for (size_t i = 0; i < count; ++i) { + if (lowered.find(needles[i]) != std::string::npos) { + return true; + } + } + return false; +} + } // namespace +const std::string * failure_unmatched_allocation(const std::vector & messages) { + if (failure_looks_like_allocation(messages)) { + return nullptr; // it matched: nothing to warn about + } + for (const std::string & message : messages) { + if (contains_any_of_ci(message, kAllocationHints, sizeof(kAllocationHints) / sizeof(kAllocationHints[0]))) { + return &message; + } + } + return nullptr; +} + bool failure_looks_like_allocation(const std::vector & messages) { for (const std::string & message : messages) { for (const char * needle : kResourceNeedles) { diff --git a/ggml/src/ggml-hrx/runtime/failure-status.h b/ggml/src/ggml-hrx/runtime/failure-status.h index f544a36b9800..1b1f8c41abb3 100644 --- a/ggml/src/ggml-hrx/runtime/failure-status.h +++ b/ggml/src/ggml-hrx/runtime/failure-status.h @@ -20,8 +20,16 @@ namespace ggml::hrx { // survive rewording of the prose around them. bool failure_looks_like_allocation(const std::vector & messages); +// A message that *looks* like an allocation failure (it mentions memory or allocation) but matched neither needle, so it is +// about to be classified as a compute failure (engine #1618 review). Returns nullptr when nothing looks suspicious. This exists +// so the next spelling of an out-of-memory status -- `hipErrorOutOfMemory`, `HSA_STATUS_ERROR_INVALID_ALLOCATION`, whatever the +// runtime renders tomorrow -- costs a log line rather than another day of misdiagnosis: the fallback stays the safe direction +// (a false OOM is worse), but it stops being silent. +const std::string * failure_unmatched_allocation(const std::vector & messages); + // The ggml status to return for these failure messages: GGML_STATUS_ALLOC_FAILED when they describe a resource failure, -// |fallback| otherwise (so a caller that already has a code keeps it). +// |fallback| otherwise. graph_compute() passes its own code (so a caller that already has one keeps it); the default is what a +// caller with nothing better to say gets, and it is what the test exercises for the preserved-code case. enum ggml_status failure_status(const std::vector & messages, enum ggml_status fallback = GGML_STATUS_FAILED); } // namespace ggml::hrx diff --git a/tests/test-hrx-failure-status.cpp b/tests/test-hrx-failure-status.cpp index 6e9640f404d7..4e019cff68ee 100644 --- a/tests/test-hrx-failure-status.cpp +++ b/tests/test-hrx-failure-status.cpp @@ -68,6 +68,22 @@ int main() { // Any message in the list counts, not just the last: graph_compute prints the last reason but classifies the whole set. REQUIRE(failure_looks_like_allocation({ "plan rejected: operand layout", "x: RESOURCE_EXHAUSTED" })); + // The uncovered set is as visible as the covered one (review of #261): a hip OOM spelling that matches neither needle + // stays a compute failure -- the safe direction -- but it must be NAMED, so the next person gets a log line instead of a + // lost day. `hipErrorOutOfMemory` is a real hip name and matches neither needle on purpose. + using ggml::hrx::failure_unmatched_allocation; + const std::vector unmatched = { "hip.qsa_attn_select: hipErrorOutOfMemory" }; + REQUIRE(!failure_looks_like_allocation(unmatched)); + REQUIRE(failure_status(unmatched) == GGML_STATUS_FAILED); + const std::string * hint = failure_unmatched_allocation(unmatched); + REQUIRE(hint != nullptr && hint->find("hipErrorOutOfMemory") != std::string::npos); + + // ...and a matched failure produces no warning, so the signal means "uncovered", not "allocation-ish". + REQUIRE(failure_unmatched_allocation(pool_exhausted) == nullptr); + // A plain kernel fault is not allocation-looking and stays quiet. + REQUIRE(failure_unmatched_allocation(kernel_fault) == nullptr); + REQUIRE(failure_unmatched_allocation({}) == nullptr); + std::printf("test-hrx-failure-status: all requirements passed\n"); return 0; } diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index ec32d5898f39..3a19ac65fc72 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3864,7 +3864,7 @@ struct server_context_impl { "(another process's resident model, a smaller -c/-ngl, or a smaller -ub) and retry."; } - if (ret < -1 && ret != -2) { + if (ret < -2) { // TODO: update slot state based on llama_memory_seq_pos_min() and llama_memory_seq_pos_max() err = "Compute error."; }