hrx: an allocation failure leaves the backend as ALLOC_FAILED, and the server says out of memory (fixes engine #1618)

ggml/src/ggml-hrx/CMakeLists.txt

@@ -367,6 +367,7 @@ ggml_add_backend_library(ggml-hrx
     runtime/host-memory.cpp
     runtime/host-memory.h
     runtime/host-memory-available.cpp
+    runtime/failure-status.cpp
     runtime/host-memory-available.h
     runtime/host-memory-ternary.cpp
     runtime/host-memory-ternary.h

ggml/src/ggml-hrx/ggml-hrx.cpp

@@ -2,6 +2,7 @@
 #include "dispatch_registration/common/moe-placement-guard.h"
 #include "ggml-hrx.h"
 #include "hrx-1bit-safety.h"
+#include "runtime/failure-status.h"
 #include "runtime/host-memory-available.h"
 #include "ggml-hrx-dmabuf.h"
 
@@ -826,12 +827,36 @@ static enum ggml_status graph_compute(ggml_backend_t backend, ggml_cgraph * grap
     auto *                                context  = static_cast<ggml_backend_hrx_context *>(backend->context);
     const ggml::hrx::GraphExecutor        executor = ggml::hrx::GraphExecutor(*context);
     const ggml::hrx::GraphExecutionResult result   = executor.execute(*graph);
+    // engine #1618: a resource failure must not leave this backend as a generic failure. llama.cpp maps ALLOC_FAILED to
+    // its own -2 and FAILED to -3, and the server prints one sentence for "out of memory" and another for "Compute error"
+    // - so without this classification a host-memory allocation the pool could not serve reads to the client as a kernel
+    // fault, which is what made #1618 look like a flash-attention defect for a day.
+    enum ggml_status code = result.success() ? GGML_STATUS_SUCCESS
+                                             : ggml::hrx::failure_status(result.status.errors(), result.code);
     if (!result.success()) {
         // Print the LAST reason: a matcher that declined also logs why, and those lines come first, so
         // errors().front() named a matcher while the graph failed on the node left without a dispatch
         // (2026-10-20 census: a q8_0 KV cache printed the qwen postprocess refusal for a graph that broke
         // on another node). The earlier reasons follow, bounded, for context.
         const std::vector<std::string> & reasons = result.status.errors();
+        // The classification above reads the WHOLE set, while the headline below prints the last reason. When the two differ,
+        // a resource failure would return ALLOC_FAILED while the log's first sentence points at a dispatch decision - the
+        // same "the log knew, the client did not" split in miniature (engine #1618 review), so print the classifying message
+        // as the headline and keep the rest as context.
+        if (const std::string * suspicious = ggml::hrx::failure_unmatched_allocation(reasons)) {
+            GGML_LOG_WARN("%s: an allocation-looking failure did not match the resource needles, so it is reported as a "
+                          "compute failure: %s (extend kResourceNeedles if this is an out-of-memory status)\n", __func__,
+                          suspicious->c_str());
+        }
+        const std::string * classifying = nullptr;
+        for (const std::string & reason : reasons) {
+            if (ggml::hrx::failure_looks_like_allocation({ reason })) {
+                classifying = &reason;
+            }
+        }
+        if (classifying != nullptr && !reasons.empty() && classifying != &reasons.back()) {
+            GGML_LOG_ERROR("%s: %s\n", __func__, classifying->c_str());
+        }
         if (reasons.empty()) {
             GGML_LOG_ERROR("%s: %s\n", __func__, status_first_error(result.status));
         } else {
@@ -845,7 +870,7 @@ static enum ggml_status graph_compute(ggml_backend_t backend, ggml_cgraph * grap
             }
         }
     }
-    return result.code;
+    return code;
 }
 
 static const ggml_backend_i backend_i = {

ggml/src/ggml-hrx/runtime/failure-status.cpp

@@ -0,0 +1,63 @@
+#include "runtime/failure-status.h"
+
+#include <cctype>
+#include <cstring>
+
+namespace ggml::hrx {
+
+namespace {
+
+// iree's status name for a pool that cannot serve the request, and HSA's for the same condition one layer down. Both appear
+// verbatim in the failures this exists for:
+//   al...: RESOURCE_EXHAUSTED; [hsa_amd_memory_pool_allocate] HSA_STATUS_ERROR_OUT_OF_RESOURCES
+constexpr const char * kResourceNeedles[] = { "RESOURCE_EXHAUSTED", "OUT_OF_RESOURCES" };
+
+// Case-insensitive on purpose: the *classification* needles are exact because they are identifiers as iree and HSA render
+// them, but the *hint* exists to catch spellings we have not seen -- `hipErrorOutOfMemory` is real hip and mixed case, and a
+// case-sensitive match would have missed exactly the case this probe was added for (the test caught that).
+constexpr const char * kAllocationHints[] = { "memory", "alloc" };
+
+bool contains_any_of_ci(const std::string & message, const char * const * needles, size_t count) {
+    std::string lowered;
+    lowered.reserve(message.size());
+    for (char c : message) {
+        lowered.push_back(static_cast<char>(std::tolower(static_cast<unsigned char>(c))));
+    }
+    for (size_t i = 0; i < count; ++i) {
+        if (lowered.find(needles[i]) != std::string::npos) {
+            return true;
+        }
+    }
+    return false;
+}
+
+}  // namespace
+
+const std::string * failure_unmatched_allocation(const std::vector<std::string> & messages) {
+    if (failure_looks_like_allocation(messages)) {
+        return nullptr;  // it matched: nothing to warn about
+    }
+    for (const std::string & message : messages) {
+        if (contains_any_of_ci(message, kAllocationHints, sizeof(kAllocationHints) / sizeof(kAllocationHints[0]))) {
+            return &message;
+        }
+    }
+    return nullptr;
+}
+
+bool failure_looks_like_allocation(const std::vector<std::string> & messages) {
+    for (const std::string & message : messages) {
+        for (const char * needle : kResourceNeedles) {
+            if (std::strstr(message.c_str(), needle) != nullptr) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+enum ggml_status failure_status(const std::vector<std::string> & messages, enum ggml_status fallback) {
+    return failure_looks_like_allocation(messages) ? GGML_STATUS_ALLOC_FAILED : fallback;
+}
+
+}  // namespace ggml::hrx

ggml/src/ggml-hrx/runtime/failure-status.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "ggml.h"
+
+#include <string>
+#include <vector>
+
+namespace ggml::hrx {
+
+// Whether a backend failure is a *resource* failure rather than a compute one (engine #1618).
+//
+// The distinction matters outside this backend: `src/llama-context.cpp` maps GGML_STATUS_ALLOC_FAILED to llama_decode's -2
+// and GGML_STATUS_FAILED to -3, and the server prints a different sentence for each. Before this, every HRX failure left
+// `GraphExecutionResult`'s default GGML_STATUS_FAILED, so a host-memory allocation the pool could not serve reached the client
+// as "Compute error." - i.e. a resource failure read as a kernel fault, which is how #1618 was filed and how it stayed
+// misdiagnosed for a day (the log said HSA_STATUS_ERROR_OUT_OF_RESOURCES while the client said "Compute error").
+//
+// Text-based on purpose: the code lives in libhrx's `iree_status_t`, and `status.h` keeps only the rendered messages by the
+// time a failure reaches this backend. The two strings matched below are iree's own status name and HSA's error name, so they
+// survive rewording of the prose around them.
+bool failure_looks_like_allocation(const std::vector<std::string> & messages);
+
+// A message that *looks* like an allocation failure (it mentions memory or allocation) but matched neither needle, so it is
+// about to be classified as a compute failure (engine #1618 review). Returns nullptr when nothing looks suspicious. This exists
+// so the next spelling of an out-of-memory status -- `hipErrorOutOfMemory`, `HSA_STATUS_ERROR_INVALID_ALLOCATION`, whatever the
+// runtime renders tomorrow -- costs a log line rather than another day of misdiagnosis: the fallback stays the safe direction
+// (a false OOM is worse), but it stops being silent.
+const std::string * failure_unmatched_allocation(const std::vector<std::string> & messages);
+
+// The ggml status to return for these failure messages: GGML_STATUS_ALLOC_FAILED when they describe a resource failure,
+// |fallback| otherwise. graph_compute() passes its own code (so a caller that already has one keeps it); the default is what a
+// caller with nothing better to say gets, and it is what the test exercises for the preserved-code case.
+enum ggml_status failure_status(const std::vector<std::string> & messages, enum ggml_status fallback = GGML_STATUS_FAILED);
+
+}  // namespace ggml::hrx

tests/CMakeLists.txt

@@ -363,6 +363,12 @@ if (TARGET ggml-hrx)
     add_test(NAME test-hrx-graph-program-budget COMMAND test-hrx-graph-program-budget)
 
     # the host-weight arena refusal's wording: pure helper, no device and no HRX link (engine #900)
+    # the backend's failure classification: resource failure vs compute failure, pure (engine #1618)
+    add_executable(test-hrx-failure-status test-hrx-failure-status.cpp
+        ../ggml/src/ggml-hrx/runtime/failure-status.cpp)
+    target_include_directories(test-hrx-failure-status PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)
+    add_test(NAME test-hrx-failure-status COMMAND test-hrx-failure-status)
+
     add_executable(test-hrx-pool-refusal-budget test-hrx-pool-refusal-budget.cpp
                    ../ggml/src/ggml-hrx/hrx-1bit-safety.cpp)
     target_include_directories(test-hrx-pool-refusal-budget PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)

tests/test-hrx-failure-status.cpp

@@ -0,0 +1,89 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Host test for failure-status.h: whether a backend failure is a resource failure (engine #1618). Pure - no device, no
+// runtime library - because the rule is a function of the messages libhrx renders.
+//
+// The case this exists for: a Flash-Next decode with flash attention on failed with
+// `al...: RESOURCE_EXHAUSTED; [hsa_amd_memory_pool_allocate] HSA_STATUS_ERROR_OUT_OF_RESOURCES` while materializing a host
+// weight, and the client was told "Compute error." - a resource failure served as a kernel fault.
+
+#include "runtime/failure-status.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+#include <vector>
+
+#define REQUIRE(condition)                                                                           \
+    do {                                                                                             \
+        if (!(condition)) {                                                                          \
+            std::fprintf(stderr, "%s:%d: requirement failed: %s\n", __FILE__, __LINE__, #condition); \
+            std::abort();                                                                            \
+        }                                                                                            \
+    } while (false)
+
+using ggml::hrx::failure_looks_like_allocation;
+using ggml::hrx::failure_status;
+
+int main() {
+    // The issue's own line, verbatim from the log that was misread.
+    const std::vector<std::string> pool_exhausted = {
+        "allocate host memory device buffer: hrx-system/runtime/src/iree/hal/drivers/amdgpu/slab_provider.c:663: "
+        "RESOURCE_EXHAUSTED; [hsa_amd_memory_pool_allocate] HSA_STATUS_ERROR_OUT_OF_RESOURCES",
+    };
+    REQUIRE(failure_looks_like_allocation(pool_exhausted));
+    REQUIRE(failure_status(pool_exhausted) == GGML_STATUS_ALLOC_FAILED);
+
+    // HSA's name alone is enough: libhrx renders both, and a caller may see only one of them.
+    REQUIRE(failure_looks_like_allocation({ "materialize host weight value 3 failed: HSA_STATUS_ERROR_OUT_OF_RESOURCES" }));
+    REQUIRE(failure_status({ "x: RESOURCE_EXHAUSTED" }) == GGML_STATUS_ALLOC_FAILED);
+
+    // A real compute failure must stay a compute failure: this is the direction that would be a regression, because a
+    // kernel error reported as "out of memory" sends the reader to free memory that was never the problem.
+    const std::vector<std::string> kernel_fault = {
+        "execute HRX command program failed: compute status: -1",
+        "hip.qsa_attn_select rejected node 20: unsupported layout",
+    };
+    REQUIRE(!failure_looks_like_allocation(kernel_fault));
+    REQUIRE(failure_status(kernel_fault) == GGML_STATUS_FAILED);
+
+    // Empty reasons keep the caller's code rather than inventing one.
+    REQUIRE(!failure_looks_like_allocation({}));
+    REQUIRE(failure_status({}, GGML_STATUS_FAILED) == GGML_STATUS_FAILED);
+    REQUIRE(failure_status({}, GGML_STATUS_ABORTED) == GGML_STATUS_ABORTED);
+
+    // Any message in the list counts, not just the last: graph_compute prints the last reason but classifies the whole set.
+    REQUIRE(failure_looks_like_allocation({ "plan rejected: operand layout", "x: RESOURCE_EXHAUSTED" }));
+
+    // The uncovered set is as visible as the covered one (review of #261): a hip OOM spelling that matches neither needle
+    // stays a compute failure -- the safe direction -- but it must be NAMED, so the next person gets a log line instead of a
+    // lost day. `hipErrorOutOfMemory` is a real hip name and matches neither needle on purpose.
+    using ggml::hrx::failure_unmatched_allocation;
+    const std::vector<std::string> unmatched = { "hip.qsa_attn_select: hipErrorOutOfMemory" };
+    REQUIRE(!failure_looks_like_allocation(unmatched));
+    REQUIRE(failure_status(unmatched) == GGML_STATUS_FAILED);
+    const std::string * hint = failure_unmatched_allocation(unmatched);
+    REQUIRE(hint != nullptr && hint->find("hipErrorOutOfMemory") != std::string::npos);
+
+    // ...and a matched failure produces no warning, so the signal means "uncovered", not "allocation-ish".
+    REQUIRE(failure_unmatched_allocation(pool_exhausted) == nullptr);
+    // A plain kernel fault is not allocation-looking and stays quiet.
+    REQUIRE(failure_unmatched_allocation(kernel_fault) == nullptr);
+    REQUIRE(failure_unmatched_allocation({}) == nullptr);
+
+    std::printf("test-hrx-failure-status: all requirements passed\n");
+    return 0;
+}

tools/server/server-context.cpp

@@ -3856,7 +3856,15 @@ struct server_context_impl {
                     err = "Invalid input batch.";
                 }
 
-                if (ret < -1) {
+                if (ret == -2) {
+                    // llama_decode's -2 is GGML_STATUS_ALLOC_FAILED: the backend could not get the memory a node needed
+                    // (engine #1618). Saying "Compute error" here is what made a pool exhaustion read as a kernel fault
+                    // to every client, so name the resource and the way out.
+                    err = "Out of memory: the device could not allocate the memory this step needs. Free the device "
+                          "(another process's resident model, a smaller -c/-ngl, or a smaller -ub) and retry.";
+                }
+
+                if (ret < -2) {
                     // TODO: update slot state based on llama_memory_seq_pos_min() and llama_memory_seq_pos_max()
                     err = "Compute error.";
                 }