From 18568c7c62b0e83efa3f324683a0f62903700cfe Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 17:05:07 -0400 Subject: [PATCH] ggml: a failed graph compute is reported as one by the backend comparison, not scored (engine #1997) ggml_backend_compare_graph_backend ignored the status of both computes. A backend that refuses or fails a graph leaves its outputs unwritten, and the callback then compared those against the other backend: an HRX program refused for an unflagged weight came out of test-backend-ops as "ERR = 1358" - k/3 + 1 against the output tensor's random fill - which reads like a kernel fault. The helper now says which backend failed, with its status, compares nothing, and returns false. Co-Authored-By: Claude Fable 5.2 --- ggml/src/ggml-backend.cpp | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 7b7c203dcaad..72fa7b987547 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -2235,8 +2235,17 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t if (num_test_nodes != 0) { GGML_ASSERT(test_nodes); // Compute the whole graph and only test the output for specific tensors - ggml_backend_graph_compute(backend1, g1); - ggml_backend_graph_compute(backend2, g2); + // engine #1997: a backend that refuses or fails the graph leaves its outputs unwritten, and comparing those + // against the other backend reports a refusal as a numerical error (an HRX program refused for an unflagged + // weight scored "ERR = k/3 + 1" against whatever the output tensor happened to hold). Say which backend failed. + const enum ggml_status status1 = ggml_backend_graph_compute(backend1, g1); + const enum ggml_status status2 = ggml_backend_graph_compute(backend2, g2); + if (status1 != GGML_STATUS_SUCCESS || status2 != GGML_STATUS_SUCCESS) { + GGML_LOG_ERROR("%s: graph compute failed (%s: status %d, %s: status %d); nothing was compared\n", __func__, + ggml_backend_name(backend1), (int) status1, ggml_backend_name(backend2), (int) status2); + ggml_backend_graph_copy_free(copy); + return false; + } bool verified = false; for (int i = 0; i < g1->n_nodes; i++) { @@ -2258,8 +2267,16 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t struct ggml_cgraph g1v = ggml_graph_view(g1, i, i + 1); struct ggml_cgraph g2v = ggml_graph_view(g2, i, i + 1); - ggml_backend_graph_compute(backend1, &g1v); - ggml_backend_graph_compute(backend2, &g2v); + const enum ggml_status status1 = ggml_backend_graph_compute(backend1, &g1v); + const enum ggml_status status2 = ggml_backend_graph_compute(backend2, &g2v); + if (status1 != GGML_STATUS_SUCCESS || status2 != GGML_STATUS_SUCCESS) { + // engine #1997: see above - an unwritten output is not a result to compare + GGML_LOG_ERROR("%s: graph compute failed at node %d (%s) (%s: status %d, %s: status %d); nothing was " + "compared\n", __func__, i, ggml_op_name(t1->op), ggml_backend_name(backend1), (int) status1, + ggml_backend_name(backend2), (int) status2); + ggml_backend_graph_copy_free(copy); + return false; + } if (ggml_is_view_op(t1->op)) { continue;