Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions ggml/src/ggml-hrx/dispatch/dispatch-scheduler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,20 @@ static std::string unsupported_node_message(const Graph & graph, size_t index, c
append_node_summary(stream, graph, consumers[i]);
}
stream << "]";
// A layout-alias node is refused here for one of two completely different reasons, and the shape cannot tell them
// apart: it is an alias (one input, same storage) whose input was not available yet, or it is NOT an alias at all
// (a strided view of the same shape) and therefore needs a dispatch that no registration provides, because no
// registration names a layout-alias op as its root. engine #1467's projector is refused at
// VIEW output=f32[2048,2,1,1] <- f32[2048,2,1,1]
// and that line alone did not say which case it was, so the next reader had to reproduce it. This costs one line
// in the message and answers it in the log.
if (is_layout_alias_op(node.op)) {
// is_layout_alias_node() is this predicate exactly (graph/graph.cpp): a message that restates the scheduler's rule
// is a message that will describe a scheduler that no longer exists. Same reasoning as rope-claim-params.h sharing
// its predicates with the matchers in #257.
stream << " alias=" << (is_layout_alias_node(graph, node) ? "same_storage" : "not_aliasing_input")
<< " inputs=" << node.inputs.size();
}
return stream.str();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

#include "dispatch-layout-utils.h"
#include "dispatch-rope-utils.h"
#include "rope-claim-params.h"
#include "ggml.h"
#include "qwen4exp-hrx-support.h"
#include "graph/graph-matcher.h"
Expand Down Expand Up @@ -129,12 +130,10 @@ static bool is_supported_rope_input_layout(const Value & value, size_t & span_el
return required <= span_elements;
}

// The condition itself lives in ggml/src/ggml-hrx/rope-claim-params.h, so the claim in
// fused-context-claim.h cannot drift away from what this matcher accepts (engine #1467).
static bool is_supported_rope_params(const RopeParams & params, int64_t head_size) {
const bool supported_mode = params.mode == GGML_ROPE_TYPE_NORMAL || params.mode == GGML_ROPE_TYPE_NEOX ||
rope_mode_is_collapsible_imrope(params);
return params.n_dims >= 4 && params.n_dims <= head_size && params.n_dims % 4 == 0 && supported_mode &&
std::isfinite(params.freq_base) && params.freq_base > 0.0f && std::isfinite(params.freq_scale) &&
params.freq_scale > 0.0f && std::isfinite(params.ext_factor) && std::isfinite(params.attn_factor);
return rope_params_supported_by_rope_set_rows(params, head_size);
}

static bool build_rope_theta_table(const GraphNode & rope,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "ggml.h"
#include "graph/graph-matcher.h"
#include "kernel-corpus/kernel-corpus-catalog-verify.h"
#include "rope-claim-params.h"

#include <cmath>
#include <cstdint>
Expand Down Expand Up @@ -104,10 +105,10 @@ static std::string to_config_value(int64_t value) {
return std::to_string(value);
}

// The condition itself lives in ggml/src/ggml-hrx/rope-claim-params.h, because the claim in
// fused-context-claim.h has to assert exactly what this matcher accepts (engine #1467).
static bool is_supported_attention_rope_params(const RopeParams & params, int64_t head_size) {
return params.n_dims == head_size && params.mode == GGML_ROPE_TYPE_NORMAL && std::isfinite(params.freq_base) &&
params.freq_base > 0.0f && std::isfinite(params.freq_scale) && params.freq_scale > 0.0f &&
params.ext_factor == 0.0f && params.attn_factor == 1.0f;
return rope_params_supported_by_attention_fusion(params, head_size);
}

static bool build_rope_theta_table(const GraphNode & rope, int64_t head_size, std::vector<uint8_t> & data) {
Expand Down
19 changes: 18 additions & 1 deletion ggml/src/ggml-hrx/fused-context-claim.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
// standalone test.

#include "ggml.h"
#include "rope-claim-params.h"
#include "ubatch-token-limits.h"

#include <cstring>
Expand Down Expand Up @@ -155,8 +156,24 @@ inline bool fused_context_claim(const ggml_tensor * op) {
op->ne[2] <= 3 && op->ne[0] >= 32 && op->ne[0] <= 65536 && op->ne[0] % 32 == 0;
// per-head attention normalisation and rotation
case GGML_OP_RMS_NORM:
case GGML_OP_ROPE:
return fused_context_is_head_row(op) && fused_context_fed_by_op(op);
case GGML_OP_ROPE: {
// A head-row rotation is claimed only when some fused registration accepts its PARAMETERS as well as
// its shape: a claim that is broader than the matchers leaves the node with no dispatch, and the
// whole graph fails instead of the node falling back to the CPU. engine #1467 is that failure: the
// Qwen3-VL projector's ROPE (n_dims 32 against a 64-wide head, mode GGML_ROPE_TYPE_VISION) is a
// head row fed by an op, so it was claimed, and both matchers then rejected it - "ROPE matcher
// rejected node: ROPE params or positions shape is unsupported" - which made clip_image_batch_encode
// fail on HRX0 at chunk 25 instead of running that node on the CPU. The predicates are shared with the
// matchers (rope-claim-params.h) rather than restated here, because restating them is what broke text
// decode twice: text rotations satisfy the attention fusion's set, not the ROPE+SET_ROWS one.
if (!fused_context_is_head_row(op) || !fused_context_fed_by_op(op)) {
return false;
}
const OpParams imported = import_op_params(*op);
const RopeParams * params = op_params_as<RopeParams>(imported);
return params != nullptr && rope_params_supported_by_any_fusion(*params, op->ne[0]);
}
default:
return false;
}
Expand Down
69 changes: 69 additions & 0 deletions ggml/src/ggml-hrx/rope-claim-params.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once

// Which ROPE parameter sets can a fused registration actually execute?
//
// The claim in `fused-context-claim.h` says "this node will run inside a fused dispatch", and
// `device_supports_op` uses that to keep the node on this backend. If the claim is broader than the
// matchers, the node is claimed and then refused when the dispatch is built, and the whole graph
// fails instead of the node falling back to the CPU - engine #1467: a vision projector's ROPE
// (`n_dims = 32`, `mode = GGML_ROPE_TYPE_VISION`) was claimed by the head-row rule and then rejected
// by both matchers, so `clip_image_batch_encode` could not compute at all on HRX0.
//
// Two matchers accept rotations and their parameter sets differ:
//
// * the attention fusion (llm/dispatch-attention-qkv.cpp): n_dims == head_size, mode NORMAL;
// * the ROPE + SET_ROWS registration (common/dispatch-rope-set-rows.cpp): 4 <= n_dims <= head_size,
// n_dims a multiple of 4, mode NORMAL / NEOX / a collapsible IMROPE.
//
// Their union is what the claim may assert. Both predicates live here, the two matchers call them and
// the claim calls the union, so a matcher can be narrowed without the claim silently going stale -
// which is how the two earlier attempts at this fix broke text decode: they re-derived the condition
// and used only the SET_ROWS one, which text rotations do not satisfy (they run inside the attention
// fusion).

#include "graph/op-params.h"
#include "qwen4exp-hrx-support.h" // rope_mode_is_collapsible_imrope

#include <cmath>

namespace ggml::hrx {

// llm/dispatch-attention-qkv.cpp: the per-head rotation consumed by the attention fusions.
inline bool rope_params_supported_by_attention_fusion(const RopeParams & params, int64_t head_size) {
return params.n_dims == head_size && params.mode == GGML_ROPE_TYPE_NORMAL && std::isfinite(params.freq_base) &&
params.freq_base > 0.0f && std::isfinite(params.freq_scale) && params.freq_scale > 0.0f &&
params.ext_factor == 0.0f && params.attn_factor == 1.0f;
}

// common/dispatch-rope-set-rows.cpp: the rotation that registration builds a theta table for.
inline bool rope_params_supported_by_rope_set_rows(const RopeParams & params, int64_t head_size) {
const bool supported_mode = params.mode == GGML_ROPE_TYPE_NORMAL || params.mode == GGML_ROPE_TYPE_NEOX ||
rope_mode_is_collapsible_imrope(params);
return params.n_dims >= 4 && params.n_dims <= head_size && params.n_dims % 4 == 0 && supported_mode &&
std::isfinite(params.freq_base) && params.freq_base > 0.0f && std::isfinite(params.freq_scale) &&
params.freq_scale > 0.0f && std::isfinite(params.ext_factor) && std::isfinite(params.attn_factor);
}

// What the claim may assert: at least one fused registration accepts these parameters. Anything else
// (a vision rotation, a mode nobody implements) must fall back to the CPU rather than be claimed.
inline bool rope_params_supported_by_any_fusion(const RopeParams & params, int64_t head_size) {
return rope_params_supported_by_attention_fusion(params, head_size) ||
rope_params_supported_by_rope_set_rows(params, head_size);
}

} // namespace ggml::hrx
14 changes: 14 additions & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,13 @@ add_test(NAME test-server-health COMMAND test-server-health)

if (TARGET ggml-hrx)
# the program cache's byte budget and LRU order: pure policy, no device and no HRX link
# Which rotations the fused registrations accept, i.e. what the claim may assert (engine #1467).
# Header-only: no device, no model, and the vision rotation must be rejected here.
add_executable(test-hrx-rope-claim-params test-hrx-rope-claim-params.cpp)
target_include_directories(test-hrx-rope-claim-params PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)
add_test(NAME test-hrx-rope-claim-params COMMAND test-hrx-rope-claim-params)
target_link_libraries(test-hrx-rope-claim-params PRIVATE ggml)

add_executable(test-hrx-graph-program-budget test-hrx-graph-program-budget.cpp
../ggml/src/ggml-hrx/runtime/graph-program-budget.cpp)
target_include_directories(test-hrx-graph-program-budget PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)
Expand Down Expand Up @@ -392,6 +399,13 @@ if (TARGET ggml-hrx)
target_include_directories(test-hrx-loom-jit PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)
add_test(NAME test-hrx-loom-jit COMMAND test-hrx-loom-jit)

# How a layout-alias node is judged when its input is PRODUCED vs EXTERNAL: engine #1467's projector is
# refused at a no-op VIEW that the standalone oracle calls supported because it feeds the view from a leaf.
add_executable(test-hrx-view-alias-elision test-hrx-view-alias-elision.cpp)
target_link_libraries(test-hrx-view-alias-elision PRIVATE ggml-hrx ggml hrx::hrx loomc::loomc)
target_include_directories(test-hrx-view-alias-elision PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)
add_test(NAME test-hrx-view-alias-elision COMMAND test-hrx-view-alias-elision)

add_executable(test-hrx-ops test-hrx-ops.cpp)
target_link_libraries(test-hrx-ops PRIVATE ggml-hrx ggml hrx::hrx loomc::loomc)
target_include_directories(test-hrx-ops PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)
Expand Down
Loading