diff --git a/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-gated-delta-net.cpp b/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-gated-delta-net.cpp index d70fee16e761..e252ed518758 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-gated-delta-net.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-gated-delta-net.cpp @@ -86,6 +86,24 @@ static bool append_covered_node(const DispatchMatchContext & context, const Grap return append_covered_node_index_once(context.graph, context.covered_nodes, node, match.covered_nodes); } +// A node of the fused pattern that only renames storage (the state RESHAPE, the cache VIEW) has nothing to execute, and +// the scheduler may already have passed it: it visits nodes in dependency order, not index order, and elides a layout +// alias as soon as its input exists. The state RESHAPE's input is the recurrent-state gather, which depends on no +// other node of the layer, so in a graph where the layer's projections wait on earlier work the RESHAPE is visited +// (and elided) before the L2_NORM this pattern is rooted at. Refusing the match for that left the L2_NORM with no +// dispatch and failed the whole graph (engine #1925). Such a node needs no cover from this match; any other node that +// is already covered still refuses it. +static bool append_covered_pattern_node(const DispatchMatchContext & context, + const GraphNode * node, + DispatchMatch & match) { + size_t index = 0; + if (node != nullptr && context.graph.index().node_index(node, index) && index < context.covered_nodes.size() && + context.covered_nodes[index] && is_layout_alias_node(context.graph, *node)) { + return true; + } + return append_covered_node(context, node, match); +} + struct GatedDeltaNetMatch { std::vector covered; const Value * alpha_raw = nullptr; @@ -779,7 +797,7 @@ static bool match_gated_delta_net_dispatch(const DispatchMatchContext & context, } for (const GraphNode * node : match.covered) { - if (!append_covered_node(context, node, dispatch_match)) { + if (!append_covered_pattern_node(context, node, dispatch_match)) { return false; } } diff --git a/ggml/src/ggml-hrx/graph/graph-traversal.cpp b/ggml/src/ggml-hrx/graph/graph-traversal.cpp index 22edc6bba011..2e83355e7c88 100644 --- a/ggml/src/ggml-hrx/graph/graph-traversal.cpp +++ b/ggml/src/ggml-hrx/graph/graph-traversal.cpp @@ -100,6 +100,16 @@ GraphTraversalOrder GraphTraversalOrder::build(const Graph & graph) { std::vector pending_inputs(nodes.size(), 0); for (size_t i = 0; i < nodes.size(); ++i) { for (ValueId input : nodes[i].inputs) { + // An in-place op lists its own destination as an input: the CPY of a ggml_cast, or of the empty logits + // copy, has its dst as its own output value. That input is produced by this very node, so it must not + // make the node wait for itself: counted, the node never became ready, the walk stopped there, and + // everything downstream of it was appended in index order instead of dependency order. In index order + // the root of a fused pattern can come before a projection the pattern reads (the L2_NORM of the + // gated delta net before its gate MUL_MAT), which fails the graph (engine #1925). The scheduler's + // node_inputs_are_available() makes the same exception. + if (input == nodes[i].output) { + continue; + } if (graph.index().producer(input) != nullptr) { ++pending_inputs[i]; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7111758c8825..869eb4e867bc 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -438,6 +438,13 @@ if (TARGET ggml-hrx) target_include_directories(test-hrx-external-window-view PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) add_test(NAME test-hrx-external-window-view COMMAND test-hrx-external-window-view) + # the fused GATED_DELTA_NET pattern when the scheduler has already elided one of its layout aliases (the state + # RESHAPE visited before the L2_NORM root, engine #1925): host only + add_executable(test-hrx-gdn-passed-alias test-hrx-gdn-passed-alias.cpp) + target_link_libraries(test-hrx-gdn-passed-alias PRIVATE ggml-hrx ggml hrx::hrx loomc::loomc) + target_include_directories(test-hrx-gdn-passed-alias PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) + add_test(NAME test-hrx-gdn-passed-alias COMMAND test-hrx-gdn-passed-alias) + add_executable(test-hrx-ops test-hrx-ops.cpp) target_link_libraries(test-hrx-ops PRIVATE ggml-hrx ggml hrx::hrx loomc::loomc) target_include_directories(test-hrx-ops PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) diff --git a/tests/test-hrx-gdn-passed-alias.cpp b/tests/test-hrx-gdn-passed-alias.cpp new file mode 100644 index 000000000000..3379b8a13bd5 --- /dev/null +++ b/tests/test-hrx-gdn-passed-alias.cpp @@ -0,0 +1,226 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The fused GATED_DELTA_NET pattern in a graph the scheduler walks out of order (engine #1925). +// +// Qwen3.8-Flash-Next, the first 512-token batch past 2,048 tokens of context, with the f16 mask kernels on: +// +// unsupported HRX node 160: L2_NORM output=f32[128,16,512,1]<-L2_NORM inputs=[f32[128,16,512,1]<-VIEW] +// consumers=[174:GATED_DELTA_NET] +// +// The cause is the walk. The scheduler visits nodes in dependency order with MUL_MAT and the other root ops first, +// which is what puts the gate and beta projections of a layer ahead of the L2_NORM its fused pattern is rooted at. An +// in-place CPY (the one ggml_cast builds) lists its own output as an input; counted as a dependency, it never became +// ready, the walk stopped at it, and every node downstream was appended in index order. The sparse path's mask chain +// has such casts, so every later layer was walked in index order, where the L2_NORM comes before the projections: +// +// fusion order: llm.gated_delta_net.f32_wmma_head128 at node 160 (L2_NORM) refused: ... alpha_raw reads value 206 +// produced by node 164 (MUL_MAT) that is not scheduled yet +// +// Nodes that depend on nothing in the stalled part were still walked early, among them the RESHAPE that gives the +// gathered recurrent state its [128,128,heads,seqs] shape. It is a layout alias, so it was elided, and the pattern +// then could not cover a node that was already covered. That was the first refusal the model showed. +// +// The cases build the layer's graph at that model's head counts (16 q/k heads, 48 v heads, width 128, so one token's +// q|k|v row is 10,240 floats) and schedule it for gfx1151 on the host. check_walk puts the layer behind a ggml_cast; +// check puts the state RESHAPE ahead of the L2_NORM. Each plan must exist and must carry the same gated-delta-net +// dispatches (kernel, parameters, binding ranges) as the same graph without the disturbance. + +#include "dispatch/dispatch-scheduler.h" +#include "ggml.h" +#include "graph/graph.h" +#include "kernel-corpus/kernel-corpus.h" + +#include +#include +#include +#include +#include +#include + +#define REQUIRE(condition) \ + do { \ + if (!(condition)) { \ + std::fprintf(stderr, "%s:%d: requirement failed: %s\n", __FILE__, __LINE__, #condition); \ + std::abort(); \ + } \ + } while (false) + +namespace { + +using namespace ggml::hrx; + +std::string kernel_name(uint64_t id) { + const KernelResolveResult resolved = resolve_kernel_definition(get_qwen_kernel_corpus(), "gfx1151", id); + REQUIRE(resolved.found()); + return kernel_definition_name(*resolved.definition); +} + +struct Planned { + bool ok = false; + std::string error; + std::vector gdn; // one line per gated-delta-net dispatch: kernel, parameters, binding ranges +}; + +// state_leaf: the state gather ran in another split, so the RESHAPE's input has no producer in this graph. +Planned plan_layer(int64_t tokens, bool state_first, bool state_leaf, bool model_order = false, bool cast_upstream = false) { + ggml_init_params params = {}; + params.mem_size = 8 * 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + const int64_t width = 128, qheads = 16, heads = 48, inputs = 5120; + const int64_t hidden = width * (2 * qheads + heads); + const int64_t state_elements = width * width * heads; + const size_t state_bytes = state_elements * sizeof(float); + + // cast_upstream: the layer's input comes through a ggml_cast, whose CPY writes into its own output value (the + // sparse path's mask chain has such casts ahead of every later layer). + ggml_tensor * source = cast_upstream ? + ggml_cast(ctx, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, inputs, tokens), GGML_TYPE_F32) : + ggml_new_tensor_2d(ctx, GGML_TYPE_F32, inputs, tokens); + ggml_tensor * input = ggml_scale(ctx, source, 0.5f); + ggml_tensor * w_alpha = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_K, inputs, heads); + ggml_tensor * w_beta = ggml_dup_tensor(ctx, w_alpha); + ggml_tensor * alpha = ggml_mul_mat(ctx, w_alpha, input); + ggml_tensor * beta_raw = ggml_mul_mat(ctx, w_beta, input); + ggml_tensor * bias = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, heads); + ggml_tensor * scale = ggml_dup_tensor(ctx, bias); + ggml_tensor * gate = ggml_mul( + ctx, ggml_softplus(ctx, ggml_add(ctx, ggml_reshape_3d(ctx, alpha, heads, tokens, 1), bias)), scale); + gate = ggml_reshape_4d(ctx, gate, 1, heads, tokens, 1); + ggml_tensor * beta = ggml_sigmoid(ctx, ggml_reshape_4d(ctx, beta_raw, 1, heads, tokens, 1)); + // the q|k|v row: with model_order a projection of the layer input, as in the model (so the L2_NORM depends on a + // MUL_MAT of the same input the gate and beta projections read) + ggml_tensor * raw = model_order ? + ggml_mul_mat(ctx, ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_K, inputs, hidden), input) : + ggml_scale(ctx, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden, tokens), 0.5f); + const auto view = [&](int64_t h, size_t offset) { + return ggml_view_4d(ctx, raw, width, h, tokens, 1, width * sizeof(float), hidden * sizeof(float), + hidden * tokens * sizeof(float), offset); + }; + ggml_tensor * q = ggml_l2_norm(ctx, view(qheads, 0), 1.e-6f); + ggml_tensor * k = ggml_l2_norm(ctx, view(qheads, width * qheads * sizeof(float)), 1.e-6f); + ggml_tensor * v = view(heads, 2 * width * qheads * sizeof(float)); + ggml_tensor * cache = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, state_elements, 20); + ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); + ggml_tensor * gathered = ggml_get_rows(ctx, cache, ids); + ggml_tensor * state = ggml_reshape_4d( + ctx, state_leaf ? ggml_new_tensor_2d(ctx, GGML_TYPE_F32, state_elements, 1) : gathered, width, width, heads, 1); + ggml_tensor * gdn = ggml_gated_delta_net(ctx, q, k, v, gate, beta, state, 5); + const int64_t count = std::min(tokens, 5); + const size_t attention_bytes = width * heads * tokens * sizeof(float); + ggml_tensor * attention = ggml_view_4d(ctx, gdn, width, heads, tokens, 1, width * sizeof(float), + width * heads * sizeof(float), attention_bytes, 0); + ggml_tensor * snapshots = + ggml_view_3d(ctx, gdn, state_elements, 1, count, state_bytes, state_bytes, attention_bytes); + ggml_tensor * target = ggml_view_3d(ctx, cache, state_elements, 1, count, state_bytes, 4 * state_bytes, 0); + + ggml_cgraph * graph = ggml_new_graph(ctx); + if (!model_order) { + ggml_build_forward_expand(graph, raw); + ggml_build_forward_expand(graph, alpha); + ggml_build_forward_expand(graph, beta_raw); + } + ggml_build_forward_expand(graph, gathered); + if (state_first) { + ggml_build_forward_expand(graph, state); + } + ggml_build_forward_expand(graph, ggml_cpy(ctx, snapshots, target)); + ggml_build_forward_expand(graph, ggml_scale(ctx, attention, 0.5f)); + + Planned result; + auto imported = import_ggml_graph(*graph); + REQUIRE(imported.valid()); + DispatchScheduler scheduler; + result.ok = scheduler.schedule_graph(imported.graph, DispatchTarget{ "gfx1151" }); + if (!result.ok) { + for (const std::string & e : scheduler.plan().status.errors()) { + result.error += e + "\n"; + } + } else { + for (const Dispatch & d : scheduler.plan().dispatches) { + const std::string name = kernel_name(d.kernel.kernel_id); + if (name.find("gated_delta_net") == std::string::npos) { + continue; + } + std::string line = name; + for (const auto & p : d.kernel.compile_parameters) { + line += " " + p.first + "=" + p.second; + } + for (const auto & p : d.kernel.integer_parameters) { + line += " " + p.first + "=" + std::to_string(p.second); + } + for (const DispatchBinding & b : d.bindings) { + line += " [" + std::to_string(b.offset) + "+" + std::to_string(b.length) + "]"; + } + result.gdn.push_back(std::move(line)); + } + } + ggml_free(ctx); + return result; +} + +void check(const char * what, int64_t tokens, bool state_leaf) { + const Planned ordinary = plan_layer(tokens, false, false); + REQUIRE(ordinary.ok); + REQUIRE(!ordinary.gdn.empty()); + const Planned passed = plan_layer(tokens, true, state_leaf); + std::printf("%-58s tokens=%-3lld : %s, %zu gated-delta-net dispatches (ordinary order: %zu)\n", what, + (long long) tokens, passed.ok ? "plans" : "REFUSED", passed.gdn.size(), ordinary.gdn.size()); + if (!passed.ok) { + std::printf(" %s", passed.error.c_str()); + } + REQUIRE(passed.ok); + REQUIRE(passed.gdn == ordinary.gdn); +} + +// The walk itself (engine #1925, the cause). With model_order the graph is expanded from its outputs only, as llama.cpp +// builds it, so the L2_NORM has a lower index than the gate and beta projections the pattern reads. The scheduler's +// dependency walk still reaches the projections first (MUL_MAT is a root op). With cast_upstream the layer's inputs +// come through a ggml_cast: its CPY lists its own output as an input, the walk used to stop at it and append the rest +// in index order, and the pattern at the L2_NORM was refused because the projection was "not scheduled yet". +void check_walk(int64_t tokens) { + const Planned ordinary = plan_layer(tokens, false, false, true, false); + if (!ordinary.ok) { + std::printf("model order, no cast, tokens=%lld REFUSED\n %s", (long long) tokens, ordinary.error.c_str()); + } + REQUIRE(ordinary.ok); + REQUIRE(!ordinary.gdn.empty()); + const Planned behind_cast = plan_layer(tokens, false, false, true, true); + std::printf("%-58s tokens=%-3lld : %s, %zu gated-delta-net dispatches (no cast: %zu)\n", + "layer behind an in-place CPY (ggml_cast)", (long long) tokens, behind_cast.ok ? "plans" : "REFUSED", + behind_cast.gdn.size(), ordinary.gdn.size()); + if (!behind_cast.ok) { + std::printf(" %s", behind_cast.error.c_str()); + } + REQUIRE(behind_cast.ok); + REQUIRE(behind_cast.gdn == ordinary.gdn); +} + +} // namespace + +int main() { + setvbuf(stdout, nullptr, _IOLBF, 0); + check_walk(512); + check_walk(64); + check_walk(2); + check("state RESHAPE ahead of the L2_NORM", 512, false); + check("state RESHAPE ahead of the L2_NORM, gather in another split", 512, true); + check("state RESHAPE ahead of the L2_NORM", 64, false); + check("state RESHAPE ahead of the L2_NORM", 2, false); + return 0; +}