From fcb91d8c14bc02be54728d1670178af778a65506 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 12:56:39 -0400 Subject: [PATCH 1/2] hrx: the multi-section rotation of Qwen3-VL runs on the device, not on the CPU (engine #1467) Qwen3-VL's text rotation is GGML_ROPE_TYPE_IMROPE with sections that leave some pairs on the fourth position component, so it does not collapse to the plain rotation and no registration took it: 56 ROPE nodes per text graph ran on the CPU, which cut the graph into 114 splits. The two generic rotation kernels (rope_f32, rope_set_rows_f32) gain mode 3: each rotated pair reads the position section its index selects, as ggml_mrope_cache_init does. A launch over a token chunk keeps the whole positions buffer and is told where its tokens start. Modes 0 and 2 compile to the same code as before. The claim and the matcher share the new predicate. Co-Authored-By: Claude Fable 5.2 --- .../common/dispatch-rope-set-rows.cpp | 45 ++++++++- .../kernels/loom-libs/ops/rope_f32.loom | 98 ++++++++++++++++++- .../loom-libs/ops/rope_set_rows_f32.loom | 98 ++++++++++++++++++- ggml/src/ggml-hrx/qwen4exp-hrx-support.h | 19 ++++ ggml/src/ggml-hrx/rope-claim-params.h | 2 +- tests/test-backend-ops.cpp | 3 + 6 files changed, 258 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-rope-set-rows.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-rope-set-rows.cpp index 7360dafc65ed..5b7879b0640b 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-rope-set-rows.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-rope-set-rows.cpp @@ -246,6 +246,10 @@ struct RopeMatch { size_t input_span_bytes = 0; float rope_mscale = 1.0f; int64_t mode = 0; + int64_t section_dims = 1; // mode 3 only: the position sections each pair selects from + int64_t section_t3 = 0; + int64_t section_h3 = 0; + int64_t section_w3 = 0; size_t theta_bytes = 0; size_t freq_factors_bytes = 0; std::vector theta_data; @@ -362,7 +366,16 @@ static RopeMatch match_rope_f32(const Graph & graph, const GraphNode * node, Sta match.input_span = input_span; match.input_span_bytes = input_span * sizeof(float); match.rope_mscale = rope_mscale; - match.mode = rope_kernel_mode(*params); + match.mode = rope_generic_kernel_mode(*params); + // Mode 3 reads the position section each pair selects (qwen4exp-hrx-support.h); the other modes ignore these, and + // the kernels declare section_dims >= 1. + if (match.mode == kRopeKernelModeSectioned) { + match.section_dims = static_cast(params->sections[0]) + params->sections[1] + params->sections[2] + + params->sections[3]; + match.section_t3 = 3 * static_cast(params->sections[0]); + match.section_h3 = 3 * static_cast(params->sections[1]); + match.section_w3 = 3 * static_cast(params->sections[2]); + } match.theta_bytes = theta_data.size(); match.freq_factors_bytes = freq_factors_bytes; match.theta_data = std::move(theta_data); @@ -441,6 +454,12 @@ static void add_rope_compile_parameters(Dispatch & dispatch, const RopeMatch & m dispatch.kernel.compile_parameters.emplace("ggml.rope_f32.input_stride2", to_config_value(match.input_stride2)); dispatch.kernel.compile_parameters.emplace("ggml.rope_f32.mscale", rope_mscale_config_value(match.rope_mscale)); dispatch.kernel.compile_parameters.emplace("ggml.rope_f32.mode", to_config_value(match.mode)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_f32.section_dims", to_config_value(match.section_dims)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_f32.section_t3", to_config_value(match.section_t3)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_f32.section_h3", to_config_value(match.section_h3)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_f32.section_w3", to_config_value(match.section_w3)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_f32.section_stride", to_config_value(match.token_count)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_f32.token_base", to_config_value(int64_t(0))); } static void add_set_rows_compile_parameters(Dispatch & dispatch, const SetRowsMatch & match) { @@ -541,6 +560,12 @@ static Dispatch make_rope_set_rows_dispatch(const DispatchMatchContext & context dispatch.kernel.compile_parameters.emplace("ggml.rope_set_rows_f32.output_format", to_config_value(match.set_rows.output_format)); dispatch.kernel.compile_parameters.emplace("ggml.rope_set_rows_f32.mode", to_config_value(match.rope.mode)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_set_rows_f32.section_dims", to_config_value(match.rope.section_dims)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_set_rows_f32.section_t3", to_config_value(match.rope.section_t3)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_set_rows_f32.section_h3", to_config_value(match.rope.section_h3)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_set_rows_f32.section_w3", to_config_value(match.rope.section_w3)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_set_rows_f32.section_stride", to_config_value(match.rope.token_count)); + dispatch.kernel.compile_parameters.emplace("ggml.rope_set_rows_f32.token_base", to_config_value(int64_t(0))); dispatch.bindings.push_back({ match.rope.positions->id, 0, match.rope.positions->byte_count }); dispatch.bindings.push_back({ match.set_rows.indices->id, 0, match.set_rows.indices->byte_count }); dispatch.bindings.push_back( @@ -629,13 +654,25 @@ static bool match_rope_f32_dispatch(const DispatchMatchContext & context, Dispat const size_t head_span = static_cast(match.head_count - 1) * static_cast(match.input_stride1) + static_cast(match.head_size); const size_t out_row = static_cast(match.head_count * match.head_size) * sizeof(float); + // Mode 3 reads four position sections of the WHOLE token count, so a chunk keeps the whole positions buffer and is + // told where its tokens start; the other modes take their own slice of the one position per token. + const bool sectioned = match.mode == kRopeKernelModeSectioned; + std::vector chunked = { { 1, static_cast(match.input_stride2) * sizeof(float), + head_span * sizeof(float) }, + { 4, out_row, out_row } }; + if (!sectioned) { + chunked.insert(chunked.begin(), { 0, sizeof(int32_t), sizeof(int32_t) }); + } + int64_t first_token = 0; return append_token_chunked_dispatches( make_rope_dispatch(context, match, dispatch_match), match.token_count, query_chunk_tokens(kRopeKernelMaxTokens), - { { 0, sizeof(int32_t), sizeof(int32_t) }, - { 1, static_cast(match.input_stride2) * sizeof(float), head_span * sizeof(float) }, - { 4, out_row, out_row } }, + chunked, "token_count", [&](Dispatch & part, int64_t count) { + if (sectioned) { + part.kernel.compile_parameters["ggml.rope_f32.token_base"] = to_config_value(first_token); + } + first_token += count; part.kernel.compile_parameters["ggml.rope_f32.token_capacity"] = to_config_value(count); part.kernel.compile_parameters["ggml.rope_f32.input_span"] = to_config_value(static_cast((count - 1) * match.input_stride2) + static_cast(head_span)); diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rope_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rope_f32.loom index ca704780e7ab..c5d396369b9e 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rope_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rope_f32.loom @@ -19,7 +19,25 @@ config.decl @ggml.rope_f32.input_span : %value: index where [range(%value, 1, 10 config.decl @ggml.rope_f32.mscale : f32 -config.decl @ggml.rope_f32.mode : %value: index where [range(%value, 0, 2)] +// mode 3: the interleaved multi-section rotation (GGML_ROPE_TYPE_IMROPE when it does not collapse to mode 2). The +// positions buffer holds four sections of token_count values (t, h, w, e) and each rotated pair reads the section +// its index selects, as ggml_mrope_cache_init does: sector = pair rem section_dims; sector rem 3 picks h (1), w (2) +// or t (0) while sector < 3 x that section's size, and e otherwise. The pairing is mode 2's. +config.decl @ggml.rope_f32.mode : %value: index where [range(%value, 0, 3)] + +config.decl @ggml.rope_f32.section_dims : %value: index where [range(%value, 1, 1024)] + +config.decl @ggml.rope_f32.section_t3 : %value: index where [range(%value, 0, 3072)] + +config.decl @ggml.rope_f32.section_h3 : %value: index where [range(%value, 0, 3072)] + +config.decl @ggml.rope_f32.section_w3 : %value: index where [range(%value, 0, 3072)] + +// mode 3: each position section holds section_stride values, and this launch's token 0 is token_base of them (a +// launch over a token chunk still reads the whole positions buffer). +config.decl @ggml.rope_f32.section_stride : %value: index where [range(%value, 1, 268435456)] + +config.decl @ggml.rope_f32.token_base : %value: index where [range(%value, 0, 268435456)] func.decl @ggml_rope_f32_pair_packet(%position: f32, %theta: vector<2xf32>, %freq_factors: vector<2xf32>, %x_values: vector<2xf32>, %y_values: vector<2xf32>) -> (vector<2xf32>, vector<2xf32>) @@ -60,6 +78,16 @@ template.def<@ggml.rope_f32.body> device @ggml_rope_f32_body(%token_count: index %token = scf.select %valid_token, %token0, %c0 : index %is_neox = index.cmp eq, %mode, %mode_neox : index %is_normal = index.cmp eq, %mode, %c0 : index + %mode_sectioned = index.constant 3 : index + %is_sectioned = index.cmp eq, %mode, %mode_sectioned : index + %section_dims0 = config.get @ggml.rope_f32.section_dims : index + %section_dims = index.assume %section_dims0 [range(%section_dims0, 1, 1024)] : index + %section_t3 = config.get @ggml.rope_f32.section_t3 : index + %section_h3 = config.get @ggml.rope_f32.section_h3 : index + %section_w3 = config.get @ggml.rope_f32.section_w3 : index + %section_stride0 = config.get @ggml.rope_f32.section_stride : index + %section_stride = index.assume %section_stride0 [range(%section_stride0, 1, 268435456)] : index + %token_base = config.get @ggml.rope_f32.token_base : index %positions_noalias, %input_noalias, %theta_noalias, %freq_factors_noalias, %output_noalias = buffer.assume.noalias %positions, %input, %theta, %freq_factors, %output : buffer, buffer, buffer, buffer, buffer scf.if %publish_rotary { %channel = index.assume %channel0 [lt(%channel0, %pair_packet_count)] : index @@ -95,6 +123,74 @@ template.def<@ggml.rope_f32.body> device @ggml_rope_f32_body(%token_count: index vector.store %scaled_low, %output_view[%token, %head, %pair_channel] : vector<2xf32>, view<[%bounded_token_count]x[%head_count]x[%head_size]xf32> vector.store %scaled_high, %output_view[%token, %head, %paired_channel] : vector<2xf32>, view<[%bounded_token_count]x[%head_count]x[%head_size]xf32> } + scf.if %is_sectioned { + %sc0 = index.constant 0 : index + %sc1 = index.constant 1 : index + %sc2 = index.constant 2 : index + %sc3 = index.constant 3 : index + %sc4 = index.constant 4 : index + %unit_position = scalar.constant 1.0 : f32 + %next_pair = index.add %pair_channel, %sc1 : index + %section_token = index.add %token_base, %token : index + %sector_a = index.rem %pair_channel, %section_dims : index + %kind_a = index.rem %sector_a, %sc3 : index + %kind_h_a = index.cmp eq, %kind_a, %sc1 : index + %kind_w_a = index.cmp eq, %kind_a, %sc2 : index + %kind_t_a = index.cmp eq, %kind_a, %sc0 : index + %in_h_a = index.cmp ult, %sector_a, %section_h3 : index + %in_w_a = index.cmp ult, %sector_a, %section_w3 : index + %in_t_a = index.cmp ult, %sector_a, %section_t3 : index + %is_h_a = scalar.andi %kind_h_a, %in_h_a : i1 + %is_w_a = scalar.andi %kind_w_a, %in_w_a : i1 + %is_t_a = scalar.andi %kind_t_a, %in_t_a : i1 + %section_te_a = scf.select %is_t_a, %sc0, %sc3 : index + %section_wte_a = scf.select %is_w_a, %sc2, %section_te_a : index + %section_a = scf.select %is_h_a, %sc1, %section_wte_a : index + %section_base_a = index.mul %section_a, %section_stride : index + %position_index_a0 = index.add %section_base_a, %section_token : index + %sector_b = index.rem %next_pair, %section_dims : index + %kind_b = index.rem %sector_b, %sc3 : index + %kind_h_b = index.cmp eq, %kind_b, %sc1 : index + %kind_w_b = index.cmp eq, %kind_b, %sc2 : index + %kind_t_b = index.cmp eq, %kind_b, %sc0 : index + %in_h_b = index.cmp ult, %sector_b, %section_h3 : index + %in_w_b = index.cmp ult, %sector_b, %section_w3 : index + %in_t_b = index.cmp ult, %sector_b, %section_t3 : index + %is_h_b = scalar.andi %kind_h_b, %in_h_b : i1 + %is_w_b = scalar.andi %kind_w_b, %in_w_b : i1 + %is_t_b = scalar.andi %kind_t_b, %in_t_b : i1 + %section_te_b = scf.select %is_t_b, %sc0, %sc3 : index + %section_wte_b = scf.select %is_w_b, %sc2, %section_te_b : index + %section_b = scf.select %is_h_b, %sc1, %section_wte_b : index + %section_base_b = index.mul %section_b, %section_stride : index + %position_index_b0 = index.add %section_base_b, %section_token : index + %section_positions0 = index.mul %section_stride, %sc4 : index + %position_index_a, %position_index_b, %section_positions = index.assume %position_index_a0, %position_index_b0, %section_positions0 [lt(%position_index_a0, %section_positions0), lt(%position_index_b0, %section_positions0)] : index, index, index + %section_positions_view = buffer.view %positions_noalias[%c0_offset] : buffer -> view<[%section_positions]xi32> + %position_a_i32 = view.load %section_positions_view[%position_index_a] : view<[%section_positions]xi32> -> i32 + %position_b_i32 = view.load %section_positions_view[%position_index_b] : view<[%section_positions]xi32> -> i32 + %position_a = scalar.sitofp %position_a_i32 : i32 to f32 + %position_b = scalar.sitofp %position_b_i32 : i32 to f32 + %section_position_packet = vector.from_elements %position_a, %position_b : vector<2xf32> + %section_theta_packet = vector.mulf %theta_packet, %section_position_packet : vector<2xf32> + %mscale_vector = vector.splat %mscale : vector<2xf32> + %paired_channel = index.add %pair_channel, %half_n_dims : index + %low_index0 = index.add %input_base, %pair_channel : index + %high_index0 = index.add %input_base, %paired_channel : index + %low_end0 = index.add %low_index0, %c2 : index + %high_end0 = index.add %high_index0, %c2 : index + %low_index, %low_end = index.assume %low_index0, %low_end0 [le(%low_end0, %bounded_input_span)] : index, index + %high_index, %high_end = index.assume %high_index0, %high_end0 [le(%high_end0, %bounded_input_span)] : index, index + %low_input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%low_end]xf32> + %high_input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%high_end]xf32> + %low_values = vector.load %low_input_view[%low_index] : view<[%low_end]xf32> -> vector<2xf32> + %high_values = vector.load %high_input_view[%high_index] : view<[%high_end]xf32> -> vector<2xf32> + %rotated_low, %rotated_high = func.call @ggml_rope_f32_pair_packet(%unit_position, %section_theta_packet, %freq_factors_packet, %low_values, %high_values) : (f32, vector<2xf32>, vector<2xf32>, vector<2xf32>, vector<2xf32>) -> (vector<2xf32>, vector<2xf32>) + %scaled_low = vector.mulf %rotated_low, %mscale_vector : vector<2xf32> + %scaled_high = vector.mulf %rotated_high, %mscale_vector : vector<2xf32> + vector.store %scaled_low, %output_view[%token, %head, %pair_channel] : vector<2xf32>, view<[%bounded_token_count]x[%head_count]x[%head_size]xf32> + vector.store %scaled_high, %output_view[%token, %head, %paired_channel] : vector<2xf32>, view<[%bounded_token_count]x[%head_count]x[%head_size]xf32> + } scf.if %is_normal { %mscale_vector = vector.splat %mscale : vector<2xf32> %normal_index0 = index.add %input_base, %normal_channel : index diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rope_set_rows_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rope_set_rows_f32.loom index d167255c2449..84d2472dd16b 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rope_set_rows_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/rope_set_rows_f32.loom @@ -25,7 +25,25 @@ config.decl @ggml.rope_set_rows_f32.mscale : f32 config.decl @ggml.rope_set_rows_f32.output_format : %value: index where [range(%value, 16, 32)] -config.decl @ggml.rope_set_rows_f32.mode : %value: index where [range(%value, 0, 2)] +// mode 3: the interleaved multi-section rotation (GGML_ROPE_TYPE_IMROPE when it does not collapse to mode 2). The +// positions buffer holds four sections of token_count values (t, h, w, e) and each rotated pair reads the section +// its index selects, as ggml_mrope_cache_init does: sector = pair rem section_dims; sector rem 3 picks h (1), w (2) +// or t (0) while sector < 3 x that section's size, and e otherwise. The pairing is mode 2's. +config.decl @ggml.rope_set_rows_f32.mode : %value: index where [range(%value, 0, 3)] + +config.decl @ggml.rope_set_rows_f32.section_dims : %value: index where [range(%value, 1, 1024)] + +config.decl @ggml.rope_set_rows_f32.section_t3 : %value: index where [range(%value, 0, 3072)] + +config.decl @ggml.rope_set_rows_f32.section_h3 : %value: index where [range(%value, 0, 3072)] + +config.decl @ggml.rope_set_rows_f32.section_w3 : %value: index where [range(%value, 0, 3072)] + +// mode 3: each position section holds section_stride values, and this launch's token 0 is token_base of them (a +// launch over a token chunk still reads the whole positions buffer). +config.decl @ggml.rope_set_rows_f32.section_stride : %value: index where [range(%value, 1, 268435456)] + +config.decl @ggml.rope_set_rows_f32.token_base : %value: index where [range(%value, 0, 268435456)] func.decl @ggml_rope_f32_pair_packet(%position: f32, %theta: vector<2xf32>, %freq_factors: vector<2xf32>, %x_values: vector<2xf32>, %y_values: vector<2xf32>) -> (vector<2xf32>, vector<2xf32>) @@ -107,6 +125,16 @@ template.def<@ggml.rope_set_rows_f32.body> device @ggml_rope_set_rows_f32_body(% %token = scf.select %valid_token, %token0, %c0 : index %is_neox = index.cmp eq, %mode, %mode_neox : index %is_normal = index.cmp eq, %mode, %c0 : index + %mode_sectioned = index.constant 3 : index + %is_sectioned = index.cmp eq, %mode, %mode_sectioned : index + %section_dims0 = config.get @ggml.rope_set_rows_f32.section_dims : index + %section_dims = index.assume %section_dims0 [range(%section_dims0, 1, 1024)] : index + %section_t3 = config.get @ggml.rope_set_rows_f32.section_t3 : index + %section_h3 = config.get @ggml.rope_set_rows_f32.section_h3 : index + %section_w3 = config.get @ggml.rope_set_rows_f32.section_w3 : index + %section_stride0 = config.get @ggml.rope_set_rows_f32.section_stride : index + %section_stride = index.assume %section_stride0 [range(%section_stride0, 1, 268435456)] : index + %token_base = config.get @ggml.rope_set_rows_f32.token_base : index %positions_noalias, %indices_noalias, %input_noalias, %theta_noalias, %freq_factors_noalias, %cache_noalias = buffer.assume.noalias %positions, %indices, %input, %theta, %freq_factors, %cache : buffer, buffer, buffer, buffer, buffer, buffer %positions_view = buffer.view %positions_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]xi32> %indices_view = buffer.view %indices_noalias[%c0_offset] : buffer -> view<[%bounded_token_count]xi64> @@ -155,6 +183,74 @@ template.def<@ggml.rope_set_rows_f32.body> device @ggml_rope_set_rows_f32_body(% template.apply<@ggml.rope_set_rows_f32.store>(%output_format, %publish, %cache_row, %head, %pair_channel, %bounded_cache_row_count, %head_count, %head_size, %scaled_low, %cache_noalias) : (index, i1, index, index, index, index, index, index, vector<2xf32>, buffer) template.apply<@ggml.rope_set_rows_f32.store>(%output_format, %publish, %cache_row, %head, %paired_channel, %bounded_cache_row_count, %head_count, %head_size, %scaled_high, %cache_noalias) : (index, i1, index, index, index, index, index, index, vector<2xf32>, buffer) } + scf.if %is_sectioned { + %sc0 = index.constant 0 : index + %sc1 = index.constant 1 : index + %sc2 = index.constant 2 : index + %sc3 = index.constant 3 : index + %sc4 = index.constant 4 : index + %unit_position = scalar.constant 1.0 : f32 + %next_pair = index.add %pair_channel, %sc1 : index + %section_token = index.add %token_base, %token : index + %sector_a = index.rem %pair_channel, %section_dims : index + %kind_a = index.rem %sector_a, %sc3 : index + %kind_h_a = index.cmp eq, %kind_a, %sc1 : index + %kind_w_a = index.cmp eq, %kind_a, %sc2 : index + %kind_t_a = index.cmp eq, %kind_a, %sc0 : index + %in_h_a = index.cmp ult, %sector_a, %section_h3 : index + %in_w_a = index.cmp ult, %sector_a, %section_w3 : index + %in_t_a = index.cmp ult, %sector_a, %section_t3 : index + %is_h_a = scalar.andi %kind_h_a, %in_h_a : i1 + %is_w_a = scalar.andi %kind_w_a, %in_w_a : i1 + %is_t_a = scalar.andi %kind_t_a, %in_t_a : i1 + %section_te_a = scf.select %is_t_a, %sc0, %sc3 : index + %section_wte_a = scf.select %is_w_a, %sc2, %section_te_a : index + %section_a = scf.select %is_h_a, %sc1, %section_wte_a : index + %section_base_a = index.mul %section_a, %section_stride : index + %position_index_a0 = index.add %section_base_a, %section_token : index + %sector_b = index.rem %next_pair, %section_dims : index + %kind_b = index.rem %sector_b, %sc3 : index + %kind_h_b = index.cmp eq, %kind_b, %sc1 : index + %kind_w_b = index.cmp eq, %kind_b, %sc2 : index + %kind_t_b = index.cmp eq, %kind_b, %sc0 : index + %in_h_b = index.cmp ult, %sector_b, %section_h3 : index + %in_w_b = index.cmp ult, %sector_b, %section_w3 : index + %in_t_b = index.cmp ult, %sector_b, %section_t3 : index + %is_h_b = scalar.andi %kind_h_b, %in_h_b : i1 + %is_w_b = scalar.andi %kind_w_b, %in_w_b : i1 + %is_t_b = scalar.andi %kind_t_b, %in_t_b : i1 + %section_te_b = scf.select %is_t_b, %sc0, %sc3 : index + %section_wte_b = scf.select %is_w_b, %sc2, %section_te_b : index + %section_b = scf.select %is_h_b, %sc1, %section_wte_b : index + %section_base_b = index.mul %section_b, %section_stride : index + %position_index_b0 = index.add %section_base_b, %section_token : index + %section_positions0 = index.mul %section_stride, %sc4 : index + %position_index_a, %position_index_b, %section_positions = index.assume %position_index_a0, %position_index_b0, %section_positions0 [lt(%position_index_a0, %section_positions0), lt(%position_index_b0, %section_positions0)] : index, index, index + %section_positions_view = buffer.view %positions_noalias[%c0_offset] : buffer -> view<[%section_positions]xi32> + %position_a_i32 = view.load %section_positions_view[%position_index_a] : view<[%section_positions]xi32> -> i32 + %position_b_i32 = view.load %section_positions_view[%position_index_b] : view<[%section_positions]xi32> -> i32 + %position_a = scalar.sitofp %position_a_i32 : i32 to f32 + %position_b = scalar.sitofp %position_b_i32 : i32 to f32 + %section_position_packet = vector.from_elements %position_a, %position_b : vector<2xf32> + %section_theta_packet = vector.mulf %theta_packet, %section_position_packet : vector<2xf32> + %mscale_vector = vector.splat %mscale : vector<2xf32> + %paired_channel = index.add %pair_channel, %half_n_dims : index + %low_index0 = index.add %input_base, %pair_channel : index + %high_index0 = index.add %input_base, %paired_channel : index + %low_end0 = index.add %low_index0, %c2 : index + %high_end0 = index.add %high_index0, %c2 : index + %low_index, %low_end = index.assume %low_index0, %low_end0 [le(%low_end0, %bounded_input_span)] : index, index + %high_index, %high_end = index.assume %high_index0, %high_end0 [le(%high_end0, %bounded_input_span)] : index, index + %low_input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%low_end]xf32> + %high_input_view = buffer.view %input_noalias[%c0_offset] : buffer -> view<[%high_end]xf32> + %low_values = vector.load %low_input_view[%low_index] : view<[%low_end]xf32> -> vector<2xf32> + %high_values = vector.load %high_input_view[%high_index] : view<[%high_end]xf32> -> vector<2xf32> + %rotated_low, %rotated_high = func.call @ggml_rope_f32_pair_packet(%unit_position, %section_theta_packet, %freq_factors_packet, %low_values, %high_values) : (f32, vector<2xf32>, vector<2xf32>, vector<2xf32>, vector<2xf32>) -> (vector<2xf32>, vector<2xf32>) + %scaled_low = vector.mulf %rotated_low, %mscale_vector : vector<2xf32> + %scaled_high = vector.mulf %rotated_high, %mscale_vector : vector<2xf32> + template.apply<@ggml.rope_set_rows_f32.store>(%output_format, %publish, %cache_row, %head, %pair_channel, %bounded_cache_row_count, %head_count, %head_size, %scaled_low, %cache_noalias) : (index, i1, index, index, index, index, index, index, vector<2xf32>, buffer) + template.apply<@ggml.rope_set_rows_f32.store>(%output_format, %publish, %cache_row, %head, %paired_channel, %bounded_cache_row_count, %head_count, %head_size, %scaled_high, %cache_noalias) : (index, i1, index, index, index, index, index, index, vector<2xf32>, buffer) + } scf.if %is_normal { %mscale_vector = vector.splat %mscale : vector<2xf32> %normal_index0 = index.add %input_base, %normal_channel : index diff --git a/ggml/src/ggml-hrx/qwen4exp-hrx-support.h b/ggml/src/ggml-hrx/qwen4exp-hrx-support.h index 8d423a81e93f..ca7e3f3e068b 100644 --- a/ggml/src/ggml-hrx/qwen4exp-hrx-support.h +++ b/ggml/src/ggml-hrx/qwen4exp-hrx-support.h @@ -81,6 +81,25 @@ inline int rope_kernel_mode(const RopeParams & params) { return params.mode == GGML_ROPE_TYPE_IMROPE ? GGML_ROPE_TYPE_NEOX : params.mode; } +// Mode 3 of the generic rope_f32 / rope_set_rows_f32 kernels: an IMROPE whose sections leave some pairs on the fourth +// position component (Qwen3-VL), so it does not collapse to NEOX. Each rotated pair reads the position section its index +// selects; the pairing is NEOX's (engine #1467). +constexpr int kRopeKernelModeSectioned = 3; + +inline bool rope_mode_is_sectioned_imrope(const RopeParams & params) { + if (params.mode != GGML_ROPE_TYPE_IMROPE || rope_mode_is_collapsible_imrope(params)) { + return false; + } + const int64_t sect_dims = static_cast(params.sections[0]) + params.sections[1] + params.sections[2] + params.sections[3]; + return params.sections[0] >= 0 && params.sections[1] >= 0 && params.sections[2] >= 0 && params.sections[3] >= 0 && + sect_dims >= 1 && sect_dims <= 1024 && sect_dims <= params.n_dims; +} + +// The mode for the two generic kernels, which have the sectioned path; the fused kernels keep rope_kernel_mode(). +inline int rope_generic_kernel_mode(const RopeParams & params) { + return rope_mode_is_sectioned_imrope(params) ? kRopeKernelModeSectioned : rope_kernel_mode(params); +} + // Routed-expert weight types that the HRX MUL_MAT_ID kernels cannot execute // correctly, so the scheduler must keep the expert GEMM on CPU: // * IQ4_NL -- the shared f16 dequantizer mis-applies the 4-bit XOR-12 nibble diff --git a/ggml/src/ggml-hrx/rope-claim-params.h b/ggml/src/ggml-hrx/rope-claim-params.h index b2517de047bb..a284eedeca41 100644 --- a/ggml/src/ggml-hrx/rope-claim-params.h +++ b/ggml/src/ggml-hrx/rope-claim-params.h @@ -53,7 +53,7 @@ inline bool rope_params_supported_by_attention_fusion(const RopeParams & params, // common/dispatch-rope-set-rows.cpp: the rotation that registration builds a theta table for. inline bool rope_params_supported_by_rope_set_rows(const RopeParams & params, int64_t head_size) { const bool supported_mode = params.mode == GGML_ROPE_TYPE_NORMAL || params.mode == GGML_ROPE_TYPE_NEOX || - rope_mode_is_collapsible_imrope(params); + rope_mode_is_collapsible_imrope(params) || rope_mode_is_sectioned_imrope(params); return params.n_dims >= 4 && params.n_dims <= head_size && params.n_dims % 4 == 0 && supported_mode && std::isfinite(params.freq_base) && params.freq_base > 0.0f && std::isfinite(params.freq_scale) && params.freq_scale > 0.0f && std::isfinite(params.ext_factor) && std::isfinite(params.attn_factor); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 306217b7841a..392a5917a071 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9632,6 +9632,9 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rope(type, {128, 28, 2, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,imrope (qwen3vl 7B) test_cases.emplace_back(new test_rope(type, {128, 12, 2, 1}, 20, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); test_cases.emplace_back(new test_rope(type, {128, 28, 2, 1}, 32, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); + // engine #1467: a token count between the 2-token cases above and the chunked ones below, so + // the per-section position read is checked on a batch-sized launch + test_cases.emplace_back(new test_rope(type, {128, 16, 300, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); test_cases.emplace_back(new test_rope(type, { 80, 16, 2, 1}, 80, GGML_ROPE_TYPE_VISION, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen2vl ViT) test_cases.emplace_back(new test_rope(type, {128, 16, 2, 1}, 128, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); // rope_multi,m-rope (qwen3vl) test_cases.emplace_back(new test_rope(type, {16, 16, 8192, 1}, 16, GGML_ROPE_TYPE_IMROPE, 512, fs, ef, af, ff, v, fw)); From d690d6859f8a30d72ac8ae5735789f22ff48c1cc Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:41:08 -0400 Subject: [PATCH 2/2] hrx: claim the multi-section rotation only for section sums that are measured (review of #286) rope_mode_is_sectioned_imrope() accepted any section sum up to n_dims. A sum below the pair count makes the sector wrap, and neither test-backend-ops (3 x n_dims / 3) nor Qwen3-VL (one section entry per pair) runs that shape. The kernel takes the remainder ggml takes, but an unmeasured shape is not claimed: it now needs n_dims / 2 <= sum <= n_dims, and anything else stays on the CPU. Co-Authored-By: Claude Fable 5.2 --- ggml/src/ggml-hrx/qwen4exp-hrx-support.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-hrx/qwen4exp-hrx-support.h b/ggml/src/ggml-hrx/qwen4exp-hrx-support.h index ca7e3f3e068b..01b8dd3ebc09 100644 --- a/ggml/src/ggml-hrx/qwen4exp-hrx-support.h +++ b/ggml/src/ggml-hrx/qwen4exp-hrx-support.h @@ -91,8 +91,11 @@ inline bool rope_mode_is_sectioned_imrope(const RopeParams & params) { return false; } const int64_t sect_dims = static_cast(params.sections[0]) + params.sections[1] + params.sections[2] + params.sections[3]; + // Claimed only where it is measured: section sums from one per rotated pair (Qwen3-VL: 24 + 20 + 20 = 64 pairs of + // n_dims 128) up to n_dims (test-backend-ops: 3 x n_dims / 3). A sum below the pair count makes the sector wrap; the + // kernel takes the same remainder ggml does, but no case here runs that shape, so it stays on the CPU. return params.sections[0] >= 0 && params.sections[1] >= 0 && params.sections[2] >= 0 && params.sections[3] >= 0 && - sect_dims >= 1 && sect_dims <= 1024 && sect_dims <= params.n_dims; + sect_dims >= 1 && sect_dims <= 1024 && sect_dims >= params.n_dims / 2 && sect_dims <= params.n_dims; } // The mode for the two generic kernels, which have the sectioned path; the fused kernels keep rope_kernel_mode().