From 81fb5494e24b2c0bf81cd4cae0437c69665bb568 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 19:16:40 -0400 Subject: [PATCH 1/3] hrx: conv4 prefill leaves a Q4_K weight decode holds stored to the stored kernels (engine #2137) The fused projection+conv kernel reads Q4_K only in the packed rows. For a file-mapped Q4_K weight that already has a stored device copy, decline the fused matcher so the projection runs on the stored 8-bit-block prefill kernel and the conv runs separately: one device copy per weight. Co-Authored-By: Claude Opus 5.5 --- .../dispatch_registration/llm/dispatch-ssm-conv.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp b/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp index b4603376dc4f..3e8f6907f0fb 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp @@ -3,6 +3,7 @@ #include "ubatch-token-limits.h" #include "../common/dispatch-mul-mat-common.h" +#include "../common/dispatch-q4k-packed-tail.h" #include "ggml.h" #include "graph/graph-matcher.h" @@ -608,6 +609,12 @@ static bool match_mul_mat_conv4_dispatch(const DispatchMatchContext & context, D !common_mul_mat_uses_k16_major_f16(mat.weight_format, mat.input_size, mat.output_size, mat.token_count)) { return false; } + // engine #2137: this fused kernel reads Q4_K only in the packed rows. For a Q4_K weight that decode already holds + // stored, the plain matchers take the projection instead (the stored 8-bit-block prefill kernel, then the separate + // conv), so the weight keeps one device copy. + if (q4k_aligned_prefill_reads_stored(*mat.weight)) { + return false; + } const Value * value = mat.output; std::vector layouts; From 96012527f6c06be5edd23c1a1f8cf3bf7d273575 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 19:33:53 -0400 Subject: [PATCH 2/3] tests: the ssm_conv q4_k rows expect the unfused path when Q4_K is forced stored (engine #2137) Co-Authored-By: Claude Opus 5.5 --- tests/test-hrx-ops.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test-hrx-ops.cpp b/tests/test-hrx-ops.cpp index 6e228ff280c7..49e8395d3157 100644 --- a/tests/test-hrx-ops.cpp +++ b/tests/test-hrx-ops.cpp @@ -5336,8 +5336,13 @@ static SsmConvPrefillGraph make_ssm_conv_prefill_graph(ggml_type type, int64_t k const auto sequence = scheduled_kernel_sequence(c.graph); const char * fused = gathered ? "loom_libs:ggml_mul_mat_quantized_f16_wmma_prefill_conv4_interior" : "loom_libs:ggml_mul_mat_quantized_f16_wmma_prefill_conv4"; - REQUIRE((std::find(sequence.begin(), sequence.end(), fused) != sequence.end()) == !expose); - if (gathered && !expose) { + // engine #2137: the fused kernel reads Q4_K only packed, so GGML_HRX_Q4K_PACKED_TAIL=0 (stored for every Q4_K + // weight) sends the projection to the stored kernels and the conv runs on its own. + const char * q4k_layout = std::getenv("GGML_HRX_Q4K_PACKED_TAIL"); + const bool q4k_stored = type == GGML_TYPE_Q4_K && q4k_layout != nullptr && std::string(q4k_layout) == "0"; + const bool expect_fused = !expose && !q4k_stored; + REQUIRE((std::find(sequence.begin(), sequence.end(), fused) != sequence.end()) == expect_fused); + if (gathered && expect_fused) { REQUIRE(std::find(sequence.begin(), sequence.end(), "loom_libs:llm_ssm_conv_dconv4_silu_prefill_finish_f32") != sequence.end()); } From a591023dc2e25d184d63e16541e46869946eeb27 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 11 Nov 2026 19:53:00 -0400 Subject: [PATCH 3/3] hrx: conv4 prefill reads a Q6_K weight in the packed rows where prefill reads it packed (engine #2137) The fused kernel shares the prefill body that already takes Q6_K as format 46 (packed rows). Bind that layout when q6k_packed_rows_enabled() says prefill reads this weight packed and no GET_ROWS reads it stored, so the weight keeps one device copy. Co-Authored-By: Claude Opus 5.5 --- .../llm/dispatch-ssm-conv.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp b/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp index 3e8f6907f0fb..c450ea7bc3dd 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp @@ -687,11 +687,25 @@ static bool match_mul_mat_conv4_dispatch(const DispatchMatchContext & context, D dispatch.kernel.integer_parameters.emplace("token_count", mat.token_count); set_compile_parameter(dispatch.kernel, "ggml.mul_mat.input_size", mat.input_size); set_compile_parameter(dispatch.kernel, "ggml.mul_mat.output_size", mat.output_size); - set_compile_parameter(dispatch.kernel, "ggml.mul_mat.weight_format", mat.weight->type == GGML_TYPE_Q4_K ? 4 : 6); + // engine #2137: Q6_K in the packed rows (format 46) where prefill reads this weight packed elsewhere, so the fused + // kernel shares that copy instead of adding a stored one. Not for a weight a GET_ROWS also reads stored. + bool q6_packed = mat.weight->type == GGML_TYPE_Q6_K && + q6k_packed_rows_weight_ok(*mat.weight, mat.input_size, mat.output_size) && + q6k_packed_rows_enabled(*mat.weight); + for (const GraphNode * consumer : context.graph.index().consumers(mat.weight->id)) { + if (consumer != nullptr && consumer->op == GGML_OP_GET_ROWS) { + q6_packed = false; + } + } + const int64_t weight_format = mat.weight->type == GGML_TYPE_Q4_K ? 4 : q6_packed ? 46 : 6; + set_compile_parameter(dispatch.kernel, "ggml.mul_mat.weight_format", weight_format); dispatch.bindings.push_back(activation); if (mat.weight->type == GGML_TYPE_Q4_K) { dispatch.bindings.push_back({mat.weight->id, 0, mat.weight->byte_count, kQ4KPackedK256Row64Layout, mat.weight->type, mat.input_size, mat.output_size, mat.weight->byte_count}); + } else if (q6_packed) { + dispatch.bindings.push_back({mat.weight->id, 0, mat.weight->byte_count, kQ6KPackedK256Row64ScaleRowLayout, + mat.weight->type, mat.input_size, mat.output_size, mat.weight->byte_count}); } else { dispatch.bindings.push_back({mat.weight->id, 0, mat.weight->byte_count}); }