From 33a0d4bb2e4c371e01d3b919cc6b52a668953b43 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 15:48:10 -0500 Subject: [PATCH 01/13] Add opaque matcher --- src/include/migraphx/matcher.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/include/migraphx/matcher.hpp b/src/include/migraphx/matcher.hpp index dd88747ef56..2adfad2c0e0 100644 --- a/src/include/migraphx/matcher.hpp +++ b/src/include/migraphx/matcher.hpp @@ -269,6 +269,12 @@ struct any_matcher : any_matcher_base } }; +template +auto opaque(M m) +{ + return any_matcher{m}; +} + /// Create a basic matcher from a matcher template typename type_erased_matcher::type make_basic_matcher(M m) From d3613de4131b034b8449de1b99e0233757cf6fe5 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 17:29:04 -0500 Subject: [PATCH 02/13] Add opaque --- src/fuse_attention.cpp | 91 +++++++++++++----------- src/include/migraphx/match/gelu_erf.hpp | 11 +-- src/include/migraphx/match/gelu_tanh.hpp | 31 ++++---- src/rewrite_gelu.cpp | 7 +- src/simplify_algebra.cpp | 83 ++++++++++----------- 5 files changed, 122 insertions(+), 101 deletions(-) diff --git a/src/fuse_attention.cpp b/src/fuse_attention.cpp index 716ea3cca4f..ec347584018 100644 --- a/src/fuse_attention.cpp +++ b/src/fuse_attention.cpp @@ -141,13 +141,14 @@ struct find_quant_attention { auto matcher() const { - auto gemm1 = + auto gemm1 = match::opaque( match::name("dequantizelinear")(match::arg(0)(match::name("quant_dot").bind("qgemm1"))) - .bind("deq1"); - auto softmax = match::softmax_input(match::skip(match::name("convert"))(gemm1)); - auto probs = match::name("quantizelinear")( - match::arg(0)(match::skip(match::name("convert"))(softmax))); - auto gemm2 = match::name("quant_dot")(match::arg(0)(probs)).bind("qgemm2"); + .bind("deq1")); + auto softmax = + match::opaque(match::softmax_input(match::skip(match::name("convert"))(gemm1))); + auto probs = match::opaque(match::name("quantizelinear")( + match::arg(0)(match::skip(match::name("convert"))(softmax)))); + auto gemm2 = match::opaque(match::name("quant_dot")(match::arg(0)(probs)).bind("qgemm2")); return match::name("dequantizelinear")(match::arg(0)(gemm2)).bind("deq2"); } @@ -193,8 +194,10 @@ struct find_transposed_attention { auto matcher() const { - auto gemm1 = match::any_of[pointwise_inputs()](match::name("dot").bind("dot1")); - auto softmax = match::skip(match::name("convert"))(match::softmax_input(gemm1)); + auto gemm1 = + match::opaque(match::any_of[pointwise_inputs()](match::name("dot").bind("dot1"))); + auto softmax = + match::opaque(match::skip(match::name("convert"))(match::softmax_input(gemm1))); auto swap_last_two = match::make_basic_pred_matcher([](instruction_ref ins) { auto perm = ins->get_operator().to_value()["permutation"].to_vector(); if(perm.size() < 2) @@ -204,8 +207,9 @@ struct find_transposed_attention perm[perm.size() - 2] == static_cast(perm.size() - 1) and perm[perm.size() - 1] == static_cast(perm.size() - 2); }); - auto transposed_softmax = match::name("transpose")(swap_last_two, match::arg(0)(softmax)) - .bind("transposed_softmax"); + auto transposed_softmax = + match::opaque(match::name("transpose")(swap_last_two, match::arg(0)(softmax)) + .bind("transposed_softmax")); auto input_of_dot2 = match::any().bind("input_of_dot2"); return match::name("dot")(match::arg(0)(input_of_dot2), match::arg(1)(transposed_softmax)); } @@ -248,8 +252,10 @@ struct find_attention auto matcher() const { - auto gemm1 = match::any_of[pointwise_inputs()](match::name("dot").bind("dot1")); - auto softmax = match::skip(match::name("convert"))(match::softmax_input(gemm1)); + auto gemm1 = + match::opaque(match::any_of[pointwise_inputs()](match::name("dot").bind("dot1"))); + auto softmax = + match::opaque(match::skip(match::name("convert"))(match::softmax_input(gemm1))); return match::name("dot")(match::arg(0)(softmax)); } @@ -888,36 +894,41 @@ struct find_kv_cache_attention static const std::unordered_set skip_set = { "multibroadcast", "broadcast", "reshape", "unsqueeze", "squeeze"}; - auto keys = - match::skip(match::name(skip_set))(match::name("concat_past_present")).bind("pres_k"); - auto k_transpose = - match::skip(match::name(skip_set))(match::name("transpose")(match::arg(0)(keys))); + auto keys = match::opaque( + match::skip(match::name(skip_set))(match::name("concat_past_present")).bind("pres_k")); + auto k_transpose = match::opaque( + match::skip(match::name(skip_set))(match::name("transpose")(match::arg(0)(keys)))); auto queries = match::name("slice"); - auto gemm1 = match::name("dot")(match::arg(0)(queries), match::arg(1)(k_transpose)); - auto gemm1_maybe_cvt = match::skip(match::name("convert"))(gemm1); - auto scale = match::name("mul")(match::any_arg(0, 1)(gemm1_maybe_cvt)); - auto broadcasted_const = match::name("multibroadcast")(match::arg(0)(match::is_constant())); - auto attn_scores = match::any_of(scale, gemm1_maybe_cvt); - auto causal_mask = - match::name("where")(match::arg(0)(broadcasted_const), match::arg(2)(attn_scores)); - auto conv_grtr = match::name("convert")(match::arg(0)(match::name("greater"))); - auto local_window_comp = match::skip(match::name(skip_set))(conv_grtr); - auto local_window_mask = - match::name("where")(match::arg(0)(match::any_of(local_window_comp, broadcasted_const)), - match::arg(2)(match::any_of(causal_mask, scale, gemm1_maybe_cvt))); - auto greater = match::name("greater")(match::arg(1)(match::any().bind("total_sl"))); - auto conv_greater = - match::skip(match::name("unsqueeze"))(match::name("convert")(match::arg(0)(greater))); - auto bc_greater = match::name("multibroadcast")(match::arg(0)(conv_greater)); - auto mask = match::name("where")( + auto gemm1 = + match::opaque(match::name("dot")(match::arg(0)(queries), match::arg(1)(k_transpose))); + auto gemm1_maybe_cvt = match::opaque(match::skip(match::name("convert"))(gemm1)); + auto scale = match::opaque(match::name("mul")(match::any_arg(0, 1)(gemm1_maybe_cvt))); + auto broadcasted_const = + match::opaque(match::name("multibroadcast")(match::arg(0)(match::is_constant()))); + auto attn_scores = match::opaque(match::any_of(scale, gemm1_maybe_cvt)); + auto causal_mask = match::opaque( + match::name("where")(match::arg(0)(broadcasted_const), match::arg(2)(attn_scores))); + auto conv_grtr = + match::opaque(match::name("convert")(match::arg(0)(match::name("greater")))); + auto local_window_comp = match::opaque(match::skip(match::name(skip_set))(conv_grtr)); + auto local_window_mask = match::opaque(match::name("where")( + match::arg(0)(match::any_of(local_window_comp, broadcasted_const)), + match::arg(2)(match::any_of(causal_mask, scale, gemm1_maybe_cvt)))); + auto greater = + match::opaque(match::name("greater")(match::arg(1)(match::any().bind("total_sl")))); + auto conv_greater = match::opaque( + match::skip(match::name("unsqueeze"))(match::name("convert")(match::arg(0)(greater)))); + auto bc_greater = match::opaque(match::name("multibroadcast")(match::arg(0)(conv_greater))); + auto mask = match::opaque(match::name("where")( match::arg(0)(bc_greater), - match::arg(2)(match::any_of(local_window_mask, causal_mask, scale, gemm1_maybe_cvt))); - auto attn_probabilities = match::skip(match::name("convert"))( - match::softmax_input(match::skip(match::name("convert"))(mask))); - auto values = - match::skip(match::name(skip_set))(match::name("concat_past_present")).bind("pres_v"); - auto gemm2 = match::name("dot")(match::arg(0)(attn_probabilities), match::arg(1)(values)); - auto transpose_out = match::name("transpose")(match::arg(0)(gemm2)); + match::arg(2)(match::any_of(local_window_mask, causal_mask, scale, gemm1_maybe_cvt)))); + auto attn_probabilities = match::opaque(match::skip(match::name("convert"))( + match::softmax_input(match::skip(match::name("convert"))(mask)))); + auto values = match::opaque( + match::skip(match::name(skip_set))(match::name("concat_past_present")).bind("pres_v")); + auto gemm2 = match::opaque( + match::name("dot")(match::arg(0)(attn_probabilities), match::arg(1)(values))); + auto transpose_out = match::opaque(match::name("transpose")(match::arg(0)(gemm2))); return match::name("reshape")(match::arg(0)(transpose_out)); } diff --git a/src/include/migraphx/match/gelu_erf.hpp b/src/include/migraphx/match/gelu_erf.hpp index c83abcc561c..aa221ed00b3 100644 --- a/src/include/migraphx/match/gelu_erf.hpp +++ b/src/include/migraphx/match/gelu_erf.hpp @@ -38,15 +38,16 @@ struct gelu_erf_matcher F f; auto erf_fn() const { - auto mul_1_sqrt_2 = f("mul")( - either_arg(0, 1)(none_of(has_value(M_SQRT1_2)).bind("x"), has_value(M_SQRT1_2))); - auto div_sqrt_2 = f("div")(args(none_of(has_value(M_SQRT2)).bind("x"), has_value(M_SQRT2))); - return f("erf")(used_once(), arg(0)(used_once(), any_of(mul_1_sqrt_2, div_sqrt_2))); + auto mul_1_sqrt_2 = opaque(f("mul")( + either_arg(0, 1)(none_of(has_value(M_SQRT1_2)).bind("x"), has_value(M_SQRT1_2)))); + auto div_sqrt_2 = + opaque(f("div")(args(none_of(has_value(M_SQRT2)).bind("x"), has_value(M_SQRT2)))); + return opaque(f("erf")(used_once(), arg(0)(used_once(), any_of(mul_1_sqrt_2, div_sqrt_2)))); } auto add_erf() const { - return f("add")(used_once(), either_arg(0, 1)(erf_fn(), has_value(1.0))); + return opaque(f("add")(used_once(), either_arg(0, 1)(erf_fn(), has_value(1.0)))); } auto one_half() const { return has_value(0.5); } diff --git a/src/include/migraphx/match/gelu_tanh.hpp b/src/include/migraphx/match/gelu_tanh.hpp index 2e462e56822..8f9afaa7a79 100644 --- a/src/include/migraphx/match/gelu_tanh.hpp +++ b/src/include/migraphx/match/gelu_tanh.hpp @@ -38,22 +38,23 @@ struct gelu_tanh_matcher F f; /// x ^ 3 - auto pow_fn() const { return f("pow")(used_once(), arg(1)(has_value(3.0))); } + auto pow_fn() const { return opaque(f("pow")(used_once(), arg(1)(has_value(3.0)))); } auto tanh_fn() const { /// Gelu tanh approximation /// tanh( sqrt(2/M_PI) * (x + 0.044715 * x ^ 3 ) - auto mul_const_pow1 = f("mul")(either_arg(0, 1)(has_value(0.044715), pow_fn())); - auto add_any_mul = f("add")(any_arg(0, 1)(mul_const_pow1)); - auto mul_sqrt2rpi_add = f("mul")(either_arg(0, 1)(has_value(sqrt(M_2_PI)), add_any_mul)); + auto mul_const_pow1 = opaque(f("mul")(either_arg(0, 1)(has_value(0.044715), pow_fn()))); + auto add_any_mul = opaque(f("add")(any_arg(0, 1)(mul_const_pow1))); + auto mul_sqrt2rpi_add = + opaque(f("mul")(either_arg(0, 1)(has_value(sqrt(M_2_PI)), add_any_mul))); /// FastGelu tanh approximation /// tanh( 0.797885 * x + 0.035677 * x ^ 3 ) - auto mul_const_pow2 = f("mul")(either_arg(0, 1)(has_value(0.035677), pow_fn())); - auto mul_const_x = f("mul")(any_arg(0, 1)(has_value(0.797885))); - auto add_mul_x_mul_pow = f("add")(either_arg(0, 1)(mul_const_pow2, mul_const_x)); - return f("tanh")(used_once(), arg(0)(any_of(add_mul_x_mul_pow, mul_sqrt2rpi_add))); + auto mul_const_pow2 = opaque(f("mul")(either_arg(0, 1)(has_value(0.035677), pow_fn()))); + auto mul_const_x = opaque(f("mul")(any_arg(0, 1)(has_value(0.797885)))); + auto add_mul_x_mul_pow = opaque(f("add")(either_arg(0, 1)(mul_const_pow2, mul_const_x))); + return opaque(f("tanh")(used_once(), arg(0)(any_of(add_mul_x_mul_pow, mul_sqrt2rpi_add)))); } /// x * (0.5? + 0.5 * tanh( sqrt(2/M_PI) * (x? + 0.044715 * x? ^ 3) ) ) or @@ -61,18 +62,20 @@ struct gelu_tanh_matcher /// ? question mark means it doesn't explicitly match that item (anything will work) auto matcher_v0() const { - auto mul_half_tanh = f("mul")(either_arg(0, 1)(has_value(0.5), tanh_fn())); - auto add_any_mul = f("add")(any_arg(0, 1)(mul_half_tanh)); - return f("mul")(either_arg(0, 1)(any().bind("x"), add_any_mul)); + auto mul_half_tanh = opaque(f("mul")(either_arg(0, 1)(has_value(0.5), tanh_fn()))); + auto add_any_mul = opaque(f("add")(any_arg(0, 1)(mul_half_tanh))); + return opaque(f("mul")(either_arg(0, 1)(any().bind("x"), add_any_mul))); } /// x * 0.5 * (1.0 + tanh( sqrt(2/M_PI) * (x + 0.044715 * x ^ 3) ) ) or /// x * 0.5 * (1.0 + tanh( 0.797885 * x + 0.035677 * x ^ 3 ) ) ) auto matcher_v1() const { - auto add_one_tanh = f("add")(used_once(), either_arg(0, 1)(has_value(1.0), tanh_fn())); - auto mul_half_x = f("mul")(used_once(), either_arg(0, 1)(has_value(0.5), any().bind("x"))); - return f("mul")(either_arg(0, 1)(mul_half_x, add_one_tanh)); + auto add_one_tanh = + opaque(f("add")(used_once(), either_arg(0, 1)(has_value(1.0), tanh_fn()))); + auto mul_half_x = + opaque(f("mul")(used_once(), either_arg(0, 1)(has_value(0.5), any().bind("x")))); + return opaque(f("mul")(either_arg(0, 1)(mul_half_x, add_one_tanh))); } }; } // namespace detail diff --git a/src/rewrite_gelu.cpp b/src/rewrite_gelu.cpp index 132fcc0ce70..27db918ed80 100644 --- a/src/rewrite_gelu.cpp +++ b/src/rewrite_gelu.cpp @@ -66,7 +66,10 @@ static void replace_with_tanh_exp_gelu(module& m, const match::matcher_result& r */ struct find_gelu_erf { - auto matcher() const { return match::any_of(match::gelu_erf(), match::gelu_tanh()); } + auto matcher() const + { + return match::any_of(match::opaque(match::gelu_erf()), match::opaque(match::gelu_tanh())); + } void apply(module& m, const match::matcher_result& r) const { @@ -86,7 +89,7 @@ struct find_gelu_erf */ struct find_tanh_fast_gelu { - auto matcher() const { return match::gelu_tanh(); } + auto matcher() const { return match::opaque(match::gelu_tanh()); } void apply(module& m, const match::matcher_result& r) const { diff --git a/src/simplify_algebra.cpp b/src/simplify_algebra.cpp index b054d007edf..2cd294906c0 100644 --- a/src/simplify_algebra.cpp +++ b/src/simplify_algebra.cpp @@ -48,23 +48,23 @@ inline namespace MIGRAPHX_INLINE_NS { static auto lit_broadcast() { - return match::any_of(match::is_constant(), match::name("broadcast")); + return match::opaque(match::any_of(match::is_constant(), match::name("broadcast"))); } static auto not_lit_broadcast() { - return match::none_of(match::is_constant(), match::name("broadcast")); + return match::opaque(match::none_of(match::is_constant(), match::name("broadcast"))); } static auto op_lit_broadcast(std::string op, std::string x, std::string y) { - return match::name(std::move(op))(match::either_arg(0, 1)( - lit_broadcast().bind(std::move(x)), not_lit_broadcast().bind(std::move(y)))); + return match::opaque(match::name(std::move(op))(match::either_arg(0, 1)( + lit_broadcast().bind(std::move(x)), not_lit_broadcast().bind(std::move(y))))); } static auto conv_const_weights() { - return match::name("convolution")( + return match::opaque(match::name("convolution")( match::used_once(), - match::args(match::none_of(match::is_constant()), match::is_constant().bind("w"))); + match::args(match::none_of(match::is_constant()), match::is_constant().bind("w")))); } static auto from_int4() @@ -83,7 +83,7 @@ static auto from_int4() }); } -static auto not_from_int4() { return match::none_of(from_int4()); } +static auto not_from_int4() { return match::opaque(match::none_of(from_int4())); } static auto reduction() { return match::name_contains("reduce"); } @@ -166,16 +166,17 @@ struct find_mul_slice_conv { static auto conv() { - return match::name("convolution")( - match::all_of[match::outputs()](match::name("slice")), - match::args(match::any(), match::is_constant().bind("w"))); + return match::opaque( + match::name("convolution")(match::all_of[match::outputs()](match::name("slice")), + match::args(match::any(), match::is_constant().bind("w")))); } auto matcher() const { - return match::name("mul")(match::either_arg(0, 1)( + auto slice = match::opaque( match::name("slice")(match::used_once(), match::arg(0)(conv().bind("conv"))) - .bind("slice"), - match::name("broadcast")(match::is_constant()).bind("a"))); + .bind("slice")); + auto a = match::opaque(match::name("broadcast")(match::is_constant()).bind("a")); + return match::name("mul")(match::either_arg(0, 1)(slice, a)); } void apply(module& m, const match::matcher_result& r) const @@ -258,11 +259,12 @@ struct find_mul_dot { auto matcher() const { - auto constant = match::is_constant(not_from_int4()); - auto is_dot_const_inputs = - match::name("dot")(match::any_of[match::inputs()](constant), match::used_once()); - return match::name("mul")(match::either_arg(0, 1)( - is_dot_const_inputs.bind("dot"), match::name("broadcast", "multibroadcast").bind("c"))); + auto constant = match::opaque(match::is_constant(not_from_int4())); + auto is_dot_const_inputs = match::opaque( + match::name("dot")(match::any_of[match::inputs()](constant), match::used_once()) + .bind("dot")); + auto c = match::opaque(match::name("broadcast", "multibroadcast").bind("c")); + return match::name("mul")(match::either_arg(0, 1)(is_dot_const_inputs, c)); } void apply(module& m, const match::matcher_result& r) const @@ -326,8 +328,9 @@ struct find_dot_slice { auto matcher() const { - return match::name("slice")( - match::args(match::name("dot", "quant_dot")(match::used_once()).bind("dot_ins"))); + auto dot_ins = + match::opaque(match::name("dot", "quant_dot")(match::used_once()).bind("dot_ins")); + return match::name("slice")(match::args(dot_ins)); } void apply(module& m, const match::matcher_result& r) const @@ -404,13 +407,14 @@ struct find_dot_mul { auto matcher() const { - auto const_broadcast = match::name("broadcast", "multibroadcast")(match::is_constant()); - auto mul = match::name("mul")( + auto const_broadcast = + match::opaque(match::name("broadcast", "multibroadcast")(match::is_constant())); + auto mul = match::opaque(match::name("mul")( match::used_once(), match::either_arg(0, 1)(const_broadcast.bind("d"), - match::none_of(match::is_constant()).bind("z"))); - return match::name("dot")( - match::either_arg(0, 1)(mul, match::is_constant(not_from_int4()).bind("c"))); + match::none_of(match::is_constant()).bind("z")))); + auto c = match::opaque(match::is_constant(not_from_int4()).bind("c")); + return match::name("dot")(match::either_arg(0, 1)(mul, c)); } void apply(module& m, const match::matcher_result& r) const @@ -474,14 +478,13 @@ struct find_mul_add { auto matcher() const { - return match::name("mul")(match::either_arg(0, 1)( - match::name("add")( - match::either_arg(0, 1)( - match::any().bind("x"), - match::any_of(conv_const_weights(), match::is_constant()).bind("b")), - match::none_of(match::args(match::is_constant(), match::is_constant())), - match::used_once()), - match::is_constant().bind("a"))); + auto b = match::opaque(match::any_of(conv_const_weights(), match::is_constant()).bind("b")); + auto add = match::opaque(match::name("add")( + match::either_arg(0, 1)(match::any().bind("x"), b), + match::none_of(match::args(match::is_constant(), match::is_constant())), + match::used_once())); + auto a = match::opaque(match::is_constant().bind("a")); + return match::name("mul")(match::either_arg(0, 1)(add, a)); } void apply(module& m, const match::matcher_result& r) const @@ -502,13 +505,13 @@ struct find_dot_add { auto matcher() const { - return match::name("dot")(match::either_arg(0, 1)( - match::name("add")( - match::either_arg(0, 1)(match::any().bind("x"), - match::any_of(match::is_constant()).bind("b")), - match::none_of(match::args(match::is_constant(), match::is_constant())), - match::used_once()), - match::is_constant().bind("a"))); + auto b = match::opaque(match::any_of(match::is_constant()).bind("b")); + auto add = match::opaque(match::name("add")( + match::either_arg(0, 1)(match::any().bind("x"), b), + match::none_of(match::args(match::is_constant(), match::is_constant())), + match::used_once())); + auto a = match::opaque(match::is_constant().bind("a")); + return match::name("dot")(match::either_arg(0, 1)(add, a)); } void apply(module& m, const match::matcher_result& r) const From 313dbc5556d2a82a40e02c29b3255b7bd7c5b7e9 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 17:43:37 -0500 Subject: [PATCH 03/13] Type erased opaque for windows --- src/CMakeLists.txt | 2 +- src/include/migraphx/matcher.hpp | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4b18a01803d..5bb02c4ecfa 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -160,7 +160,7 @@ target_sources(migraphx PRIVATE ${BUILDER_SRCS}) if(WIN32) # Due to compilation crashing, we need to use type-erased matchers on Windows. - target_compile_definitions(migraphx PUBLIC MIGRAPHX_USE_TYPE_ERASED_MATCHERS=1) + target_compile_definitions(migraphx PUBLIC MIGRAPHX_USE_TYPE_ERASED_OPAQUE=1) endif() configure_file(version.h.in include/migraphx/version.h) diff --git a/src/include/migraphx/matcher.hpp b/src/include/migraphx/matcher.hpp index 2adfad2c0e0..5b63bd5e4c3 100644 --- a/src/include/migraphx/matcher.hpp +++ b/src/include/migraphx/matcher.hpp @@ -46,6 +46,10 @@ #define MIGRAPHX_USE_TYPE_ERASED_MATCHERS 0 #endif +#ifndef MIGRAPHX_USE_TYPE_ERASED_OPAQUE +#define MIGRAPHX_USE_TYPE_ERASED_OPAQUE 0 +#endif + namespace migraphx { inline namespace MIGRAPHX_INLINE_NS { @@ -272,7 +276,11 @@ struct any_matcher : any_matcher_base template auto opaque(M m) { +#ifdef MIGRAPHX_USE_TYPE_ERASED_OPAQUE return any_matcher{m}; +#else + return m; +#endif } /// Create a basic matcher from a matcher From 828c62ca0e148a09b04bb70abb921b05ebc793e8 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 17:44:31 -0500 Subject: [PATCH 04/13] On debug builds --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0654b9f320b..206a9e8763f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -454,7 +454,7 @@ jobs: env: CMAKE_PREFIX_PATH: ${{ github.workspace }}/cget CCACHE_LOGFILE: /tmp/ccache.log - CXXFLAGS: -Werror -pthread -fdebug-prefix-map=$PWD=. -fdebug-types-section -DMIGRAPHX_USE_TYPE_ERASED_MATCHERS=1 --param ggc-min-expand=5 --param ggc-min-heapsize=8192 + CXXFLAGS: -Werror -pthread -fdebug-prefix-map=$PWD=. -fdebug-types-section -DMIGRAPHX_USE_TYPE_ERASED_OPAQUE=1 --param ggc-min-expand=5 --param ggc-min-heapsize=8192 run: | echo "leak:dnnl::impl::malloc" > suppressions.txt export LSAN_OPTIONS="suppressions=$(pwd)/suppressions.txt" From 0285c37dafcb7c37f4240c86f3dde803b739059d Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 18:25:36 -0500 Subject: [PATCH 05/13] Type erased opaque for debug builds --- src/include/migraphx/matcher.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/include/migraphx/matcher.hpp b/src/include/migraphx/matcher.hpp index 5b63bd5e4c3..2d9f98c89c7 100644 --- a/src/include/migraphx/matcher.hpp +++ b/src/include/migraphx/matcher.hpp @@ -47,7 +47,11 @@ #endif #ifndef MIGRAPHX_USE_TYPE_ERASED_OPAQUE +#if MIGRAPHX_USE_TYPE_ERASED_MATCHERS || defined (NDEBUG) #define MIGRAPHX_USE_TYPE_ERASED_OPAQUE 0 +#else +#define MIGRAPHX_USE_TYPE_ERASED_OPAQUE 1 +#endif #endif namespace migraphx { From fce7fbf24a89ee946eac51bd1eca4c3da1fffa64 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 18:25:46 -0500 Subject: [PATCH 06/13] Format --- src/include/migraphx/matcher.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/migraphx/matcher.hpp b/src/include/migraphx/matcher.hpp index 2d9f98c89c7..cc9ced3d5a1 100644 --- a/src/include/migraphx/matcher.hpp +++ b/src/include/migraphx/matcher.hpp @@ -47,7 +47,7 @@ #endif #ifndef MIGRAPHX_USE_TYPE_ERASED_OPAQUE -#if MIGRAPHX_USE_TYPE_ERASED_MATCHERS || defined (NDEBUG) +#if MIGRAPHX_USE_TYPE_ERASED_MATCHERS || defined(NDEBUG) #define MIGRAPHX_USE_TYPE_ERASED_OPAQUE 0 #else #define MIGRAPHX_USE_TYPE_ERASED_OPAQUE 1 From b36f1f8d1b3a7d28c9be43a974568f7a2841b66f Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 18:26:12 -0500 Subject: [PATCH 07/13] Remove extra flag --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 206a9e8763f..d829276a956 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -454,7 +454,7 @@ jobs: env: CMAKE_PREFIX_PATH: ${{ github.workspace }}/cget CCACHE_LOGFILE: /tmp/ccache.log - CXXFLAGS: -Werror -pthread -fdebug-prefix-map=$PWD=. -fdebug-types-section -DMIGRAPHX_USE_TYPE_ERASED_OPAQUE=1 --param ggc-min-expand=5 --param ggc-min-heapsize=8192 + CXXFLAGS: -Werror -pthread -fdebug-prefix-map=$PWD=. -fdebug-types-section --param ggc-min-expand=5 --param ggc-min-heapsize=8192 run: | echo "leak:dnnl::impl::malloc" > suppressions.txt export LSAN_OPTIONS="suppressions=$(pwd)/suppressions.txt" From 2caaea7d2ca94ea7bbf4c88ed6d19f33f24529d3 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 18:26:36 -0500 Subject: [PATCH 08/13] Update year --- src/include/migraphx/match/gelu_erf.hpp | 2 +- src/include/migraphx/match/gelu_tanh.hpp | 2 +- src/rewrite_gelu.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/include/migraphx/match/gelu_erf.hpp b/src/include/migraphx/match/gelu_erf.hpp index aa221ed00b3..c6bcf049e33 100644 --- a/src/include/migraphx/match/gelu_erf.hpp +++ b/src/include/migraphx/match/gelu_erf.hpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/src/include/migraphx/match/gelu_tanh.hpp b/src/include/migraphx/match/gelu_tanh.hpp index 8f9afaa7a79..9023f8aeec4 100644 --- a/src/include/migraphx/match/gelu_tanh.hpp +++ b/src/include/migraphx/match/gelu_tanh.hpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2024 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/src/rewrite_gelu.cpp b/src/rewrite_gelu.cpp index 27db918ed80..85bc2c83cf9 100644 --- a/src/rewrite_gelu.cpp +++ b/src/rewrite_gelu.cpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal From 8b2706f6aaf87b22b8a1bc9ec1eaced3d84dc554 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 20:07:41 -0500 Subject: [PATCH 09/13] Update layernorm --- src/include/migraphx/match/layernorm.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/include/migraphx/match/layernorm.hpp b/src/include/migraphx/match/layernorm.hpp index 10353fe4a8f..750436c8207 100644 --- a/src/include/migraphx/match/layernorm.hpp +++ b/src/include/migraphx/match/layernorm.hpp @@ -51,31 +51,31 @@ struct layernorm_matcher }); } - auto reduce_mean() const { return f("reduce_mean")(last_axis()); } + auto reduce_mean() const { return opaque(f("reduce_mean")(last_axis())); } auto x_minus_mean() const { - return f("sub")(arg(0)(any().bind("x")), arg(1)(skip_broadcasts(reduce_mean()))); + return opaque(f("sub")(arg(0)(any().bind("x")), arg(1)(skip_broadcasts(reduce_mean())))); } auto variance() const { - return reduce_mean()(arg(0)(any_of( + return opaque(reduce_mean()(arg(0)(any_of( f("pow")(arg(0)(x_minus_mean()), arg(1)(has_value(2.0f))), f("mul")(arg(0)(x_minus_mean()), arg(1)(x_minus_mean())), - f("sqdiff")(either_arg(0, 1)(any().bind("x"), skip_broadcasts(reduce_mean())))))); + f("sqdiff")(either_arg(0, 1)(any().bind("x"), skip_broadcasts(reduce_mean()))))))); } auto sqrt_add_eps(const std::string& name) const { - auto add_eps = f("add")(either_arg(0, 1)(variance(), is_constant().bind("eps"))); - return skip_broadcasts(f(name)(arg(0)(any_of(add_eps, variance())))); + auto add_eps = opaque(f("add")(either_arg(0, 1)(variance(), is_constant().bind("eps")))); + return opaque(skip_broadcasts(f(name)(arg(0)(any_of(add_eps, variance()))))); } auto layernorm_onnx() const { - auto div_sqrt = f("div")(arg(0)(x_minus_mean()), arg(1)(sqrt_add_eps("sqrt"))); - auto mul_rsqrt = f("mul")(either_arg(0, 1)(x_minus_mean(), sqrt_add_eps("rsqrt"))); + auto div_sqrt = opaque(f("div")(arg(0)(x_minus_mean()), arg(1)(sqrt_add_eps("sqrt")))); + auto mul_rsqrt = opaque(f("mul")(either_arg(0, 1)(x_minus_mean(), sqrt_add_eps("rsqrt")))); return any(any_of(div_sqrt, mul_rsqrt)); } From 29f7b15d85be20510e3a9a6a30df972dbe11eb72 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 7 Aug 2026 20:08:23 -0500 Subject: [PATCH 10/13] Update year --- src/include/migraphx/match/layernorm.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/migraphx/match/layernorm.hpp b/src/include/migraphx/match/layernorm.hpp index 750436c8207..01f7aa0d74a 100644 --- a/src/include/migraphx/match/layernorm.hpp +++ b/src/include/migraphx/match/layernorm.hpp @@ -1,7 +1,7 @@ /* * The MIT License (MIT) * - * Copyright (c) 2015-2025 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal From d4f1e353972fc6b9a6844e3f070f49b6766ede00 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 13 Aug 2026 18:31:33 -0500 Subject: [PATCH 11/13] Fix tidy warnings and fix a bug --- src/CMakeLists.txt | 2 +- src/fuse_attention.cpp | 36 ++++++++++++++++++-------------- src/include/migraphx/matcher.hpp | 8 +++---- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5bb02c4ecfa..7fd85dffc0b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -160,7 +160,7 @@ target_sources(migraphx PRIVATE ${BUILDER_SRCS}) if(WIN32) # Due to compilation crashing, we need to use type-erased matchers on Windows. - target_compile_definitions(migraphx PUBLIC MIGRAPHX_USE_TYPE_ERASED_OPAQUE=1) + target_compile_definitions(migraphx PUBLIC MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER=1) endif() configure_file(version.h.in include/migraphx/version.h) diff --git a/src/fuse_attention.cpp b/src/fuse_attention.cpp index ec347584018..88dffedca63 100644 --- a/src/fuse_attention.cpp +++ b/src/fuse_attention.cpp @@ -146,8 +146,8 @@ struct find_quant_attention .bind("deq1")); auto softmax = match::opaque(match::softmax_input(match::skip(match::name("convert"))(gemm1))); - auto probs = match::opaque(match::name("quantizelinear")( - match::arg(0)(match::skip(match::name("convert"))(softmax)))); + auto softmax_cvt = match::opaque(match::skip(match::name("convert"))(softmax)); + auto probs = match::opaque(match::name("quantizelinear")(match::arg(0)(softmax_cvt))); auto gemm2 = match::opaque(match::name("quant_dot")(match::arg(0)(probs)).bind("qgemm2")); return match::name("dequantizelinear")(match::arg(0)(gemm2)).bind("deq2"); } @@ -896,34 +896,38 @@ struct find_kv_cache_attention auto keys = match::opaque( match::skip(match::name(skip_set))(match::name("concat_past_present")).bind("pres_k")); - auto k_transpose = match::opaque( - match::skip(match::name(skip_set))(match::name("transpose")(match::arg(0)(keys)))); + auto keys_transpose = match::opaque(match::name("transpose")(match::arg(0)(keys))); + auto k_transpose = match::opaque(match::skip(match::name(skip_set))(keys_transpose)); auto queries = match::name("slice"); auto gemm1 = match::opaque(match::name("dot")(match::arg(0)(queries), match::arg(1)(k_transpose))); auto gemm1_maybe_cvt = match::opaque(match::skip(match::name("convert"))(gemm1)); auto scale = match::opaque(match::name("mul")(match::any_arg(0, 1)(gemm1_maybe_cvt))); + auto constant = match::opaque(match::is_constant()); auto broadcasted_const = - match::opaque(match::name("multibroadcast")(match::arg(0)(match::is_constant()))); + match::opaque(match::name("multibroadcast")(match::arg(0)(constant))); auto attn_scores = match::opaque(match::any_of(scale, gemm1_maybe_cvt)); auto causal_mask = match::opaque( match::name("where")(match::arg(0)(broadcasted_const), match::arg(2)(attn_scores))); - auto conv_grtr = - match::opaque(match::name("convert")(match::arg(0)(match::name("greater")))); + auto grtr = match::opaque(match::name("greater")); + auto conv_grtr = match::opaque(match::name("convert")(match::arg(0)(grtr))); auto local_window_comp = match::opaque(match::skip(match::name(skip_set))(conv_grtr)); + auto local_window_cond = match::opaque(match::any_of(local_window_comp, broadcasted_const)); + auto local_window_val = match::opaque(match::any_of(causal_mask, scale, gemm1_maybe_cvt)); auto local_window_mask = match::opaque(match::name("where")( - match::arg(0)(match::any_of(local_window_comp, broadcasted_const)), - match::arg(2)(match::any_of(causal_mask, scale, gemm1_maybe_cvt)))); + match::arg(0)(local_window_cond), match::arg(2)(local_window_val))); auto greater = match::opaque(match::name("greater")(match::arg(1)(match::any().bind("total_sl")))); - auto conv_greater = match::opaque( - match::skip(match::name("unsqueeze"))(match::name("convert")(match::arg(0)(greater)))); + auto greater_cvt = match::opaque(match::name("convert")(match::arg(0)(greater))); + auto conv_greater = match::opaque(match::skip(match::name("unsqueeze"))(greater_cvt)); auto bc_greater = match::opaque(match::name("multibroadcast")(match::arg(0)(conv_greater))); - auto mask = match::opaque(match::name("where")( - match::arg(0)(bc_greater), - match::arg(2)(match::any_of(local_window_mask, causal_mask, scale, gemm1_maybe_cvt)))); - auto attn_probabilities = match::opaque(match::skip(match::name("convert"))( - match::softmax_input(match::skip(match::name("convert"))(mask)))); + auto mask_val = + match::opaque(match::any_of(local_window_mask, causal_mask, scale, gemm1_maybe_cvt)); + auto mask = + match::opaque(match::name("where")(match::arg(0)(bc_greater), match::arg(2)(mask_val))); + auto mask_cvt = match::opaque(match::skip(match::name("convert"))(mask)); + auto attn_probabilities = + match::opaque(match::skip(match::name("convert"))(match::softmax_input(mask_cvt))); auto values = match::opaque( match::skip(match::name(skip_set))(match::name("concat_past_present")).bind("pres_v")); auto gemm2 = match::opaque( diff --git a/src/include/migraphx/matcher.hpp b/src/include/migraphx/matcher.hpp index cc9ced3d5a1..bf788627d41 100644 --- a/src/include/migraphx/matcher.hpp +++ b/src/include/migraphx/matcher.hpp @@ -46,11 +46,11 @@ #define MIGRAPHX_USE_TYPE_ERASED_MATCHERS 0 #endif -#ifndef MIGRAPHX_USE_TYPE_ERASED_OPAQUE +#ifndef MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER #if MIGRAPHX_USE_TYPE_ERASED_MATCHERS || defined(NDEBUG) -#define MIGRAPHX_USE_TYPE_ERASED_OPAQUE 0 +#define MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER 0 #else -#define MIGRAPHX_USE_TYPE_ERASED_OPAQUE 1 +#define MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER 1 #endif #endif @@ -280,7 +280,7 @@ struct any_matcher : any_matcher_base template auto opaque(M m) { -#ifdef MIGRAPHX_USE_TYPE_ERASED_OPAQUE +#if MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER return any_matcher{m}; #else return m; From 3fbf1659f1aece9ce7ca6b676fe8b23d5e3e5497 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 14 Aug 2026 20:05:09 -0500 Subject: [PATCH 12/13] Update check --- src/include/migraphx/matcher.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/migraphx/matcher.hpp b/src/include/migraphx/matcher.hpp index bf788627d41..6e801d3bf8d 100644 --- a/src/include/migraphx/matcher.hpp +++ b/src/include/migraphx/matcher.hpp @@ -47,7 +47,7 @@ #endif #ifndef MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER -#if MIGRAPHX_USE_TYPE_ERASED_MATCHERS || defined(NDEBUG) +#if MIGRAPHX_USE_TYPE_ERASED_MATCHERS || defined(__clang__) || not defined(__GNUC__) #define MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER 0 #else #define MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER 1 From 222e61183e1bb450bc717c6ee6cfeef65c49709a Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 14 Aug 2026 20:08:29 -0500 Subject: [PATCH 13/13] Check for windows as well --- src/CMakeLists.txt | 5 ----- src/include/migraphx/matcher.hpp | 7 +++++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7fd85dffc0b..db5f6ba752f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -158,11 +158,6 @@ enable_static_init(migraphx) file(GLOB BUILDER_SRCS CONFIGURE_DEPENDS op/builder/*.cpp op/builder/torch/*.cpp) target_sources(migraphx PRIVATE ${BUILDER_SRCS}) -if(WIN32) - # Due to compilation crashing, we need to use type-erased matchers on Windows. - target_compile_definitions(migraphx PUBLIC MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER=1) -endif() - configure_file(version.h.in include/migraphx/version.h) add_library(migraphx_version INTERFACE) rocm_install_targets( diff --git a/src/include/migraphx/matcher.hpp b/src/include/migraphx/matcher.hpp index 6e801d3bf8d..ab3d2326173 100644 --- a/src/include/migraphx/matcher.hpp +++ b/src/include/migraphx/matcher.hpp @@ -47,10 +47,13 @@ #endif #ifndef MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER -#if MIGRAPHX_USE_TYPE_ERASED_MATCHERS || defined(__clang__) || not defined(__GNUC__) +#if MIGRAPHX_USE_TYPE_ERASED_MATCHERS #define MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER 0 -#else +// Windows and gcc use an excessive amount of memory to compile deeply nested matcher types +#elif defined(_WIN32) || (defined(__GNUC__) && not defined(__clang__)) #define MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER 1 +#else +#define MIGRAPHX_USE_TYPE_ERASED_OPAQUE_MATCHER 0 #endif #endif