diff --git a/ggml/src/ggml-hrx/dispatch/command-plan-metadata.cpp b/ggml/src/ggml-hrx/dispatch/command-plan-metadata.cpp index 89d9572d66f8..8bf7c41925c8 100644 --- a/ggml/src/ggml-hrx/dispatch/command-plan-metadata.cpp +++ b/ggml/src/ggml-hrx/dispatch/command-plan-metadata.cpp @@ -14,12 +14,13 @@ static bool metadata_matches(const CommandPlanResourceMetadata & lhs, const Comm static bool generated_resource_matches(const CommandPlanGeneratedResource & lhs, const CommandPlanGeneratedResource & rhs) { return lhs.source_value == rhs.source_value && lhs.role == rhs.role && lhs.generated_value == rhs.generated_value && - lhs.byte_count == rhs.byte_count && metadata_matches(lhs.metadata, rhs.metadata); + lhs.byte_count == rhs.byte_count && lhs.scale_log2 == rhs.scale_log2 && + metadata_matches(lhs.metadata, rhs.metadata); } static bool alternate_value_matches(const CommandPlanAlternateValue & lhs, const CommandPlanAlternateValue & rhs) { return lhs.graph_value == rhs.graph_value && lhs.alternate_value == rhs.alternate_value && lhs.type == rhs.type && - lhs.byte_count == rhs.byte_count && lhs.name == rhs.name; + lhs.byte_count == rhs.byte_count && lhs.name == rhs.name && lhs.scale_log2 == rhs.scale_log2; } static bool moe_routing_bundle_matches(const CommandPlanMoeRoutingBundle & lhs, @@ -77,7 +78,7 @@ bool CommandPlanMetadata::append_generated_resource(CommandPlanGeneratedResource bool CommandPlanMetadata::append_alternate_value(CommandPlanAlternateValue alternate, Status & status) { for (const CommandPlanAlternateValue & existing : alternate_values_) { if (existing.graph_value == alternate.graph_value && existing.type == alternate.type && - existing.byte_count == alternate.byte_count) { + existing.byte_count == alternate.byte_count && existing.scale_log2 == alternate.scale_log2) { if (alternate_value_matches(existing, alternate)) { return true; } @@ -116,7 +117,7 @@ const CommandPlanGeneratedResource * CommandPlanMetadata::find_generated_resourc const CommandPlanAlternateValue * CommandPlanMetadata::find_alternate_value(ValueId graph_value) const { for (const CommandPlanAlternateValue & alternate : alternate_values_) { - if (alternate.graph_value == graph_value) { + if (alternate.graph_value == graph_value && alternate.scale_log2 == 0) { return &alternate; } } @@ -125,9 +126,11 @@ const CommandPlanAlternateValue * CommandPlanMetadata::find_alternate_value(Valu const CommandPlanAlternateValue * CommandPlanMetadata::find_alternate_value(ValueId graph_value, ggml_type type, - size_t byte_count) const { + size_t byte_count, + int scale_log2) const { for (const CommandPlanAlternateValue & alternate : alternate_values_) { - if (alternate.graph_value == graph_value && alternate.type == type && alternate.byte_count == byte_count) { + if (alternate.graph_value == graph_value && alternate.type == type && alternate.byte_count == byte_count && + alternate.scale_log2 == scale_log2) { return &alternate; } } diff --git a/ggml/src/ggml-hrx/dispatch/command-plan-metadata.h b/ggml/src/ggml-hrx/dispatch/command-plan-metadata.h index 0025b2b0fbbf..d1d725c9a745 100644 --- a/ggml/src/ggml-hrx/dispatch/command-plan-metadata.h +++ b/ggml/src/ggml-hrx/dispatch/command-plan-metadata.h @@ -76,6 +76,8 @@ struct CommandPlanGeneratedResource { ValueId generated_value; size_t byte_count = 0; CommandPlanResourceMetadata metadata; + // The stored values are the source values * 2^-scale_log2 (0 = as they are). engine #1962 + int scale_log2 = 0; }; struct CommandPlanAlternateValue { @@ -84,6 +86,9 @@ struct CommandPlanAlternateValue { ggml_type type = GGML_TYPE_COUNT; size_t byte_count = 0; std::string name; + // The stored values are the graph values * 2^-scale_log2 (0 = as they are). A reader that does not ask for a + // scale only ever finds an unscaled alternate. engine #1962 + int scale_log2 = 0; }; struct CommandPlanMoeRoutingBundle { @@ -118,7 +123,8 @@ class CommandPlanMetadata { const CommandPlanAlternateValue * find_alternate_value(ValueId graph_value, ggml_type type, - size_t byte_count) const; + size_t byte_count, + int scale_log2 = 0) const; const CommandPlanMoeRoutingBundle * find_moe_routing_bundle(ValueId route_ids) const; diff --git a/ggml/src/ggml-hrx/dispatch/command-plan.h b/ggml/src/ggml-hrx/dispatch/command-plan.h index a1f1abf62168..4f81b55e78ab 100644 --- a/ggml/src/ggml-hrx/dispatch/command-plan.h +++ b/ggml/src/ggml-hrx/dispatch/command-plan.h @@ -51,8 +51,9 @@ inline const CommandPlanAlternateValue * find_alternate_value(const CommandPlan inline const CommandPlanAlternateValue * find_alternate_value(const CommandPlan & plan, ValueId graph_value, ggml_type type, - size_t byte_count) { - return plan.metadata.find_alternate_value(graph_value, type, byte_count); + size_t byte_count, + int scale_log2 = 0) { + return plan.metadata.find_alternate_value(graph_value, type, byte_count, scale_log2); } inline bool same_full_value_range(const Value & lhs, const Value & rhs) { @@ -63,8 +64,9 @@ inline const CommandPlanAlternateValue * find_alternate_value(const Graph & const CommandPlan & plan, ValueId graph_value, ggml_type type, - size_t byte_count) { - const CommandPlanAlternateValue * exact = find_alternate_value(plan, graph_value, type, byte_count); + size_t byte_count, + int scale_log2 = 0) { + const CommandPlanAlternateValue * exact = find_alternate_value(plan, graph_value, type, byte_count, scale_log2); if (exact != nullptr) { return exact; } @@ -79,7 +81,7 @@ inline const CommandPlanAlternateValue * find_alternate_value(const Graph & if (candidate == nullptr || !same_full_value_range(*value, *candidate)) { return nullptr; } - return find_alternate_value(plan, candidate_id, type, byte_count); + return find_alternate_value(plan, candidate_id, type, byte_count, scale_log2); }; ValueId alias = value->alias_source; diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat.cpp index 35323fcbf4c5..ec7b43efc846 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat.cpp @@ -586,11 +586,12 @@ static bool match_mul_mat_swiglu_q4_q8_prefill_dispatch(const DispatchMatchConte const bool packed_input = match.token_count % 512 == 0 && (match.token_count / 512) * (match.output_size / 64) >= 64; DispatchBinding activation; + int activation_scale_log2 = 0; const bool prepared = packed_input ? common_prepare_k16_major_f16_input(context, *match.input, match.input_size, match.token_count, - dispatch_match, activation) : + dispatch_match, activation, activation_scale_log2) : common_prepare_f16_input(context, *match.input, match.input_size, match.token_count, - dispatch_match, activation); + dispatch_match, activation, activation_scale_log2); if (!prepared) { return false; } @@ -605,6 +606,7 @@ static bool match_mul_mat_swiglu_q4_q8_prefill_dispatch(const DispatchMatchConte Dispatch dispatch; dispatch.kernel = make_kernel_specialization(kMulMatSwiGLUQ4Q8PrefillKernel); + common_set_f16_prefill_activation_scale(dispatch, activation_scale_log2); dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_swiglu.f16_output_layout", packed_output ? "1" : "0"); dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_swiglu.op", common_to_config_value(static_cast(binary_kind_config_value(match.op)))); @@ -626,12 +628,16 @@ static bool match_mul_mat_swiglu_q4_q8_prefill_dispatch(const DispatchMatchConte const char * f16_name = packed_output ? "common.mul_mat_swiglu.k16_major_f16" : "common.mul_mat_swiglu.f16"; dispatch_match.transients.push_back({ f16_output, f16_name, f16_bytes, 256 }); dispatch.bindings.push_back({ f16_output, 0, f16_bytes }); + // The kernel publishes this f16 copy of the fused result as value * 2^-6 (the result is the next projection's + // activation, and a gated product is where a model leaves the f16 range: engine #1539, #1962). Status status; const bool recorded = packed_output ? dispatch_match.metadata.append_generated_resource( - { match.output->id, GeneratedResourceRole::F16K16Major, f16_output, f16_bytes, {} }, status) : + { match.output->id, GeneratedResourceRole::F16K16Major, f16_output, f16_bytes, {}, + kCommonF16ActivationScaleLog2 }, status) : dispatch_match.metadata.append_alternate_value( - { match.output->id, f16_output, GGML_TYPE_F16, f16_bytes, f16_name }, status); + { match.output->id, f16_output, GGML_TYPE_F16, f16_bytes, f16_name, kCommonF16ActivationScaleLog2 }, + status); if (!recorded) { dispatch_match.status.append(status); return false; diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-common.h b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-common.h index ddbfeaf433f6..676310b56b89 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-common.h +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-common.h @@ -552,19 +552,40 @@ inline CommonMulMatMatch common_match_mul_mat_any_format(const Graph & graph return match; } -// Share the conversion used by F16-operand matmuls. +// An f16 activation copy holds value * 2^-kCommonF16ActivationScaleLog2. A matmul narrows its f32 activation to f16 +// for WMMA, and f16 ends at 65504: a larger activation became inf and the product NaN (engine #1962, the prefill +// form of #1539). Prescaled by 2^-6 the copy represents [3.9e-3, 4.19e6] at full precision; the kernel that reads it +// scales its f32 accumulator back by 2^6, which is exact. The scale travels with the alternate value / generated +// resource, so a reader always knows which form it holds. +constexpr int kCommonF16ActivationScaleLog2 = 6; +constexpr const char * kCommonF16PrefillActivationScaleParameter = "ggml.f16_prefill.activation_scale_log2"; + +// Tell a kernel built on ggml_mul_mat_quantized_f16_prefill_* which scale its f16 activation carries. +inline void common_set_f16_prefill_activation_scale(Dispatch & dispatch, int scale_log2) { + if (scale_log2 != 0) { + dispatch.kernel.compile_parameters.emplace(kCommonF16PrefillActivationScaleParameter, + common_to_config_value(static_cast(scale_log2))); + } +} + +// Share the conversion used by F16-operand matmuls. `scale_log2` returns the scale of the bound copy: the copy made +// here (and shared by later readers) is prescaled; an unscaled F16 value another kernel published is used as it is. inline bool common_prepare_f16_input(const DispatchMatchContext & context, const Value & input, int64_t input_size, int64_t token_count, DispatchMatch & match, - DispatchBinding & binding) { + DispatchBinding & binding, + int & scale_log2) { const size_t bytes = static_cast(token_count * input_size) * sizeof(ggml_fp16_t); - const CommandPlanAlternateValue * alternate = - find_alternate_value(context.graph, context.plan, input.id, GGML_TYPE_F16, bytes); - if (alternate != nullptr) { - binding = { alternate->alternate_value, 0, bytes }; - return true; + for (const int candidate_scale : { kCommonF16ActivationScaleLog2, 0 }) { + const CommandPlanAlternateValue * alternate = + find_alternate_value(context.graph, context.plan, input.id, GGML_TYPE_F16, bytes, candidate_scale); + if (alternate != nullptr) { + binding = { alternate->alternate_value, 0, bytes }; + scale_log2 = candidate_scale; + return true; + } } constexpr KernelCatalogRef kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_copy_f32_f16"); @@ -578,11 +599,13 @@ inline bool common_prepare_f16_input(const DispatchMatchContext & context, convert.bindings.push_back({ activation, 0, bytes }); match.dispatches.push_back(std::move(convert)); Status status; - if (!match.metadata.append_alternate_value({ input.id, activation, GGML_TYPE_F16, bytes, name }, status)) { + if (!match.metadata.append_alternate_value( + { input.id, activation, GGML_TYPE_F16, bytes, name, kCommonF16ActivationScaleLog2 }, status)) { match.status.append(status); return false; } - binding = { activation, 0, bytes }; + binding = { activation, 0, bytes }; + scale_log2 = kCommonF16ActivationScaleLog2; return true; } @@ -592,18 +615,25 @@ inline bool common_prepare_k16_major_f16_input(const DispatchMatchContext & cont int64_t input_size, int64_t token_count, DispatchMatch & match, - DispatchBinding & binding) { + DispatchBinding & binding, + int & scale_log2) { constexpr KernelCatalogRef kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_copy_f16_k16_major"); constexpr const char * name = "common.mul_mat.k16_major_f16"; const size_t bytes = static_cast(token_count * input_size) * sizeof(ggml_fp16_t); const CommandPlanGeneratedResource * generated = context.plan.metadata.find_generated_resource(input.id, GeneratedResourceRole::F16K16Major); if (generated != nullptr && generated->byte_count == bytes) { - binding = { generated->generated_value, 0, bytes }; + binding = { generated->generated_value, 0, bytes }; + scale_log2 = generated->scale_log2; return true; } - const CommandPlanAlternateValue * alternate = - find_alternate_value(context.graph, context.plan, input.id, GGML_TYPE_F16, bytes); + // The pack keeps the scale of the F16 value it repacks; packed from the f32 value, the kernel prescales it. + const CommandPlanAlternateValue * alternate = find_alternate_value( + context.graph, context.plan, input.id, GGML_TYPE_F16, bytes, kCommonF16ActivationScaleLog2); + if (alternate == nullptr) { + alternate = find_alternate_value(context.graph, context.plan, input.id, GGML_TYPE_F16, bytes); + } + const int packed_scale_log2 = alternate != nullptr ? alternate->scale_log2 : kCommonF16ActivationScaleLog2; const ValueId packed(context.next_plan_value.value + static_cast(match.transients.size())); match.transients.push_back({ packed, name, bytes, 256 }); Dispatch copy; @@ -618,11 +648,12 @@ inline bool common_prepare_k16_major_f16_input(const DispatchMatchContext & cont match.dispatches.push_back(std::move(copy)); Status status; if (!match.metadata.append_generated_resource( - { input.id, GeneratedResourceRole::F16K16Major, packed, bytes, {} }, status)) { + { input.id, GeneratedResourceRole::F16K16Major, packed, bytes, {}, packed_scale_log2 }, status)) { match.status.append(status); return false; } - binding = { packed, 0, bytes }; + binding = { packed, 0, bytes }; + scale_log2 = packed_scale_log2; return true; } diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp index a612d4962113..3785c64b441e 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp @@ -653,6 +653,7 @@ static bool match_q6_k_i8_prepacked_dispatch(const DispatchMatchContext & contex static bool build_q6_k_prefill_wave32_dispatch(const CommonMulMatMatch & match, ValueId input, size_t input_bytes, + int input_scale_log2, KernelCatalogRef kernel, size_t root_index, DispatchMatch & dispatch_match) { @@ -666,6 +667,7 @@ static bool build_q6_k_prefill_wave32_dispatch(const CommonMulMatMatch & match, dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_q6_k_packed.output_accumulation", "0"); dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_q6_k_packed.token_capacity", common_to_config_value(match.token_count)); + common_set_f16_prefill_activation_scale(dispatch, input_scale_log2); dispatch.bindings.push_back({ input, 0, input_bytes }); dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count }); dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count }); @@ -688,15 +690,16 @@ static bool match_q6_k_prefill_wave32_dispatch(const DispatchMatchContext & cont const bool packed_input = common_mul_mat_uses_k16_major_f16( match.weight_format, match.input_size, match.output_size, match.token_count); DispatchBinding activation; + int activation_scale_log2 = 0; const bool prepared = packed_input ? common_prepare_k16_major_f16_input(context, *match.input, match.input_size, match.token_count, - dispatch_match, activation) : + dispatch_match, activation, activation_scale_log2) : common_prepare_f16_input(context, *match.input, match.input_size, match.token_count, - dispatch_match, activation); + dispatch_match, activation, activation_scale_log2); if (!prepared) { return false; } - return build_q6_k_prefill_wave32_dispatch(match, activation.value, activation.length, + return build_q6_k_prefill_wave32_dispatch(match, activation.value, activation.length, activation_scale_log2, q6k_packed_prefill_enabled() ? kMulMatQ6KF16WmmaPrefillFromPackedKernel : kMulMatQ6KF16WmmaPrefillWave32Kernel, context.root_index, @@ -863,15 +866,16 @@ static bool match_packed_q8_1_x4_prefill_dispatch(const DispatchMatchContext & c const bool use_f16 = !split_tail && match.weight->type == GGML_TYPE_Q4_K && match.weight->alias_source.value < 0 && match.output_size >= match.input_size / 4; DispatchBinding activation; + int activation_scale_log2 = 0; const bool packed_input = use_f16 && common_mul_mat_uses_k16_major_f16( match.weight_format, match.input_size, match.output_size, match.token_count, context.plan.metadata.find_generated_resource(match.input->id, GeneratedResourceRole::F16K16Major) != nullptr); if (use_f16) { const bool prepared = packed_input ? common_prepare_k16_major_f16_input(context, *match.input, match.input_size, match.token_count, - dispatch_match, activation) : + dispatch_match, activation, activation_scale_log2) : common_prepare_f16_input(context, *match.input, match.input_size, match.token_count, - dispatch_match, activation); + dispatch_match, activation, activation_scale_log2); if (!prepared) { return false; } @@ -900,6 +904,7 @@ static bool match_packed_q8_1_x4_prefill_dispatch(const DispatchMatchContext & c match.weight->alias_source.value < 0; if (use_f16) { dispatch.kernel.compile_parameters.emplace("ggml.mul_mat.f16_input_layout", packed_input ? "1" : "0"); + common_set_f16_prefill_activation_scale(dispatch, activation_scale_log2); } else { dispatch.kernel.compile_parameters.emplace( "ggml.mul_mat_q8_1_x4.weight_format", diff --git a/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp b/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp index f4619ba999a2..b4603376dc4f 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-ssm-conv.cpp @@ -669,12 +669,14 @@ static bool match_mul_mat_conv4_dispatch(const DispatchMatchContext & context, D } DispatchBinding activation; + int activation_scale_log2 = 0; if (!common_prepare_k16_major_f16_input(context, *mat.input, mat.input_size, mat.token_count, - dispatch_match, activation)) { + dispatch_match, activation, activation_scale_log2)) { return false; } Dispatch dispatch; dispatch.kernel = make_kernel_specialization(deferred_state ? kMulMatConv4InteriorKernel : kMulMatConv4Kernel); + common_set_f16_prefill_activation_scale(dispatch, activation_scale_log2); dispatch.kernel.integer_parameters.emplace("token_count", mat.token_count); set_compile_parameter(dispatch.kernel, "ggml.mul_mat.input_size", mat.input_size); set_compile_parameter(dispatch.kernel, "ggml.mul_mat.output_size", mat.output_size); diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/mul_mat_quantized_f16_prefill.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/mul_mat_quantized_f16_prefill.loom index 86a71706509a..396873ca8f68 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/mul_mat_quantized_f16_prefill.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/mul_mat_quantized_f16_prefill.loom @@ -1,6 +1,11 @@ // Copyright 2026 The HRX Authors // SPDX-License-Identifier: Apache-2.0 +// The f16 activation this body reads is stored as value * 2^-activation_scale_log2 (0 or 6): a copy the dispatch +// narrows from f32 is prescaled by 2^-6 so an activation above the f16 maximum (65504) stays finite, and the f32 +// accumulators are scaled back here. An f16 value another kernel published is read as it is (0). engine #1962. +config.def @ggml.f16_prefill.activation_scale_log2 = 0 : index + template.decl @ggml.binary_f32.apply_vector8(%op: index, %lhs: vector<8xf32>, %rhs: vector<8xf32>) -> (vector<8xf32>) func.decl @ggml_q4k_native_row64_f16_pair(%half_packet: i1, %paired: i1, %weight: buffer, %peer: buffer, %is_up: i1, %row_base: offset, %group0: index, %packet: index) -> (vector<16xf16>, vector<16xf16>) @@ -614,7 +619,7 @@ func.def inline @ggml_mul_mat_quantized_f16_prefill_wave32_body(%weight_format: func.call @ggml_f16_prefill_publish_weights(%weight_format, %paired, %packed_input, %short_weight_packets, %true, %bounded_input_size, %bounded_output_size, %channel_tile, %c0, %weight_noalias, %up_weight, %weight_stage, %c0_offset) : (index, i1, i1, i1, i1, index, index, index, index, buffer, buffer, buffer, offset) kernel.barrier scope(workgroup) ordering(acq_rel) } - %result00, %result01, %result02, %result03, %result10, %result11, %result12, %result13, %final_a0, %final_a1 = scf.for %weight_epoch = [%c0 to %weight_epoch_count step %c1](%block_acc00 = %init00 : vector<8xf32>, %block_acc01 = %init01 : vector<8xf32>, %block_acc02 = %init02 : vector<8xf32>, %block_acc03 = %init03 : vector<8xf32>, %block_acc10 = %init10 : vector<8xf32>, %block_acc11 = %init11 : vector<8xf32>, %block_acc12 = %init12 : vector<8xf32>, %block_acc13 = %init13 : vector<8xf32>, %block_a0 = %initial_a0 : vector<8xf16>, %block_a1 = %initial_a1 : vector<8xf16>) -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf16>, vector<8xf16>) { + %sum00, %sum01, %sum02, %sum03, %sum10, %sum11, %sum12, %sum13, %final_a0, %final_a1 = scf.for %weight_epoch = [%c0 to %weight_epoch_count step %c1](%block_acc00 = %init00 : vector<8xf32>, %block_acc01 = %init01 : vector<8xf32>, %block_acc02 = %init02 : vector<8xf32>, %block_acc03 = %init03 : vector<8xf32>, %block_acc10 = %init10 : vector<8xf32>, %block_acc11 = %init11 : vector<8xf32>, %block_acc12 = %init12 : vector<8xf32>, %block_acc13 = %init13 : vector<8xf32>, %block_a0 = %initial_a0 : vector<8xf16>, %block_a1 = %initial_a1 : vector<8xf16>) -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf16>, vector<8xf16>) { %quant_block = index.div %weight_epoch, %epochs_per_block : index %weight_epoch_half = index.rem %weight_epoch, %epochs_per_block : index %groups_per_epoch = scf.select %wide_tile, %c2, %c4 : index @@ -651,6 +656,22 @@ func.def inline @ggml_mul_mat_quantized_f16_prefill_wave32_body(%weight_format: } scf.yield %block_result00, %block_result01, %block_result02, %block_result03, %block_result10, %block_result11, %block_result12, %block_result13, %block_next_a0, %block_next_a1 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf16>, vector<8xf16> } + %activation_scale_log2 = config.get @ggml.f16_prefill.activation_scale_log2 : index + %activation_scaled = index.cmp ne, %activation_scale_log2, %c0 : index + %result00, %result01, %result02, %result03, %result10, %result11, %result12, %result13 = scf.if %activation_scaled -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) { + %activation_unscale = vector.constant 64.0 : vector<8xf32> + %unscaled00 = vector.mulf %sum00, %activation_unscale : vector<8xf32> + %unscaled01 = vector.mulf %sum01, %activation_unscale : vector<8xf32> + %unscaled02 = vector.mulf %sum02, %activation_unscale : vector<8xf32> + %unscaled03 = vector.mulf %sum03, %activation_unscale : vector<8xf32> + %unscaled10 = vector.mulf %sum10, %activation_unscale : vector<8xf32> + %unscaled11 = vector.mulf %sum11, %activation_unscale : vector<8xf32> + %unscaled12 = vector.mulf %sum12, %activation_unscale : vector<8xf32> + %unscaled13 = vector.mulf %sum13, %activation_unscale : vector<8xf32> + scf.yield %unscaled00, %unscaled01, %unscaled02, %unscaled03, %unscaled10, %unscaled11, %unscaled12, %unscaled13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> + } else { + scf.yield %sum00, %sum01, %sum02, %sum03, %sum10, %sum11, %sum12, %sum13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> + } %token_subgroups = scf.select %packed_input, %c16, %c8 : index %token_subgroup = index.rem %subgroup, %token_subgroups : index %subgroup_token_add = index.mul %token_subgroup, %c32 : index @@ -681,11 +702,15 @@ func.def inline @ggml_mul_mat_quantized_f16_prefill_wave32_body(%weight_format: %f16_output_view0 = buffer.view %f16_output[%f16_base0] : buffer -> view<[%bounded_token_count]x[%f16_columns]xf16> %f16_output_view1 = buffer.view %f16_output[%f16_base1] : buffer -> view<[%bounded_token_count]x[%f16_columns]xf16> scf.if %paired { + // The f16 copy of the fused result is published as value * 2^-6, like the copies the dispatch makes: the + // projection that reads it scales back (activation_scale_log2 = 6). The f32 output is the plain value. + %f16_output_prescale = vector.constant 0.015625 : vector<8xf32> %out_m00 = index.add %token_tile_base, %c0 : index %out_n00 = index.add %channel_output_base, %c0 : index %fused00 = template.apply<@ggml.binary_f32.apply_vector8>(%binary_op, %result00, %result02) : (index, vector<8xf32>, vector<8xf32>) -> (vector<8xf32>) vector.fragment.store %fused00, %output_view[%out_m00, %out_n00] shape [%m, %n] : vector<8xf32>, view<[%bounded_token_count]x[%bounded_output_size]xf32> - %narrow00 = vector.fptrunc %fused00 : vector<8xf32> to vector<8xf16> + %prescaled00 = vector.mulf %fused00, %f16_output_prescale : vector<8xf32> + %narrow00 = vector.fptrunc %prescaled00 : vector<8xf32> to vector<8xf16> %f16_n00 = scf.select %packed_output, %c0, %out_n00 : index vector.fragment.store %narrow00, %f16_output_view0[%out_m00, %f16_n00] shape [%m, %n] : vector<8xf16>, view<[%bounded_token_count]x[%f16_columns]xf16> scf.schedule.fence @@ -693,7 +718,8 @@ func.def inline @ggml_mul_mat_quantized_f16_prefill_wave32_body(%weight_format: %out_n01 = index.add %channel_output_base, %c16 : index %fused01 = template.apply<@ggml.binary_f32.apply_vector8>(%binary_op, %result01, %result03) : (index, vector<8xf32>, vector<8xf32>) -> (vector<8xf32>) vector.fragment.store %fused01, %output_view[%out_m01, %out_n01] shape [%m, %n] : vector<8xf32>, view<[%bounded_token_count]x[%bounded_output_size]xf32> - %narrow01 = vector.fptrunc %fused01 : vector<8xf32> to vector<8xf16> + %prescaled01 = vector.mulf %fused01, %f16_output_prescale : vector<8xf32> + %narrow01 = vector.fptrunc %prescaled01 : vector<8xf32> to vector<8xf16> %f16_n01 = scf.select %packed_output, %c0, %out_n01 : index vector.fragment.store %narrow01, %f16_output_view1[%out_m01, %f16_n01] shape [%m, %n] : vector<8xf16>, view<[%bounded_token_count]x[%f16_columns]xf16> scf.schedule.fence @@ -701,7 +727,8 @@ func.def inline @ggml_mul_mat_quantized_f16_prefill_wave32_body(%weight_format: %out_n10 = index.add %channel_output_base, %c0 : index %fused10 = template.apply<@ggml.binary_f32.apply_vector8>(%binary_op, %result10, %result12) : (index, vector<8xf32>, vector<8xf32>) -> (vector<8xf32>) vector.fragment.store %fused10, %output_view[%out_m10, %out_n10] shape [%m, %n] : vector<8xf32>, view<[%bounded_token_count]x[%bounded_output_size]xf32> - %narrow10 = vector.fptrunc %fused10 : vector<8xf32> to vector<8xf16> + %prescaled10 = vector.mulf %fused10, %f16_output_prescale : vector<8xf32> + %narrow10 = vector.fptrunc %prescaled10 : vector<8xf32> to vector<8xf16> %f16_n10 = scf.select %packed_output, %c0, %out_n10 : index vector.fragment.store %narrow10, %f16_output_view0[%out_m10, %f16_n10] shape [%m, %n] : vector<8xf16>, view<[%bounded_token_count]x[%f16_columns]xf16> scf.schedule.fence @@ -709,7 +736,8 @@ func.def inline @ggml_mul_mat_quantized_f16_prefill_wave32_body(%weight_format: %out_n11 = index.add %channel_output_base, %c16 : index %fused11 = template.apply<@ggml.binary_f32.apply_vector8>(%binary_op, %result11, %result13) : (index, vector<8xf32>, vector<8xf32>) -> (vector<8xf32>) vector.fragment.store %fused11, %output_view[%out_m11, %out_n11] shape [%m, %n] : vector<8xf32>, view<[%bounded_token_count]x[%bounded_output_size]xf32> - %narrow11 = vector.fptrunc %fused11 : vector<8xf32> to vector<8xf16> + %prescaled11 = vector.mulf %fused11, %f16_output_prescale : vector<8xf32> + %narrow11 = vector.fptrunc %prescaled11 : vector<8xf32> to vector<8xf16> %f16_n11 = scf.select %packed_output, %c0, %out_n11 : index vector.fragment.store %narrow11, %f16_output_view1[%out_m11, %f16_n11] shape [%m, %n] : vector<8xf16>, view<[%bounded_token_count]x[%f16_columns]xf16> scf.schedule.fence diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/copy_f32.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/copy_f32.loom index 5f9d043acadd..31347b0eb45a 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/copy_f32.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/copy_f32.loom @@ -91,7 +91,10 @@ kernel.def target(@ggml_copy_f16_k16_major_target) export("ggml_copy_f16_k16_maj } else { %source = buffer.view %input_noalias[%c0] : buffer -> view<[%tokens]x[%width]xf32> %loaded = vector.load %source[%bounded_row, %bounded_k] : view<[%tokens]x[%width]xf32> -> vector<8xf32> - %converted = vector.fptrunc %loaded : vector<8xf32> to vector<8xf16> + // Stored as value * 2^-6: an activation above the f16 maximum stays finite, the reader scales back (engine #1962). + %f16_range_prescale = vector.constant 0.015625 : vector<8xf32> + %prescaled = vector.mulf %loaded, %f16_range_prescale : vector<8xf32> + %converted = vector.fptrunc %prescaled : vector<8xf32> to vector<8xf16> scf.yield %converted : vector<8xf16> } vector.store %values, %dest[%linear] : vector<8xf16>, view<[%count]xf16> @@ -178,7 +181,8 @@ check.case public @ggml_copy_f32_small_case { check.benchmark<@ggml_copy_f32_small_case> @ggml_copy_f32_small -// Aligned tensor conversions used by existing F16-consuming matmul entries. +// Aligned tensor conversion for the F16-consuming matmul entries. The copy is value * 2^-6 (engine #1962); the +// dispatch records that scale with the alternate value and the consuming kernel scales its result back. kernel.def target(@ggml_copy_f32_gfx11_wave64) export("ggml_copy_f32_f16") @ggml_copy_f32_f16(%element_count: index) { %one = index.constant 1 : index %threads = index.constant 256 : index @@ -204,7 +208,10 @@ kernel.def target(@ggml_copy_f32_gfx11_wave64) export("ggml_copy_f32_f16") @ggml %output_view = buffer.view %output_noalias[%zero] : buffer -> view<[%count]xf16> scf.if %valid { %values = vector.load %source_view[%linear] : view<[%count]xf32> -> vector<4xf32> - %converted = vector.fptrunc %values : vector<4xf32> to vector<4xf16> + // Stored as value * 2^-6: an activation above the f16 maximum stays finite, the reader scales back (engine #1962). + %f16_range_prescale = vector.constant 0.015625 : vector<4xf32> + %prescaled = vector.mulf %values, %f16_range_prescale : vector<4xf32> + %converted = vector.fptrunc %prescaled : vector<4xf32> to vector<4xf16> vector.store %converted, %output_view[%linear] : vector<4xf16>, view<[%count]xf16> } kernel.return @@ -214,7 +221,7 @@ check.case public @ggml_copy_f32_f16_small_case { %count = check.literal value(12) : index %source = check.generate.iota offset(-2.0) step(0.5) : tensor<12xf32> %output = check.generate.fill value(0.0) : tensor<12xf16> - %expected = check.generate.iota offset(-2.0) step(0.5) : tensor<12xf16> + %expected = check.generate.iota offset(-0.03125) step(0.0078125) : tensor<12xf16> kernel.launch @ggml_copy_f32_f16[%count](%count, %source, %output) : [index](index, tensor<12xf32>, tensor<12xf16>) check.expect.close actual(%output) expected(%expected) atol(0.0) rtol(0.0) nan(same) : tensor<12xf16> check.return diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom index f1ae5b6344c6..52a683fa2b8c 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom @@ -10,6 +10,9 @@ config.decl @ggml.mul_mat_q6_k_packed.weight_offset : %value: index where [range config.decl @ggml.mul_mat_q6_k_packed.token_capacity : %value: index where [range(%value, 128, 2048), multiple_of(%value, 128)] +// Defined (default 0) with the shared f16 prefill body in motifs/mul_mat_quantized_f16_prefill.loom. +config.decl @ggml.f16_prefill.activation_scale_log2 : %value: index where [range(%value, 0, 6)] + config.decl @ggml.mul_mat_q6_k_shortlist.input_size : %value: index where [range(%value, 256, 32768), multiple_of(%value, 256)] config.decl @ggml.mul_mat_q6_k_shortlist.output_size : %value: index where [range(%value, 64, 262144), multiple_of(%value, 64)] @@ -1293,6 +1296,7 @@ template.def<@ggml.mul_mat_q6_k_packed.publish_f32_wave32> device @ggml_mul_mat_ } template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat_q6_k_packed_prefill_wave32_body(%input_is_f16: i1, %weight_layout: index, %token_count: index, %input_size0: index, %output_size0: index, %channel_tile: index, %token_tile: index, %input: buffer, %weight: buffer, %output: buffer) { + %f16_range_prescale = vector.constant 0.015625 : vector<8xf32> %output_accumulation = config.get @ggml.mul_mat_q6_k_packed.output_accumulation : index %bounded_token_count = index.assume %token_count [range(%token_count, 1, 2048)] : index %bounded_input_size = index.assume %input_size0 [range(%input_size0, 256, 32768), multiple_of(%input_size0, 256)] : index @@ -1365,7 +1369,7 @@ template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat %init11 = vector.fragment %zero_accumulator shape [%m, %n] : vector<8xf32> %init12 = vector.fragment %zero_accumulator shape [%m, %n] : vector<8xf32> %init13 = vector.fragment %zero_accumulator shape [%m, %n] : vector<8xf32> - %result00, %result01, %result02, %result03, %result10, %result11, %result12, %result13 = scf.for %quant_block = [%c0 to %quant_block_count step %c1](%block_acc00 = %init00 : vector<8xf32>, %block_acc01 = %init01 : vector<8xf32>, %block_acc02 = %init02 : vector<8xf32>, %block_acc03 = %init03 : vector<8xf32>, %block_acc10 = %init10 : vector<8xf32>, %block_acc11 = %init11 : vector<8xf32>, %block_acc12 = %init12 : vector<8xf32>, %block_acc13 = %init13 : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) { + %sum00, %sum01, %sum02, %sum03, %sum10, %sum11, %sum12, %sum13 = scf.for %quant_block = [%c0 to %quant_block_count step %c1](%block_acc00 = %init00 : vector<8xf32>, %block_acc01 = %init01 : vector<8xf32>, %block_acc02 = %init02 : vector<8xf32>, %block_acc03 = %init03 : vector<8xf32>, %block_acc10 = %init10 : vector<8xf32>, %block_acc11 = %init11 : vector<8xf32>, %block_acc12 = %init12 : vector<8xf32>, %block_acc13 = %init13 : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) { %block_result00, %block_result01, %block_result02, %block_result03, %block_result10, %block_result11, %block_result12, %block_result13 = scf.for %quant_group = [%c0 to %c8 step %c2](%acc00 = %block_acc00 : vector<8xf32>, %acc01 = %block_acc01 : vector<8xf32>, %acc02 = %block_acc02 : vector<8xf32>, %acc03 = %block_acc03 : vector<8xf32>, %acc10 = %block_acc10 : vector<8xf32>, %acc11 = %block_acc11 : vector<8xf32>, %acc12 = %block_acc12 : vector<8xf32>, %acc13 = %block_acc13 : vector<8xf32>) -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) { %block_k_base = index.mul %quant_block, %c256 : index %group_k_add = index.mul %quant_group, %c32 : index @@ -1442,7 +1446,8 @@ template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat scf.yield %narrow0 : vector<8xf16> } else { %wide0 = vector.load %input_f32_view[%bounded_token0, %input_k0] : view<[%bounded_token_count]x[%bounded_input_size]xf32> -> vector<8xf32> - %narrow0 = vector.fptrunc %wide0 : vector<8xf32> to vector<8xf16> + %prescaled0 = vector.mulf %wide0, %f16_range_prescale : vector<8xf32> + %narrow0 = vector.fptrunc %prescaled0 : vector<8xf32> to vector<8xf16> scf.yield %narrow0 : vector<8xf16> } scf.yield %loaded0 : vector<8xf16> @@ -1480,7 +1485,8 @@ template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat scf.yield %narrow1 : vector<8xf16> } else { %wide1 = vector.load %input_f32_view[%bounded_token1, %input_k1] : view<[%bounded_token_count]x[%bounded_input_size]xf32> -> vector<8xf32> - %narrow1 = vector.fptrunc %wide1 : vector<8xf32> to vector<8xf16> + %prescaled1 = vector.mulf %wide1, %f16_range_prescale : vector<8xf32> + %narrow1 = vector.fptrunc %prescaled1 : vector<8xf32> to vector<8xf16> scf.yield %narrow1 : vector<8xf16> } scf.yield %loaded1 : vector<8xf16> @@ -1509,6 +1515,29 @@ template.def<@ggml.mul_mat_q6_k_packed.prefill_wave32.body> device @ggml_mul_mat } scf.yield %block_result00, %block_result01, %block_result02, %block_result03, %block_result10, %block_result11, %block_result12, %block_result13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> } + // The activation enters WMMA as value * 2^-6 so that one above the f16 maximum (65504) stays finite: an f32 + // activation is prescaled where it is narrowed above, an f16 one arrives prescaled when the dispatch says so + // (activation_scale_log2). The f32 sums are scaled back here, which is exact. engine #1962 + %f16_range_scale_log2 = config.get @ggml.f16_prefill.activation_scale_log2 : index + %f16_range_c0 = index.constant 0 : index + %f16_range_true = scalar.constant true : i1 + %f16_input_scaled = index.cmp ne, %f16_range_scale_log2, %f16_range_c0 : index + %f16_range_f32_input = scalar.xori %input_is_f16, %f16_range_true : i1 + %f16_range_scaled = scalar.ori %f16_range_f32_input, %f16_input_scaled : i1 + %result00, %result01, %result02, %result03, %result10, %result11, %result12, %result13 = scf.if %f16_range_scaled -> (vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>) { + %f16_range_unscale = vector.constant 64.0 : vector<8xf32> + %unscaled00 = vector.mulf %sum00, %f16_range_unscale : vector<8xf32> + %unscaled01 = vector.mulf %sum01, %f16_range_unscale : vector<8xf32> + %unscaled02 = vector.mulf %sum02, %f16_range_unscale : vector<8xf32> + %unscaled03 = vector.mulf %sum03, %f16_range_unscale : vector<8xf32> + %unscaled10 = vector.mulf %sum10, %f16_range_unscale : vector<8xf32> + %unscaled11 = vector.mulf %sum11, %f16_range_unscale : vector<8xf32> + %unscaled12 = vector.mulf %sum12, %f16_range_unscale : vector<8xf32> + %unscaled13 = vector.mulf %sum13, %f16_range_unscale : vector<8xf32> + scf.yield %unscaled00, %unscaled01, %unscaled02, %unscaled03, %unscaled10, %unscaled11, %unscaled12, %unscaled13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> + } else { + scf.yield %sum00, %sum01, %sum02, %sum03, %sum10, %sum11, %sum12, %sum13 : vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32>, vector<8xf32> + } template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result00, %c0, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result01, %c16, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) template.apply<@ggml.mul_mat_q6_k_packed.publish_f32_wave32>(%result02, %c32, %token_tile_base, %bounded_token_count, %bounded_output_size, %channel_tile_base, %subgroup_channel_add, %output_noalias, %output_accumulation) : (vector<8xf32>, index, index, index, index, index, index, buffer, index) diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index 8fffb7d6f07f..a6c76b65525a 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -10777,8 +10777,9 @@ static void run_tiled_matmul_alternate_publish_checks() { const size_t alternate_bytes = test.alternate_type == GGML_TYPE_F16 ? projection_value->byte_count / 2 : static_cast(256) * ggml_row_size(GGML_TYPE_Q8_1, 2048); - const auto * alternate = - plan.metadata.find_alternate_value(projection_value->id, test.alternate_type, alternate_bytes); + // engine #1962: an f16 activation copy holds value * 2^-6 and is found under that scale only + const auto * alternate = plan.metadata.find_alternate_value(projection_value->id, test.alternate_type, alternate_bytes, + test.alternate_type == GGML_TYPE_F16 ? 6 : 0); REQUIRE(alternate != nullptr); // engine #449: neither publish producer this table named is in the corpus any more. The @@ -10926,8 +10927,10 @@ static void run_rmsnorm_binary_f16_output_checks() { // and a consumer reads it - i.e. the f16 operand path is still there. const ggml::hrx::Value * norm_value = imported.graph.values().find_tensor(norm); REQUIRE(norm_value != nullptr); + // engine #1962: the copy holds value * 2^-6; a reader that does not ask for the scale must not find it + REQUIRE(plan.metadata.find_alternate_value(norm_value->id, GGML_TYPE_F16, norm_value->byte_count / 2) == nullptr); const auto * f16 = - plan.metadata.find_alternate_value(norm_value->id, GGML_TYPE_F16, norm_value->byte_count / 2); + plan.metadata.find_alternate_value(norm_value->id, GGML_TYPE_F16, norm_value->byte_count / 2, 6); REQUIRE(f16 != nullptr); const auto publisher = std::find_if(plan.dispatches.begin(), plan.dispatches.end(), [&](const auto & dispatch) { return !dispatch.bindings.empty() && dispatch.bindings.back().value == f16->alternate_value && @@ -10979,7 +10982,7 @@ static void run_rmsnorm_binary_f16_output_checks() { const auto * q8 = plan.metadata.find_alternate_value(norm_value->id, GGML_TYPE_Q8_1, qwen_q8_1_x4_size(512, 2048)); const auto * f16 = - plan.metadata.find_alternate_value(norm_value->id, GGML_TYPE_F16, norm_value->byte_count / 2); + plan.metadata.find_alternate_value(norm_value->id, GGML_TYPE_F16, norm_value->byte_count / 2, 6); REQUIRE(q8 != nullptr); REQUIRE(f16 != nullptr); for (const ggml::hrx::CommandPlanAlternateValue * publication : { q8, f16 }) { @@ -11388,7 +11391,8 @@ static void run_packed_f16_producer_consumer_checks() { REQUIRE(value != nullptr); const auto * generated = plan.metadata.find_generated_resource(value->id, ggml::hrx::GeneratedResourceRole::F16K16Major); - const auto * alternate = plan.metadata.find_alternate_value(value->id, GGML_TYPE_F16, value->byte_count / 2); + // engine #1962: the gated product's f16 form is published at 2^-6 + const auto * alternate = plan.metadata.find_alternate_value(value->id, GGML_TYPE_F16, value->byte_count / 2, 6); REQUIRE((generated != nullptr) == (test.packed || test.packed_copy)); // engine #449: a row that used to get a plain f16 alternate now gets the packed k16-major form // (or the other way round). What the row asserts is that the producer's f16 output is published. @@ -11406,10 +11410,14 @@ static void run_packed_f16_producer_consumer_checks() { const auto consumer = std::next(producer); REQUIRE(consumer != plan.dispatches.end()); REQUIRE(consumer->bindings.front().value == generated->generated_value); + // the consumer is told the scale of the pack it reads + REQUIRE(generated->scale_log2 == 6); + require_compile_parameter(*consumer, "ggml.f16_prefill.activation_scale_log2", "6"); if (test.type == GGML_TYPE_Q4_K) { require_compile_parameter(*consumer, "ggml.mul_mat.f16_input_layout", "1"); } } else if (test.packed_copy) { + REQUIRE(generated->scale_log2 == 6); const auto copy = std::next(producer); REQUIRE(copy != plan.dispatches.end()); REQUIRE(kernel_name_for_id(copy->kernel.kernel_id) == "loom_libs:ggml_copy_f16_k16_major"); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 83e90abdf59b..5e6a5939cbb8 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9527,8 +9527,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_resident_weight(GGML_TYPE_Q6_K, 2048, 1, 4096)); test_cases.emplace_back(new test_mul_mat_resident_weight(GGML_TYPE_Q5_K, 2048, 512, 4096)); // engine #1962: the two small-batch Q6_K kernels with activations of the magnitude measured in flint-40b, and the - // lower edge (the smallest dense-input column maximum measured on four models). The 512-token prefill kernels for - // Q4_K and Q6_K still overflow above 65504 and are not listed until they pass. + // lower edge (the smallest dense-input column maximum measured on four models). test_cases.emplace_back(new test_mul_mat_resident_weight_large_activation(GGML_TYPE_Q6_K, 2048, 8, 4096, 9.1e4f)); test_cases.emplace_back(new test_mul_mat_resident_weight_large_activation(GGML_TYPE_Q6_K, 4096, 8, 512, 9.1e4f)); test_cases.emplace_back(new test_mul_mat_resident_weight_large_activation(GGML_TYPE_Q6_K, 2048, 8, 4096, 2.4e-2f)); @@ -9542,6 +9541,16 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_resident_weight_large_activation(GGML_TYPE_Q5_K, 2048, 512, 4096, 9.1e4f)); test_cases.emplace_back(new test_mul_mat_resident_weight_large_activation(GGML_TYPE_Q5_K, 2048, 512, 4096, 1.0e6f)); test_cases.emplace_back(new test_mul_mat_resident_weight_large_activation(GGML_TYPE_Q5_K, 2048, 512, 4096, 2.4e-2f)); + // engine #1962: the prefill kernels, which read an f16 copy of the activation. Q4_K at 256 tokens (staged copy) and + // 512 (packed copy), Q6_K at 128 (the narrow-tile body), 256 and 512. The feed-forward hand-over (gate/up kernel + // to down-projection) needs the fused kernels and is in test-hrx-ops (ffn.q4_k.prefill.large_activation.*). + for (float magnitude : { 9.1e4f, 2.4e-2f }) { + test_cases.emplace_back(new test_mul_mat_resident_weight_large_activation(GGML_TYPE_Q4_K, 2048, 256, 4096, magnitude)); + test_cases.emplace_back(new test_mul_mat_resident_weight_large_activation(GGML_TYPE_Q4_K, 2048, 512, 4096, magnitude)); + test_cases.emplace_back(new test_mul_mat_resident_weight_large_activation(GGML_TYPE_Q6_K, 2048, 128, 4096, magnitude)); + test_cases.emplace_back(new test_mul_mat_resident_weight_large_activation(GGML_TYPE_Q6_K, 2048, 256, 4096, magnitude)); + test_cases.emplace_back(new test_mul_mat_resident_weight_large_activation(GGML_TYPE_Q6_K, 2048, 512, 4096, magnitude)); + } test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 576, 512, 576, {1,1}, {1,1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 1, 2048, 8192, {1, 1}, {1, 1})); diff --git a/tests/test-hrx-ops.cpp b/tests/test-hrx-ops.cpp index be17e1e2fff4..72638e97dae0 100644 --- a/tests/test-hrx-ops.cpp +++ b/tests/test-hrx-ops.cpp @@ -422,7 +422,11 @@ static void append_match_to_plan(ggml::hrx::CommandPlan & plan, for (ggml::hrx::CommandPlanCompletionCounterRequest & request : match.completion_counter_requests) { plan.completion_counter_requests.push_back(std::move(request)); } - REQUIRE(plan.metadata.append(std::move(match.metadata), plan.status)); + const bool metadata_appended = plan.metadata.append(std::move(match.metadata), plan.status); + for (const std::string & error : plan.status.errors()) { + std::fprintf(stderr, "plan status: %s\n", error.c_str()); + } + REQUIRE(metadata_appended); plan.status.append(match.status); for (size_t covered_node : match.covered_nodes) { REQUIRE(covered_node < covered_nodes.size()); @@ -3336,6 +3340,111 @@ static void run_dense_matmul_swiglu_cpu_reference_case(ggml_type gate_weight_ ggml_backend_free(hrx_backend); } +// engine #1962: down(swiglu(gate(x), up(x))) at prefill, with a gated product above the f16 maximum. +// The gate/up kernel hands the f16 form of its result to the down-projection. +static void run_ffn_prefill_large_activation_case(int64_t token_count, float input_scale) { + constexpr int64_t input_size = 1024; + constexpr int64_t ff_size = 4096; + ggml_backend_t cpu_backend = init_cpu_backend(); + ggml_backend_t hrx_backend = ggml_backend_hrx_init(0); + REQUIRE(hrx_backend != nullptr); + + ggml_init_params params = {}; + params.mem_size = static_cast(48 * 1024 * 1024); + params.no_alloc = true; + ggml_context * cpu_ctx = ggml_init(params); + ggml_context * hrx_ctx = ggml_init(params); + REQUIRE(cpu_ctx != nullptr); + REQUIRE(hrx_ctx != nullptr); + + struct Graph { + ggml_tensor * gate_weight; + ggml_tensor * up_weight; + ggml_tensor * down_weight; + ggml_tensor * input; + ggml_tensor * gated; + ggml_tensor * output; + }; + auto build = [&](ggml_context * ctx) { + Graph g; + g.gate_weight = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_K, input_size, ff_size); + g.up_weight = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_K, input_size, ff_size); + g.down_weight = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_K, ff_size, input_size); + g.input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, input_size, token_count); + g.gated = ggml_glu_split(ctx, ggml_mul_mat(ctx, g.gate_weight, g.input), ggml_mul_mat(ctx, g.up_weight, g.input), + GGML_GLU_OP_SWIGLU); + g.output = ggml_mul_mat(ctx, g.down_weight, g.gated); + return g; + }; + const Graph cpu = build(cpu_ctx); + const Graph hrx = build(hrx_ctx); + + auto hrx_gate_buffer = allocate_test_weight(hrx_backend, hrx.gate_weight); + auto hrx_up_buffer = allocate_test_weight(hrx_backend, hrx.up_weight); + auto hrx_down_buffer = allocate_test_weight(hrx_backend, hrx.down_weight); + + ggml_cgraph * cpu_graph = ggml_new_graph(cpu_ctx); + ggml_cgraph * hrx_graph = ggml_new_graph(hrx_ctx); + ggml_build_forward_expand(cpu_graph, cpu.output); + ggml_build_forward_expand(hrx_graph, hrx.output); + + const std::vector sequence = scheduled_kernel_sequence(hrx_graph); + REQUIRE(test_sequence_has(sequence, "loom_libs:ggml_mul_mat_swiglu_q4_k_f16_wmma_prefill_wave32")); + REQUIRE(test_sequence_has(sequence, "loom_libs:ggml_mul_mat_q4_k_f16_wmma_prefill_wave32")); + + ggml_backend_buffer_t cpu_buffer = ggml_backend_alloc_ctx_tensors(cpu_ctx, cpu_backend); + ggml_backend_buffer_t hrx_buffer = ggml_backend_alloc_ctx_tensors(hrx_ctx, hrx_backend); + REQUIRE(cpu_buffer != nullptr); + REQUIRE(hrx_buffer != nullptr); + + const std::vector gate_weight = make_matmul_weight_bytes(GGML_TYPE_Q4_K, input_size, ff_size, 6); + const std::vector up_weight = make_matmul_weight_bytes(GGML_TYPE_Q4_K, input_size, ff_size, 11); + const std::vector down_weight = make_matmul_weight_bytes(GGML_TYPE_Q4_K, ff_size, input_size, 17); + std::vector input(static_cast(input_size * token_count)); + for (size_t i = 0; i < input.size(); ++i) { + input[i] = input_scale * (static_cast((i * 37) % 17) - 8.0f) / 8.0f; + } + set_tensor_pair_bytes(cpu_backend, cpu.gate_weight, hrx_backend, hrx.gate_weight, gate_weight.data(), gate_weight.size()); + set_tensor_pair_bytes(cpu_backend, cpu.up_weight, hrx_backend, hrx.up_weight, up_weight.data(), up_weight.size()); + set_tensor_pair_bytes(cpu_backend, cpu.down_weight, hrx_backend, hrx.down_weight, down_weight.data(), down_weight.size()); + set_tensor_pair_bytes(cpu_backend, cpu.input, hrx_backend, hrx.input, input.data(), input.size() * sizeof(float)); + + REQUIRE(ggml_backend_graph_compute(cpu_backend, cpu_graph) == GGML_STATUS_SUCCESS); + REQUIRE(ggml_backend_graph_compute(hrx_backend, hrx_graph) == GGML_STATUS_SUCCESS); + ggml_backend_synchronize(cpu_backend); + ggml_backend_synchronize(hrx_backend); + + // the case must put the gated product past the f16 maximum and inside the prescaled window + float gated_max = 0.0f; + for (const float value : get_f32_tensor(cpu_backend, cpu.gated)) { + gated_max = std::max(gated_max, std::fabs(value)); + } + const std::vector expected = get_f32_tensor(cpu_backend, cpu.output); + const std::vector actual = get_f32_tensor(hrx_backend, hrx.output); + double error = 0.0; + double norm = 0.0; + size_t non_finite = 0; + for (size_t i = 0; i < expected.size(); ++i) { + non_finite += std::isfinite(actual[i]) ? 0 : 1; + const double diff = static_cast(actual[i]) - static_cast(expected[i]); + error += diff * diff; + norm += static_cast(expected[i]) * static_cast(expected[i]); + } + std::fprintf(stderr, "ffn prefill: tokens=%lld gated max=%g non-finite=%zu nmse=%g\n", + static_cast(token_count), static_cast(gated_max), non_finite, error / norm); + REQUIRE(gated_max > 2.0f * 65504.0f); + REQUIRE(gated_max < 2.0e6f); + REQUIRE(non_finite == 0); + REQUIRE(error / norm < 1.0e-3); + + ggml_backend_buffer_free(cpu_buffer); + ggml_backend_buffer_free(hrx_buffer); + ggml_free(cpu_ctx); + ggml_free(hrx_ctx); + ggml_backend_free(cpu_backend); + ggml_backend_free(hrx_backend); +} + static void run_dense_matmul_packed_glu_cpu_reference_case(ggml_type weight_type, const char * expected_kernel, int64_t token_count, @@ -4991,6 +5100,11 @@ static void add_cases(test_runner::Suite & suite) { 4, 4096, GGML_GLU_OP_SWIGLU); }); + for (const int64_t tokens : { 256, 512 }) { + // an input scale of 640 gives a gated maximum of 4.0e5 + add(suite, "ffn.q4_k.prefill.large_activation." + std::to_string(tokens), + [=] { run_ffn_prefill_large_activation_case(tokens, 640.0f); }); + } add(suite, "swiglu.q4_k.q4_k.swiglu_q4_k_f16_wmma_prefill_wave32.256.128.swiglu", [=] { run_dense_matmul_swiglu_cpu_reference_case( GGML_TYPE_Q4_K, GGML_TYPE_Q4_K,