From 54c8b5bc53d0234dbc4c59dae1a1863063a84835 Mon Sep 17 00:00:00 2001 From: GF Date: Thu, 24 Sep 2026 16:46:08 -0400 Subject: [PATCH 1/6] perf: improve codec batching and Metal kernels --- Cargo.lock | 5 +- Cargo.toml | 2 +- crates/j2k-compare/src/common.rs | 37 +- crates/j2k-compare/src/encode_compare.rs | 44 +- .../j2k-compare/src/fixture_compare/tests.rs | 42 - crates/j2k-jpeg-metal/Cargo.toml | 1 + crates/j2k-jpeg-metal/README.md | 9 +- crates/j2k-jpeg-metal/benches/compare.rs | 98 ++ .../benches/support/distinct_batch.rs | 83 +- .../j2k-jpeg-metal/docs/routing-benchmarks.md | 11 +- crates/j2k-jpeg-metal/src/abi.rs | 66 +- .../src/codec_batch/owner_accounting.rs | 84 + .../j2k-jpeg-metal/src/compute/batch_full.rs | 18 + .../src/compute/batch_full/rgb.rs | 208 ++- .../src/compute/batch_full/rgb_grouped.rs | 232 ++- .../src/compute/batch_full/texture.rs | 135 +- .../src/compute/batch_full/texture/staged.rs | 41 +- .../src/compute/batch_full/texture_grouped.rs | 217 ++- .../j2k-jpeg-metal/src/compute/batch_plan.rs | 98 +- .../src/compute/batch_region/rgb.rs | 354 +++-- crates/j2k-jpeg-metal/src/compute/command.rs | 51 +- .../src/compute/fast_packets/descriptors.rs | 11 +- .../src/compute/fast_packets/pipelines.rs | 5 +- crates/j2k-jpeg-metal/src/compute/mod.rs | 9 +- .../compute/pack_dispatch/grouped_output.rs | 3 +- .../src/compute/pipeline_registry.rs | 13 +- crates/j2k-jpeg-metal/src/compute/runtime.rs | 170 +++ .../src/compute/scratch_pool.rs | 80 +- .../src/compute/surface_output_pool.rs | 86 ++ crates/j2k-jpeg-metal/src/compute/tests.rs | 88 +- .../compute/tests/decode_kernel_harness.rs | 1349 +++++++++++++++++ .../src/compute/tests/surface_output_reuse.rs | 224 +++ .../src/compute/texture_tuning.rs | 14 + crates/j2k-jpeg-metal/src/decode_surface.rs | 11 +- crates/j2k-jpeg-metal/src/lib.rs | 2 + crates/j2k-jpeg-metal/src/routing.rs | 14 - crates/j2k-jpeg-metal/src/session.rs | 9 +- crates/j2k-jpeg-metal/src/session/tests.rs | 22 + .../src/shaders_decode_fast420.metal | 26 +- .../src/shaders_decode_fast422_regions.metal | 18 +- .../src/shaders_decode_fast444.metal | 75 +- .../j2k-jpeg-metal/src/shaders_encode.metal | 513 +++---- .../j2k-jpeg-metal/src/shaders_shared.metal | 14 +- .../src/surface/batch_texture.rs | 8 + crates/j2k-jpeg-metal/src/tests.rs | 194 ++- crates/j2k-jpeg-metal/src/tests/textures.rs | 87 +- crates/j2k-jpeg-metal/tests/batch.rs | 66 + crates/j2k-jpeg/benches/decode_cpu.rs | 78 +- crates/j2k-jpeg/fuzz/Cargo.lock | 4 +- .../src/adapter/baseline_encode/tests.rs | 102 -- crates/j2k-jpeg/src/adapter/device_plan.rs | 8 +- crates/j2k-jpeg/src/adapter/mod.rs | 5 + crates/j2k-jpeg/src/baseline_entropy.rs | 40 +- crates/j2k-jpeg/src/batch_session.rs | 115 +- crates/j2k-jpeg/src/batch_session/planning.rs | 65 +- crates/j2k-jpeg/src/batch_session/runtime.rs | 29 - crates/j2k-jpeg/src/batch_session/tests.rs | 39 + crates/j2k-jpeg/src/batch_session/worker.rs | 6 - crates/j2k-jpeg/src/color/upsample.rs | 27 + crates/j2k-jpeg/src/color/ycbcr.rs | 6 +- crates/j2k-jpeg/src/context.rs | 5 +- crates/j2k-jpeg/src/decoder.rs | 19 +- crates/j2k-jpeg/src/decoder/allocation.rs | 41 +- .../j2k-jpeg/src/decoder/extended12/tests.rs | 221 ++- .../src/decoder/extended12/upsample.rs | 37 +- .../src/decoder/extended12/writers.rs | 36 +- crates/j2k-jpeg/src/decoder/plan.rs | 12 +- .../j2k-jpeg/src/decoder/routing/dispatch.rs | 2 +- crates/j2k-jpeg/src/decoder/sequential.rs | 8 +- crates/j2k-jpeg/src/decoder/tests.rs | 45 + crates/j2k-jpeg/src/decoder/tile.rs | 7 +- crates/j2k-jpeg/src/decoder/tile/tests.rs | 20 + crates/j2k-jpeg/src/decoder/view.rs | 2 +- crates/j2k-jpeg/src/encoded_output/tests.rs | 9 - crates/j2k-jpeg/src/encoder/entropy.rs | 12 - .../j2k-jpeg/src/encoder/entropy/restart.rs | 6 - .../j2k-jpeg/src/encoder/entropy/workspace.rs | 9 - crates/j2k-jpeg/src/encoder/transform.rs | 158 +- crates/j2k-jpeg/src/entropy/huffman.rs | 58 +- .../src/entropy/progressive/render.rs | 12 +- .../j2k-jpeg/src/entropy/sequential/emit.rs | 2 - .../entropy/sequential/emit/four_component.rs | 28 +- .../src/entropy/sequential/emit/output.rs | 20 +- .../src/entropy/sequential/emit/region420.rs | 5 +- .../src/entropy/sequential/emit/rgb.rs | 25 +- .../sequential/emit/structure_tests.rs | 83 - .../src/entropy/sequential/emit/upsample.rs | 43 +- .../src/entropy/sequential/generic/row.rs | 17 +- .../j2k-jpeg/src/entropy/sequential/plan.rs | 4 +- .../j2k-jpeg/src/entropy/sequential/tests.rs | 257 ++-- crates/j2k-jpeg/src/idct/scalar.rs | 72 +- crates/j2k-jpeg/src/output/rgba8.rs | 37 +- crates/j2k-jpeg/src/simd/x86.rs | 2 +- .../j2k-jpeg/tests/libjpeg_turbo_compare.rs | 509 ++++++- .../src/batch_decoder/encoder_count_tests.rs | 16 +- crates/j2k-metal/src/buffer_pool/state.rs | 34 +- .../buffer_pool/tests/production_limits.rs | 57 +- crates/j2k-metal/src/decoder/routes.rs | 16 + .../j2k-metal/src/encode/tests/dwt_parity.rs | 106 +- .../src/encode/tests/resident_batches.rs | 118 +- crates/j2k-metal/src/engine/abi.rs | 42 +- .../j2k-metal/src/engine/decode_dispatch.rs | 2 +- .../engine/decode_dispatch/classic_subband.rs | 31 +- .../decode_dispatch/ht_chunks/execution.rs | 4 +- .../engine/decode_dispatch/ht_chunks/tests.rs | 1 + .../ht_chunks/tests/cooperative.rs | 335 ++++ .../decode_dispatch/idwt/irreversible.rs | 157 +- .../idwt/irreversible/interleave_tests.rs | 7 +- .../idwt/irreversible/parity_tests.rs | 277 ++-- .../idwt/irreversible/performance.rs | 3 +- .../src/engine/decode_dispatch/mct.rs | 5 + .../src/engine/decode_dispatch/store.rs | 3 + crates/j2k-metal/src/engine/direct_execute.rs | 6 +- .../color_destination/store.rs | 6 + .../component_plane/execution.rs | 3 + .../destination/submission.rs | 2 + .../engine/direct_grayscale_execute/single.rs | 1 + .../single/execution.rs | 89 +- .../j2k-metal/src/engine/direct_plan_types.rs | 18 + .../src/engine/direct_prepare/color.rs | 9 +- .../src/engine/direct_prepare/grayscale.rs | 1 + .../src/engine/direct_prepare/referenced.rs | 1 + .../command_submission.rs | 2 + .../command_submission/final_store.rs | 1 + .../command_submission/reconstruction.rs | 22 +- .../repeated_grayscale/execution.rs | 2 + .../execution/final_store.rs | 2 + .../engine/direct_stacked_batch/validation.rs | 1 + .../j2k-metal/src/engine/forward_transform.rs | 12 +- .../src/engine/lossless_prepare/commands.rs | 8 + .../engine/resident_codestream/ht_cleanup.rs | 228 ++- crates/j2k-metal/src/engine/runtime/decode.rs | 27 +- crates/j2k-metal/src/engine/shader_source.rs | 1 + crates/j2k-metal/src/engine/surface_decode.rs | 6 +- crates/j2k-metal/src/engine/test_counters.rs | 66 + crates/j2k-metal/src/engine/tests.rs | 1 + .../src/engine/tests/decode_stage_profile.rs | 186 +++ crates/j2k-metal/src/fdwt.metal | 67 +- crates/j2k-metal/src/generated/promotion.rs | 111 +- crates/j2k-metal/src/ht.rs | 66 +- crates/j2k-metal/src/ht_cleanup.metal | 65 +- crates/j2k-metal/src/ht_cleanup_simd.metal | 906 +++++++++++ crates/j2k-metal/src/idwt.metal | 212 ++- crates/j2k-metal/src/mct.metal | 8 +- crates/j2k-metal/src/mct_abi.metal | 1 + crates/j2k-metal/src/pack.metal | 54 +- crates/j2k-metal/src/routing/mod.rs | 4 +- crates/j2k-metal/src/routing/promotion.rs | 24 + crates/j2k-metal/src/routing/tests.rs | 260 ++-- crates/j2k-metal/src/store.metal | 63 +- crates/j2k-metal/tests/auto_routing_parity.rs | 115 ++ crates/j2k-metal/tests/device.rs | 2 + .../tests/device/auto_single_routing.rs | 103 +- .../j2k-metal/tests/device/auto_tile_batch.rs | 19 +- .../j2k-metal/tests/device/direct_repeated.rs | 85 ++ .../device/irreversible_output_rounding.rs | 211 +++ crates/j2k-metal/tests/shader_integrity.rs | 4 +- crates/j2k-native/src/direct_cpu.rs | 3 +- crates/j2k-native/src/direct_cpu/component.rs | 7 + .../j2k-native/src/direct_cpu/referenced.rs | 1 + .../src/direct_cpu/referenced/component.rs | 16 +- .../src/direct_cpu/referenced_classic.rs | 21 +- .../direct_cpu/referenced_staged/finish.rs | 9 +- crates/j2k-native/src/image.rs | 103 +- crates/j2k-native/src/image/output_api.rs | 107 +- crates/j2k-native/src/j2c/mct.rs | 55 +- crates/j2k-native/src/math.rs | 19 +- crates/j2k-t803/fuzz/Cargo.lock | 4 +- crates/j2k-transcode/fuzz/Cargo.lock | 4 +- .../src/jpeg_to_htj2k/error/native_encode.rs | 8 - crates/j2k-transcode/tests/dct53_2d.rs | 15 - crates/j2k/fuzz/Cargo.lock | 4 +- crates/j2k/src/decode.rs | 7 +- crates/j2k/src/owned_batch/cpu_materialize.rs | 6 +- crates/j2k/tests/irreversible_rounding.rs | 214 +++ docs/architecture.md | 4 +- docs/benchmark-evidence.md | 88 ++ docs/env-vars.md | 1 + .../P30-P31-metal-decode-notes.md | 149 ++ .../P30-metal-ht-cooperative-cleanup.json | 542 +++++++ .../P31-metal-idwt97-fused-lifting.json | 542 +++++++ .../P32-metal-jpeg-decode-kernel-notes.md | 174 +++ .../P32-metal-jpeg-decode-kernel.json | 807 ++++++++++ .../P33-metal-jpeg-444-component-planes.json | 302 ++++ .../P34-metal-jpeg-surface-output-pool.json | 806 ++++++++++ .../P35-metal-buffer-pool-cap.json | 358 +++++ .../P36-metal-idwt97-batched-large.json | 484 ++++++ .../P37-metal-jpeg-default-session.json | 550 +++++++ docs/routing-promotion-evidence.json | 29 +- ...able-api-1.0.implementation-public-api.txt | 5 + xtask/src/promotion_codegen.rs | 46 +- 191 files changed, 14572 insertions(+), 2507 deletions(-) create mode 100644 crates/j2k-jpeg-metal/src/compute/surface_output_pool.rs create mode 100644 crates/j2k-jpeg-metal/src/compute/tests/decode_kernel_harness.rs create mode 100644 crates/j2k-jpeg-metal/src/compute/tests/surface_output_reuse.rs delete mode 100644 crates/j2k-jpeg/src/entropy/sequential/emit/structure_tests.rs create mode 100644 crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/tests/cooperative.rs create mode 100644 crates/j2k-metal/src/engine/tests/decode_stage_profile.rs create mode 100644 crates/j2k-metal/src/ht_cleanup_simd.metal create mode 100644 crates/j2k-metal/tests/auto_routing_parity.rs create mode 100644 crates/j2k-metal/tests/device/irreversible_output_rounding.rs create mode 100644 crates/j2k/tests/irreversible_rounding.rs create mode 100644 docs/performance-experiments/P30-P31-metal-decode-notes.md create mode 100644 docs/performance-experiments/P30-metal-ht-cooperative-cleanup.json create mode 100644 docs/performance-experiments/P31-metal-idwt97-fused-lifting.json create mode 100644 docs/performance-experiments/P32-metal-jpeg-decode-kernel-notes.md create mode 100644 docs/performance-experiments/P32-metal-jpeg-decode-kernel.json create mode 100644 docs/performance-experiments/P33-metal-jpeg-444-component-planes.json create mode 100644 docs/performance-experiments/P34-metal-jpeg-surface-output-pool.json create mode 100644 docs/performance-experiments/P35-metal-buffer-pool-cap.json create mode 100644 docs/performance-experiments/P36-metal-idwt97-batched-large.json create mode 100644 docs/performance-experiments/P37-metal-jpeg-default-session.json diff --git a/Cargo.lock b/Cargo.lock index 4e27e2a68..951b87026 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1735,9 +1735,9 @@ dependencies = [ [[package]] name = "fearless_simd" -version = "0.7.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" +checksum = "f3772b63c40606beea8fe1f7b06a60de27ceb9bfc2d9dad3427368ca41dc7e62" [[package]] name = "filetime" @@ -2662,6 +2662,7 @@ name = "j2k-jpeg-metal" version = "0.11.1" dependencies = [ "criterion", + "dispatch2", "j2k-core", "j2k-jpeg", "j2k-metal-support", diff --git a/Cargo.toml b/Cargo.toml index 3ab6b880f..1d0d2a4cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,7 +111,7 @@ jpeg-encoder = "0.7.0" jpeg-decoder = { version = "0.3.2", default-features = false } zune-core = "0.5.1" zune-jpeg = "0.5.15" -fearless_simd = "0.7" +fearless_simd = "1.0" libm = { version = "0.2", default-features = false } openjpeg-sys = "1.0.12" proc-macro2 = "1.0.106" diff --git a/crates/j2k-compare/src/common.rs b/crates/j2k-compare/src/common.rs index 1c68ddb7d..879c8c7ab 100644 --- a/crates/j2k-compare/src/common.rs +++ b/crates/j2k-compare/src/common.rs @@ -432,7 +432,42 @@ pub fn mib_per_second(bytes: usize, elapsed_us: f64) -> f64 { mod tests { use std::path::Path; - use super::infer_corpus_category; + use super::{batch_size_config_from_values, infer_corpus_category, BatchSizeConfig}; + + #[test] + fn batch_size_config_preserves_defaults_and_independent_legacy_overrides() { + for (case_sizes, mixed_sizes, legacy, expected_case, expected_mixed) in [ + (None, None, None, vec![1], vec![1, 16, 256, 1024]), + (Some("3"), None, Some(vec![2, 4]), vec![3], vec![2, 4]), + ( + None, + Some("8,16"), + Some(vec![2, 4]), + vec![2, 4], + vec![8, 16], + ), + ] { + let config = batch_size_config_from_values( + case_sizes, + mixed_sizes, + legacy, + "case batch sizes", + "mixed batch sizes", + &[1], + &[1, 16, 256, 1024], + ) + .expect("batch size config parses"); + + assert_eq!( + config, + BatchSizeConfig { + case_batch_sizes: expected_case, + mixed_batch_sizes: expected_mixed, + }, + "case sizes {case_sizes:?}, mixed sizes {mixed_sizes:?}" + ); + } + } #[test] fn corpus_category_rules_cover_every_supported_needle() { diff --git a/crates/j2k-compare/src/encode_compare.rs b/crates/j2k-compare/src/encode_compare.rs index 647cf8743..6a57c4445 100644 --- a/crates/j2k-compare/src/encode_compare.rs +++ b/crates/j2k-compare/src/encode_compare.rs @@ -137,52 +137,10 @@ mod tests { }; use super::{ canonicalize_manifest_row_path, measurement_row, EncoderKind, EncoderTool, ImageCase, - Measurement, MetadataInput, MixedImageBatch, DEFAULT_CASE_BATCH_SIZES, - DEFAULT_MIXED_BATCH_SIZES, + Measurement, MetadataInput, MixedImageBatch, }; - use crate::common; use std::path::Path; - fn test_batch_size_config_from_values( - case_batch_sizes: Option<&str>, - mixed_batch_sizes: Option<&str>, - legacy: Option>, - ) -> Result { - common::batch_size_config_from_values( - case_batch_sizes, - mixed_batch_sizes, - legacy, - "J2K_ENCODE_COMPARE_CASE_BATCH_SIZES", - "J2K_ENCODE_COMPARE_MIXED_BATCH_SIZES", - DEFAULT_CASE_BATCH_SIZES, - DEFAULT_MIXED_BATCH_SIZES, - ) - } - - #[test] - fn encode_batch_config_defaults_keep_large_batches_mixed_only() { - let config = test_batch_size_config_from_values(None, None, None) - .expect("default batch config parses"); - - assert_eq!(config.case_batch_sizes, DEFAULT_CASE_BATCH_SIZES); - assert_eq!(config.mixed_batch_sizes, DEFAULT_MIXED_BATCH_SIZES); - } - - #[test] - fn encode_batch_config_split_env_overrides_legacy_independently() { - let config = test_batch_size_config_from_values(Some("3"), None, Some(vec![2, 4])) - .expect("case override with legacy config parses"); - - assert_eq!(config.case_batch_sizes, vec![3]); - assert_eq!(config.mixed_batch_sizes, vec![2, 4]); - - let config = test_batch_size_config_from_values(None, Some("8,16"), Some(vec![2, 4])) - .expect("mixed override with legacy config parses"); - - assert_eq!(config.case_batch_sizes, vec![2, 4]); - assert_eq!(config.mixed_batch_sizes, vec![8, 16]); - } - #[test] fn encode_manifest_path_remaps_to_supplied_fixture_root_by_suffix() { let root = std::env::current_dir() diff --git a/crates/j2k-compare/src/fixture_compare/tests.rs b/crates/j2k-compare/src/fixture_compare/tests.rs index 21da1c8c8..c4632632e 100644 --- a/crates/j2k-compare/src/fixture_compare/tests.rs +++ b/crates/j2k-compare/src/fixture_compare/tests.rs @@ -8,52 +8,10 @@ use super::metadata::{ use super::{ canonicalize_manifest_row_path, publication_blockers, unique_input_count, BenchmarkMode, Codec, Container, DecoderKind, FixtureCase, MixedFixtureBatch, Operation, OperationClass, - DEFAULT_CASE_BATCH_SIZES, DEFAULT_MIXED_BATCH_SIZES, }; -use crate::common; use j2k_core::{Downscale, PixelFormat, Rect}; use std::path::Path; -fn test_batch_size_config_from_values( - case_batch_sizes: Option<&str>, - mixed_batch_sizes: Option<&str>, - legacy: Option>, -) -> Result { - common::batch_size_config_from_values( - case_batch_sizes, - mixed_batch_sizes, - legacy, - "J2K_FIXTURE_COMPARE_CASE_BATCH_SIZES", - "J2K_FIXTURE_COMPARE_MIXED_BATCH_SIZES", - DEFAULT_CASE_BATCH_SIZES, - DEFAULT_MIXED_BATCH_SIZES, - ) -} - -#[test] -fn decode_batch_config_defaults_keep_large_batches_mixed_only() { - let config = - test_batch_size_config_from_values(None, None, None).expect("default batch config parses"); - - assert_eq!(config.case_batch_sizes, DEFAULT_CASE_BATCH_SIZES); - assert_eq!(config.mixed_batch_sizes, DEFAULT_MIXED_BATCH_SIZES); -} - -#[test] -fn decode_batch_config_split_env_overrides_legacy_independently() { - let config = test_batch_size_config_from_values(Some("3"), None, Some(vec![2, 4])) - .expect("case override with legacy config parses"); - - assert_eq!(config.case_batch_sizes, vec![3]); - assert_eq!(config.mixed_batch_sizes, vec![2, 4]); - - let config = test_batch_size_config_from_values(None, Some("8,16"), Some(vec![2, 4])) - .expect("mixed override with legacy config parses"); - - assert_eq!(config.case_batch_sizes, vec![2, 4]); - assert_eq!(config.mixed_batch_sizes, vec![8, 16]); -} - #[test] fn decode_manifest_path_remaps_to_supplied_fixture_root_by_suffix() { let root = std::env::current_dir() diff --git a/crates/j2k-jpeg-metal/Cargo.toml b/crates/j2k-jpeg-metal/Cargo.toml index 1775de8f4..0cb8a57c2 100644 --- a/crates/j2k-jpeg-metal/Cargo.toml +++ b/crates/j2k-jpeg-metal/Cargo.toml @@ -26,6 +26,7 @@ j2k-profile = { path = "../j2k-profile", version = "=0.11.1" } thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] +dispatch2 = { version = "=0.3.1", default-features = false, features = ["std"] } objc2 = { workspace = true } objc2-metal = { workspace = true } diff --git a/crates/j2k-jpeg-metal/README.md b/crates/j2k-jpeg-metal/README.md index 50d04957b..7c246b04a 100644 --- a/crates/j2k-jpeg-metal/README.md +++ b/crates/j2k-jpeg-metal/README.md @@ -23,9 +23,12 @@ fast 4:2:0, 4:2:2, or 4:4:4 baseline packet and only `Gray8`, `Rgb8`, or unsupported output formats return `UnsupportedMetalRequest` instead of silently falling back. -`BackendRequest::Auto` stays conservative. Single-image decode remains CPU even -when fast-packet capabilities match. Batched and resident-output paths are the -places to look for Metal wins, and any future Auto widening should be backed by +`BackendRequest::Auto` keeps single-image decode on the CPU. Full RGB batches +can use Metal for at least 16 compatible non-restart 4:2:0 or 4:2:2 tiles of at +least 256×256 pixels, including distinct inputs. The tiles must share dimensions, +tables, and checkpoint count. Smaller, mixed-table, restart-coded, and scaled +batches remain on the CPU. This threshold follows completed +submission measurements on an M4 Pro; further widening should be backed by the benchmark groups documented in [`docs/routing-benchmarks.md`](docs/routing-benchmarks.md). diff --git a/crates/j2k-jpeg-metal/benches/compare.rs b/crates/j2k-jpeg-metal/benches/compare.rs index cd074cb29..789cef088 100644 --- a/crates/j2k-jpeg-metal/benches/compare.rs +++ b/crates/j2k-jpeg-metal/benches/compare.rs @@ -608,6 +608,36 @@ fn auto_decode_tile_batch(bytes: &[u8], batch_size: usize) { device_decode_tile_batch(bytes, batch_size, BackendRequest::Auto); } +/// Decode context, scratch, and Metal session kept across iterations, as a +/// long-running tile server would, unlike `device_decode_tile_batch`. +#[derive(Default)] +struct RetainedTileBatchState { + ctx: JpegDecoderContext, + pool: ScratchPool, + session: MetalSession, +} + +impl RetainedTileBatchState { + fn decode(&mut self, bytes: &[u8], batch_size: usize, backend: BackendRequest) { + let submissions = (0..batch_size) + .map(|_| { + ::submit_tile_to_device( + &mut self.ctx, + &mut self.session, + &mut self.pool, + bytes, + PixelFormat::Rgb8, + backend, + ) + .expect("submit") + }) + .collect::>(); + for submission in submissions { + std::hint::black_box(submission.wait().expect("surface")); + } + } +} + fn device_decode_tile_batch(bytes: &[u8], batch_size: usize, backend: BackendRequest) { let mut ctx = JpegDecoderContext::default(); let mut pool = ScratchPool::new(); @@ -630,6 +660,65 @@ fn device_decode_tile_batch(bytes: &[u8], batch_size: usize, backend: BackendReq } } +#[cfg(target_os = "macos")] +fn bench_distinct_batch_routing(c: &mut Criterion) { + let mut group = c.benchmark_group("jpeg_distinct_batch_routing"); + for (family, sampling) in [ + ("420", SamplingFactor::F_2_2), + ("422", SamplingFactor::F_2_1), + ] { + for side in [64_u16, 256] { + let inputs = (0..64) + .map(|index| generated_rgb_jpeg_variant(side, side, sampling, None, index)) + .collect::>(); + for count in [16, 64] { + let decode = |backend| { + let mut context = JpegDecoderContext::default(); + let mut session = MetalSession::default(); + let mut pool = ScratchPool::new(); + let submissions = inputs[..count] + .iter() + .map(|bytes| { + ::submit_tile_to_device( + &mut context, + &mut session, + &mut pool, + bytes, + PixelFormat::Rgb8, + backend, + ) + .expect("distinct routing submission") + }) + .collect::>(); + submissions + .into_iter() + .map(|submission| submission.wait().expect("distinct routing result")) + .collect::>() + }; + let metal = decode(BackendRequest::Metal); + let cpu = decode(BackendRequest::Cpu); + for (actual, expected) in metal.iter().zip(&cpu) { + assert_metal_surface_pixels( + actual, + expected.as_bytes().expect("CPU pixels").as_ref(), + ); + } + for (label, backend) in [ + ("cpu", BackendRequest::Cpu), + ("metal", BackendRequest::Metal), + ("auto", BackendRequest::Auto), + ] { + group.bench_function( + format!("{family}/{side}x{side}/batch{count}/{label}"), + |b| b.iter(|| std::hint::black_box(decode(backend))), + ); + } + } + } + } + group.finish(); +} + fn metal_decode_tile_batch_scaled(bytes: &[u8], batch_size: usize, factor: Downscale) { device_decode_tile_batch_scaled(bytes, batch_size, factor, BackendRequest::Metal); } @@ -885,6 +974,13 @@ fn bench_full_and_tile_decode_groups(c: &mut Criterion, inputs: &[BenchInput], h wsi_tile_batch_rgb.bench_function(format!("{}/metal", input.name), |b| { b.iter(|| metal_decode_tile_batch(&input.bytes, 64)); }); + let mut retained = RetainedTileBatchState::default(); + wsi_tile_batch_rgb.bench_function( + format!("{}/metal_retained_session", input.name), + |b| { + b.iter(|| retained.decode(&input.bytes, 64, BackendRequest::Metal)); + }, + ); } wsi_tile_batch_rgb.bench_function(format!("{}/auto", input.name), |b| { b.iter(|| auto_decode_tile_batch(&input.bytes, 64)); @@ -1634,6 +1730,8 @@ fn bench_compare(c: &mut Criterion) { #[cfg(target_os = "macos")] if has_metal { distinct_batch::bench(c); + distinct_batch::bench_mixed(c); + bench_distinct_batch_routing(c); representative_matrix::bench(c); } bench_resident_texture_batches(c, &inputs, has_metal); diff --git a/crates/j2k-jpeg-metal/benches/support/distinct_batch.rs b/crates/j2k-jpeg-metal/benches/support/distinct_batch.rs index 004874ec2..4f87636c3 100644 --- a/crates/j2k-jpeg-metal/benches/support/distinct_batch.rs +++ b/crates/j2k-jpeg-metal/benches/support/distinct_batch.rs @@ -13,14 +13,91 @@ use criterion::{Criterion, Throughput}; use j2k_core::PixelFormat; use j2k_jpeg::DecodeRequest; use j2k_jpeg_metal::{ - Codec, Decoder, MetalBackendSession, MetalBatchOutputBuffer, MetalBufferBatchTarget, - Rgb8MetalBatchOp, Rgb8MetalBatchRequest, Rgb8MetalBatchSource, + Codec, Decoder, MetalBackendSession, MetalBatchOutputBuffer, MetalBatchTextureOutput, + MetalBufferBatchTarget, Rgb8MetalBatchOp, Rgb8MetalBatchRequest, Rgb8MetalBatchSource, }; -use jpeg_encoder::SamplingFactor; +use jpeg_encoder::{ColorType, Encoder, SamplingFactor}; use std::{hint::black_box, sync::Barrier, time::Instant}; const BATCH_SIZE: usize = 4; +pub(super) fn bench_mixed(c: &mut Criterion) { + const SIDE: u16 = 128; + const COUNT: usize = 16; + let dimensions = (u32::from(SIDE), u32::from(SIDE)); + let pixels = j2k_test_support::gpu_bench_rgb8(dimensions.0, dimensions.1); + let mut group = c.benchmark_group("jpeg_metal_mixed_tables"); + for (family, sampling) in [ + ("420", SamplingFactor::F_2_2), + ("444", SamplingFactor::F_1_1), + ] { + for table_count in [2, 4] { + let inputs = [95, 85, 70, 50][..table_count] + .iter() + .map(|&quality| { + let mut bytes = Vec::new(); + let mut encoder = Encoder::new(&mut bytes, quality); + encoder.set_sampling_factor(sampling); + encoder + .encode(&pixels, SIDE, SIDE, ColorType::Rgb) + .expect("mixed-table JPEG"); + bytes + }) + .collect::>(); + let expected = inputs + .iter() + .map(|bytes| native_request_pixels(bytes, DecodeRequest::full(PixelFormat::Rgb8))) + .collect::>(); + let decoders = inputs + .iter() + .map(|bytes| Decoder::new(bytes).expect("mixed-table decoder")) + .collect::>(); + let refs = (0..COUNT) + .map(|index| &decoders[index % table_count]) + .collect::>(); + let session = MetalBackendSession::system_default().expect("mixed-table Metal session"); + let output = MetalBatchOutputBuffer::new_rgb8_tiles(&session, dimensions, COUNT) + .expect("mixed-table RGB output"); + let decode = || { + Codec::decode_rgb8_batch_into_buffer_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(&refs), + op: Rgb8MetalBatchOp::Full, + }, + MetalBufferBatchTarget::Reusable(&output), + &session, + ) + .expect("mixed-table RGB decode") + }; + for (index, surface) in decode().into_iter().enumerate() { + assert_metal_surface_pixels( + &surface.expect("mixed-table surface"), + &expected[index % table_count], + ); + } + group.bench_function( + format!("{family}/tables{table_count}/batch{COUNT}/rgb"), + |b| { + b.iter(|| black_box(decode())); + }, + ); + let mut textures = + MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, COUNT) + .expect("mixed-table textures"); + group.bench_function(format!("{family}/tables{table_count}/batch{COUNT}/textures"), |b| { + b.iter(|| { + let tiles = Codec::decode_rgb8_decoder_batch_into_resizable_metal_textures_with_session( + &refs, &mut textures, &session, + ).expect("mixed-table texture decode"); + assert_eq!(tiles.len(), COUNT); + for tile in tiles { black_box(tile.expect("mixed-table texture")); } + }); + }); + } + } + group.finish(); +} + pub(super) fn bench(c: &mut Criterion) { let mut group = c.benchmark_group("jpeg_metal_distinct_batch"); for side in [128_u16, 512] { diff --git a/crates/j2k-jpeg-metal/docs/routing-benchmarks.md b/crates/j2k-jpeg-metal/docs/routing-benchmarks.md index 93212a689..da568893e 100644 --- a/crates/j2k-jpeg-metal/docs/routing-benchmarks.md +++ b/crates/j2k-jpeg-metal/docs/routing-benchmarks.md @@ -22,9 +22,10 @@ Explicit `BackendRequest::Metal` is strict: - Single-image full, region, scaled, and region-scaled requests stay on CPU even when a fast packet exists. -- Small restart-coded tile batches stay CPU. -- Existing restart-coded batch threshold tests cover the current macOS Auto path - that can use Metal for coalesced WSI-style batches. +- Full RGB batches can use Metal for at least 16 compatible non-restart 4:2:0 + or 4:2:2 tiles at least 256×256 pixels, including distinct inputs. Dimensions, + tables, and checkpoint counts must match across the batch. +- Smaller, mixed-table, restart-coded, and scaled tile batches stay CPU. - Sparse viewport workloads stay CPU for scheduled surface output; contiguous restart-coded viewports may use the hybrid path on macOS. Reusable resident viewport outputs may use direct contiguous decode or resident composition. @@ -47,6 +48,10 @@ Use these groups to decide where Metal makes sense: check for CPU-preferred single decode. - `wsi_tile_batch_rgb` and `wsi_tile_batch_scaled_rgb_q4`: repeated tile batches comparing CPU, explicit Metal, and Auto. +- `jpeg_distinct_batch_routing`: completed submissions for distinct 4:2:0 and + 4:2:2 inputs, including plan construction, at 64×64 and 256×256 dimensions. +- `jpeg_metal_mixed_tables`: interleaved table groups targeting reusable RGB + buffers and textures, to measure the cost of submitting multiple groups. - `wsi_tile_batch_region_scaled_coalesced_rgb_q4`: coalesced region+scaled batch candidate where Metal can amortize setup. - `wsi_tile_batch_region_scaled_distinct_rgb_q4`: low-coalescing control case. diff --git a/crates/j2k-jpeg-metal/src/abi.rs b/crates/j2k-jpeg-metal/src/abi.rs index 7e32202c3..1a05d4d62 100644 --- a/crates/j2k-jpeg-metal/src/abi.rs +++ b/crates/j2k-jpeg-metal/src/abi.rs @@ -361,12 +361,21 @@ pub(crate) struct PreparedHuffmanHost { pub(crate) max_code: [i32; 17], pub(crate) val_offset: [i32; 17], pub(crate) values: [u8; 256], - pub(crate) fast_symbol: [u8; 512], - pub(crate) fast_len: [u8; 512], + /// `(code length << 8) | symbol` for each 9-bit lookahead whose code is + /// at most 9 bits long; 0 when the code is longer. + pub(crate) fast: [u16; 512], + /// `(value << 8) | (run << 4) | total length` when an AC code and its + /// extra bits both fit in the 9-bit lookahead and the extended value fits + /// in `i8`; 0 otherwise. A valid entry is never 0 (total length >= 2). + pub(crate) fast_ac: [i16; 512], pub(crate) values_len: u16, pub(crate) reserved: u16, } +/// Width of the Huffman lookahead tables in `PreparedHuffmanHost`. +#[cfg(target_os = "macos")] +pub(crate) const HUFFMAN_LOOKAHEAD_BITS: usize = 9; + #[cfg(target_os = "macos")] impl From<&PacketHuffmanTable> for PreparedHuffmanHost { fn from(value: &PacketHuffmanTable) -> Self { @@ -374,23 +383,19 @@ impl From<&PacketHuffmanTable> for PreparedHuffmanHost { .derive_canonical() .expect("backend packet Huffman table must be canonicalizable"); let mut values = [0u8; 256]; - let mut fast_symbol = [0u8; 512]; - let mut fast_len = [0u8; 512]; let values_len = usize::from(value.values_len); values[..values_len].copy_from_slice(&value.values[..values_len]); + let mut fast = [0u16; 1 << HUFFMAN_LOOKAHEAD_BITS]; for (idx, &symbol) in values.iter().enumerate().take(canonical.huffsize_len) { let len = usize::from(canonical.huffsize[idx]); - if len == 0 || len > 9 { + if len == 0 || len > HUFFMAN_LOOKAHEAD_BITS { continue; } let code = usize::from(canonical.huffcode[idx]); - let prefix = code << (9 - len); - let fill = 1usize << (9 - len); - for suffix in 0..fill { - fast_symbol[prefix | suffix] = symbol; - fast_len[prefix | suffix] = canonical.huffsize[idx]; - } + let prefix = code << (HUFFMAN_LOOKAHEAD_BITS - len); + let entry = (u16::from(canonical.huffsize[idx]) << 8) | u16::from(symbol); + fast[prefix..prefix + (1 << (HUFFMAN_LOOKAHEAD_BITS - len))].fill(entry); } Self { @@ -398,14 +403,45 @@ impl From<&PacketHuffmanTable> for PreparedHuffmanHost { max_code: canonical.max_code, val_offset: canonical.val_offset, values, - fast_symbol, - fast_len, + fast_ac: fast_ac_table(&fast), + fast, values_len: value.values_len, reserved: 0, } } } +/// Derives the fused AC lookahead table from the symbol table, mirroring the +/// shader's `huff_extend` and the `receive_extend` bit order. +#[cfg(target_os = "macos")] +pub(crate) fn fast_ac_table(fast: &[u16; 1 << HUFFMAN_LOOKAHEAD_BITS]) -> [i16; 512] { + let mut fast_ac = [0i16; 1 << HUFFMAN_LOOKAHEAD_BITS]; + for (lookahead, (&entry, slot)) in fast.iter().zip(fast_ac.iter_mut()).enumerate() { + let code_len = usize::from(entry >> 8); + let run = entry & 0xf0; + let ssss = usize::from(entry & 0x0f); + let total = code_len + ssss; + if code_len == 0 || ssss == 0 || total > HUFFMAN_LOOKAHEAD_BITS { + continue; + } + let extra = (lookahead >> (HUFFMAN_LOOKAHEAD_BITS - total)) & ((1 << ssss) - 1); + let extra = i32::try_from(extra).expect("extra bits fit in i32"); + let value = if extra < 1 << (ssss - 1) { + extra - (1 << ssss) + 1 + } else { + extra + }; + let Ok(value) = i8::try_from(value) else { + continue; + }; + let packed = (i32::from(value) << 8) + | i32::from(run) + | i32::try_from(total).expect("lookahead length fits in i32"); + *slot = i16::try_from(packed).expect("packed fast AC entry fits in i16"); + } + fast_ac +} + #[cfg(target_os = "macos")] #[repr(C)] #[derive(Clone, Copy, Default)] @@ -709,8 +745,8 @@ impl_gpu_readback_abi!( max_code: [i32; 17], val_offset: [i32; 17], values: [u8; 256], - fast_symbol: [u8; 512], - fast_len: [u8; 512], + fast: [u16; 512], + fast_ac: [i16; 512], values_len: u16, reserved: u16, }, diff --git a/crates/j2k-jpeg-metal/src/codec_batch/owner_accounting.rs b/crates/j2k-jpeg-metal/src/codec_batch/owner_accounting.rs index d3c1345ed..1cd3f6cc8 100644 --- a/crates/j2k-jpeg-metal/src/codec_batch/owner_accounting.rs +++ b/crates/j2k-jpeg-metal/src/codec_batch/owner_accounting.rs @@ -117,4 +117,88 @@ mod tests { } ); } + + #[test] + fn decoder_owner_bytes_exclude_the_shared_context_reserve() { + // One `DecoderContext` reserve is charged once per decode budget, not + // once per prepared decoder, so 64-tile batches stay far below the cap. + let decoders = (0..64) + .map(|_| Decoder::new(BASELINE_420).expect("decoder")) + .collect::>(); + let decoder_refs = decoders.iter().collect::>(); + let one = distinct_decoder_retained_bytes(&decoder_refs[..1]).expect("one owner"); + let all = distinct_decoder_retained_bytes(&decoder_refs).expect("64 owners"); + assert!(one < 1024 * 1024, "one decoder retains {one} bytes"); + assert_eq!(all, 64 * one); + assert!(all < DEFAULT_MAX_HOST_ALLOCATION_BYTES / 16); + } + + #[cfg(target_os = "macos")] + #[test] + fn resident_texture_batch_of_64_distinct_decoders_matches_cpu() { + use j2k_jpeg::{encode_jpeg_baseline, JpegEncodeOptions, JpegSamples, JpegSubsampling}; + + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let (width, height) = (64_u32, 64_u32); + let jpegs = (0..64_u8) + .map(|seed| { + let mut pixels = j2k_test_support::patterned_rgb8(width, height); + for sample in &mut pixels { + *sample = sample.wrapping_add(seed.wrapping_mul(37)); + } + encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &pixels, + width, + height, + }, + JpegEncodeOptions { + subsampling: JpegSubsampling::Ybr420, + ..Default::default() + }, + ) + .expect("encode tile") + .data + }) + .collect::>(); + let decoders = jpegs + .iter() + .map(|jpeg| Decoder::new(jpeg).expect("Metal decoder")) + .collect::>(); + let decoder_refs = decoders.iter().collect::>(); + let mut output = crate::MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1) + .expect("texture output"); + + let tiles = crate::Codec::decode_rgb8_batch_into_textures_with_session( + crate::Rgb8MetalBatchRequest { + source: crate::Rgb8MetalBatchSource::Decoders(&decoder_refs), + op: crate::Rgb8MetalBatchOp::Full, + }, + crate::MetalTextureBatchTarget::Resizable(&mut output), + &session, + ) + .expect("64-decoder resident texture batch"); + + assert_eq!(tiles.len(), jpegs.len()); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + let actual = crate::tests::download_rgba8_texture( + &session, + tile.texture_trusted(), + (width, height), + ); + let (rgb, _) = j2k_jpeg::Decoder::new(&jpegs[index]) + .expect("CPU decoder") + .decode_request(j2k_jpeg::DecodeRequest::full(j2k_core::PixelFormat::Rgb8)) + .expect("CPU decode"); + assert_eq!( + actual, + crate::tests::rgb_to_rgba_opaque(&rgb), + "tile {index}" + ); + } + } } diff --git a/crates/j2k-jpeg-metal/src/compute/batch_full.rs b/crates/j2k-jpeg-metal/src/compute/batch_full.rs index 8d100d1ee..9ac5b3961 100644 --- a/crates/j2k-jpeg-metal/src/compute/batch_full.rs +++ b/crates/j2k-jpeg-metal/src/compute/batch_full.rs @@ -18,3 +18,21 @@ pub(super) use rgb::{ try_decode_fast_subsampled_full_rgb_batch_to_surfaces_into_output, }; pub(super) use texture::try_decode_fast_subsampled_full_rgba_batch_to_textures; + +/// Unwraps per-tile results merged from grouped sub-batches, in request order. +/// A slot no group filled is an internal error naming the tile. +#[cfg(target_os = "macos")] +fn ordered_grouped_results( + budget: &mut crate::batch_allocation::BatchMetadataBudget, + merged: Vec>>, + phase: &'static str, + missing: impl Fn(usize) -> String, +) -> Result>, super::Error> { + let mut results = budget.try_vec(merged.len(), phase)?; + for (index, result) in merged.into_iter().enumerate() { + results.push(result.ok_or_else(|| super::Error::MetalKernel { + message: missing(index), + })?); + } + Ok(results) +} diff --git a/crates/j2k-jpeg-metal/src/compute/batch_full/rgb.rs b/crates/j2k-jpeg-metal/src/compute/batch_full/rgb.rs index bd7e8b6eb..4bb196ff8 100644 --- a/crates/j2k-jpeg-metal/src/compute/batch_full/rgb.rs +++ b/crates/j2k-jpeg-metal/src/compute/batch_full/rgb.rs @@ -16,9 +16,10 @@ use super::super::{ new_command_buffer, new_compute_command_encoder, packed_pair_extent, surface_batch_error_results, surface_batch_success_results, wait_for_completion_jpeg, BatchEntropyBufferKeys, BatchEntropyLabels, BatchEntropyMetadata, BatchedFastPacket, Buffer, - CommandBufferRef, Error, FastBatchDecodeMode, FastBatchTiming, FastDecodeEntropyInputs, - FastSubsampledMetal, JpegDecodeStatus, JpegEntropyCheckpointHost, JpegFast420BatchParams, - MetalBatchScratch, MetalRuntime, PixelFormat, PreparedHuffmanHost, Surface, + CommandBuffer, CommandBufferRef, Error, FastBatchDecodeMode, FastBatchTiming, + FastDecodeEntropyInputs, FastSubsampledMetal, JpegDecodeStatus, JpegEntropyCheckpointHost, + JpegFast420BatchParams, MetalBatchScratch, MetalRuntime, PixelFormat, PreparedHuffmanHost, + Surface, }; #[cfg(test)] use super::super::{encode_split_coeff_idct_passes, new_private_buffer, SplitCoeffIdctPasses}; @@ -60,10 +61,6 @@ pub(in crate::compute) fn try_decode_fast_subsampled_full_rgb_batch_to_surfaces_ } #[cfg(target_os = "macos")] -#[expect( - clippy::too_many_lines, - reason = "the full-batch GPU path keeps eligibility, scratch allocation, command encoding, and completion timing in dispatch order" -)] pub(in crate::compute) fn try_decode_fast_subsampled_full_rgb_batch_to_surfaces_with_mode_and_output< P: FastSubsampledMetal, >( @@ -73,11 +70,6 @@ pub(in crate::compute) fn try_decode_fast_subsampled_full_rgb_batch_to_surfaces_ decode_mode: FastBatchDecodeMode, output: Option<&crate::MetalBatchOutputBuffer>, ) -> Result>>, Error> { - let timing_enabled = - decode_mode == FastBatchDecodeMode::Fused && P::full_rgb_batch_timing_enabled(); - let timing_total_start = timing_enabled.then(Instant::now); - let mut timing = FastBatchTiming::default(); - if requests.is_empty() || requests .iter() @@ -124,6 +116,37 @@ pub(in crate::compute) fn try_decode_fast_subsampled_full_rgb_batch_to_surfaces_ ); } + let Some(pending) = try_submit_compatible_full_rgb_batch::

( + runtime, + requests, + &family_packets, + decode_mode, + output, + runtime.batch_scratch()?, + )? + else { + return Ok(None); + }; + pending.finish(requests, output).map(Some) +} + +#[cfg(target_os = "macos")] +pub(super) fn try_submit_compatible_full_rgb_batch<'runtime, 'packet, P: FastSubsampledMetal>( + runtime: &'runtime MetalRuntime, + requests: &[batch::QueuedRequest], + family_packets: &[&'packet P], + decode_mode: FastBatchDecodeMode, + output: Option<&crate::MetalBatchOutputBuffer>, + mut batch_scratch: BatchScratchLease<'runtime>, +) -> Result>, Error> { + let timing_enabled = + decode_mode == FastBatchDecodeMode::Fused && P::full_rgb_batch_timing_enabled(); + let timing_total_start = timing_enabled.then(Instant::now); + let mut timing = FastBatchTiming::default(); + + let Some(first) = family_packets.first().copied() else { + return Ok(None); + }; let segment_count = first.entropy_checkpoints().len(); if !family_packets.iter().all(|packet| { fast_subsampled_packets_share_full_rgb_batch_shape(first, packet, segment_count) @@ -163,13 +186,12 @@ pub(in crate::compute) fn try_decode_fast_subsampled_full_rgb_batch_to_surfaces_ } let timing_buffer_start = timing_enabled.then(Instant::now); - let mut batch_scratch = runtime.batch_scratch()?; let buffers = full_rgb_surface_batch_buffers::

( runtime, requests, &mut batch_scratch, output, - &family_packets, + family_packets, shape, &entropy_metadata, )?; @@ -193,25 +215,31 @@ pub(in crate::compute) fn try_decode_fast_subsampled_full_rgb_batch_to_surfaces_ timing_enabled, &mut timing, )?; - Ok(Some(finish_fast_subsampled_full_rgb_batch::

( - FullRgbFinishState { - runtime, - requests, - first, - command_buffer: &command_buffer, - batch_scratch, - buffers: &buffers, - shape, - split_scratch, - }, - FullRgbFinishTiming { + encode_fast_subsampled_full_rgb_pack::

( + runtime, + &command_buffer, + &buffers, + shape, + timing_enabled, + &mut timing, + )?; + command_buffer.commit(); + + Ok(Some(PendingFullRgbBatch { + first, + command_buffer, + _batch_scratch: batch_scratch, + buffers, + shape, + split_scratch, + timing: FullRgbFinishTiming { enabled: timing_enabled, total_start: timing_total_start, timing, segment_count, }, - output, - )?)) + waited: false, + })) } #[cfg(target_os = "macos")] @@ -256,15 +284,15 @@ struct FullRgbDecodePass<'a, P> { } #[cfg(target_os = "macos")] -struct FullRgbFinishState<'a, 'scratch, P> { - runtime: &'a MetalRuntime, - requests: &'a [batch::QueuedRequest], - first: &'a P, - command_buffer: &'a CommandBufferRef, - batch_scratch: BatchScratchLease<'scratch>, - buffers: &'a FullRgbSurfaceBatchBuffers, +pub(super) struct PendingFullRgbBatch<'runtime, 'packet, P> { + first: &'packet P, + command_buffer: CommandBuffer, + _batch_scratch: BatchScratchLease<'runtime>, + buffers: FullRgbSurfaceBatchBuffers, shape: FullRgbSurfaceBatchShape, split_scratch: Option<(Buffer, Buffer)>, + timing: FullRgbFinishTiming, + waited: bool, } #[cfg(target_os = "macos")] @@ -574,22 +602,15 @@ fn encode_fast_subsampled_full_rgb_split_decode( } #[cfg(target_os = "macos")] -fn finish_fast_subsampled_full_rgb_batch( - state: FullRgbFinishState<'_, '_, P>, - mut timing: FullRgbFinishTiming, - output: Option<&crate::MetalBatchOutputBuffer>, -) -> Result>, Error> { - let FullRgbFinishState { - runtime, - requests, - first, - command_buffer, - batch_scratch: _batch_scratch, - buffers, - shape, - split_scratch, - } = state; - let timing_pack_encode_start = timing.enabled.then(Instant::now); +fn encode_fast_subsampled_full_rgb_pack( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + buffers: &FullRgbSurfaceBatchBuffers, + shape: FullRgbSurfaceBatchShape, + timing_enabled: bool, + timing: &mut FastBatchTiming, +) -> Result<(), Error> { + let timing_pack_encode_start = timing_enabled.then(Instant::now); let pack_pipeline = P::pack_full_rgb_batch_pipeline(runtime); let pack_encoder = new_compute_command_encoder(command_buffer)?; pack_encoder.setComputePipelineState(pack_pipeline); @@ -614,42 +635,63 @@ fn finish_fast_subsampled_full_rgb_batch( ); pack_encoder.endEncoding(); if let Some(start) = timing_pack_encode_start { - timing.timing.encode_pack = start.elapsed(); + timing.encode_pack = start.elapsed(); } + Ok(()) +} - command_buffer.commit(); - if timing.enabled { - let timing_wait_start = Instant::now(); - wait_for_completion_jpeg(command_buffer)?; - timing.timing.wait_pack = timing_wait_start.elapsed(); - if let Some(start) = timing.total_start { - timing.timing.total = start.elapsed(); +impl PendingFullRgbBatch<'_, '_, P> { + pub(super) fn finish( + mut self, + requests: &[batch::QueuedRequest], + output: Option<&crate::MetalBatchOutputBuffer>, + ) -> Result>, Error> { + let timing_wait_start = self.timing.enabled.then(Instant::now); + let completion = wait_for_completion_jpeg(&self.command_buffer); + self.waited = true; + completion?; + if self.timing.enabled { + let timing = &mut self.timing; + if let Some(start) = timing_wait_start { + timing.timing.wait_pack = start.elapsed(); + } + if let Some(start) = timing.total_start { + timing.timing.total = start.elapsed(); + } + timing.timing.log( + P::FULL_RGB_BATCH_TIMING_TAG, + "fused-stages", + self.shape.tile_count, + self.first.dimensions(), + timing.segment_count, + ); } - timing.timing.log( - P::FULL_RGB_BATCH_TIMING_TAG, - "fused-stages", - shape.tile_count, - first.dimensions(), - timing.segment_count, - ); - } else { - wait_for_completion_jpeg(command_buffer)?; + let _ = self.split_scratch.take(); + // Keep scratch leased until the CPU has consumed the GPU status below. + + if let Some(results) = surface_batch_error_results( + requests, + &self.buffers.status_buffer, + self.shape.total_decode_threads, + )? { + return Ok(results); + } + surface_batch_success_results( + requests, + &self.buffers.out_buffer, + self.first.dimensions(), + PixelFormat::Rgb8, + requests.len(), + self.shape.out_tile_len, + output, + ) } - drop(split_scratch); - // Keep scratch leased until the CPU has consumed the GPU status below. +} - if let Some(results) = - surface_batch_error_results(requests, &buffers.status_buffer, shape.total_decode_threads)? - { - return Ok(results); +impl

Drop for PendingFullRgbBatch<'_, '_, P> { + fn drop(&mut self) { + if !self.waited { + self.command_buffer.waitUntilCompleted(); + } } - surface_batch_success_results( - requests, - &buffers.out_buffer, - first.dimensions(), - PixelFormat::Rgb8, - requests.len(), - shape.out_tile_len, - output, - ) } diff --git a/crates/j2k-jpeg-metal/src/compute/batch_full/rgb_grouped.rs b/crates/j2k-jpeg-metal/src/compute/batch_full/rgb_grouped.rs index a0b1cf1b0..f017744a3 100644 --- a/crates/j2k-jpeg-metal/src/compute/batch_full/rgb_grouped.rs +++ b/crates/j2k-jpeg-metal/src/compute/batch_full/rgb_grouped.rs @@ -1,10 +1,152 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use super::super::scratch_pool::BatchScratchLease; use super::super::{ batch, batch_output_buffer_or_new, copy_grouped_surfaces_to_output, Error, FastBatchDecodeMode, FastSubsampledMetal, MetalRuntime, PixelFormat, Surface, }; -use super::rgb::try_decode_fast_subsampled_full_rgb_batch_to_surfaces_with_mode_and_output; +use super::ordered_grouped_results; +use super::rgb::{try_submit_compatible_full_rgb_batch, PendingFullRgbBatch}; + +struct PendingFullRgbGroup<'runtime, 'packet, P> { + indices: Vec, + requests: Vec, + pending: PendingFullRgbBatch<'runtime, 'packet, P>, +} + +impl PendingFullRgbGroup<'_, '_, P> { + fn finish( + mut self, + runtime: &MetalRuntime, + output: Option<&crate::MetalBatchOutputBuffer>, + family_packets: &[&P], + merged_results: &mut [Option>], + external_live_bytes: usize, + ) -> Result<(), Error> { + let mut completion_budget = + crate::batch_allocation::BatchMetadataBudget::with_external_live( + "JPEG Metal pending full RGB completion", + external_live_bytes, + ); + completion_budget.account_capacity::(self.indices.capacity())?; + completion_budget.account_capacity::(self.requests.capacity())?; + batch::stamp_execution_owner_baseline( + &mut self.requests, + 0, + completion_budget.live_bytes(), + ); + let group_results = self.pending.finish(&self.requests, None)?; + merge_group_results::

( + runtime, + output, + family_packets, + self.indices, + group_results, + merged_results, + completion_budget.live_bytes(), + ) + } +} + +fn pending_full_rgb_group_live_bytes

( + base_live_bytes: usize, + pending_groups: &[PendingFullRgbGroup<'_, '_, P>], +) -> Result { + let mut budget = crate::batch_allocation::BatchMetadataBudget::with_external_live( + "JPEG Metal live pending full RGB groups", + base_live_bytes, + ); + for group in pending_groups { + budget.account_capacity::(group.indices.capacity())?; + budget.account_capacity::(group.requests.capacity())?; + } + Ok(budget.live_bytes()) +} + +/// Waits for the oldest in-flight group and merges its results; the groups +/// still in flight stay charged to the completion budget. +#[cfg(target_os = "macos")] +fn finish_oldest_full_rgb_group( + runtime: &MetalRuntime, + output: Option<&crate::MetalBatchOutputBuffer>, + family_packets: &[&P], + result_live_bytes: usize, + pending_groups: &mut Vec>, + merged_results: &mut [Option>], +) -> Result<(), Error> { + let group = pending_groups.remove(0); + let external_live_bytes = pending_full_rgb_group_live_bytes(result_live_bytes, pending_groups)?; + group.finish( + runtime, + output, + family_packets, + merged_results, + external_live_bytes, + ) +} + +/// Keeps at most two groups in flight and leases scratch for the next one. +/// Never waits for another lease while retaining one: concurrent batches may +/// each own one of the pool's two slots, so the oldest group finishes first. +#[cfg(target_os = "macos")] +fn lease_scratch_for_next_full_rgb_group<'runtime, P: FastSubsampledMetal>( + runtime: &'runtime MetalRuntime, + output: Option<&crate::MetalBatchOutputBuffer>, + family_packets: &[&P], + result_live_bytes: usize, + pending_groups: &mut Vec>, + merged_results: &mut [Option>], +) -> Result, Error> { + if pending_groups.len() == 2 { + finish_oldest_full_rgb_group( + runtime, + output, + family_packets, + result_live_bytes, + pending_groups, + merged_results, + )?; + } + if pending_groups.is_empty() { + return runtime.batch_scratch(); + } + if let Some(scratch) = runtime.try_batch_scratch()? { + return Ok(scratch); + } + finish_oldest_full_rgb_group( + runtime, + output, + family_packets, + result_live_bytes, + pending_groups, + merged_results, + )?; + runtime.batch_scratch() +} + +/// Checks (or grows, for a resizable output) the caller's buffer for every +/// packet shape before any group is submitted. +#[cfg(target_os = "macos")] +fn check_grouped_rgb_output( + runtime: &MetalRuntime, + output: &crate::MetalBatchOutputBuffer, + family_packets: &[&P], + tile_count: usize, +) -> Result<(), Error> { + for packet in family_packets { + let out_stride = packet.dimensions().0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let out_tile_len = out_stride * packet.dimensions().1 as usize; + batch_output_buffer_or_new( + runtime, + Some(output), + packet.dimensions(), + tile_count, + out_stride, + out_tile_len, + )?; + } + Ok(()) +} #[cfg(target_os = "macos")] pub(super) fn try_decode_grouped_fast_subsampled_full_rgb_batch_to_surfaces_with_output< @@ -18,18 +160,7 @@ pub(super) fn try_decode_grouped_fast_subsampled_full_rgb_batch_to_surfaces_with groups: Vec>, ) -> Result>>, Error> { if let Some(output) = output { - for packet in family_packets { - let out_stride = packet.dimensions().0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let out_tile_len = out_stride * packet.dimensions().1 as usize; - batch_output_buffer_or_new( - runtime, - Some(output), - packet.dimensions(), - requests.len(), - out_stride, - out_tile_len, - )?; - } + check_grouped_rgb_output(runtime, output, family_packets, requests.len())?; } let mut result_budget = crate::plan_owner_ledger::batch_execution_budget( @@ -41,59 +172,72 @@ pub(super) fn try_decode_grouped_fast_subsampled_full_rgb_batch_to_surfaces_with None, "JPEG Metal grouped full RGB result slots", )?; + let mut pending_groups: Vec> = + result_budget.try_vec(2, "JPEG Metal pending full RGB groups")?; for group_indices in groups { - let mut group_budget = crate::plan_owner_ledger::batch_execution_budget( - "JPEG Metal grouped full RGB sub-batch", - requests, + let scratch = lease_scratch_for_next_full_rgb_group( + runtime, + output, + family_packets, + result_budget.live_bytes(), + &mut pending_groups, + &mut merged_results, )?; + let pending_live_bytes = + pending_full_rgb_group_live_bytes(result_budget.live_bytes(), &pending_groups)?; + let mut group_budget = crate::batch_allocation::BatchMetadataBudget::with_external_live( + "JPEG Metal grouped full RGB sub-batch", + pending_live_bytes, + ); + group_budget.account_capacity::(group_indices.capacity())?; let mut group_requests = group_budget.try_vec(group_indices.len(), "JPEG Metal grouped full RGB requests")?; group_requests.extend(group_indices.iter().map(|&index| requests[index].clone())); let mut group_packets = group_budget.try_vec(group_indices.len(), "JPEG Metal grouped full RGB packets")?; - group_packets.extend( - group_indices - .iter() - .map(|&index| family_packets[index].to_batched()), - ); + group_packets.extend(group_indices.iter().map(|&index| family_packets[index])); batch::stamp_execution_owner_baseline(&mut group_requests, 0, group_budget.live_bytes()); - let Some(group_results) = - try_decode_fast_subsampled_full_rgb_batch_to_surfaces_with_mode_and_output::

( - runtime, - &group_requests, - &group_packets, - decode_mode, - None, - )? + let Some(pending) = try_submit_compatible_full_rgb_batch::

( + runtime, + &group_requests, + &group_packets, + decode_mode, + None, + scratch, + )? else { return Ok(None); }; - - merge_group_results::

( + pending_groups.push(PendingFullRgbGroup { + indices: group_indices, + requests: group_requests, + pending, + }); + } + while !pending_groups.is_empty() { + finish_oldest_full_rgb_group( runtime, output, family_packets, - group_indices, - group_results, - &mut merged_results, result_budget.live_bytes(), + &mut pending_groups, + &mut merged_results, )?; } - let mut results = result_budget.try_vec( - requests.len(), + ordered_grouped_results( + &mut result_budget, + merged_results, "JPEG Metal ordered grouped full RGB results", - )?; - for (index, result) in merged_results.into_iter().enumerate() { - results.push(result.ok_or_else(|| Error::MetalKernel { - message: format!( + |index| { + format!( "JPEG Metal grouped {} buffer result for tile {index} was missing", P::FAMILY_NAME - ), - })?); - } - Ok(Some(results)) + ) + }, + ) + .map(Some) } #[cfg(target_os = "macos")] diff --git a/crates/j2k-jpeg-metal/src/compute/batch_full/texture.rs b/crates/j2k-jpeg-metal/src/compute/batch_full/texture.rs index 1b2fd2b4c..5b2da391e 100644 --- a/crates/j2k-jpeg-metal/src/compute/batch_full/texture.rs +++ b/crates/j2k-jpeg-metal/src/compute/batch_full/texture.rs @@ -1,20 +1,21 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use crate::metal_types::prelude::*; +use objc2_metal::MTLCommandBuffer; use super::super::scratch_pool::BatchScratchLease; use super::super::{ batch, batch_entropy_buffers, bind_fast_decode_entropy_inputs, checked_u32, - commit_and_wait_jpeg, dispatch_1d_pipeline, dispatch_rgba_texture_pack, - fast_packet_huffman_tables, fast_subsampled_full_rgb_batch_groups, new_command_buffer, - new_compute_command_encoder, packed_pair_extent, plane_mode_to_u32, - texture_batch_error_results, texture_batch_success_results, validate_rgba_texture_batch_output, - BatchEntropyBufferKeys, BatchEntropyBufferPlan, BatchEntropyBuffers, BatchedFastPacket, Buffer, + dispatch_1d_pipeline, dispatch_rgba_texture_pack, fast_packet_huffman_tables, + fast_subsampled_full_rgb_batch_groups, new_command_buffer, new_compute_command_encoder, + plane_mode_to_u32, texture_batch_error_results, texture_batch_success_results, + validate_rgba_texture_batch_output, wait_for_completion_jpeg, BatchEntropyBufferKeys, + BatchEntropyBufferPlan, BatchEntropyBuffers, BatchedFastPacket, Buffer, CommandBuffer, CommandBufferRef, Error, FastBatchDecodeMode, FastDecodeEntropyInputs, FastSubsampledMetal, FastTextureRepairCtx, JpegDecodeStatus, JpegFast420BatchParams, JpegFast420TextureBatchParams, JpegFast444TextureBatchParams, JpegTexturePackBatchParams, MetalBatchScratch, MetalRuntime, - PixelFormat, PlaneMode, PreparedHuffmanHost, MODE_YCBCR, + PixelFormat, PlaneMode, PreparedHuffmanHost, }; #[cfg(target_os = "macos")] mod staged; @@ -23,10 +24,6 @@ use self::staged::decode_fast_subsampled_full_rgba_staged_texture_batch; use super::texture_grouped::try_decode_grouped_fast_subsampled_full_rgba_batch_to_textures; #[cfg(target_os = "macos")] -#[expect( - clippy::too_many_lines, - reason = "ordered Metal texture command and resource lifetime" -)] pub(in crate::compute) fn try_decode_fast_subsampled_full_rgba_batch_to_textures< P: FastSubsampledMetal, >( @@ -97,20 +94,41 @@ pub(in crate::compute) fn try_decode_fast_subsampled_full_rgba_batch_to_textures ); } + let Some(pending) = try_submit_compatible_full_rgba_texture_batch::

( + runtime, + requests, + &family_packets, + family_mode.unwrap_or(PlaneMode::YCbCr), + output, + decode_mode, + runtime.batch_scratch()?, + )? + else { + return Ok(None); + }; + pending.finish(requests, output).map(Some) +} + +pub(super) fn try_submit_compatible_full_rgba_texture_batch<'runtime, P: FastSubsampledMetal>( + runtime: &'runtime MetalRuntime, + requests: &[batch::QueuedRequest], + family_packets: &[&P], + family_mode: PlaneMode, + output: &crate::MetalBatchTextureOutput, + decode_mode: FastBatchDecodeMode, + mut batch_scratch: BatchScratchLease<'runtime>, +) -> Result>, Error> { + let Some(first) = family_packets.first().copied() else { + return Ok(None); + }; let segment_count = first.entropy_checkpoints().len(); let tile_count = family_packets.len(); - let shape = full_rgba_texture_batch_shape::

( - first, - tile_count, - segment_count, - family_mode.unwrap_or(PlaneMode::YCbCr), - )?; + let shape = full_rgba_texture_batch_shape::

(first, tile_count, segment_count, family_mode)?; validate_rgba_texture_batch_output(output, first.dimensions(), tile_count, shape.out_tile_len)?; #[cfg(test)] let total_blocks = full_rgba_texture_total_blocks::

(shape.total_mcus, shape.tile_count)?; - let mut batch_scratch = runtime.batch_scratch()?; let Some(entropy_buffers) = batch_entropy_buffers( runtime, requests, @@ -134,16 +152,14 @@ pub(in crate::compute) fn try_decode_fast_subsampled_full_rgba_batch_to_textures return Ok(None); }; - // Subsampled batches distribute entropy/IDCT across all tiles, then pack - // private component planes. This avoids the direct shader's per-tile - // decode and chroma-repair passes. The 4:4:4 path has no chroma repair and - // uses a different plane-decode ABI, so it retains its direct kernel. - let direct_texture = P::USE_FAST444_TEXTURE_PARAMS; + // Every family distributes entropy/IDCT across all tiles in one dispatch, + // then packs private component planes. This avoids the direct shaders' + // per-tile decode dispatches and chroma-repair passes. The direct kernels + // stay reachable from tests for A/B comparison. + let direct_texture = false; #[cfg(test)] - let direct_texture = super::super::texture_tuning::component_planes() - .map_or(direct_texture, |planes| { - !planes || P::USE_FAST444_TEXTURE_PARAMS - }); + let direct_texture = + super::super::texture_tuning::component_planes().map_or(direct_texture, |planes| !planes); if decode_mode == FastBatchDecodeMode::Fused && direct_texture { return Ok(Some( decode_fast_subsampled_full_rgba_fused_texture_batch::

(FullRgbaTextureBatchCtx { @@ -176,6 +192,49 @@ pub(in crate::compute) fn try_decode_fast_subsampled_full_rgba_batch_to_textures )) } +// Completion owns the scratch lease even when a later group fails to submit. +// Returning the lease before the GPU finishes would let another batch overwrite it. +pub(super) struct PendingTextureBatch<'runtime> { + command_buffer: CommandBuffer, + _batch_scratch: BatchScratchLease<'runtime>, + status_buffer: Buffer, + shape: FullRgbaTextureBatchShape, + waited: bool, +} + +impl PendingTextureBatch<'_> { + pub(super) fn finish( + mut self, + requests: &[batch::QueuedRequest], + output: &crate::MetalBatchTextureOutput, + ) -> Result>, Error> { + let completion = wait_for_completion_jpeg(&self.command_buffer); + self.waited = true; + completion?; + if let Some(results) = texture_batch_error_results( + requests, + &self.status_buffer, + self.shape.total_decode_threads, + )? { + return Ok(results); + } + texture_batch_success_results( + requests, + output, + (self.shape.width, self.shape.height), + requests.len(), + ) + } +} + +impl Drop for PendingTextureBatch<'_> { + fn drop(&mut self) { + if !self.waited { + self.command_buffer.waitUntilCompleted(); + } + } +} + #[cfg(target_os = "macos")] #[derive(Clone, Copy)] struct FullRgbaTextureBatchShape { @@ -341,7 +400,7 @@ fn full_rgba_texture_batch_shape( )?; let width = first.dimensions().0; let height = first.dimensions().1; - let chroma_width = width.div_ceil(2); + let chroma_width = P::chroma_width(width); let chroma_height = P::chroma_height(height); let y_len = width as usize * height as usize; let chroma_len = chroma_width as usize * chroma_height as usize; @@ -599,9 +658,9 @@ fn encode_fast_subsampled_full_rgba_texture_boundary_passes( - ctx: FullRgbaTextureBatchCtx<'_, '_, P>, -) -> Result>, Error> { +fn decode_fast_subsampled_full_rgba_fused_texture_batch<'runtime, P: FastSubsampledMetal>( + ctx: FullRgbaTextureBatchCtx<'_, 'runtime, P>, +) -> Result, Error> { let FullRgbaTextureBatchCtx { runtime, requests, @@ -664,12 +723,12 @@ fn decode_fast_subsampled_full_rgba_fused_texture_batch( }, )?; - commit_and_wait_jpeg(&command_buffer)?; - // Keep scratch leased until the CPU has consumed the GPU status below. - if let Some(results) = - texture_batch_error_results(requests, &repair.status, shape.total_decode_threads)? - { - return Ok(results); - } - texture_batch_success_results(requests, output, first.dimensions(), requests.len()) + command_buffer.commit(); + Ok(PendingTextureBatch { + command_buffer, + _batch_scratch: batch_scratch, + status_buffer: repair.status, + shape, + waited: false, + }) } diff --git a/crates/j2k-jpeg-metal/src/compute/batch_full/texture/staged.rs b/crates/j2k-jpeg-metal/src/compute/batch_full/texture/staged.rs index da0a95e75..d7307d7e9 100644 --- a/crates/j2k-jpeg-metal/src/compute/batch_full/texture/staged.rs +++ b/crates/j2k-jpeg-metal/src/compute/batch_full/texture/staged.rs @@ -1,17 +1,17 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use crate::metal_types::prelude::*; +use objc2_metal::MTLCommandBuffer; #[cfg(test)] use super::MetalBatchScratch; use super::{ - bind_fast_decode_entropy_inputs, commit_and_wait_jpeg, dispatch_1d_pipeline, - dispatch_rgba_texture_pack, fast_packet_huffman_tables, full_rgba_texture_status_buffer, - new_command_buffer, new_compute_command_encoder, packed_pair_extent, - texture_batch_error_results, texture_batch_success_results, BatchEntropyBuffers, Buffer, - CommandBufferRef, Error, FastBatchDecodeMode, FastDecodeEntropyInputs, FastSubsampledMetal, + bind_fast_decode_entropy_inputs, dispatch_1d_pipeline, dispatch_rgba_texture_pack, + fast_packet_huffman_tables, full_rgba_texture_status_buffer, new_command_buffer, + new_compute_command_encoder, plane_mode_to_u32, BatchEntropyBuffers, Buffer, CommandBufferRef, + Error, FastBatchDecodeMode, FastDecodeEntropyInputs, FastSubsampledMetal, FullRgbaTextureBatchCtx, FullRgbaTextureBatchShape, JpegFast420BatchParams, - JpegTexturePackBatchParams, PreparedHuffmanHost, MODE_YCBCR, + JpegTexturePackBatchParams, PendingTextureBatch, PreparedHuffmanHost, }; #[cfg(test)] use crate::compute::{encode_split_coeff_idct_passes, SplitCoeffIdctPasses}; @@ -37,11 +37,14 @@ struct FullRgbaStagedDecodePass<'a, P> { clippy::similar_names, reason = "Cb and Cr are normative JPEG component names" )] -pub(super) fn decode_fast_subsampled_full_rgba_staged_texture_batch( - ctx: FullRgbaTextureBatchCtx<'_, '_, P>, +pub(super) fn decode_fast_subsampled_full_rgba_staged_texture_batch< + 'runtime, + P: FastSubsampledMetal, +>( + ctx: FullRgbaTextureBatchCtx<'_, 'runtime, P>, decode_mode: FastBatchDecodeMode, #[cfg(test)] total_blocks: Option, -) -> Result>, Error> { +) -> Result, Error> { let FullRgbaTextureBatchCtx { runtime, requests, @@ -97,7 +100,7 @@ pub(super) fn decode_fast_subsampled_full_rgba_staged_texture_batch( diff --git a/crates/j2k-jpeg-metal/src/compute/batch_full/texture_grouped.rs b/crates/j2k-jpeg-metal/src/compute/batch_full/texture_grouped.rs index c3e4986e7..74baf4bb0 100644 --- a/crates/j2k-jpeg-metal/src/compute/batch_full/texture_grouped.rs +++ b/crates/j2k-jpeg-metal/src/compute/batch_full/texture_grouped.rs @@ -1,10 +1,119 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use super::super::scratch_pool::BatchScratchLease; use super::super::{ batch, validate_rgba_texture_batch_output, Error, FastBatchDecodeMode, FastSubsampledMetal, MetalRuntime, PixelFormat, PlaneMode, }; -use super::texture::try_decode_fast_subsampled_full_rgba_batch_to_textures; +use super::ordered_grouped_results; +use super::texture::{try_submit_compatible_full_rgba_texture_batch, PendingTextureBatch}; + +struct PendingTextureGroup<'runtime> { + indices: Vec, + requests: Vec, + output: crate::MetalBatchTextureOutput, + pending: PendingTextureBatch<'runtime>, +} + +impl PendingTextureGroup<'_> { + fn account_retained_metadata( + &self, + budget: &mut crate::batch_allocation::BatchMetadataBudget, + ) -> Result<(), Error> { + budget.account_capacity::(self.indices.capacity())?; + budget.account_capacity::(self.requests.capacity())?; + self.output.account_texture_handle_capacity(budget) + } + + fn finish( + mut self, + merged: &mut [Option>], + external_live_bytes: usize, + ) -> Result<(), Error> { + batch::stamp_execution_owner_baseline(&mut self.requests, 0, external_live_bytes); + let results = self.pending.finish(&self.requests, &self.output)?; + if results.len() != self.indices.len() { + return Err(Error::MetalKernel { + message: format!( + "JPEG Metal grouped {} texture result count mismatch", + P::FAMILY_NAME + ), + }); + } + for (index, result) in self.indices.into_iter().zip(results) { + merged[index] = Some(result); + } + Ok(()) + } +} + +fn pending_texture_metadata_budget( + phase: &'static str, + result_live_bytes: usize, + pending_groups: &[PendingTextureGroup<'_>], +) -> Result { + let mut budget = + crate::batch_allocation::BatchMetadataBudget::with_external_live(phase, result_live_bytes); + budget.preflight(&[])?; + for group in pending_groups { + group.account_retained_metadata(&mut budget)?; + } + Ok(budget) +} + +fn finish_oldest_pending_texture_group( + pending_groups: &mut Vec>, + merged: &mut [Option>], + result_live_bytes: usize, +) -> Result<(), Error> { + let finish_budget = pending_texture_metadata_budget( + "JPEG Metal pending full texture group completion", + result_live_bytes, + pending_groups, + )?; + pending_groups + .remove(0) + .finish::

(merged, finish_budget.live_bytes()) +} + +/// Keeps at most two groups in flight and leases scratch for the next one. +/// Never waits for another lease while retaining one: concurrent batches may +/// each own one of the pool's two slots, so the oldest group finishes first. +#[cfg(target_os = "macos")] +fn lease_scratch_for_next_texture_group<'runtime, P: FastSubsampledMetal>( + runtime: &'runtime MetalRuntime, + pending_groups: &mut Vec>, + merged: &mut [Option>], + result_live_bytes: usize, +) -> Result, Error> { + if pending_groups.len() == 2 { + finish_oldest_pending_texture_group::

(pending_groups, merged, result_live_bytes)?; + } + if pending_groups.is_empty() { + return runtime.batch_scratch(); + } + if let Some(scratch) = runtime.try_batch_scratch()? { + return Ok(scratch); + } + finish_oldest_pending_texture_group::

(pending_groups, merged, result_live_bytes)?; + runtime.batch_scratch() +} + +/// Checks the caller's texture set against every packet shape before any group +/// is submitted. +#[cfg(target_os = "macos")] +fn check_grouped_texture_output( + output: &crate::MetalBatchTextureOutput, + family_packets: &[&P], + tile_count: usize, +) -> Result<(), Error> { + for packet in family_packets { + let out_stride = packet.dimensions().0 as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let out_tile_len = out_stride * packet.dimensions().1 as usize; + validate_rgba_texture_batch_output(output, packet.dimensions(), tile_count, out_tile_len)?; + } + Ok(()) +} #[cfg(target_os = "macos")] pub(super) fn try_decode_grouped_fast_subsampled_full_rgba_batch_to_textures< @@ -18,16 +127,7 @@ pub(super) fn try_decode_grouped_fast_subsampled_full_rgba_batch_to_textures< decode_mode: FastBatchDecodeMode, groups: Vec>, ) -> Result>>, Error> { - for packet in family_packets { - let out_stride = packet.dimensions().0 as usize * PixelFormat::Rgba8.bytes_per_pixel(); - let out_tile_len = out_stride * packet.dimensions().1 as usize; - validate_rgba_texture_batch_output( - output, - packet.dimensions(), - requests.len(), - out_tile_len, - )?; - } + check_grouped_texture_output(output, family_packets, requests.len())?; let mut result_budget = crate::plan_owner_ledger::batch_execution_budget( "JPEG Metal grouped full texture results", @@ -38,62 +138,91 @@ pub(super) fn try_decode_grouped_fast_subsampled_full_rgba_batch_to_textures< None, "JPEG Metal grouped full texture result slots", )?; + let mut pending_groups: Vec> = + result_budget.try_vec(2, "JPEG Metal pending full texture groups")?; for group_indices in groups { - let group_output = output.clone_slots(&group_indices)?; - let mut group_budget = crate::plan_owner_ledger::batch_execution_budget( - "JPEG Metal grouped full texture sub-batch", - requests, + let scratch = lease_scratch_for_next_texture_group::

( + runtime, + &mut pending_groups, + &mut merged_results, + result_budget.live_bytes(), )?; - let mut group_requests = group_budget.try_vec( + let mut retained_budget = pending_texture_metadata_budget( + "JPEG Metal grouped full texture retained sub-batch", + result_budget.live_bytes(), + &pending_groups, + )?; + retained_budget.account_capacity::(group_indices.capacity())?; + retained_budget.preflight(&[ + crate::batch_allocation::BatchMetadataRequest::of::( + group_indices.len(), + ), + crate::batch_allocation::BatchMetadataRequest::of::( + group_indices.len(), + ), + ])?; + let group_output = output.clone_slots(&group_indices)?; + group_output.account_texture_handle_capacity(&mut retained_budget)?; + let mut group_requests = retained_budget.try_vec( group_indices.len(), "JPEG Metal grouped full texture requests", )?; group_requests.extend(group_indices.iter().map(|&index| requests[index].clone())); - let mut group_packets = group_budget.try_vec( + let mut submission_budget = + crate::batch_allocation::BatchMetadataBudget::with_external_live( + "JPEG Metal grouped full texture submission", + retained_budget.live_bytes(), + ); + let mut group_packets = submission_budget.try_vec( group_indices.len(), "JPEG Metal grouped full texture packets", )?; - group_packets.extend( - group_indices.iter().map(|&index| { - family_packets[index].to_batched_with_texture_mode(family_modes[index]) - }), + group_packets.extend(group_indices.iter().map(|&index| family_packets[index])); + batch::stamp_execution_owner_baseline( + &mut group_requests, + 0, + submission_budget.live_bytes(), ); - batch::stamp_execution_owner_baseline(&mut group_requests, 0, group_budget.live_bytes()); - let Some(group_results) = try_decode_fast_subsampled_full_rgba_batch_to_textures::

( + let Some(pending) = try_submit_compatible_full_rgba_texture_batch::

( runtime, &group_requests, &group_packets, + family_modes[group_indices[0]], &group_output, decode_mode, + scratch, )? else { return Ok(None); }; - if group_results.len() != group_indices.len() { - return Err(Error::MetalKernel { - message: format!( - "JPEG Metal grouped {} texture result count mismatch", - P::FAMILY_NAME - ), - }); - } - for (original_index, result) in group_indices.into_iter().zip(group_results) { - merged_results[original_index] = Some(result); - } + drop(group_packets); + batch::stamp_execution_owner_baseline(&mut group_requests, 0, retained_budget.live_bytes()); + pending_groups.push(PendingTextureGroup { + indices: group_indices, + requests: group_requests, + output: group_output, + pending, + }); + } + while !pending_groups.is_empty() { + finish_oldest_pending_texture_group::

( + &mut pending_groups, + &mut merged_results, + result_budget.live_bytes(), + )?; } - let mut results = result_budget.try_vec( - requests.len(), + ordered_grouped_results( + &mut result_budget, + merged_results, "JPEG Metal ordered grouped full texture results", - )?; - for (index, result) in merged_results.into_iter().enumerate() { - results.push(result.ok_or_else(|| Error::MetalKernel { - message: format!( + |index| { + format!( "JPEG Metal grouped {} texture result for tile {index} was missing", P::FAMILY_NAME - ), - })?); - } - Ok(Some(results)) + ) + }, + ) + .map(Some) } diff --git a/crates/j2k-jpeg-metal/src/compute/batch_plan.rs b/crates/j2k-jpeg-metal/src/compute/batch_plan.rs index 589ccd142..71526d6c9 100644 --- a/crates/j2k-jpeg-metal/src/compute/batch_plan.rs +++ b/crates/j2k-jpeg-metal/src/compute/batch_plan.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use crate::metal_types::Buffer; -use j2k_core::{BackendRequest, Rect}; +use j2k_core::{BackendRequest, PixelFormat, Rect}; use j2k_jpeg::adapter::{ JpegEntropyCheckpointV1, JpegFast420PacketV1, JpegFast422PacketV1, JpegFast444PacketV1, }; @@ -79,16 +79,63 @@ impl BatchDeviceBufferCache { } } -fn request_allows_batched_packet( - _requests: &[batch::QueuedRequest], - request: &batch::QueuedRequest, - _restart_interval_mcus: u32, - _dimensions: (u32, u32), -) -> bool { - match request.backend { - BackendRequest::Metal => true, - BackendRequest::Auto | BackendRequest::Cpu | BackendRequest::Cuda => false, +fn auto_batch_is_profitable(requests: &[batch::QueuedRequest]) -> bool { + // Auto already prepares fast packets before flush. Completed-submission + // measurements favor Metal for compatible 256x256 RGB batches of 16+, + // including distinct inputs; smaller work stays on the CPU. + if requests.len() < 16 + || requests.iter().any(|request| { + !matches!( + request.backend, + BackendRequest::Auto | BackendRequest::Metal + ) || request.fmt != PixelFormat::Rgb8 + || request.op != batch::BatchOp::Full + }) + { + return false; } + let Some(packet) = &requests[0].fast_packet else { + return false; + }; + if let Some(first) = packet.fast420() { + return auto_packets_share_profitable_shape( + requests, + first, + crate::SharedJpegFastPacket::fast420, + ); + } + if let Some(first) = packet.fast422() { + return auto_packets_share_profitable_shape( + requests, + first, + crate::SharedJpegFastPacket::fast422, + ); + } + false +} + +fn auto_packets_share_profitable_shape( + requests: &[batch::QueuedRequest], + first: &P, + family: for<'a> fn(&'a crate::SharedJpegFastPacket) -> Option<&'a P>, +) -> bool { + let dimensions = first.dimensions(); + dimensions.0 >= 256 + && dimensions.1 >= 256 + && requests.iter().all(|request| { + request + .fast_packet + .as_ref() + .and_then(family) + .is_some_and(|packet| { + packet.restart_interval_mcus() == 0 + && super::fast_subsampled_packets_share_full_rgb_batch_shape( + first, + packet, + first.entropy_checkpoints().len(), + ) + }) + }) } pub(super) fn batched_fast_packets( @@ -97,6 +144,13 @@ pub(super) fn batched_fast_packets( if requests.is_empty() { return Ok(None); } + if requests + .iter() + .any(|request| request.backend == BackendRequest::Auto) + && !auto_batch_is_profitable(requests) + { + return Ok(None); + } let mut budget = crate::plan_owner_ledger::batch_execution_budget( "JPEG Metal fast packet batch plan", @@ -136,14 +190,6 @@ pub(super) fn batched_fast_packets( .as_ref() .and_then(crate::SharedJpegFastPacket::fast420) { - if !request_allows_batched_packet( - requests, - request, - packet.restart_interval_mcus, - packet.dimensions, - ) { - return Ok(None); - } packets.push(BatchedFastPacket::Fast420(packet)); continue; } @@ -153,14 +199,6 @@ pub(super) fn batched_fast_packets( .as_ref() .and_then(crate::SharedJpegFastPacket::fast422) { - if !request_allows_batched_packet( - requests, - request, - packet.restart_interval_mcus, - packet.dimensions, - ) { - return Ok(None); - } packets.push(BatchedFastPacket::Fast422(packet)); continue; } @@ -170,14 +208,6 @@ pub(super) fn batched_fast_packets( .as_ref() .and_then(crate::SharedJpegFastPacket::fast444) { - if !request_allows_batched_packet( - requests, - request, - packet.restart_interval_mcus, - packet.dimensions, - ) { - return Ok(None); - } let mode = match request.plane_mode_hint() { batch::PlaneModeHint::YCbCr => PlaneMode::YCbCr, batch::PlaneModeHint::Rgb => PlaneMode::Rgb, diff --git a/crates/j2k-jpeg-metal/src/compute/batch_region/rgb.rs b/crates/j2k-jpeg-metal/src/compute/batch_region/rgb.rs index 33fea9421..8b8ef3109 100644 --- a/crates/j2k-jpeg-metal/src/compute/batch_region/rgb.rs +++ b/crates/j2k-jpeg-metal/src/compute/batch_region/rgb.rs @@ -1,21 +1,24 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use crate::metal_types::prelude::*; +use objc2_metal::MTLCommandBuffer; +use super::super::scratch_pool::BatchScratchLease; use super::super::{ batch, batch_entropy_buffers, batch_output_buffer_or_new, bind_three_plane_pack, checked_u32, - commit_and_wait_jpeg, copy_grouped_surfaces_to_output, dispatch_3d_pipeline, - fast444_scaled_region_params, fast_subsampled_region_scaled_batch_groups, - fast_subsampled_region_scaled_batch_plan, new_command_buffer, new_compute_command_encoder, - region_scaled_batch_error_results, surface_batch_success_results, BatchEntropyBufferKeys, - BatchEntropyBufferPlan, BatchedFastPacket, Error, FastRegionScaledMetal, JpegDecodeStatus, - JpegFast420PacketV1, JpegFast422PacketV1, JpegFast444PacketV1, JpegWindowedPackBatchParams, - MetalRuntime, PixelFormat, PlaneMode, Rect, RegionScaledBatchPlan, Surface, + copy_grouped_surfaces_to_output, dispatch_3d_pipeline, fast444_scaled_region_params, + fast_subsampled_region_scaled_batch_groups, fast_subsampled_region_scaled_batch_plan, + new_command_buffer, new_compute_command_encoder, region_scaled_batch_error_results, + surface_batch_success_results, wait_for_completion_jpeg, BatchEntropyBufferKeys, + BatchEntropyBufferPlan, BatchedFastPacket, Buffer, CommandBuffer, CommandBufferRef, Error, + FastRegionScaledMetal, JpegDecodeStatus, JpegFast420PacketV1, JpegFast422PacketV1, + JpegFast444PacketV1, JpegWindowedPackBatchParams, MetalRuntime, PixelFormat, PlaneMode, Rect, + RegionScaledBatchPlan, Surface, }; use super::common::{ decode_region_scaled_packet_surface, encode_subsampled_region_rgb_decode, first_region_scaled_op, region_plane_buffers, subsampled_region_rgb_batch_shape, - subsampled_region_rgb_packets, + subsampled_region_rgb_packets, SubsampledRegionBatchShape, }; #[cfg(target_os = "macos")] @@ -226,14 +229,6 @@ fn try_decode_fast_subsampled_restart_region_scaled_rgb_batch_to_surfaces_with_o } #[cfg(target_os = "macos")] -#[expect( - clippy::too_many_lines, - reason = "the region-scaled GPU path keeps batch eligibility, scratch allocation, command encoding, and output copies in dispatch order" -)] -#[expect( - clippy::similar_names, - reason = "Cb and Cr are normative JPEG component names" -)] pub(in crate::compute) fn try_decode_fast_subsampled_region_scaled_rgb_batch_to_surfaces_with_output< P: FastRegionScaledMetal, >( @@ -246,12 +241,12 @@ pub(in crate::compute) fn try_decode_fast_subsampled_region_scaled_rgb_batch_to_ return Ok(None); }; - let Some((first, first_mode)) = family_packets.first().copied() else { + let Some((first, _)) = family_packets.first().copied() else { return Ok(None); }; - let Some((first_roi, first_scale)) = first_region_scaled_op(requests) else { + if first_region_scaled_op(requests).is_none() { return Ok(None); - }; + } if family_packets .iter() .any(|(packet, _)| packet.restart_interval_mcus() != 0) @@ -281,9 +276,40 @@ pub(in crate::compute) fn try_decode_fast_subsampled_region_scaled_rgb_batch_to_ ); } - let Some(shape) = subsampled_region_rgb_batch_shape::

( + let Some(pending) = try_submit_compatible_region_scaled_rgb_batch::

( + runtime, requests, &family_packets, + output, + runtime.batch_scratch()?, + )? + else { + return Ok(None); + }; + pending.finish(requests, output).map(Some) +} + +#[cfg(target_os = "macos")] +#[expect( + clippy::similar_names, + reason = "Cb and Cr are normative JPEG component names" +)] +fn try_submit_compatible_region_scaled_rgb_batch<'runtime, P: FastRegionScaledMetal>( + runtime: &'runtime MetalRuntime, + requests: &[batch::QueuedRequest], + family_packets: &[(&P, PlaneMode)], + output: Option<&crate::MetalBatchOutputBuffer>, + mut batch_scratch: BatchScratchLease<'runtime>, +) -> Result>, Error> { + let Some((first, first_mode)) = family_packets.first().copied() else { + return Ok(None); + }; + let Some((first_roi, first_scale)) = first_region_scaled_op(requests) else { + return Ok(None); + }; + let Some(shape) = subsampled_region_rgb_batch_shape::

( + requests, + family_packets, first, first_mode, first_roi, @@ -293,7 +319,6 @@ pub(in crate::compute) fn try_decode_fast_subsampled_region_scaled_rgb_batch_to_ return Ok(None); }; - let mut batch_scratch = runtime.batch_scratch()?; let Some(entropy_buffers) = batch_entropy_buffers( runtime, requests, @@ -360,17 +385,47 @@ pub(in crate::compute) fn try_decode_fast_subsampled_region_scaled_rgb_batch_to_ shape, )?; - let pack_encoder = new_compute_command_encoder(&command_buffer)?; - pack_encoder.setComputePipelineState(P::pack_windowed_rgb_batch_pipeline(runtime)); + encode_subsampled_region_rgb_pack::

( + runtime, + &command_buffer, + [&y_plane, &cb_plane, &cr_plane], + &out_buffer, + shape, + )?; + + command_buffer.commit(); + Ok(Some(PendingRegionScaledRgbBatch { + command_buffer, + _batch_scratch: batch_scratch, + status_buffer, + out_buffer, + shape, + waited: false, + })) +} + +/// Encodes the windowed pack that upsamples and color-converts the decoded +/// component planes into the batch's RGB output. +#[cfg(target_os = "macos")] +fn encode_subsampled_region_rgb_pack( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + planes: [&Buffer; 3], + out_buffer: &Buffer, + shape: SubsampledRegionBatchShape, +) -> Result<(), Error> { + let pipeline = P::pack_windowed_rgb_batch_pipeline(runtime); + let pack_encoder = new_compute_command_encoder(command_buffer)?; + pack_encoder.setComputePipelineState(pipeline); bind_three_plane_pack::( &pack_encoder, - [Some(&y_plane), Some(&cb_plane), Some(&cr_plane)], - &out_buffer, + planes.map(Some), + out_buffer, &shape.plan.pack_params, ); dispatch_3d_pipeline( &pack_encoder, - P::pack_windowed_rgb_batch_pipeline(runtime), + pipeline, ( shape.plan.out_dims.0, shape.plan.out_dims.1, @@ -378,25 +433,52 @@ pub(in crate::compute) fn try_decode_fast_subsampled_region_scaled_rgb_batch_to_ ), ); pack_encoder.endEncoding(); + Ok(()) +} - commit_and_wait_jpeg(&command_buffer)?; - // Keep scratch leased until the CPU has consumed the GPU status below. +struct PendingRegionScaledRgbBatch<'runtime> { + command_buffer: CommandBuffer, + _batch_scratch: BatchScratchLease<'runtime>, + status_buffer: Buffer, + out_buffer: Buffer, + shape: SubsampledRegionBatchShape, + waited: bool, +} - if let Some(results) = - region_scaled_batch_error_results(requests, &status_buffer, shape.total_decode_threads)? - { - return Ok(Some(results)); +impl PendingRegionScaledRgbBatch<'_> { + fn finish( + mut self, + requests: &[batch::QueuedRequest], + output: Option<&crate::MetalBatchOutputBuffer>, + ) -> Result>, Error> { + let completion = wait_for_completion_jpeg(&self.command_buffer); + self.waited = true; + completion?; + if let Some(results) = region_scaled_batch_error_results( + requests, + &self.status_buffer, + self.shape.total_decode_threads, + )? { + return Ok(results); + } + surface_batch_success_results( + requests, + &self.out_buffer, + self.shape.plan.out_dims, + PixelFormat::Rgb8, + requests.len(), + self.shape.plan.out_tile_len, + output, + ) } +} - Ok(Some(surface_batch_success_results( - requests, - &out_buffer, - shape.plan.out_dims, - PixelFormat::Rgb8, - requests.len(), - shape.plan.out_tile_len, - output, - )?)) +impl Drop for PendingRegionScaledRgbBatch<'_> { + fn drop(&mut self) { + if !self.waited { + self.command_buffer.waitUntilCompleted(); + } + } } fn try_decode_fast420_region_scaled_rgb_batch_to_surfaces_with_output( @@ -410,6 +492,78 @@ fn try_decode_fast420_region_scaled_rgb_batch_to_surfaces_with_output( ) } +struct PendingRegionScaledRgbGroup<'runtime> { + indices: Vec, + requests: Vec, + pending: PendingRegionScaledRgbBatch<'runtime>, +} + +impl PendingRegionScaledRgbGroup<'_> { + fn finish( + mut self, + runtime: &MetalRuntime, + output: Option<&crate::MetalBatchOutputBuffer>, + merged_results: &mut [Option>], + external_live_bytes: usize, + ) -> Result<(), Error> { + let mut completion_budget = + crate::batch_allocation::BatchMetadataBudget::with_external_live( + "JPEG Metal pending region-scaled RGB completion", + external_live_bytes, + ); + completion_budget.account_capacity::(self.indices.capacity())?; + completion_budget.account_capacity::(self.requests.capacity())?; + batch::stamp_execution_owner_baseline( + &mut self.requests, + 0, + completion_budget.live_bytes(), + ); + let plan = self.pending.shape.plan; + let group_results = self.pending.finish(&self.requests, None)?; + if let Some(output) = output { + for (original_index, result) in copy_grouped_surfaces_to_output( + runtime, + output, + plan.out_dims, + plan.out_tile_len, + &self.indices, + group_results, + completion_budget.live_bytes(), + )? { + merged_results[original_index] = Some(result); + } + } else { + if group_results.len() != self.indices.len() { + return Err(Error::MetalKernel { + message: format!( + "JPEG Metal grouped {} region scaled buffer result count mismatch", + P::FAMILY_NAME + ), + }); + } + for (original_index, result) in self.indices.into_iter().zip(group_results) { + merged_results[original_index] = Some(result); + } + } + Ok(()) + } +} + +fn pending_region_scaled_rgb_group_live_bytes( + base_live_bytes: usize, + pending_groups: &[PendingRegionScaledRgbGroup<'_>], +) -> Result { + let mut budget = crate::batch_allocation::BatchMetadataBudget::with_external_live( + "JPEG Metal live pending region-scaled RGB groups", + base_live_bytes, + ); + for group in pending_groups { + budget.account_capacity::(group.indices.capacity())?; + budget.account_capacity::(group.requests.capacity())?; + } + Ok(budget.live_bytes()) +} + #[cfg(target_os = "macos")] #[expect( clippy::too_many_lines, @@ -466,11 +620,41 @@ fn try_decode_grouped_fast_subsampled_region_scaled_rgb_batch_to_surfaces_with_o None, "JPEG Metal grouped region-scaled buffer result slots", )?; + let mut pending_groups: Vec> = + result_budget.try_vec(2, "JPEG Metal pending region-scaled RGB groups")?; for group_indices in groups { - let mut group_budget = crate::plan_owner_ledger::batch_execution_budget( - "JPEG Metal grouped region-scaled buffer sub-batch", - requests, + if pending_groups.len() == 2 { + let group = pending_groups.remove(0); + let external_live_bytes = pending_region_scaled_rgb_group_live_bytes( + result_budget.live_bytes(), + &pending_groups, + )?; + group.finish::

(runtime, output, &mut merged_results, external_live_bytes)?; + } + let scratch = if pending_groups.is_empty() { + runtime.batch_scratch()? + } else if let Some(scratch) = runtime.try_batch_scratch()? { + scratch + } else { + // Never wait for another lease while retaining one: concurrent + // batches may each own one of the pool's two slots. + let group = pending_groups.remove(0); + let external_live_bytes = pending_region_scaled_rgb_group_live_bytes( + result_budget.live_bytes(), + &pending_groups, + )?; + group.finish::

(runtime, output, &mut merged_results, external_live_bytes)?; + runtime.batch_scratch()? + }; + let pending_live_bytes = pending_region_scaled_rgb_group_live_bytes( + result_budget.live_bytes(), + &pending_groups, )?; + let mut group_budget = crate::batch_allocation::BatchMetadataBudget::with_external_live( + "JPEG Metal grouped region-scaled buffer sub-batch", + pending_live_bytes, + ); + group_budget.account_capacity::(group_indices.capacity())?; let mut group_requests = group_budget.try_vec( group_indices.len(), "JPEG Metal grouped region-scaled buffer requests", @@ -480,72 +664,32 @@ fn try_decode_grouped_fast_subsampled_region_scaled_rgb_batch_to_surfaces_with_o group_indices.len(), "JPEG Metal grouped region-scaled buffer packets", )?; - group_packets.extend(group_indices.iter().map(|&index| { - let (packet, mode) = family_packets[index]; - packet.to_region_scaled_batched(mode) - })); + group_packets.extend(group_indices.iter().map(|&index| family_packets[index])); batch::stamp_execution_owner_baseline(&mut group_requests, 0, group_budget.live_bytes()); - let Some(group_results) = - try_decode_fast_subsampled_region_scaled_rgb_batch_to_surfaces_with_output::

( - runtime, - &group_requests, - &group_packets, - None, - )? + let Some(pending) = try_submit_compatible_region_scaled_rgb_batch::

( + runtime, + &group_requests, + &group_packets, + None, + scratch, + )? else { return Ok(None); }; - - if let Some(output) = output { - let Some(&first_group_index) = group_indices.first() else { - continue; - }; - let batch::BatchOp::RegionScaled { roi, scale } = requests[first_group_index].op else { - return Ok(None); - }; - let (packet, mode) = family_packets[first_group_index]; - let segment_count_u32 = checked_u32( - packet.entropy_checkpoints().len(), - &format!( - "{} grouped region scaled buffer segment count", - P::FAMILY_NAME - ), - )?; - let Some(plan) = fast_subsampled_region_scaled_batch_plan( - packet, - roi, - scale, - 1, - segment_count_u32, - mode, - ) else { - return Ok(None); - }; - for (original_index, result) in copy_grouped_surfaces_to_output( - runtime, - output, - plan.out_dims, - plan.out_tile_len, - &group_indices, - group_results, - result_budget.live_bytes(), - )? { - merged_results[original_index] = Some(result); - } - } else { - if group_results.len() != group_indices.len() { - return Err(Error::MetalKernel { - message: format!( - "JPEG Metal grouped {} region scaled buffer result count mismatch", - P::FAMILY_NAME - ), - }); - } - for (original_index, result) in group_indices.into_iter().zip(group_results) { - merged_results[original_index] = Some(result); - } - } + pending_groups.push(PendingRegionScaledRgbGroup { + indices: group_indices, + requests: group_requests, + pending, + }); + } + while !pending_groups.is_empty() { + let group = pending_groups.remove(0); + let external_live_bytes = pending_region_scaled_rgb_group_live_bytes( + result_budget.live_bytes(), + &pending_groups, + )?; + group.finish::

(runtime, output, &mut merged_results, external_live_bytes)?; } let mut results = result_budget.try_vec( diff --git a/crates/j2k-jpeg-metal/src/compute/command.rs b/crates/j2k-jpeg-metal/src/compute/command.rs index d28888263..d2fd2e6ae 100644 --- a/crates/j2k-jpeg-metal/src/compute/command.rs +++ b/crates/j2k-jpeg-metal/src/compute/command.rs @@ -39,13 +39,56 @@ pub(in crate::compute) fn new_blit_command_encoder( pub(in crate::compute) fn commit_and_wait_jpeg( command_buffer: &CommandBufferRef, ) -> Result<(), Error> { - commit_and_wait(command_buffer) - .map_err(|error| metal_kernel_support_error(error.to_string(), error)) + let result = commit_and_wait(command_buffer) + .map_err(|error| metal_kernel_support_error(error.to_string(), error)); + #[cfg(test)] + gpu_time::record(command_buffer); + result } pub(in crate::compute) fn wait_for_completion_jpeg( command_buffer: &CommandBufferRef, ) -> Result<(), Error> { - wait_for_completion(command_buffer) - .map_err(|error| metal_kernel_support_error(error.to_string(), error)) + let result = wait_for_completion(command_buffer) + .map_err(|error| metal_kernel_support_error(error.to_string(), error)); + #[cfg(test)] + gpu_time::record(command_buffer); + result +} + +/// Test-only GPU execution time of the command buffers this thread waited on. +#[cfg(test)] +pub(in crate::compute) mod gpu_time { + use std::cell::Cell; + + use objc2_metal::MTLCommandBuffer as _; + + use crate::metal_types::CommandBufferRef; + + thread_local! { + static GPU_SECONDS: Cell = const { Cell::new(0.0) }; + static SUBMIT_SECONDS: Cell<(f64, f64)> = const { Cell::new((0.0, 0.0)) }; + } + + pub(super) fn record(command_buffer: &CommandBufferRef) { + let seconds = command_buffer.GPUEndTime() - command_buffer.GPUStartTime(); + GPU_SECONDS.with(|total| total.set(total.get() + seconds)); + let scheduling = command_buffer.kernelEndTime() - command_buffer.kernelStartTime(); + let queued = command_buffer.GPUStartTime() - command_buffer.kernelEndTime(); + SUBMIT_SECONDS.with(|total| { + let (sum_scheduling, sum_queued) = total.get(); + total.set((sum_scheduling + scheduling, sum_queued + queued)); + }); + } + + /// Returns the accumulated GPU seconds and resets the counter. + pub(in crate::compute) fn take_seconds() -> f64 { + GPU_SECONDS.with(|total| total.replace(0.0)) + } + + /// Returns the accumulated CPU-side scheduling seconds (kernel start to + /// kernel end) and queue seconds (kernel end to GPU start), then resets. + pub(in crate::compute) fn take_submit_seconds() -> (f64, f64) { + SUBMIT_SECONDS.with(|total| total.replace((0.0, 0.0))) + } } diff --git a/crates/j2k-jpeg-metal/src/compute/fast_packets/descriptors.rs b/crates/j2k-jpeg-metal/src/compute/fast_packets/descriptors.rs index 29c3c8321..c5111807d 100644 --- a/crates/j2k-jpeg-metal/src/compute/fast_packets/descriptors.rs +++ b/crates/j2k-jpeg-metal/src/compute/fast_packets/descriptors.rs @@ -57,6 +57,11 @@ pub(in crate::compute) trait FastSubsampledPacket { /// Vertical dispatch extent for the full-frame pack kernels: 4:2:0 packs /// 2x2 pixel quads per thread, 4:2:2 packs 2x1 pairs (full-height rows). fn packed_height_extent(height: u32) -> u32; + /// Horizontal dispatch extent for the full-frame pack kernels: the + /// subsampled families pack horizontal pairs, 4:4:4 one pixel per thread. + fn packed_width_extent(width: u32) -> u32 { + width.div_ceil(2).max(1) + } } macro_rules! impl_fast_subsampled_packet_accessors { @@ -203,6 +208,9 @@ impl FastSubsampledPacket for JpegFast444PacketV1 { fn packed_height_extent(height: u32) -> u32 { height } + fn packed_width_extent(width: u32) -> u32 { + width + } } /// Scratch-pool cache keys for one batch driver's buffers; keys stay @@ -253,9 +261,6 @@ pub(in crate::compute) trait FastSubsampledMetal: FastSubsampledPacket { fn from_batched<'a>(packet: &BatchedFastPacket<'a>) -> Option<&'a Self>; fn to_batched(&self) -> BatchedFastPacket<'_>; - fn to_batched_with_texture_mode(&self, _mode: PlaneMode) -> BatchedFastPacket<'_> { - self.to_batched() - } fn texture_plane_mode_from_batched(packet: &BatchedFastPacket<'_>) -> Option { Self::from_batched(packet).map(|_| PlaneMode::YCbCr) } diff --git a/crates/j2k-jpeg-metal/src/compute/fast_packets/pipelines.rs b/crates/j2k-jpeg-metal/src/compute/fast_packets/pipelines.rs index 5e9d3371e..ee93b4345 100644 --- a/crates/j2k-jpeg-metal/src/compute/fast_packets/pipelines.rs +++ b/crates/j2k-jpeg-metal/src/compute/fast_packets/pipelines.rs @@ -442,9 +442,6 @@ impl FastSubsampledMetal for JpegFast444PacketV1 { fn to_batched(&self) -> BatchedFastPacket<'_> { BatchedFastPacket::Fast444(self, PlaneMode::YCbCr) } - fn to_batched_with_texture_mode(&self, mode: PlaneMode) -> BatchedFastPacket<'_> { - BatchedFastPacket::Fast444(self, mode) - } fn texture_plane_mode_from_batched(packet: &BatchedFastPacket<'_>) -> Option { match packet { BatchedFastPacket::Fast444(_, mode) => Some(*mode), @@ -489,7 +486,7 @@ impl FastSubsampledMetal for JpegFast444PacketV1 { &runtime.pipelines.pack_444_rgba_texture } fn full_rgb_batch_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { - &runtime.pipelines.fast444_scaled_region_batch_decode + &runtime.pipelines.fast444_batch_decode } fn pack_full_rgb_batch_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { &runtime.pipelines.pack_444_rgb_batch diff --git a/crates/j2k-jpeg-metal/src/compute/mod.rs b/crates/j2k-jpeg-metal/src/compute/mod.rs index c7b5f86cf..f16194912 100644 --- a/crates/j2k-jpeg-metal/src/compute/mod.rs +++ b/crates/j2k-jpeg-metal/src/compute/mod.rs @@ -64,6 +64,8 @@ mod region_scaled_plan; mod runtime; #[cfg(target_os = "macos")] mod scratch_pool; +#[cfg(target_os = "macos")] +mod surface_output_pool; #[cfg(all(target_os = "macos", test))] use self::pipeline_registry::SHADER_SOURCE; #[cfg(target_os = "macos")] @@ -182,13 +184,16 @@ use self::command::{ commit_and_wait_jpeg, new_blit_command_encoder, new_command_buffer, new_compute_command_encoder, wait_for_completion_jpeg, }; +pub use self::runtime::release_default_session_buffers; +#[cfg(target_os = "macos")] +pub(crate) use self::runtime::{ + default_tile_backend_session, runtime_initialization_error, MetalRuntime, +}; #[cfg(target_os = "macos")] use self::runtime::{ private_jpeg_tile_from_fast_rgb_buffer, with_runtime, with_runtime_for_session, FastRgbDecodeBuffer, }; -#[cfg(target_os = "macos")] -pub(crate) use self::runtime::{runtime_initialization_error, MetalRuntime}; #[cfg(target_os = "macos")] const REGION_SCALED_BATCH_CHUNK: usize = 8; diff --git a/crates/j2k-jpeg-metal/src/compute/pack_dispatch/grouped_output.rs b/crates/j2k-jpeg-metal/src/compute/pack_dispatch/grouped_output.rs index 0d6dfe9b4..2c983d462 100644 --- a/crates/j2k-jpeg-metal/src/compute/pack_dispatch/grouped_output.rs +++ b/crates/j2k-jpeg-metal/src/compute/pack_dispatch/grouped_output.rs @@ -1,6 +1,5 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use crate::buffers::new_shared_buffer; use objc2::Message; use objc2_metal::{MTLBlitCommandEncoder, MTLCommandEncoder}; @@ -24,7 +23,7 @@ pub(in crate::compute) fn batch_output_buffer_or_new( .ok_or(BufferError::SizeOverflow { what: "JPEG Metal batch output bytes", })?; - return new_shared_buffer(&runtime.device, byte_len); + return runtime.surface_output_buffer(byte_len); }; if output.dimensions() != dimensions diff --git a/crates/j2k-jpeg-metal/src/compute/pipeline_registry.rs b/crates/j2k-jpeg-metal/src/compute/pipeline_registry.rs index 2fa4bb6e9..ec27053f2 100644 --- a/crates/j2k-jpeg-metal/src/compute/pipeline_registry.rs +++ b/crates/j2k-jpeg-metal/src/compute/pipeline_registry.rs @@ -150,6 +150,7 @@ pub(in crate::compute) struct JpegPipelineRegistry { pub(in crate::compute) fast444_scaled_decode: ComputePipelineState, pub(in crate::compute) fast444_scaled_region_decode: ComputePipelineState, pub(in crate::compute) fast444_scaled_region_batch_decode: ComputePipelineState, + pub(in crate::compute) fast444_batch_decode: ComputePipelineState, pub(in crate::compute) fast444_rgba_texture_batch_decode: ComputePipelineState, pub(in crate::compute) rgb8_to_rgba_texture: ComputePipelineState, } @@ -166,7 +167,16 @@ impl JpegPipelineRegistry { } pub(in crate::compute) fn load(device: &Device) -> Result { - let loader = MetalPipelineLoader::new(device, SHADER_SOURCE)?; + Self::load_from_source(device, SHADER_SOURCE) + } + + /// Builds every pipeline from `source`. Production passes + /// [`SHADER_SOURCE`]; the kernel harness passes baseline or probe variants. + pub(in crate::compute) fn load_from_source( + device: &Device, + source: &str, + ) -> Result { + let loader = MetalPipelineLoader::new(device, source)?; let pipeline = |name: &str| loader.pipeline(name); Ok(Self { pack: pipeline("jpeg_pack")?, @@ -236,6 +246,7 @@ impl JpegPipelineRegistry { fast444_scaled_region_batch_decode: pipeline( "jpeg_decode_fast444_scaled_region_batch", )?, + fast444_batch_decode: pipeline("jpeg_decode_fast444_batch")?, fast444_rgba_texture_batch_decode: pipeline("jpeg_decode_fast444_rgba_texture_batch")?, rgb8_to_rgba_texture: pipeline("jpeg_copy_rgb8_to_rgba_texture")?, }) diff --git a/crates/j2k-jpeg-metal/src/compute/runtime.rs b/crates/j2k-jpeg-metal/src/compute/runtime.rs index dc6365717..ee5e089f0 100644 --- a/crates/j2k-jpeg-metal/src/compute/runtime.rs +++ b/crates/j2k-jpeg-metal/src/compute/runtime.rs @@ -9,6 +9,7 @@ use std::{ use super::pipeline_registry::JpegPipelineRegistry; use super::scratch_pool::{BatchScratchLease, BatchScratchPool}; +use super::surface_output_pool::SurfaceOutputPool; use super::viewport_cache::{ CachedViewportPlanes, ViewportPlaneCacheGate, ViewportPlaneCacheLease, }; @@ -21,11 +22,99 @@ thread_local! { static DEFAULT_METAL_SESSION: RefCell>> = const { RefCell::new(None) }; } +/// Backend session shared by every `MetalSession` created without one, so +/// one-shot tile batches reuse one command queue and warm buffer pools instead +/// of building a runtime per session. Its pooled buffers are released under +/// memory pressure and by `release_default_session_buffers`. +static DEFAULT_TILE_SESSION: Mutex> = Mutex::new(None); + +fn default_tile_session_slot( +) -> Result>, Error> { + DEFAULT_TILE_SESSION + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "JPEG Metal default tile session", + }) +} + +/// The process-wide backend session for `MetalSession`s created without one. +pub(crate) fn default_tile_backend_session() -> Result { + let mut slot = default_tile_session_slot()?; + if let Some(session) = slot.as_ref() { + return Ok(session.clone()); + } + let session = crate::MetalBackendSession::system_default()?; + *slot = Some(session.clone()); + install_memory_pressure_release(); + Ok(session) +} + +/// Releases the buffers that the default session of `MetalSession`s created +/// without a backend session keeps pooled between calls. +/// +/// Buffers still referenced by surfaces or in-flight work stay alive until +/// those references drop. The session itself, its queue and compiled +/// pipelines are kept. This also runs automatically when macOS reports +/// memory pressure. +/// +/// # Errors +/// +/// Returns an error if a pool lock was poisoned by a panic. +pub fn release_default_session_buffers() -> Result<(), Error> { + let session = default_tile_session_slot()?.clone(); + let Some(session) = session else { + return Ok(()); + }; + match session.initialized_runtime() { + Some(Ok(runtime)) => runtime.release_pooled_buffers(), + Some(Err(_)) | None => Ok(()), + } +} + +/// Registers a libdispatch memory-pressure source (warning and critical +/// levels) that calls `release_default_session_buffers`, once per process. +fn install_memory_pressure_release() { + use dispatch2::{ + dispatch_source_memorypressure_flags_t as PressureFlags, DispatchObject, DispatchQoS, + DispatchQueue, DispatchSource, GlobalQueueIdentifier, + }; + + extern "C" fn release_on_memory_pressure(_context: *mut core::ffi::c_void) { + // The handler has no caller to report to; a poisoned pool keeps its + // buffers and the next explicit release returns the error. + let _ = release_default_session_buffers(); + } + + static INSTALLED: std::sync::Once = std::sync::Once::new(); + INSTALLED.call_once(|| { + let queue = DispatchQueue::global_queue(GlobalQueueIdentifier::QualityOfService( + DispatchQoS::Utility, + )); + let levels = PressureFlags::DISPATCH_MEMORYPRESSURE_WARN.0 + | PressureFlags::DISPATCH_MEMORYPRESSURE_CRITICAL.0; + // SAFETY: a memory-pressure source takes no handle and a mask of + // pressure levels; the source type is libdispatch's static descriptor. + let source = unsafe { + DispatchSource::new( + (&raw const dispatch2::_dispatch_source_type_memorypressure).cast_mut(), + 0, + usize::try_from(levels).expect("pressure mask fits usize"), + Some(&queue), + ) + }; + source.set_event_handler_f(release_on_memory_pressure); + source.activate(); + // The source lives for the rest of the process. + core::mem::forget(source); + }); +} + pub(crate) struct MetalRuntime { pub(in crate::compute) device: Device, pub(in crate::compute) queue: CommandQueue, pub(in crate::compute) pipelines: Arc, batch_scratch: BatchScratchPool, + surface_outputs: SurfaceOutputPool, viewport_plane_cache: Mutex>, viewport_plane_cache_gate: Arc, } @@ -45,14 +134,36 @@ impl MetalRuntime { Self::new_with_device(device) } + /// Builds an uncached runtime whose pipelines are compiled from + /// `shader_source`, so the kernel harness can A/B shader variants. + #[cfg(test)] + pub(in crate::compute) fn new_with_shader_source( + shader_source: &str, + ) -> Result { + let device = system_default_device()?; + let pipelines = Arc::new(JpegPipelineRegistry::load_from_source( + &device, + shader_source, + )?); + Self::from_pipelines(device, pipelines) + } + pub(crate) fn new_with_device(device: Device) -> Result { let pipelines = JpegPipelineRegistry::shared(&device)?; + Self::from_pipelines(device, pipelines) + } + + fn from_pipelines( + device: Device, + pipelines: Arc, + ) -> Result { let queue = checked_command_queue(&device)?; Ok(Self { device, queue, pipelines, batch_scratch: BatchScratchPool::default(), + surface_outputs: SurfaceOutputPool::default(), viewport_plane_cache: Mutex::new(None), viewport_plane_cache_gate: ViewportPlaneCacheGate::new(), }) @@ -62,6 +173,30 @@ impl MetalRuntime { self.batch_scratch.acquire() } + pub(in crate::compute) fn try_batch_scratch( + &self, + ) -> Result>, Error> { + self.batch_scratch.try_acquire() + } + + /// Shared output buffer of at least `bytes` for batch surfaces that the + /// caller keeps after this call; see `surface_output_pool`. + pub(in crate::compute) fn surface_output_buffer(&self, bytes: usize) -> Result { + self.surface_outputs.acquire(&self.device, bytes) + } + + /// Drops pooled surface outputs and idle batch scratch; see + /// `release_default_session_buffers`. + pub(in crate::compute) fn release_pooled_buffers(&self) -> Result<(), Error> { + self.surface_outputs.release()?; + self.batch_scratch.release_idle() + } + + #[cfg(test)] + pub(in crate::compute) fn pooled_surface_outputs_for_test(&self) -> usize { + self.surface_outputs.pooled_count_for_test() + } + #[cfg(test)] pub(in crate::compute) fn batch_scratch_in_use_for_test(&self) -> bool { self.batch_scratch.in_use() @@ -153,6 +288,41 @@ pub(in crate::compute) fn private_jpeg_tile_from_fast_rgb_buffer( mod tests { use super::*; + #[test] + fn release_forgets_pooled_outputs_and_keeps_leased_scratch() { + use objc2_metal::MTLBuffer as _; + + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let runtime = MetalRuntime::new().expect("runtime"); + let output = runtime + .surface_output_buffer(2 * 1024 * 1024) + .expect("pooled output"); + let kept_output = output.clone(); + drop(output); + assert_eq!(runtime.pooled_surface_outputs_for_test(), 1); + + let mut lease = runtime.batch_scratch().expect("scratch lease"); + let status = lease + .shared_buffer_with_bytes(&runtime.device, "release status", &[7, 9]) + .expect("leased status"); + runtime.release_pooled_buffers().expect("release"); + assert_eq!(runtime.pooled_surface_outputs_for_test(), 0); + // A buffer the caller still holds survives the release. + assert!(kept_output.length() >= 2 * 1024 * 1024); + // Leased scratch is untouched and returns to its slot on drop. + assert!(runtime.batch_scratch_in_use_for_test()); + let bytes = crate::buffers::checked_buffer_slice::(&status, 2, "leased status") + .expect("read leased status"); + assert_eq!(bytes, [7, 9]); + drop(lease); + runtime + .release_pooled_buffers() + .expect("release idle scratch"); + assert!(!runtime.batch_scratch_in_use_for_test()); + } + #[test] fn scratch_slots_allow_two_independent_batches() { if !j2k_test_support::metal_runtime_gate(module_path!()) { diff --git a/crates/j2k-jpeg-metal/src/compute/scratch_pool.rs b/crates/j2k-jpeg-metal/src/compute/scratch_pool.rs index 4cf6f7723..353218e1e 100644 --- a/crates/j2k-jpeg-metal/src/compute/scratch_pool.rs +++ b/crates/j2k-jpeg-metal/src/compute/scratch_pool.rs @@ -39,23 +39,8 @@ impl BatchScratchPool { if *availability { return Err(poisoned()); } - for slot in &self.slots { - match slot.try_lock() { - Ok(mut entry) => { - if let Some(scratch) = entry.take() { - drop(entry); - drop(availability); - return Ok(BatchScratchLease { - scratch: Some(scratch), - slot, - pool: self, - panicking_on_acquire: std::thread::panicking(), - }); - } - } - Err(TryLockError::WouldBlock) => {} - Err(TryLockError::Poisoned(_)) => return Err(poisoned()), - } + if let Some(lease) = self.take_available()? { + return Ok(lease); } // The availability mutex joins this check to release notifications, // so neither a free slot nor a wakeup can be missed between them. @@ -63,6 +48,52 @@ impl BatchScratchPool { } } + pub(super) fn try_acquire(&self) -> Result>, Error> { + let availability = self.availability.lock().map_err(|_| poisoned())?; + if *availability { + return Err(poisoned()); + } + self.take_available() + } + + // Callers hold the availability gate until a slot has been taken. + fn take_available(&self) -> Result>, Error> { + for slot in &self.slots { + match slot.try_lock() { + Ok(mut entry) => { + if let Some(scratch) = entry.take() { + return Ok(Some(BatchScratchLease { + scratch: Some(scratch), + slot, + pool: self, + panicking_on_acquire: std::thread::panicking(), + })); + } + } + Err(TryLockError::WouldBlock) => {} + Err(TryLockError::Poisoned(_)) => return Err(poisoned()), + } + } + Ok(None) + } + + /// Replaces idle scratch with empty scratch, dropping its buffers. Leased + /// or locked slots are left alone; their scratch returns intact. + pub(super) fn release_idle(&self) -> Result<(), Error> { + for slot in &self.slots { + match slot.try_lock() { + Ok(mut entry) => { + if entry.is_some() { + *entry = Some(MetalBatchScratch::default()); + } + } + Err(TryLockError::WouldBlock) => {} + Err(TryLockError::Poisoned(_)) => return Err(poisoned()), + } + } + Ok(()) + } + #[cfg(test)] pub(super) fn in_use(&self) -> bool { self.slots.iter().any(|slot| match slot.try_lock() { @@ -144,6 +175,21 @@ mod tests { assert!(!pool.in_use()); } + #[test] + fn extra_scratch_acquisition_never_waits_while_both_slots_are_owned() { + let pool = BatchScratchPool::default(); + let first = pool.acquire().expect("first lease"); + let second = pool + .try_acquire() + .expect("try second slot") + .expect("second lease"); + assert!(pool.try_acquire().expect("full pool").is_none()); + drop(first); + assert!(pool.try_acquire().expect("released slot").is_some()); + drop(second); + assert!(!pool.in_use()); + } + #[test] fn scratch_slots_fail_closed_after_panicking_owner() { let pool = BatchScratchPool::default(); diff --git a/crates/j2k-jpeg-metal/src/compute/surface_output_pool.rs b/crates/j2k-jpeg-metal/src/compute/surface_output_pool.rs new file mode 100644 index 000000000..25f1aca71 --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute/surface_output_pool.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Reuse of batch output buffers whose surfaces have all been dropped. +//! +//! Batch decodes that return [`Surface`](crate::Surface)s write every tile into +//! one shared buffer, and each surface retains that buffer. Allocating a fresh +//! buffer per call costs the driver about 0.26 ms of command-buffer scheduling +//! for 16 RGB tiles of 512x512 (mapping the new pages), which is a quarter of +//! the call. The pool hands a buffer out again only when it holds the sole +//! reference: every live surface and every in-flight command buffer that used +//! the buffer retains it, so no output a caller can still read is overwritten. + +use std::sync::Mutex; + +use objc2::runtime::NSObjectProtocol as _; +use objc2_metal::MTLBuffer as _; + +use crate::buffers::new_shared_buffer; +use crate::metal_types::{Buffer, DeviceRef}; +use crate::Error; + +/// Two slots let a caller keep one batch's surfaces while decoding the next. +const SURFACE_OUTPUT_POOL_SLOTS: usize = 2; +/// Smaller outputs are allocated per call, as before: the driver serves them +/// without a page-mapping cost, and reusing one measured slower (64 RGB tiles +/// of 16x16, 49 KiB: +9% to +17% per batch). +const SURFACE_OUTPUT_POOL_MIN_BUFFER_BYTES: usize = 1024 * 1024; +/// Larger outputs are allocated per call, as before, so an idle session +/// retains at most `SLOTS x 64 MiB` of output buffers (64 RGB tiles of 512x512 +/// need 48 MiB). +const SURFACE_OUTPUT_POOL_MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024; + +#[derive(Default)] +pub(super) struct SurfaceOutputPool { + slots: Mutex>, +} + +impl SurfaceOutputPool { + /// Returns a shared buffer of at least `bytes`, reusing a pooled buffer + /// that nothing else references. + pub(super) fn acquire(&self, device: &DeviceRef, bytes: usize) -> Result { + if !(SURFACE_OUTPUT_POOL_MIN_BUFFER_BYTES..=SURFACE_OUTPUT_POOL_MAX_BUFFER_BYTES) + .contains(&bytes) + { + return new_shared_buffer(device, bytes); + } + let mut slots = self.slots.lock().map_err(|_| Error::MetalStatePoisoned { + state: "JPEG Metal surface output pool", + })?; + // The count is read under the pool lock. New references to a pooled + // buffer come only from this pool or from cloning a surface that + // already holds one, so a count of one cannot rise concurrently. + let is_free = |buffer: &Buffer| buffer.retainCount() == 1; + if let Some(buffer) = slots + .iter() + .find(|buffer| is_free(buffer) && buffer.length() >= bytes) + { + return Ok(buffer.clone()); + } + let buffer = new_shared_buffer(device, bytes)?; + if slots.len() < SURFACE_OUTPUT_POOL_SLOTS { + slots.push(buffer.clone()); + } else if let Some(slot) = slots.iter_mut().find(|buffer| is_free(buffer)) { + // A free slot was too small; keep the larger buffer instead. + *slot = buffer.clone(); + } + Ok(buffer) + } + + /// Forgets every pooled buffer. Surfaces keep their own references, so a + /// buffer still in use is freed only when its last surface drops. + pub(super) fn release(&self) -> Result<(), Error> { + self.slots + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "JPEG Metal surface output pool", + })? + .clear(); + Ok(()) + } + + #[cfg(test)] + pub(super) fn pooled_count_for_test(&self) -> usize { + self.slots.lock().map_or(0, |slots| slots.len()) + } +} diff --git a/crates/j2k-jpeg-metal/src/compute/tests.rs b/crates/j2k-jpeg-metal/src/compute/tests.rs index 1348f892e..1711b4624 100644 --- a/crates/j2k-jpeg-metal/src/compute/tests.rs +++ b/crates/j2k-jpeg-metal/src/compute/tests.rs @@ -345,7 +345,9 @@ fn shader_source_keeps_entropy_fast_paths() { assert!(SHADER_SOURCE.contains("inline bool refill_four_bytes(")); assert!(SHADER_SOURCE.contains("return refill_four_bytes(br, bytes, len) || refill_one_byte")); assert!(SHADER_SOURCE.contains("ensure_bits_padded(br, bytes, len, 9)")); - assert!(SHADER_SOURCE.contains("table.fast_len[fast_index]")); + assert!(SHADER_SOURCE.contains("const uint fast = uint(table.fast[lookahead]);")); + assert!(SHADER_SOURCE.contains("const int fast_ac = int(ac_table.fast_ac[lookahead]);")); + assert!(SHADER_SOURCE.contains("for (uint length = 10; length <= 16; ++length)")); assert!(SHADER_SOURCE.contains("inline bool decode_block_skip(")); assert!(SHADER_SOURCE.contains("skip_receive_extend(br, bytes, len, ssss, status)")); assert!(SHADER_SOURCE.contains("inline bool configure_batch_entropy_thread(")); @@ -519,6 +521,87 @@ fn auto_batched_packets_reject_restart_batch_without_validated_promotion_evidenc .is_none()); } +#[test] +fn auto_batched_packets_promote_only_large_compatible_full_rgb_batches() { + let input = Arc::<[u8]>::from(generated_rgb_jpeg(256)); + let packet = Arc::new(j2k_jpeg::adapter::build_fast420_packet(&input).expect("packet")); + let request = batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Auto, + batch::BatchOp::Full, + None, + None, + Some(Arc::clone(&packet)), + ); + let mut requests = vec![request.clone(); 16]; + assert!(batched_fast_packets(&requests) + .expect("promoted batch") + .is_some()); + assert!(batched_fast_packets(&requests[..15]) + .expect("small batch") + .is_none()); + let small = batch::QueuedRequest::new( + Arc::<[u8]>::from(BASELINE_420), + PixelFormat::Rgb8, + BackendRequest::Auto, + batch::BatchOp::Full, + None, + None, + Some(Arc::new( + j2k_jpeg::adapter::build_fast420_packet(BASELINE_420).expect("small packet"), + )), + ); + assert!(batched_fast_packets(&vec![small; 16]) + .expect("small tiles") + .is_none()); + requests[15] = batch::QueuedRequest::new( + Arc::<[u8]>::from(input.as_ref()), + PixelFormat::Rgb8, + BackendRequest::Auto, + batch::BatchOp::Full, + None, + None, + Some(Arc::clone(&packet)), + ); + assert!(batched_fast_packets(&requests) + .expect("distinct owners") + .is_some()); + let mut different_tables = j2k_jpeg::adapter::build_fast420_packet(&input).expect("packet"); + different_tables.y_quant[0] += 1; + requests[15] = request.clone(); + requests[15].fast_packet = Some( + crate::SharedJpegFastPacket::try_new(j2k_jpeg::adapter::JpegFastPacket::Fast420( + different_tables, + )) + .expect("different table owner"), + ); + assert!(batched_fast_packets(&requests) + .expect("incompatible table group") + .is_none()); + for (backend, fmt, op) in [ + (BackendRequest::Cpu, PixelFormat::Rgb8, batch::BatchOp::Full), + ( + BackendRequest::Auto, + PixelFormat::Rgba8, + batch::BatchOp::Full, + ), + ( + BackendRequest::Auto, + PixelFormat::Rgb8, + batch::BatchOp::Scaled(j2k_core::Downscale::Half), + ), + ] { + requests[15] = request.clone(); + requests[15].backend = backend; + requests[15].fmt = fmt; + requests[15].op = op; + assert!(batched_fast_packets(&requests) + .expect("unpromoted batch") + .is_none()); + } +} + #[test] fn auto_batched_packets_reject_large_batch_without_validated_promotion_evidence() { let input = Arc::<[u8]>::from(generated_rgb_jpeg(512)); @@ -2091,3 +2174,6 @@ fn single_scratch_recovers_after_gpu_entropy_failure() { assert!(!runtime.batch_scratch_in_use_for_test()); } } + +mod decode_kernel_harness; +mod surface_output_reuse; diff --git a/crates/j2k-jpeg-metal/src/compute/tests/decode_kernel_harness.rs b/crates/j2k-jpeg-metal/src/compute/tests/decode_kernel_harness.rs new file mode 100644 index 000000000..20e2ac4ac --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute/tests/decode_kernel_harness.rs @@ -0,0 +1,1349 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! JPEG decode kernel harness. +//! +//! Correctness: every Metal consumer of the shared `decode_block` and IDCT +//! routines must match the CPU decoder bit for bit on entropy-stress fixtures: +//! all three samplings, quality 50 to 100, noise that forces long Huffman +//! codes, large magnitudes and 0xFF stuffing, odd dimensions, restart +//! intervals, batch 1 and 16, and every batch operation and output kind. +//! +//! Profiling (`#[ignore]`d): GPU time per decode stage for the buffer-output +//! batch kernels. Probe variants replace `decode_idct_deposit_block` with a +//! body that stops after Huffman decoding, coefficient materialization, or the +//! IDCT, so successive differences split the fused kernel's time. Setting +//! `J2K_JPEG_HARNESS_BASELINE_SHADERS` to a directory holding another copy of +//! the shader files A/Bs those kernels against the in-tree ones in one process. +//! +//! ```sh +//! J2K_REQUIRE_METAL_RUNTIME=1 cargo test --profile gpu-quick -p j2k-jpeg-metal --lib \ +//! -- decode_kernel_harness --include-ignored --nocapture --test-threads=1 +//! ``` + +use super::*; +use crate::compute::command::gpu_time; +use crate::compute::pipeline_registry::SHADER_SOURCE; +use j2k_core::{DeviceSubmission, Downscale, ImageDecodeSubmit, Rect}; +use std::time::Instant; + +/// Shader files in `SHADER_SOURCE` concatenation order. +const SHADER_FILES: [&str; 9] = [ + "shaders_shared.metal", + "shaders_encode.metal", + "shaders_encode_staged.metal", + "shaders_decode_helpers.metal", + "shaders_pack_444.metal", + "shaders_decode_fast420.metal", + "shaders_decode_fast422_regions.metal", + "shaders_decode_fast444.metal", + "shaders_pack_subsampled.metal", +]; + +const BASELINE_SHADERS_ENV: &str = "J2K_JPEG_HARNESS_BASELINE_SHADERS"; +const PROFILE_BATCH: usize = 16; +const PROFILE_WARMUP: usize = 3; +const PROFILE_SAMPLES: usize = 21; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Sampling { + S420, + S422, + S444, +} + +impl Sampling { + fn factor(self) -> jpeg_encoder::SamplingFactor { + match self { + Self::S420 => jpeg_encoder::SamplingFactor::F_2_2, + Self::S422 => jpeg_encoder::SamplingFactor::F_2_1, + Self::S444 => jpeg_encoder::SamplingFactor::F_1_1, + } + } + + fn label(self) -> &'static str { + match self { + Self::S420 => "420", + Self::S422 => "422", + Self::S444 => "444", + } + } +} + +#[derive(Clone, Copy, Debug)] +enum Content { + /// The shared GPU benchmark image (sawtooth ramps, sharp edges). + Bench, + /// Smooth gradients plus ±16 noise: photographic high-frequency detail. + Textured, + /// Uniform noise: long AC codes, large magnitudes, frequent 0xFF stuffing. + Noise, +} + +impl Content { + fn label(self) -> &'static str { + match self { + Self::Bench => "bench", + Self::Textured => "textured", + Self::Noise => "noise", + } + } +} + +#[derive(Clone, Copy, Debug)] +struct FixtureSpec { + sampling: Sampling, + content: Content, + quality: u8, + width: u16, + height: u16, + restart: Option, +} + +impl FixtureSpec { + fn label(&self) -> String { + let restart = self + .restart + .map_or(String::new(), |interval| format!("_rst{interval}")); + format!( + "{}_{}_q{}_{}x{}{restart}", + self.sampling.label(), + self.content.label(), + self.quality, + self.width, + self.height + ) + } + + fn dimensions(&self) -> (u32, u32) { + (u32::from(self.width), u32::from(self.height)) + } +} + +struct XorShift(u32); + +impl XorShift { + fn new(seed: u32) -> Self { + Self(seed.wrapping_mul(0x9e37_79b9) | 1) + } + + fn next(&mut self) -> u32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 17; + self.0 ^= self.0 << 5; + self.0 + } +} + +fn rgb_pixels(content: Content, width: u32, height: u32, seed: u32) -> Vec { + let mut rng = XorShift::new(seed); + match content { + Content::Bench => { + let mut rgb = j2k_test_support::gpu_bench_rgb8(width, height); + // Distinct tiles need distinct entropy streams. + let shift = (seed as usize * 3 * 7) % rgb.len().max(1); + rgb.rotate_left(shift); + rgb + } + Content::Textured => { + let span = (width + height).max(1); + let mut rgb = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + let base = [ + (x + y) * 255 / span, + (x * 2 + seed * 17) % 256, + (y * 3 + (x * y) / 64) % 256, + ]; + for channel in base { + let noise = i32::try_from(rng.next() % 33).expect("noise fits i32") - 16; + let channel = i32::try_from(channel).expect("channel fits i32"); + let sample = (channel + noise).clamp(0, 255); + rgb.push(u8::try_from(sample).expect("clamped sample fits u8")); + } + } + } + rgb + } + Content::Noise => (0..width as usize * height as usize * 3) + .map(|_| rng.next().to_le_bytes()[1]) + .collect(), + } +} + +fn encode_jpeg(spec: &FixtureSpec, seed: u32) -> Vec { + let (width, height) = spec.dimensions(); + let rgb = rgb_pixels(spec.content, width, height, seed); + let mut jpeg = Vec::new(); + let mut encoder = jpeg_encoder::Encoder::new(&mut jpeg, spec.quality); + encoder.set_sampling_factor(spec.sampling.factor()); + if let Some(interval) = spec.restart { + encoder.set_restart_interval(interval); + } + encoder + .encode(&rgb, spec.width, spec.height, jpeg_encoder::ColorType::Rgb) + .unwrap_or_else(|error| panic!("encode {}: {error}", spec.label())); + jpeg +} + +fn encode_batch(spec: &FixtureSpec, count: usize) -> Vec> { + (0..count) + .map(|seed| { + let seed = u32::try_from(seed).expect("fixture seed fits u32"); + Arc::<[u8]>::from(encode_jpeg(spec, seed + 1)) + }) + .collect() +} + +fn correctness_specs() -> Vec { + let mut specs = Vec::new(); + for sampling in [Sampling::S420, Sampling::S422, Sampling::S444] { + let spec = |content, quality, width, height, restart| FixtureSpec { + sampling, + content, + quality, + width, + height, + restart, + }; + specs.extend([ + spec(Content::Bench, 90, 128, 128, None), + spec(Content::Textured, 75, 128, 128, None), + spec(Content::Textured, 95, 133, 77, None), + spec(Content::Noise, 100, 96, 64, None), + spec(Content::Noise, 50, 131, 67, None), + spec(Content::Noise, 90, 130, 66, None), + spec(Content::Textured, 90, 96, 64, Some(3)), + ]); + } + specs +} + +fn queued_requests( + jpegs: &[Arc<[u8]>], + sampling: Sampling, + op: batch::BatchOp, +) -> Vec { + jpegs + .iter() + .map(|jpeg| { + let bytes = jpeg.as_ref(); + let (p444, p422, p420) = match sampling { + Sampling::S444 => ( + Some(Arc::new( + j2k_jpeg::adapter::build_fast444_packet(bytes).expect("fast444 packet"), + )), + None, + None, + ), + Sampling::S422 => ( + None, + Some(Arc::new( + j2k_jpeg::adapter::build_fast422_packet(bytes).expect("fast422 packet"), + )), + None, + ), + Sampling::S420 => ( + None, + None, + Some(Arc::new( + j2k_jpeg::adapter::build_fast420_packet(bytes).expect("fast420 packet"), + )), + ), + }; + batch::QueuedRequest::new( + Arc::clone(jpeg), + PixelFormat::Rgb8, + BackendRequest::Metal, + op, + p444, + p422, + p420, + ) + }) + .collect() +} + +fn jpeg_rect(rect: Rect) -> j2k_jpeg::Rect { + j2k_jpeg::Rect { + x: rect.x, + y: rect.y, + w: rect.w, + h: rect.h, + } +} + +fn cpu_request(op: batch::BatchOp) -> DecodeRequest { + match op { + batch::BatchOp::Full => DecodeRequest::full(PixelFormat::Rgb8), + batch::BatchOp::Region(roi) => DecodeRequest::region(PixelFormat::Rgb8, jpeg_rect(roi)), + batch::BatchOp::Scaled(scale) => DecodeRequest::scaled(PixelFormat::Rgb8, scale), + batch::BatchOp::RegionScaled { roi, scale } => { + DecodeRequest::region_scaled(PixelFormat::Rgb8, jpeg_rect(roi), scale) + } + } +} + +fn cpu_rgb(jpeg: &[u8], op: batch::BatchOp) -> Vec { + CpuDecoder::new(jpeg) + .expect("CPU decoder") + .decode_request(cpu_request(op)) + .expect("CPU oracle decode") + .0 +} + +/// Describes how `actual` differs from `expected` without dumping megabytes: +/// the mismatch count, the rows involved, and the first differing sample. +fn pixel_mismatch( + label: &str, + actual: &[u8], + expected: &[u8], + width: u32, + channels: usize, +) -> Option { + if actual.len() != expected.len() { + return Some(format!( + "{label}: {} output bytes, expected {}", + actual.len(), + expected.len() + )); + } + let first = actual.iter().zip(expected).position(|(a, e)| a != e)?; + let row_len = width as usize * channels; + let rows = actual + .chunks(row_len) + .zip(expected.chunks(row_len)) + .enumerate() + .filter(|(_, (a, e))| a != e) + .map(|(row, _)| row) + .collect::>(); + let mismatches = actual.iter().zip(expected).filter(|(a, e)| a != e).count(); + let pixel = first / channels; + Some(format!( + "{label}: {mismatches} samples differ in rows {rows:?} of {}; first x={} y={} c={} (GPU {} vs CPU {})", + actual.len() / row_len.max(1), + pixel % width as usize, + pixel / width as usize, + first % channels, + actual[first], + expected[first] + )) +} + +fn assert_no_mismatches(failures: &[String]) { + assert!( + failures.is_empty(), + "{} mismatching outputs:\n{}", + failures.len(), + failures.join("\n") + ); +} + +fn output_width(spec: &FixtureSpec, op: batch::BatchOp) -> u32 { + let scaled = |width: u32, scale: Downscale| width.div_ceil(scale.denominator()); + match op { + batch::BatchOp::Full => spec.dimensions().0, + batch::BatchOp::Region(roi) => roi.w, + batch::BatchOp::Scaled(scale) => scaled(spec.dimensions().0, scale), + batch::BatchOp::RegionScaled { roi, scale } => scaled(roi.w, scale), + } +} + +fn batch_ops(spec: &FixtureSpec) -> Vec { + let (width, height) = spec.dimensions(); + // An odd window that straddles MCU boundaries on both axes. + let roi = Rect { + x: 5, + y: 3, + w: width - 5 - 7, + h: height - 3 - 6, + }; + // The same columns down to the bottom edge, where 4:2:0 upsampling must + // replicate the last real chroma row rather than read MCU padding. + let bottom_roi = Rect { + y: height - 11, + h: 11, + ..roi + }; + vec![ + batch::BatchOp::Full, + batch::BatchOp::Scaled(Downscale::Half), + batch::BatchOp::Scaled(Downscale::Quarter), + batch::BatchOp::Scaled(Downscale::Eighth), + batch::BatchOp::Region(roi), + batch::BatchOp::RegionScaled { + roi, + scale: Downscale::Half, + }, + batch::BatchOp::Region(bottom_roi), + batch::BatchOp::RegionScaled { + roi: bottom_roi, + scale: Downscale::Half, + }, + ] +} + +#[test] +fn shader_file_order_matches_production_source() { + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src"); + assert_eq!(read_shader_dir(std::path::Path::new(dir)), SHADER_SOURCE); +} + +#[test] +fn batch_buffer_decodes_match_cpu_on_entropy_stress_fixtures() { + if !should_run_metal_runtime() { + return; + } + let mut failures = Vec::new(); + for spec in correctness_specs() { + let jpegs = encode_batch(&spec, PROFILE_BATCH); + for op in batch_ops(&spec) { + let expected = jpegs + .iter() + .map(|jpeg| cpu_rgb(jpeg, op)) + .collect::>(); + for batch_len in [1, PROFILE_BATCH] { + let requests = queued_requests(&jpegs[..batch_len], spec.sampling, op); + let results = decode_full_batch_to_surfaces(&requests) + .expect("Metal batch decode") + .unwrap_or_else(|| panic!("{}: {op:?} left the Metal path", spec.label())); + assert_eq!(results.len(), batch_len); + for (index, result) in results.into_iter().enumerate() { + let surface = result.expect("tile surface"); + let label = format!("{} {op:?} batch{batch_len} tile{index}", spec.label()); + failures.extend(pixel_mismatch( + &label, + &surface.as_bytes().expect("surface bytes"), + &expected[index], + output_width(&spec, op), + 3, + )); + } + } + } + } + assert_no_mismatches(&failures); +} + +#[test] +fn batch_texture_decodes_match_cpu_on_entropy_stress_fixtures() { + if !should_run_metal_runtime() { + return; + } + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let mut failures = Vec::new(); + for spec in correctness_specs() { + let jpegs = encode_batch(&spec, PROFILE_BATCH); + if spec.restart.is_some() && spec.sampling != Sampling::S420 { + // Documented limitation: resident texture batches reject + // restart-coded full-tile 4:2:2 and 4:4:4 with a typed error. + continue; + } + let decoders = jpegs + .iter() + .map(|jpeg| crate::Decoder::new(jpeg.as_ref()).expect("Metal decoder")) + .collect::>(); + let decoder_refs = decoders.iter().collect::>(); + let ops = [ + (crate::Rgb8MetalBatchOp::Full, batch::BatchOp::Full), + ( + crate::Rgb8MetalBatchOp::Scaled(Downscale::Half), + batch::BatchOp::Scaled(Downscale::Half), + ), + ]; + for (public_op, op) in ops { + let mut output = crate::MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1) + .expect("texture output"); + let tiles = crate::Codec::decode_rgb8_batch_into_textures_with_session( + crate::Rgb8MetalBatchRequest { + source: crate::Rgb8MetalBatchSource::Decoders(&decoder_refs), + op: public_op, + }, + crate::MetalTextureBatchTarget::Resizable(&mut output), + &session, + ) + .unwrap_or_else(|error| panic!("{} {op:?} textures: {error}", spec.label())); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + let actual = crate::tests::download_rgba8_texture( + &session, + tile.texture_trusted(), + tile.dimensions(), + ); + let expected = crate::tests::rgb_to_rgba_opaque(&cpu_rgb(&jpegs[index], op)); + let label = format!("{} {op:?} texture tile{index}", spec.label()); + failures.extend(pixel_mismatch( + &label, + &actual, + &expected, + tile.dimensions().0, + 4, + )); + } + } + } + assert_no_mismatches(&failures); +} + +#[test] +fn single_decodes_match_cpu_on_entropy_stress_fixtures() { + if !should_run_metal_runtime() { + return; + } + let mut failures = Vec::new(); + for spec in correctness_specs() { + let jpeg = encode_jpeg(&spec, 1); + let mut decoder = crate::Decoder::new(&jpeg).expect("Metal decoder"); + let mut session = crate::MetalSession::default(); + let surface = as ImageDecodeSubmit<'_>>::submit_to_device( + &mut decoder, + &mut session, + PixelFormat::Rgb8, + BackendRequest::Metal, + ) + .expect("single submit") + .wait() + .expect("single surface"); + assert!( + matches!(surface.storage, crate::Storage::Metal { .. }), + "{}: single decode left the Metal path", + spec.label() + ); + failures.extend(pixel_mismatch( + &format!("{} single full", spec.label()), + &surface.as_bytes().expect("surface bytes"), + &cpu_rgb(&jpeg, batch::BatchOp::Full), + spec.dimensions().0, + 3, + )); + } + assert_no_mismatches(&failures); +} + +/// Marks a 3-component JPEG as RGB with an Adobe APP14 segment (transform 0), +/// so its components decode without the YCbCr conversion. +fn with_adobe_rgb_transform(jpeg: &[u8]) -> Vec { + const APP14_ADOBE_RGB: [u8; 16] = [ + 0xFF, 0xEE, 0x00, 0x0E, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]; + let mut out = Vec::with_capacity(jpeg.len() + APP14_ADOBE_RGB.len()); + out.extend_from_slice(&jpeg[..2]); + out.extend_from_slice(&APP14_ADOBE_RGB); + out.extend_from_slice(&jpeg[2..]); + out +} + +#[test] +fn rgb_colorspace_444_batches_match_cpu() { + if !should_run_metal_runtime() { + return; + } + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let spec = FixtureSpec { + sampling: Sampling::S444, + content: Content::Textured, + quality: 90, + width: 133, + height: 77, + restart: None, + }; + let jpegs = encode_batch(&spec, PROFILE_BATCH) + .iter() + .map(|jpeg| Arc::<[u8]>::from(with_adobe_rgb_transform(jpeg))) + .collect::>(); + assert_eq!( + CpuDecoder::new(&jpegs[0]) + .expect("CPU decoder") + .info() + .color_space, + j2k_jpeg::ColorSpace::Rgb + ); + let op = batch::BatchOp::Full; + let expected = jpegs + .iter() + .map(|jpeg| cpu_rgb(jpeg, op)) + .collect::>(); + let mut failures = Vec::new(); + + let requests = queued_requests(&jpegs, spec.sampling, op); + let surfaces = decode_full_batch_to_surfaces(&requests) + .expect("Metal batch decode") + .expect("RGB 4:4:4 buffer batch stays on Metal"); + for (index, surface) in surfaces.into_iter().enumerate() { + failures.extend(pixel_mismatch( + &format!("rgb444 buffer tile{index}"), + &surface.expect("surface").as_bytes().expect("surface bytes"), + &expected[index], + spec.dimensions().0, + 3, + )); + } + + let inputs = jpegs.iter().map(AsRef::as_ref).collect::>(); + let mut output = crate::MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1) + .expect("texture output"); + let tiles = crate::Codec::decode_rgb8_batch_into_textures_with_session( + crate::Rgb8MetalBatchRequest { + source: crate::Rgb8MetalBatchSource::Bytes(&inputs), + op: crate::Rgb8MetalBatchOp::Full, + }, + crate::MetalTextureBatchTarget::Resizable(&mut output), + &session, + ) + .expect("RGB 4:4:4 texture batch"); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + let actual = crate::tests::download_rgba8_texture( + &session, + tile.texture_trusted(), + tile.dimensions(), + ); + failures.extend(pixel_mismatch( + &format!("rgb444 texture tile{index}"), + &actual, + &crate::tests::rgb_to_rgba_opaque(&expected[index]), + tile.dimensions().0, + 4, + )); + } + assert_no_mismatches(&failures); +} + +fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + format!("{:x}", Sha256::digest(bytes)) +} + +/// Evidence for the P33 record: input and output SHA-256 of the +/// `wsi_tile_batch_rgba_textures` 4:4:4 benchmark workloads through the +/// component-plane route and the direct texture kernels, plus the CPU oracle. +#[test] +#[ignore = "P33 evidence; run explicitly with --include-ignored --nocapture"] +fn p33_fast444_texture_route_output_hashes() { + if !should_run_metal_runtime() { + return; + } + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let generated = generated_bench_jpeg(jpeg_encoder::SamplingFactor::F_1_1, None); + let fixture = include_bytes!("../../../fixtures/jpeg/baseline_444_8x8.jpg").to_vec(); + let mut corpus = generated.clone(); + corpus.extend_from_slice(&fixture); + println!("input_corpus_sha256 {}", sha256_hex(&corpus)); + for (name, bytes) in [ + ("generated/fast444_256x256", &generated), + ("repo/baseline_444_8x8", &fixture), + ] { + let (rgb, _) = CpuDecoder::new(bytes) + .expect("CPU decoder") + .decode_request(DecodeRequest::full(PixelFormat::Rgb8)) + .expect("CPU decode"); + let cpu_tile = crate::tests::rgb_to_rgba_opaque(&rgb); + for batch_size in [16, 64] { + let decoders = (0..batch_size) + .map(|_| crate::Decoder::new(bytes).expect("Metal decoder")) + .collect::>(); + let decoder_refs = decoders.iter().collect::>(); + let mut output = crate::MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1) + .expect("texture output"); + let expected = cpu_tile.repeat(batch_size); + let mut hashes = Vec::new(); + for planes in [false, true] { + let tiles = crate::compute::texture_tuning::with_component_planes(planes, || { + crate::Codec::decode_rgb8_decoder_batch_into_resizable_metal_textures_with_session( + &decoder_refs, + &mut output, + &session, + ) + }) + .expect("texture batch"); + let mut actual = Vec::with_capacity(expected.len()); + for tile in tiles { + let tile = tile.expect("texture tile"); + actual.extend(crate::tests::download_rgba8_texture( + &session, + tile.texture_trusted(), + tile.dimensions(), + )); + } + assert!( + actual == expected, + "{name} batch{batch_size} planes={planes} differs from CPU" + ); + hashes.push(sha256_hex(&actual)); + } + println!( + "{name} batch{batch_size} input_sha256 {} direct {} planes {} cpu {}", + sha256_hex(bytes), + hashes[0], + hashes[1], + sha256_hex(&expected) + ); + } + } +} + +/// A generated 256x256 benchmark JPEG, as `benches/compare.rs` encodes it. +fn generated_bench_jpeg( + sampling: jpeg_encoder::SamplingFactor, + restart_interval: Option, +) -> Vec { + let rgb = j2k_test_support::gpu_bench_rgb8(256, 256); + let mut jpeg = Vec::new(); + let mut encoder = jpeg_encoder::Encoder::new(&mut jpeg, 90); + encoder.set_sampling_factor(sampling); + if let Some(interval) = restart_interval { + encoder.set_restart_interval(interval); + } + encoder + .encode(&rgb, 256, 256, jpeg_encoder::ColorType::Rgb) + .expect("encode generated benchmark JPEG"); + jpeg +} + +/// Decodes 64 tiles of `bytes` three times on one session, as the +/// `wsi_tile_batch_rgb` one-shot and retained-session rows do, and returns the +/// concatenated RGB output of each batch. +fn retained_tile_batches(bytes: &[u8]) -> Vec> { + use j2k_core::TileBatchDecodeSubmit as _; + + let mut ctx = j2k_jpeg::DecoderContext::default(); + let mut pool = j2k_jpeg::ScratchPool::new(); + let mut session = crate::MetalSession::default(); + (0..3) + .map(|_| { + let submissions = (0..64) + .map(|_| { + crate::Codec::submit_tile_to_device( + &mut ctx, + &mut session, + &mut pool, + bytes, + PixelFormat::Rgb8, + BackendRequest::Metal, + ) + .expect("submit tile") + }) + .collect::>(); + let mut output = Vec::new(); + for submission in submissions { + let surface = submission.wait().expect("tile surface"); + output.extend_from_slice(surface.as_bytes().expect("surface bytes").as_ref()); + } + output + }) + .collect() +} + +/// Decodes 16 resident RGBA textures of `bytes`, as the batch-16 +/// `wsi_tile_batch_rgba_textures` rows do, and reads them back. +fn texture_batch16(session: &crate::MetalBackendSession, bytes: &[u8]) -> Vec { + let decoders = (0..16) + .map(|_| crate::Decoder::new(bytes).expect("Metal decoder")) + .collect::>(); + let decoder_refs = decoders.iter().collect::>(); + let mut output = crate::MetalBatchTextureOutput::new_rgba8_tiles(session, (256, 256), 16) + .expect("texture output"); + let tiles = crate::Codec::decode_rgb8_decoder_batch_into_resizable_metal_textures_with_session( + &decoder_refs, + &mut output, + session, + ) + .expect("texture batch"); + let mut textures = Vec::new(); + for tile in tiles { + let tile = tile.expect("texture tile"); + textures.extend(crate::tests::download_rgba8_texture( + session, + tile.texture_trusted(), + tile.dimensions(), + )); + } + textures +} + +/// Evidence for the P32 record: input and output SHA-256 of the generated +/// 256x256 `wsi_tile_batch_rgb` workloads (64 tiles; three batches on one +/// session, as in the one-shot and retained-session rows) and the batch-16 +/// `wsi_tile_batch_rgba_textures` workloads, each checked against the CPU +/// decoder. The P32 A/B runs it once per arm. +#[test] +#[ignore = "P32 evidence; run explicitly with --include-ignored --nocapture"] +fn p32_tile_and_texture_batch_output_hashes() { + use jpeg_encoder::SamplingFactor; + + if !should_run_metal_runtime() { + return; + } + let inputs = [ + ("generated/fast420_256x256", SamplingFactor::F_2_2, None), + ( + "generated/fast420_restart2_256x256", + SamplingFactor::F_2_2, + Some(2), + ), + ("generated/fast422_256x256", SamplingFactor::F_2_1, None), + ("generated/fast444_256x256", SamplingFactor::F_1_1, None), + ]; + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let mut corpus = Vec::new(); + for (name, sampling, restart_interval) in inputs { + let bytes = generated_bench_jpeg(sampling, restart_interval); + corpus.extend_from_slice(&bytes); + let (rgb, _) = CpuDecoder::new(&bytes) + .expect("CPU decoder") + .decode_request(DecodeRequest::full(PixelFormat::Rgb8)) + .expect("CPU decode"); + + let expected_rgb = rgb.repeat(64); + for (iteration, output) in retained_tile_batches(&bytes).iter().enumerate() { + assert!( + *output == expected_rgb, + "{name} tile batch iteration {iteration} differs from CPU" + ); + } + let textures = texture_batch16(&session, &bytes); + assert!( + textures == crate::tests::rgb_to_rgba_opaque(&rgb).repeat(16), + "{name} texture batch differs from CPU" + ); + println!( + "{name} input_sha256 {} rgb_batch64_sha256 {} rgba_textures_batch16_sha256 {}", + sha256_hex(&bytes), + sha256_hex(&expected_rgb), + sha256_hex(&textures) + ); + } + println!("input_corpus_sha256 {}", sha256_hex(&corpus)); +} + +// --------------------------------------------------------------------------- +// Stage profile +// --------------------------------------------------------------------------- + +fn read_shader_dir(dir: &std::path::Path) -> String { + SHADER_FILES + .iter() + .map(|name| { + std::fs::read_to_string(dir.join(name)) + .unwrap_or_else(|error| panic!("read {}: {error}", dir.join(name).display())) + }) + .collect() +} + +const PRODUCTION_HELPER: &str = "inline bool decode_idct_deposit_block("; + +/// Replaces `decode_idct_deposit_block` for the buffer-output kernels. Every +/// stage consumes the same bits as production, except stage 0, which decodes +/// nothing and measures launch, checkpoint setup and the pack pass. Stages 2 +/// and 3 fold their whole result into one stored byte so it cannot be elided. +const PROBE_HELPER: &str = r" +inline bool decode_idct_deposit_block( + thread BitReader &br, + device const uchar *bytes, + uint len, + TABLE_SPACE PreparedHuffman &dc_table, + TABLE_SPACE PreparedHuffman &ac_table, + constant ushort *quant, + thread int &prev_dc, + device JpegDecodeStatus *status, + device uchar *plane, + uint stride, + uint width, + uint height, + uint x, + uint y, + thread short coeffs[64] +) { +#if JPEG_HARNESS_PROBE_STAGE == 0 + return true; +#elif JPEG_HARNESS_PROBE_STAGE == 1 + return decode_block_skip(br, bytes, len, dc_table, ac_table, prev_dc, status); +#else + bool dc_only = false; + if (!decode_block(br, bytes, len, dc_table, ac_table, quant, prev_dc, status, coeffs, dc_only)) { + return false; + } + uint sink = dc_only ? 1u : 0u; +#if JPEG_HARNESS_PROBE_STAGE == 2 + for (uint i = 0; i < 64; ++i) { + sink ^= uint(ushort(coeffs[i])) << (i & 15u); + } +#else + if (!dc_only) { + thread uchar pixels[64]; + idct_islow(coeffs, pixels); + for (uint i = 0; i < 64; ++i) { + sink ^= uint(pixels[i]) << (i & 23u); + } + } +#endif + if (x < width && y < height) { + plane[y * stride + x] = uchar(sink ^ (sink >> 8) ^ (sink >> 16) ^ (sink >> 24)); + } + return true; +#endif +} +"; + +/// Pack kernels of the profiled buffer paths. Probes stub them so every probe +/// column is decode-kernel time alone; `full - decode` is the pack pass. +const PROFILED_PACK_KERNELS: [&str; 2] = ["jpeg_pack_420_rgb_batch", "jpeg_pack_422_rgb_batch"]; + +/// Stage of a probe build. `Decode` keeps the production decode kernel. +#[derive(Clone, Copy)] +enum ProbeStage { + Launch, + Huffman, + Coefficients, + Idct, + Decode, +} + +impl ProbeStage { + const ALL: [Self; 5] = [ + Self::Launch, + Self::Huffman, + Self::Coefficients, + Self::Idct, + Self::Decode, + ]; + + fn label(self) -> &'static str { + match self { + Self::Launch => "launch", + Self::Huffman => "+huffman", + Self::Coefficients => "+coeffs", + Self::Idct => "+idct", + Self::Decode => "+store", + } + } +} + +fn stub_kernel(source: &str, kernel: &str) -> String { + let at = source + .find(&format!("kernel void {kernel}(")) + .unwrap_or_else(|| panic!("shader source defines {kernel}")); + let body = at + source[at..].find(") {").expect("kernel body") + 3; + format!("{}\n return;{}", &source[..body], &source[body..]) +} + +fn probe_source(source: &str, stage: ProbeStage) -> String { + let mut source = PROFILED_PACK_KERNELS + .iter() + .fold(source.to_string(), |source, kernel| { + stub_kernel(&source, kernel) + }); + let stage = match stage { + ProbeStage::Decode => return source, + ProbeStage::Launch => 0, + ProbeStage::Huffman => 1, + ProbeStage::Coefficients => 2, + ProbeStage::Idct => 3, + }; + let at = source + .find(PRODUCTION_HELPER) + .expect("shader source defines decode_idct_deposit_block"); + // Match the production helper's Huffman table address space. + let table_param = &source[at..][..source[at..] + .find("PreparedHuffman &dc_table") + .expect("helper takes a DC table")]; + let table_space = table_param[table_param.rfind('\n').expect("parameter line") + 1..].trim(); + let helper = PROBE_HELPER.replace("TABLE_SPACE", table_space); + let tail = source.split_off(at).replacen( + PRODUCTION_HELPER, + "inline bool decode_idct_deposit_block_production(", + 1, + ); + format!("{source}#define JPEG_HARNESS_PROBE_STAGE {stage}\n{helper}\n{tail}") +} + +fn run_full_rgb_batch( + runtime: &MetalRuntime, + sampling: Sampling, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], +) -> Vec> { + let results = match sampling { + Sampling::S420 => try_decode_fast_subsampled_full_rgb_batch_to_surfaces::< + JpegFast420PacketV1, + >(runtime, requests, packets), + Sampling::S422 => try_decode_fast_subsampled_full_rgb_batch_to_surfaces::< + JpegFast422PacketV1, + >(runtime, requests, packets), + Sampling::S444 => unreachable!("4:4:4 full batches use the region-scaled kernels"), + }; + results + .expect("profiled batch decode") + .expect("profiled batch stays on the fused Metal path") +} + +#[derive(Clone, Copy)] +struct StageTime { + gpu_ms: f64, + wall_ms: f64, +} + +fn median(mut samples: Vec) -> f64 { + samples.sort_by(f64::total_cmp); + samples[samples.len() / 2] +} + +/// Times `run` on every runtime. Samples are taken round-robin so GPU clock +/// ramping affects all variants alike; the first `PROFILE_WARMUP` rounds are +/// discarded. +fn profile_runtimes( + runtimes: &[&MetalRuntime], + mut run: impl FnMut(&MetalRuntime), +) -> Vec { + profile_variants(runtimes.len(), |index| run(runtimes[index])) +} + +/// Samples `variants` round-robin so clock ramping affects each alike. +fn profile_variants(variants: usize, mut run: impl FnMut(usize)) -> Vec { + let mut gpu = vec![Vec::with_capacity(PROFILE_SAMPLES); variants]; + let mut wall = vec![Vec::with_capacity(PROFILE_SAMPLES); variants]; + for round in 0..PROFILE_WARMUP + PROFILE_SAMPLES { + for index in 0..variants { + let _ = gpu_time::take_seconds(); + let start = Instant::now(); + run(index); + let elapsed = start.elapsed().as_secs_f64() * 1e3; + let gpu_ms = gpu_time::take_seconds() * 1e3; + if round >= PROFILE_WARMUP { + gpu[index].push(gpu_ms); + wall[index].push(elapsed); + } + } + } + gpu.into_iter() + .zip(wall) + .map(|(gpu, wall)| StageTime { + gpu_ms: median(gpu), + wall_ms: median(wall), + }) + .collect() +} + +fn assert_buffer_batch_matches( + runtime: &MetalRuntime, + sampling: Sampling, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + expected: &[Vec], +) { + let results = run_full_rgb_batch(runtime, sampling, requests, packets); + for (index, result) in results.into_iter().enumerate() { + let surface = result.expect("profiled tile surface"); + let bytes = surface.as_bytes().expect("surface bytes"); + assert!( + bytes.as_ref() == expected[index].as_slice(), + "profiled output differs from CPU at tile {index}" + ); + } +} + +fn run_texture_batch( + runtime: &MetalRuntime, + sampling: Sampling, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchTextureOutput, +) -> Vec> { + let _access = output + .lock_for_safe_access() + .expect("texture output access"); + let results = match sampling { + Sampling::S420 => { + try_decode_fast_subsampled_full_rgba_batch_to_textures::( + runtime, + requests, + packets, + output, + FastBatchDecodeMode::Fused, + ) + } + Sampling::S422 => { + try_decode_fast_subsampled_full_rgba_batch_to_textures::( + runtime, + requests, + packets, + output, + FastBatchDecodeMode::Fused, + ) + } + Sampling::S444 => { + try_decode_fast444_full_rgba_batch_to_textures(runtime, requests, packets, output) + } + }; + results + .expect("profiled texture batch") + .expect("profiled texture batch stays on Metal") +} + +fn print_full_and_baseline(times: &[StageTime], jpegs: &[Arc<[u8]>]) { + let full = times[0]; + let base = times.get(1); + let jpeg_kib = jpegs.iter().map(|jpeg| jpeg.len()).sum::() / 1024; + println!( + " {:>9.3} {:>9} {:>9.3} {:>9} ({jpeg_kib} KiB JPEG)", + full.gpu_ms, + base.map_or("-".to_string(), |time| format!("{:.3}", time.gpu_ms)), + full.wall_ms, + base.map_or("-".to_string(), |time| format!("{:.3}", time.wall_ms)), + ); +} + +/// Resident texture batches for every sampling. +fn texture_profile_specs() -> Vec { + let spec = |sampling, side| FixtureSpec { + sampling, + content: Content::Textured, + quality: 90, + width: side, + height: side, + restart: None, + }; + vec![ + spec(Sampling::S420, 256), + spec(Sampling::S420, 512), + spec(Sampling::S422, 512), + spec(Sampling::S444, 256), + spec(Sampling::S444, 512), + ] +} + +/// Keeps the GPU busy long enough to leave its idle clock state, so the first +/// profiled workload is not measured while clocks ramp. +fn warm_up_gpu(runtime: &MetalRuntime) { + let spec = profile_specs()[0]; + let jpegs = encode_batch(&spec, PROFILE_BATCH); + let requests = queued_requests(&jpegs, spec.sampling, batch::BatchOp::Full); + let packets = batched_fast_packets(&requests) + .expect("packet lookup") + .expect("fast packets"); + let start = Instant::now(); + while start.elapsed().as_millis() < 500 { + drop(run_full_rgb_batch( + runtime, + spec.sampling, + &requests, + &packets, + )); + } +} + +/// Fused 4:2:0 and 4:2:2 buffer batches: the kernels the probes instrument. +fn profile_specs() -> Vec { + let spec = |sampling, content, quality, side| FixtureSpec { + sampling, + content, + quality, + width: side, + height: side, + restart: None, + }; + vec![ + spec(Sampling::S420, Content::Bench, 90, 512), + spec(Sampling::S420, Content::Textured, 90, 512), + spec(Sampling::S420, Content::Textured, 75, 512), + spec(Sampling::S420, Content::Textured, 90, 256), + spec(Sampling::S420, Content::Textured, 90, 1024), + spec(Sampling::S422, Content::Textured, 90, 512), + spec(Sampling::S420, Content::Noise, 95, 512), + ] +} + +/// Runs one buffer batch in a loop for a CPU sampling profiler, for example +/// `xcrun xctrace record --template 'Time Profiler' --launch -- +/// host_overhead_loop --include-ignored --exact`. +#[test] +#[ignore = "host-overhead profiling loop; run under a sampling profiler"] +fn host_overhead_loop() { + if !should_run_metal_runtime() { + return; + } + let spec = profile_specs()[1]; + let jpegs = encode_batch(&spec, PROFILE_BATCH); + let requests = queued_requests(&jpegs, spec.sampling, batch::BatchOp::Full); + let packets = batched_fast_packets(&requests) + .expect("packet lookup") + .expect("fast packets"); + let runtime = MetalRuntime::new_with_shader_source(SHADER_SOURCE).expect("runtime"); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let output = + crate::MetalBatchOutputBuffer::new_rgb8_tiles(&session, spec.dimensions(), PROFILE_BATCH) + .expect("reusable output buffer"); + for reuse_output in [false, true] { + let _ = (gpu_time::take_seconds(), gpu_time::take_submit_seconds()); + let start = Instant::now(); + let mut iterations = 0_u32; + while start.elapsed().as_secs() < 4 { + let results = if reuse_output { + let _access = output.lock_for_safe_access().expect("output access"); + try_decode_fast_subsampled_full_rgb_batch_to_surfaces_into_output::< + JpegFast420PacketV1, + >(&runtime, &requests, &packets, &output) + .expect("batch decode") + .expect("stays on Metal") + } else { + run_full_rgb_batch(&runtime, spec.sampling, &requests, &packets) + }; + for result in results { + result.expect("batch tile"); + } + iterations += 1; + } + let per_iteration = |seconds: f64| seconds * 1e3 / f64::from(iterations); + let (scheduling, queued) = gpu_time::take_submit_seconds(); + println!( + "{} reuse_output={reuse_output}: {iterations} iterations, {:.3} ms wall each: \ + scheduling {:.3}, queued {:.3}, GPU {:.3}", + spec.label(), + per_iteration(start.elapsed().as_secs_f64()), + per_iteration(scheduling), + per_iteration(queued), + per_iteration(gpu_time::take_seconds()), + ); + } +} + +#[test] +#[ignore = "GPU stage profile; run explicitly with --include-ignored --nocapture"] +fn decode_kernel_stage_profile() { + if !should_run_metal_runtime() { + return; + } + let probes = ProbeStage::ALL + .iter() + .map(|&stage| { + MetalRuntime::new_with_shader_source(&probe_source(SHADER_SOURCE, stage)) + .expect("probe runtime") + }) + .collect::>(); + let current = MetalRuntime::new_with_shader_source(SHADER_SOURCE).expect("current runtime"); + let baseline = std::env::var_os(BASELINE_SHADERS_ENV).map(|dir| { + let source = read_shader_dir(std::path::Path::new(&dir)); + MetalRuntime::new_with_shader_source(&source).expect("baseline runtime") + }); + + warm_up_gpu(¤t); + profile_buffer_batches(&probes, ¤t, baseline.as_ref()); + profile_texture_batches(¤t, baseline.as_ref()); +} + +fn profile_buffer_batches( + probes: &[MetalRuntime], + current: &MetalRuntime, + baseline: Option<&MetalRuntime>, +) { + println!( + "GPU ms, median of {PROFILE_SAMPLES}, batch {PROFILE_BATCH}. Probe columns time the decode \ + kernel alone (pack stubbed) and are cumulative; `full` adds the pack pass." + ); + print!("{:<28}", "workload"); + for stage in ProbeStage::ALL { + print!(" {:>9}", stage.label()); + } + println!( + " {:>9} {:>9} {:>9} {:>9}", + "full", "baseline", "wall", "base wall" + ); + for spec in profile_specs() { + let jpegs = encode_batch(&spec, PROFILE_BATCH); + let requests = queued_requests(&jpegs, spec.sampling, batch::BatchOp::Full); + let packets = batched_fast_packets(&requests) + .expect("packet lookup") + .expect("fast packets"); + let expected = jpegs + .iter() + .map(|jpeg| cpu_rgb(jpeg, batch::BatchOp::Full)) + .collect::>(); + let mut runtimes = probes.iter().collect::>(); + runtimes.push(current); + runtimes.extend(baseline); + for runtime in &runtimes[probes.len()..] { + assert_buffer_batch_matches(runtime, spec.sampling, &requests, &packets, &expected); + } + let times = profile_runtimes(&runtimes, |runtime| { + for result in run_full_rgb_batch(runtime, spec.sampling, &requests, &packets) { + result.expect("probe status must stay OK"); + } + }); + print!("{:<28}", spec.label()); + for time in ×[..probes.len()] { + print!(" {:>9.3}", time.gpu_ms); + } + print_full_and_baseline(×[probes.len()..], &jpegs); + } +} + +fn profile_texture_batches(current: &MetalRuntime, baseline: Option<&MetalRuntime>) { + println!( + "\nResident RGBA texture batches, batch {PROFILE_BATCH}: whole command buffer. \ + `direct` forces the direct texture kernels instead of component planes." + ); + println!( + "{:<28} {:>9} {:>9} {:>9} {:>9} {:>9} {:>9}", + "workload", "full", "direct", "baseline", "wall", "dir wall", "base wall" + ); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + for spec in texture_profile_specs() { + let jpegs = encode_batch(&spec, PROFILE_BATCH); + let requests = queued_requests(&jpegs, spec.sampling, batch::BatchOp::Full); + let packets = batched_fast_packets(&requests) + .expect("packet lookup") + .expect("fast packets"); + let output = crate::MetalBatchTextureOutput::new_rgba8_tiles( + &session, + spec.dimensions(), + PROFILE_BATCH, + ) + .expect("texture output"); + // Variants: component planes, direct kernels, then the baseline shaders. + let mut variants = vec![(current, true), (current, false)]; + variants.extend(baseline.map(|runtime| (runtime, true))); + let run_variant = |(runtime, planes): (&MetalRuntime, bool)| { + crate::compute::texture_tuning::with_component_planes(planes, || { + run_texture_batch(runtime, spec.sampling, &requests, &packets, &output) + }) + }; + for &variant in &variants { + for (index, tile) in run_variant(variant).into_iter().enumerate() { + let tile = tile.expect("texture tile"); + let actual = crate::tests::download_rgba8_texture( + &session, + tile.texture_trusted(), + tile.dimensions(), + ); + let expected = + crate::tests::rgb_to_rgba_opaque(&cpu_rgb(&jpegs[index], batch::BatchOp::Full)); + assert!( + actual == expected, + "{} texture tile {index} differs from CPU", + spec.label() + ); + } + } + let times = profile_variants(variants.len(), |index| { + for tile in run_variant(variants[index]) { + tile.expect("texture tile"); + } + }); + let cell = |time: Option<&StageTime>, wall: bool| { + time.map_or("-".to_string(), |time| { + format!("{:.3}", if wall { time.wall_ms } else { time.gpu_ms }) + }) + }; + println!( + "{:<28} {:>9} {:>9} {:>9} {:>9} {:>9} {:>9} ({} KiB JPEG)", + spec.label(), + cell(times.first(), false), + cell(times.get(1), false), + cell(times.get(2), false), + cell(times.first(), true), + cell(times.get(1), true), + cell(times.get(2), true), + jpegs.iter().map(|jpeg| jpeg.len()).sum::() / 1024, + ); + } +} diff --git a/crates/j2k-jpeg-metal/src/compute/tests/surface_output_reuse.rs b/crates/j2k-jpeg-metal/src/compute/tests/surface_output_reuse.rs new file mode 100644 index 000000000..da1df7581 --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute/tests/surface_output_reuse.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Batch surface outputs reuse a pooled buffer only after every surface that +//! referenced it has been dropped. + +use super::*; + +/// Four 512x256 tiles: 1.5 MiB of RGB output, above the pooling threshold. +fn distinct_420_jpegs(seed: u8) -> Vec> { + distinct_420_jpegs_sized(seed, (512, 256)) +} + +fn distinct_420_jpegs_sized(seed: u8, (width, height): (u16, u16)) -> Vec> { + (0..4_u8) + .map(|tile| { + let mut rgb = j2k_test_support::patterned_rgb8(u32::from(width), u32::from(height)); + for (index, sample) in rgb.iter_mut().enumerate() { + let offset = u8::try_from(index % 251).expect("offset fits u8"); + *sample = sample.wrapping_add(offset.wrapping_mul(seed.wrapping_add(tile))); + } + let mut jpeg = Vec::new(); + let mut encoder = jpeg_encoder::Encoder::new(&mut jpeg, 85); + encoder.set_sampling_factor(jpeg_encoder::SamplingFactor::F_2_2); + encoder + .encode(&rgb, width, height, jpeg_encoder::ColorType::Rgb) + .expect("encode tile"); + Arc::<[u8]>::from(jpeg) + }) + .collect() +} + +fn decode_batch(runtime: &MetalRuntime, jpegs: &[Arc<[u8]>]) -> Vec { + let requests = jpegs + .iter() + .map(|jpeg| { + batch::QueuedRequest::new( + Arc::clone(jpeg), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + None, + None, + Some(Arc::new( + j2k_jpeg::adapter::build_fast420_packet(jpeg).expect("fast420 packet"), + )), + ) + }) + .collect::>(); + let packets = batched_fast_packets(&requests) + .expect("packet lookup") + .expect("fast packets"); + try_decode_fast_subsampled_full_rgb_batch_to_surfaces::( + runtime, &requests, &packets, + ) + .expect("batch decode") + .expect("batch stays on Metal") + .into_iter() + .map(|surface| surface.expect("tile surface")) + .collect() +} + +fn buffer_address(surface: &Surface) -> usize { + let (buffer, _) = surface + .metal_buffer_trusted() + .expect("batch surface is Metal resident"); + std::ptr::from_ref(buffer).addr() +} + +fn assert_matches_cpu(surfaces: &[Surface], jpegs: &[Arc<[u8]>], what: &str) { + for (index, (surface, jpeg)) in surfaces.iter().zip(jpegs).enumerate() { + let (expected, _) = CpuDecoder::new(jpeg) + .expect("CPU decoder") + .decode_request(DecodeRequest::full(PixelFormat::Rgb8)) + .expect("CPU decode"); + assert!( + surface.as_bytes().expect("surface bytes").as_ref() == expected.as_slice(), + "{what} tile {index} differs from the CPU decoder" + ); + } +} + +#[test] +fn live_surfaces_keep_their_output_while_later_batches_decode() { + if !should_run_metal_runtime() { + return; + } + let runtime = MetalRuntime::new().expect("runtime"); + let (a, b, c) = ( + distinct_420_jpegs(1), + distinct_420_jpegs(2), + distinct_420_jpegs(3), + ); + + let first = decode_batch(&runtime, &a); + let second = decode_batch(&runtime, &b); + let third = decode_batch(&runtime, &c); + assert_ne!(buffer_address(&first[0]), buffer_address(&second[0])); + assert_ne!(buffer_address(&second[0]), buffer_address(&third[0])); + assert_ne!(buffer_address(&first[0]), buffer_address(&third[0])); + assert_eq!(runtime.pooled_surface_outputs_for_test(), 2); + assert_matches_cpu(&first, &a, "first batch"); + assert_matches_cpu(&second, &b, "second batch"); + assert_matches_cpu(&third, &c, "third batch"); +} + +#[test] +fn dropped_surfaces_release_their_output_for_the_next_batch() { + if !should_run_metal_runtime() { + return; + } + let runtime = MetalRuntime::new().expect("runtime"); + let (a, b) = (distinct_420_jpegs(4), distinct_420_jpegs(5)); + + let first = decode_batch(&runtime, &a); + let first_buffer = buffer_address(&first[0]); + assert_matches_cpu(&first, &a, "first batch"); + drop(first); + + let second = decode_batch(&runtime, &b); + assert_eq!(buffer_address(&second[0]), first_buffer); + assert_eq!(runtime.pooled_surface_outputs_for_test(), 1); + assert_matches_cpu(&second, &b, "second batch"); + + // One surface of a batch keeps the whole output buffer in use. + let kept = second.into_iter().next().expect("one surface"); + let third = decode_batch(&runtime, &a); + assert_ne!(buffer_address(&third[0]), first_buffer); + assert_matches_cpu(std::slice::from_ref(&kept), &b[..1], "kept surface"); + assert_matches_cpu(&third, &a, "third batch"); +} + +#[test] +fn small_outputs_are_allocated_per_call() { + if !should_run_metal_runtime() { + return; + } + let runtime = MetalRuntime::new().expect("runtime"); + let small = distinct_420_jpegs_sized(6, (64, 48)); + let first = decode_batch(&runtime, &small); + drop(first); + let second = decode_batch(&runtime, &small); + assert_eq!(runtime.pooled_surface_outputs_for_test(), 0); + assert_matches_cpu(&second, &small, "small batch"); +} + +/// Evidence for the P34 record: input and output SHA-256 of the retained-session +/// `wsi_tile_batch_rgb` workloads over three consecutive 64-tile batches, so +/// later batches write a reused pooled buffer, against the CPU decoder. +#[test] +#[ignore = "P34 evidence; run explicitly with --include-ignored --nocapture"] +fn p34_retained_session_output_hashes() { + use j2k_core::{DeviceSubmission as _, TileBatchDecodeSubmit as _}; + use sha2::{Digest, Sha256}; + + if !should_run_metal_runtime() { + return; + } + let hex = |bytes: &[u8]| -> String { format!("{:x}", Sha256::digest(bytes)) }; + let generated = |sampling| { + let rgb = j2k_test_support::gpu_bench_rgb8(256, 256); + let mut jpeg = Vec::new(); + let mut encoder = jpeg_encoder::Encoder::new(&mut jpeg, 90); + encoder.set_sampling_factor(sampling); + encoder + .encode(&rgb, 256, 256, jpeg_encoder::ColorType::Rgb) + .expect("encode generated benchmark JPEG"); + jpeg + }; + let inputs: [(&str, Vec); 6] = [ + ( + "generated/fast420_256x256", + generated(jpeg_encoder::SamplingFactor::F_2_2), + ), + ( + "generated/fast422_256x256", + generated(jpeg_encoder::SamplingFactor::F_2_1), + ), + ( + "generated/fast444_256x256", + generated(jpeg_encoder::SamplingFactor::F_1_1), + ), + ("repo/baseline_420_16x16", BASELINE_420.to_vec()), + ("repo/baseline_422_16x8", BASELINE_422.to_vec()), + ("repo/baseline_444_8x8", BASELINE_444.to_vec()), + ]; + let mut corpus = Vec::new(); + for (name, bytes) in &inputs { + corpus.extend_from_slice(bytes); + let (rgb, _) = CpuDecoder::new(bytes) + .expect("CPU decoder") + .decode_request(DecodeRequest::full(PixelFormat::Rgb8)) + .expect("CPU decode"); + let expected = hex(&rgb.repeat(64)); + let mut ctx = j2k_jpeg::DecoderContext::default(); + let mut pool = j2k_jpeg::ScratchPool::new(); + let mut session = crate::MetalSession::default(); + for iteration in 0..3 { + let submissions = (0..64) + .map(|_| { + crate::Codec::submit_tile_to_device( + &mut ctx, + &mut session, + &mut pool, + bytes, + PixelFormat::Rgb8, + BackendRequest::Metal, + ) + .expect("submit") + }) + .collect::>(); + let mut output = Vec::new(); + for submission in submissions { + let surface = submission.wait().expect("surface"); + output.extend_from_slice(surface.as_bytes().expect("surface bytes").as_ref()); + } + assert_eq!(hex(&output), expected, "{name} iteration {iteration}"); + } + println!( + "{name} input_sha256 {} output_sha256 {expected}", + hex(bytes) + ); + } + println!("input_corpus_sha256 {}", hex(&corpus)); +} diff --git a/crates/j2k-jpeg-metal/src/compute/texture_tuning.rs b/crates/j2k-jpeg-metal/src/compute/texture_tuning.rs index e40a1fb96..3b891f276 100644 --- a/crates/j2k-jpeg-metal/src/compute/texture_tuning.rs +++ b/crates/j2k-jpeg-metal/src/compute/texture_tuning.rs @@ -19,6 +19,20 @@ pub(super) fn component_planes() -> Option { SETTINGS.get().1 } +/// Runs `run` with full-frame texture batches forced onto the component-plane +/// route (`true`) or the direct texture kernels (`false`). +pub(super) fn with_component_planes(planes: bool, run: impl FnOnce() -> T) -> T { + struct Restore((Option, Option)); + impl Drop for Restore { + fn drop(&mut self) { + SETTINGS.set(self.0); + } + } + let previous = SETTINGS.get(); + let _restore = Restore(SETTINGS.replace((previous.0, Some(planes)))); + run() +} + fn with_settings(width: u64, planes: bool, run: impl FnOnce() -> T) -> T { struct Restore((Option, Option)); impl Drop for Restore { diff --git a/crates/j2k-jpeg-metal/src/decode_surface.rs b/crates/j2k-jpeg-metal/src/decode_surface.rs index eefe9879b..a1accd92b 100644 --- a/crates/j2k-jpeg-metal/src/decode_surface.rs +++ b/crates/j2k-jpeg-metal/src/decode_surface.rs @@ -73,7 +73,16 @@ pub(crate) fn decode_compatible_batch_with_session( ) -> Result>>, Error> { #[cfg(target_os = "macos")] { - batch_entry::decode_full_batch_to_surfaces_with_session_state(requests, session) + match batch_entry::decode_full_batch_to_surfaces_with_session_state(requests, session) { + Err(Error::MetalUnavailable) + if requests + .iter() + .all(|request| request.backend == BackendRequest::Auto) => + { + Ok(None) + } + result => result, + } } #[cfg(not(target_os = "macos"))] { diff --git a/crates/j2k-jpeg-metal/src/lib.rs b/crates/j2k-jpeg-metal/src/lib.rs index 7b70abec1..ac5d3cf5f 100644 --- a/crates/j2k-jpeg-metal/src/lib.rs +++ b/crates/j2k-jpeg-metal/src/lib.rs @@ -45,6 +45,8 @@ pub use codec_batch::{ MetalBufferBatchTarget, MetalTextureBatchTarget, Rgb8MetalBatchOp, Rgb8MetalBatchRequest, Rgb8MetalBatchSource, }; +#[cfg(target_os = "macos")] +pub use compute::release_default_session_buffers; pub use decode_request::{MetalDecodeOp, MetalDecodeRequest}; pub use decoder::Decoder; pub use encode::{ diff --git a/crates/j2k-jpeg-metal/src/routing.rs b/crates/j2k-jpeg-metal/src/routing.rs index 78b857826..0b0d009e7 100644 --- a/crates/j2k-jpeg-metal/src/routing.rs +++ b/crates/j2k-jpeg-metal/src/routing.rs @@ -284,20 +284,6 @@ mod tests { ); } - #[cfg(not(target_os = "macos"))] - #[test] - fn auto_routes_to_cpu_host_on_non_macos_even_when_metal_would_be_preferred() { - let capabilities = JpegMetalCapabilities { - has_fast_packet: true, - supports_output_format: true, - }; - - assert_eq!( - decide_route(BackendRequest::Auto, capabilities), - RouteDecision::CpuHost - ); - } - #[cfg(not(target_os = "macos"))] #[test] fn explicit_metal_unsupported_shape_is_rejected_before_host_unavailability() { diff --git a/crates/j2k-jpeg-metal/src/session.rs b/crates/j2k-jpeg-metal/src/session.rs index 948727a0f..c24a115c3 100644 --- a/crates/j2k-jpeg-metal/src/session.rs +++ b/crates/j2k-jpeg-metal/src/session.rs @@ -60,6 +60,13 @@ impl MetalBackendSession { .get_or_init_runtime(|device| compute::MetalRuntime::new_with_device(device.clone())) } + /// The runtime if it has been initialized, without initializing it. + pub(crate) fn initialized_runtime( + &self, + ) -> Option<&Result> { + self.runtime_session.runtime_result() + } + #[cfg(test)] pub(crate) fn runtime_initialized_for_test(&self) -> bool { self.runtime_session.runtime_initialized() @@ -497,7 +504,7 @@ impl SessionState { #[cfg(target_os = "macos")] pub(crate) fn backend_session(&mut self) -> Result<&MetalBackendSession, Error> { if self.backend_session.is_none() { - self.backend_session = Some(MetalBackendSession::system_default()?); + self.backend_session = Some(crate::compute::default_tile_backend_session()?); } self.backend_session.as_ref().ok_or_else(|| { j2k_jpeg::adapter::JpegPlanCacheError::Invariant( diff --git a/crates/j2k-jpeg-metal/src/session/tests.rs b/crates/j2k-jpeg-metal/src/session/tests.rs index 8fb2dbb20..2f5f4192a 100644 --- a/crates/j2k-jpeg-metal/src/session/tests.rs +++ b/crates/j2k-jpeg-metal/src/session/tests.rs @@ -202,3 +202,25 @@ fn rewrite_first_sof_quant_table_selector(mut bytes: Vec, selector: u8) -> V } panic!("fixture must contain a supported SOF marker"); } + +#[cfg(target_os = "macos")] +#[test] +fn metal_sessions_without_a_backend_share_one_default_runtime() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let runtime_of = |session: &MetalSession| { + let mut state = session.shared.lock().expect("session state"); + let backend = state.backend_session().expect("default backend").clone(); + backend.runtime_result().as_ref().expect("runtime"); + backend.runtime_ptr_for_test().expect("initialized runtime") + }; + let first = MetalSession::default(); + let second = MetalSession::default(); + assert_eq!(runtime_of(&first), runtime_of(&second)); + let explicit = MetalSession::with_backend_session( + MetalBackendSession::system_default().expect("explicit session"), + ); + assert_ne!(runtime_of(&explicit), runtime_of(&first)); + crate::release_default_session_buffers().expect("release default buffers"); +} diff --git a/crates/j2k-jpeg-metal/src/shaders_decode_fast420.metal b/crates/j2k-jpeg-metal/src/shaders_decode_fast420.metal index 43c8ad534..27f7c9307 100644 --- a/crates/j2k-jpeg-metal/src/shaders_decode_fast420.metal +++ b/crates/j2k-jpeg-metal/src/shaders_decode_fast420.metal @@ -34,7 +34,6 @@ kernel void jpeg_decode_fast420( init_decode_status(thread_status); thread short coeffs[64]; - thread uchar pixels[64]; uint mx = 0u; uint my = 0u; @@ -45,27 +44,27 @@ kernel void jpeg_decode_fast420( const uint c_x = mx * 8u; const uint c_y = my * 8u; - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x, y_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x, y_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x + 8u, y_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x + 8u, y_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x, y_y + 8u, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x, y_y + 8u, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x + 8u, y_y + 8u, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x + 8u, y_y + 8u, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, cb_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, cb_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, cr_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, cr_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs)) { return; } advance_mcu_cursor(mx, my, params.mcus_per_row); @@ -115,7 +114,6 @@ kernel void jpeg_decode_fast420_batch( device uchar *tile_cr_plane = cr_plane + chroma_plane_base; thread short coeffs[64]; - thread uchar pixels[64]; uint mx = 0u; uint my = 0u; @@ -126,27 +124,27 @@ kernel void jpeg_decode_fast420_batch( const uint c_x = mx * 8u; const uint c_y = my * 8u; - if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x, y_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x, y_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x + 8u, y_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x + 8u, y_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x, y_y + 8u, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x, y_y + 8u, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x + 8u, y_y + 8u, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x + 8u, y_y + 8u, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, entropy_end, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, tile_cb_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, entropy_end, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, tile_cb_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, entropy_end, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, tile_cr_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, entropy_end, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, tile_cr_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs)) { return; } advance_mcu_cursor(mx, my, params.mcus_per_row); diff --git a/crates/j2k-jpeg-metal/src/shaders_decode_fast422_regions.metal b/crates/j2k-jpeg-metal/src/shaders_decode_fast422_regions.metal index c4e450fb2..f529fcb82 100644 --- a/crates/j2k-jpeg-metal/src/shaders_decode_fast422_regions.metal +++ b/crates/j2k-jpeg-metal/src/shaders_decode_fast422_regions.metal @@ -34,7 +34,6 @@ kernel void jpeg_decode_fast422( init_decode_status(thread_status); thread short coeffs[64]; - thread uchar pixels[64]; uint mx = 0u; uint my = 0u; @@ -45,19 +44,19 @@ kernel void jpeg_decode_fast422( const uint c_x = mx * 8u; const uint c_y = my * 8u; - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x, y_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x, y_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x + 8u, y_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, y_x + 8u, y_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, cb_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, cb_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, cr_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, cr_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs)) { return; } advance_mcu_cursor(mx, my, params.mcus_per_row); @@ -107,7 +106,6 @@ kernel void jpeg_decode_fast422_batch( device uchar *tile_cr_plane = cr_plane + chroma_plane_base; thread short coeffs[64]; - thread uchar pixels[64]; uint mx = 0u; uint my = 0u; @@ -118,19 +116,19 @@ kernel void jpeg_decode_fast422_batch( const uint c_x = mx * 8u; const uint c_y = my * 8u; - if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x, y_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x, y_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x + 8u, y_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, y_x + 8u, y_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, entropy_end, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, tile_cb_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, entropy_end, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, tile_cb_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, entropy_end, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, tile_cr_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, entropy_end, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, tile_cr_plane, params.chroma_width, params.chroma_width, params.chroma_height, c_x, c_y, coeffs)) { return; } advance_mcu_cursor(mx, my, params.mcus_per_row); diff --git a/crates/j2k-jpeg-metal/src/shaders_decode_fast444.metal b/crates/j2k-jpeg-metal/src/shaders_decode_fast444.metal index 18bb1d1cd..24f556cc5 100644 --- a/crates/j2k-jpeg-metal/src/shaders_decode_fast444.metal +++ b/crates/j2k-jpeg-metal/src/shaders_decode_fast444.metal @@ -33,7 +33,6 @@ kernel void jpeg_decode_fast444( init_decode_status(thread_status); thread short coeffs[64]; - thread uchar pixels[64]; uint mx = 0u; uint my = 0u; init_mcu_cursor(start_mcu, params.mcus_per_row, mx, my); @@ -41,15 +40,15 @@ kernel void jpeg_decode_fast444( const uint block_x = mx * 8u; const uint block_y = my * 8u; - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, block_x, block_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, y_plane, params.width, params.width, params.height, block_x, block_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, cb_plane, params.width, params.width, params.height, block_x, block_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, cb_plane, params.width, params.width, params.height, block_x, block_y, coeffs)) { return; } - if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, cr_plane, params.width, params.width, params.height, block_x, block_y, coeffs, pixels)) { + if (!decode_idct_deposit_block(br, entropy, params.entropy_len, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, cr_plane, params.width, params.width, params.height, block_x, block_y, coeffs)) { return; } advance_mcu_cursor(mx, my, params.mcus_per_row); @@ -289,6 +288,74 @@ kernel void jpeg_decode_fast444_scaled_region_batch( } } +// Batched full-frame plane decode on the shared `JpegFast420BatchParams` ABI +// (chroma planes are full size). One thread per entropy segment across all +// tiles; the pack pass converts the planes afterwards. +kernel void jpeg_decode_fast444_batch( + device const uchar *entropy [[buffer(0)]], + device uchar *y_plane [[buffer(1)]], + device uchar *cb_plane [[buffer(2)]], + device uchar *cr_plane [[buffer(3)]], + constant JpegFast420BatchParams ¶ms [[buffer(4)]], + constant ushort *y_quant [[buffer(5)]], + constant ushort *cb_quant [[buffer(6)]], + constant ushort *cr_quant [[buffer(7)]], + constant PreparedHuffman &y_dc [[buffer(8)]], + constant PreparedHuffman &y_ac [[buffer(9)]], + constant PreparedHuffman &cb_dc [[buffer(10)]], + constant PreparedHuffman &cb_ac [[buffer(11)]], + constant PreparedHuffman &cr_dc [[buffer(12)]], + constant PreparedHuffman &cr_ac [[buffer(13)]], + device const uint *entropy_offsets [[buffer(14)]], + device const uint *entropy_lens [[buffer(15)]], + device JpegDecodeStatus *status [[buffer(16)]], + device const JpegEntropyCheckpoint *entropy_checkpoints [[buffer(17)]], + uint gid [[thread_position_in_grid]] +) { + const uint total_mcus = params.mcus_per_row * params.mcu_rows; + JPEG_BATCH_ENTROPY_THREAD_VARS(); + if (!JPEG_CONFIGURE_BATCH_ENTROPY_THREAD( + gid, + total_mcus, + params, + entropy_offsets, + entropy_lens, + entropy_checkpoints + )) { + return; + } + device JpegDecodeStatus *thread_status = status + gid; + init_decode_status(thread_status); + + const uint plane_base = tile_index * params.width * params.height; + device uchar *tile_y_plane = y_plane + plane_base; + device uchar *tile_cb_plane = cb_plane + plane_base; + device uchar *tile_cr_plane = cr_plane + plane_base; + + thread short coeffs[64]; + + uint mx = 0u; + uint my = 0u; + init_mcu_cursor(start_mcu, params.mcus_per_row, mx, my); + for (uint mcu_index = start_mcu; mcu_index < end_mcu; ++mcu_index) { + const uint block_x = mx * 8u; + const uint block_y = my * 8u; + + if (!decode_idct_deposit_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, tile_y_plane, params.width, params.width, params.height, block_x, block_y, coeffs)) { + return; + } + + if (!decode_idct_deposit_block(br, entropy, entropy_end, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, tile_cb_plane, params.width, params.width, params.height, block_x, block_y, coeffs)) { + return; + } + + if (!decode_idct_deposit_block(br, entropy, entropy_end, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, tile_cr_plane, params.width, params.width, params.height, block_x, block_y, coeffs)) { + return; + } + advance_mcu_cursor(mx, my, params.mcus_per_row); + } +} + kernel void jpeg_decode_fast444_rgba_texture_batch( device const uchar *entropy [[buffer(0)]], constant JpegFast444TextureBatchParams ¶ms [[buffer(4)]], diff --git a/crates/j2k-jpeg-metal/src/shaders_encode.metal b/crates/j2k-jpeg-metal/src/shaders_encode.metal index 1632153bb..807c8b650 100644 --- a/crates/j2k-jpeg-metal/src/shaders_encode.metal +++ b/crates/j2k-jpeg-metal/src/shaders_encode.metal @@ -356,93 +356,28 @@ inline bool configure_batch_entropy_thread( cr_prev_dc \ ) -inline void prepare_huffman( - constant JpegHuffmanTable &raw, - thread PreparedHuffman &out -) { - uchar huffsize[256]; - ushort huffcode[256]; - ushort huffsize_len = 0; - for (uint i = 0; i < 17; ++i) { - out.min_code[i] = 0x7fffffff; - out.max_code[i] = -1; - out.val_offset[i] = 0; - } - for (uint i = 0; i < raw.values_len; ++i) { - out.values[i] = raw.values[i]; - } - for (uint i = 0; i < 512; ++i) { - out.fast_symbol[i] = 0; - out.fast_len[i] = 0; - } - out.values_len = raw.values_len; - for (uint len_minus_1 = 0; len_minus_1 < 16; ++len_minus_1) { - const uchar len = uchar(len_minus_1 + 1); - for (uchar count = 0; count < raw.bits[len_minus_1]; ++count) { - huffsize[huffsize_len] = len; - huffsize_len += 1; - } - } - - uint code = 0; - uchar si = huffsize_len == 0 ? 0 : huffsize[0]; - for (ushort k = 0; k < huffsize_len; ++k) { - const uchar s = huffsize[k]; - while (s != si) { - code <<= 1; - si += 1; - } - huffcode[k] = ushort(code); - code += 1; - } - - ushort k = 0; - for (uint len_minus_1 = 0; len_minus_1 < 16; ++len_minus_1) { - const uint len = len_minus_1 + 1; - const ushort count = raw.bits[len_minus_1]; - if (count == 0) { - continue; - } - out.min_code[len] = int(huffcode[k]); - out.max_code[len] = int(huffcode[k + count - 1]); - out.val_offset[len] = int(k) - out.min_code[len]; - k += count; - } - - for (uint idx = 0; idx < huffsize_len; ++idx) { - const uint len = uint(huffsize[idx]); - if (len == 0u || len > 9u) { - continue; - } - const uint prefix = uint(huffcode[idx]) << (9u - len); - const uint fill = 1u << (9u - len); - for (uint suffix = 0; suffix < fill; ++suffix) { - out.fast_symbol[prefix | suffix] = raw.values[idx]; - out.fast_len[prefix | suffix] = huffsize[idx]; - } - } -} - -inline bool decode_symbol( +// Decodes one symbol once at least 9 bits (real or 1-padded) are buffered. +inline bool decode_symbol_from_lookahead( thread BitReader &br, device const uchar *bytes, uint len, constant PreparedHuffman &table, device JpegDecodeStatus *status, + uint lookahead, thread uchar &symbol ) { - ensure_bits_padded(br, bytes, len, 9); - const uint fast_index = peek_bits(br, 9); - const uchar len9 = table.fast_len[fast_index]; - if (len9 != 0) { - consume_bits(br, uint(len9)); - symbol = table.fast_symbol[fast_index]; + const uint fast = uint(table.fast[lookahead]); + if (fast != 0u) { + consume_bits(br, fast >> 8); + symbol = uchar(fast); return true; } + // Every code of at most 9 bits has a lookahead entry, so the canonical + // search starts at length 10. ensure_bits_padded(br, bytes, len, 16); const int code16 = int(peek_bits(br, 16)); - for (uint length = 1; length <= 16; ++length) { + for (uint length = 10; length <= 16; ++length) { const int code = code16 >> (16 - int(length)); if (code <= table.max_code[length]) { if (code < table.min_code[length]) { @@ -464,6 +399,18 @@ inline bool decode_symbol( return false; } +inline bool decode_symbol( + thread BitReader &br, + device const uchar *bytes, + uint len, + constant PreparedHuffman &table, + device JpegDecodeStatus *status, + thread uchar &symbol +) { + ensure_bits_padded(br, bytes, len, 9); + return decode_symbol_from_lookahead(br, bytes, len, table, status, peek_bits(br, 9), symbol); +} + inline bool decode_block( thread BitReader &br, device const uchar *bytes, @@ -500,8 +447,27 @@ inline bool decode_block( dc_only = true; uint k = 1; while (k < 64) { + ensure_bits_padded(br, bytes, len, 9); + const uint lookahead = peek_bits(br, 9); + const int fast_ac = int(ac_table.fast_ac[lookahead]); + if (fast_ac != 0) { + // Code and extra bits both lie in the buffered lookahead, exactly + // the bits decode_symbol and receive_extend would consume. + consume_bits(br, uint(fast_ac & 15)); + k += uint(fast_ac >> 4) & 15u; + if (k >= 64) { + status->code = FAST420_STATUS_HUFFMAN; + status->position = br.pos; + return false; + } + coeffs[ZIGZAG[k]] = clamp_i16((fast_ac >> 8) * int(quant[k])); + dc_only = false; + k += 1; + continue; + } + uchar symbol = 0; - if (!decode_symbol(br, bytes, len, ac_table, status, symbol)) { + if (!decode_symbol_from_lookahead(br, bytes, len, ac_table, status, lookahead, symbol)) { return false; } const uint run = uint(symbol >> 4); @@ -559,8 +525,23 @@ inline bool decode_block_skip( uint k = 1; while (k < 64) { + ensure_bits_padded(br, bytes, len, 9); + const uint lookahead = peek_bits(br, 9); + const int fast_ac = int(ac_table.fast_ac[lookahead]); + if (fast_ac != 0) { + consume_bits(br, uint(fast_ac & 15)); + k += uint(fast_ac >> 4) & 15u; + if (k >= 64) { + status->code = FAST420_STATUS_HUFFMAN; + status->position = br.pos; + return false; + } + k += 1; + continue; + } + uchar symbol = 0; - if (!decode_symbol(br, bytes, len, ac_table, status, symbol)) { + if (!decode_symbol_from_lookahead(br, bytes, len, ac_table, status, lookahead, symbol)) { return false; } const uint run = uint(symbol >> 4); @@ -614,170 +595,77 @@ inline uchar descale_and_clamp(int value, int shift) { return clamp_u8(shifted + 128); } -inline void idct_1d_column( - thread const short input[64], - thread int work[64], - uint col +// libjpeg islow column pass over four columns at once. There is no per-column +// DC shortcut: with all AC terms zero the full expression is exactly p0 << PASS1_BITS, +// and i16 inputs cannot overflow it. +inline void idct_islow_columns_x4( + int4 p0, int4 p1, int4 p2, int4 p3, int4 p4, int4 p5, int4 p6, int4 p7, + thread int4 (&out)[8] ) { - const int p0 = int(input[col]); - const int p1 = int(input[col + 8]); - const int p2 = int(input[col + 16]); - const int p3 = int(input[col + 24]); - const int p4 = int(input[col + 32]); - const int p5 = int(input[col + 40]); - const int p6 = int(input[col + 48]); - const int p7 = int(input[col + 56]); - - if (p1 == 0 && p2 == 0 && p3 == 0 && p4 == 0 && p5 == 0 && p6 == 0 && p7 == 0) { - const int dc = p0 << PASS1_BITS; - work[col] = dc; - work[col + 8] = dc; - work[col + 16] = dc; - work[col + 24] = dc; - work[col + 32] = dc; - work[col + 40] = dc; - work[col + 48] = dc; - work[col + 56] = dc; - return; - } - - const int z2 = p2; - const int z3 = p6; - const int z1 = (z2 + z3) * FIX_0_541196100; - const int tmp2 = z1 - z3 * FIX_1_847759065; - const int tmp3 = z1 + z2 * FIX_0_765366865; - - const int tmp0 = (p0 + p4) << CONST_BITS; - const int tmp1 = (p0 - p4) << CONST_BITS; - - const int tmp10 = tmp0 + tmp3; - const int tmp13 = tmp0 - tmp3; - const int tmp11 = tmp1 + tmp2; - const int tmp12 = tmp1 - tmp2; - - const int z1o = p7 + p1; - const int z2o = p5 + p3; - const int z3o = p7 + p3; - const int z4o = p5 + p1; - const int z5 = (z3o + z4o) * FIX_1_175875602; - - const int tmp0o = p7 * FIX_0_298631336; - const int tmp1o = p5 * FIX_2_053119869; - const int tmp2o = p3 * FIX_3_072711026; - const int tmp3o = p1 * FIX_1_501321110; - const int z1m = z1o * -FIX_0_899976223; - const int z2m = z2o * -FIX_2_562915447; - const int z3m = z3o * -FIX_1_961570560 + z5; - const int z4m = z4o * -FIX_0_390180644 + z5; - - const int out0 = tmp0o + z1m + z3m; - const int out1 = tmp1o + z2m + z4m; - const int out2 = tmp2o + z2m + z3m; - const int out3 = tmp3o + z1m + z4m; - - const int shift = CONST_BITS - PASS1_BITS; - const int rounding = 1 << (shift - 1); - work[col] = descale(tmp10 + out3 + rounding, shift); - work[col + 56] = descale(tmp10 - out3 + rounding, shift); - work[col + 8] = descale(tmp11 + out2 + rounding, shift); - work[col + 48] = descale(tmp11 - out2 + rounding, shift); - work[col + 16] = descale(tmp12 + out1 + rounding, shift); - work[col + 40] = descale(tmp12 - out1 + rounding, shift); - work[col + 24] = descale(tmp13 + out0 + rounding, shift); - work[col + 32] = descale(tmp13 - out0 + rounding, shift); -} - -inline void idct_1d_column_bottom_half_zero( - thread const short input[64], - thread int work[64], - uint col -) { - const int p0 = int(input[col]); - const int p1 = int(input[col + 8]); - const int p2 = int(input[col + 16]); - const int p3 = int(input[col + 24]); - - if (p1 == 0 && p2 == 0 && p3 == 0) { - const int dc = p0 << PASS1_BITS; - work[col] = dc; - work[col + 8] = dc; - work[col + 16] = dc; - work[col + 24] = dc; - work[col + 32] = dc; - work[col + 40] = dc; - work[col + 48] = dc; - work[col + 56] = dc; - return; - } - - const int z1 = p2 * FIX_0_541196100; - const int tmp2 = z1; - const int tmp3 = z1 + p2 * FIX_0_765366865; - - const int tmp0 = p0 << CONST_BITS; - const int tmp1 = p0 << CONST_BITS; - - const int tmp10 = tmp0 + tmp3; - const int tmp13 = tmp0 - tmp3; - const int tmp11 = tmp1 + tmp2; - const int tmp12 = tmp1 - tmp2; - - const int z5 = (p1 + p3) * FIX_1_175875602; - const int z1m = p1 * -FIX_0_899976223; - const int z2m = p3 * -FIX_2_562915447; - const int z3m = p3 * -FIX_1_961570560 + z5; - const int z4m = p1 * -FIX_0_390180644 + z5; - - const int out0 = z1m + z3m; - const int out1 = z2m + z4m; - const int out2 = p3 * FIX_3_072711026 + z2m + z3m; - const int out3 = p1 * FIX_1_501321110 + z1m + z4m; + const int4 z1 = (p2 + p6) * FIX_0_541196100; + const int4 tmp2 = z1 - p6 * FIX_1_847759065; + const int4 tmp3 = z1 + p2 * FIX_0_765366865; + + const int4 tmp0 = (p0 + p4) << CONST_BITS; + const int4 tmp1 = (p0 - p4) << CONST_BITS; + + const int4 tmp10 = tmp0 + tmp3; + const int4 tmp13 = tmp0 - tmp3; + const int4 tmp11 = tmp1 + tmp2; + const int4 tmp12 = tmp1 - tmp2; + + const int4 z1o = p7 + p1; + const int4 z2o = p5 + p3; + const int4 z3o = p7 + p3; + const int4 z4o = p5 + p1; + const int4 z5 = (z3o + z4o) * FIX_1_175875602; + + const int4 z1m = z1o * -FIX_0_899976223; + const int4 z2m = z2o * -FIX_2_562915447; + const int4 z3m = z3o * -FIX_1_961570560 + z5; + const int4 z4m = z4o * -FIX_0_390180644 + z5; + + const int4 out0 = p7 * FIX_0_298631336 + z1m + z3m; + const int4 out1 = p5 * FIX_2_053119869 + z2m + z4m; + const int4 out2 = p3 * FIX_3_072711026 + z2m + z3m; + const int4 out3 = p1 * FIX_1_501321110 + z1m + z4m; const int shift = CONST_BITS - PASS1_BITS; const int rounding = 1 << (shift - 1); - work[col] = descale(tmp10 + out3 + rounding, shift); - work[col + 56] = descale(tmp10 - out3 + rounding, shift); - work[col + 8] = descale(tmp11 + out2 + rounding, shift); - work[col + 48] = descale(tmp11 - out2 + rounding, shift); - work[col + 16] = descale(tmp12 + out1 + rounding, shift); - work[col + 40] = descale(tmp12 - out1 + rounding, shift); - work[col + 24] = descale(tmp13 + out0 + rounding, shift); - work[col + 32] = descale(tmp13 - out0 + rounding, shift); -} - -inline void idct_1d_row( - thread const int work[64], - thread uchar output[64], - uint row -) { - const uint base = row * 8; - const int p0 = work[base]; - const int p1 = work[base + 1]; - const int p2 = work[base + 2]; - const int p3 = work[base + 3]; - const int p4 = work[base + 4]; - const int p5 = work[base + 5]; - const int p6 = work[base + 6]; - const int p7 = work[base + 7]; - - const int shift = CONST_BITS + PASS1_BITS + 3; - const int rounding = 1 << (shift - 1); + out[0] = (tmp10 + out3 + rounding) >> shift; + out[7] = (tmp10 - out3 + rounding) >> shift; + out[1] = (tmp11 + out2 + rounding) >> shift; + out[6] = (tmp11 - out2 + rounding) >> shift; + out[2] = (tmp12 + out1 + rounding) >> shift; + out[5] = (tmp12 - out1 + rounding) >> shift; + out[3] = (tmp13 + out0 + rounding) >> shift; + out[4] = (tmp13 - out0 + rounding) >> shift; +} + +// libjpeg islow row pass for one row held in registers. The AC-zero shortcut stays: the +// pass-1 DC can be large enough that the full expression would overflow. +inline void idct_islow_row(int4 left_in, int4 right_in, thread uchar4 &left, thread uchar4 &right) { + const int p0 = left_in.x; + const int p1 = left_in.y; + const int p2 = left_in.z; + const int p3 = left_in.w; + const int p4 = right_in.x; + const int p5 = right_in.y; + const int p6 = right_in.z; + const int p7 = right_in.w; if (p1 == 0 && p2 == 0 && p3 == 0 && p4 == 0 && p5 == 0 && p6 == 0 && p7 == 0) { const int dc_shift = PASS1_BITS + 3; const int dc_rounding = 1 << (dc_shift - 1); const uchar pixel = descale_and_clamp(p0 + dc_rounding, dc_shift); - for (uint i = 0; i < 8; ++i) { - output[base + i] = pixel; - } + left = uchar4(pixel); + right = uchar4(pixel); return; } - const int z2 = p2; - const int z3 = p6; - const int z1 = (z2 + z3) * FIX_0_541196100; - const int tmp2 = z1 - z3 * FIX_1_847759065; - const int tmp3 = z1 + z2 * FIX_0_765366865; + const int z1 = (p2 + p6) * FIX_0_541196100; + const int tmp2 = z1 - p6 * FIX_1_847759065; + const int tmp3 = z1 + p2 * FIX_0_765366865; const int tmp0 = (p0 + p4) << CONST_BITS; const int tmp1 = (p0 - p4) << CONST_BITS; @@ -793,52 +681,59 @@ inline void idct_1d_row( const int z4o = p5 + p1; const int z5 = (z3o + z4o) * FIX_1_175875602; - const int tmp0o = p7 * FIX_0_298631336; - const int tmp1o = p5 * FIX_2_053119869; - const int tmp2o = p3 * FIX_3_072711026; - const int tmp3o = p1 * FIX_1_501321110; const int z1m = z1o * -FIX_0_899976223; const int z2m = z2o * -FIX_2_562915447; const int z3m = z3o * -FIX_1_961570560 + z5; const int z4m = z4o * -FIX_0_390180644 + z5; - const int out0 = tmp0o + z1m + z3m; - const int out1 = tmp1o + z2m + z4m; - const int out2 = tmp2o + z2m + z3m; - const int out3 = tmp3o + z1m + z4m; + const int out0 = p7 * FIX_0_298631336 + z1m + z3m; + const int out1 = p5 * FIX_2_053119869 + z2m + z4m; + const int out2 = p3 * FIX_3_072711026 + z2m + z3m; + const int out3 = p1 * FIX_1_501321110 + z1m + z4m; - output[base] = descale_and_clamp(tmp10 + out3 + rounding, shift); - output[base + 7] = descale_and_clamp(tmp10 - out3 + rounding, shift); - output[base + 1] = descale_and_clamp(tmp11 + out2 + rounding, shift); - output[base + 6] = descale_and_clamp(tmp11 - out2 + rounding, shift); - output[base + 2] = descale_and_clamp(tmp12 + out1 + rounding, shift); - output[base + 5] = descale_and_clamp(tmp12 - out1 + rounding, shift); - output[base + 3] = descale_and_clamp(tmp13 + out0 + rounding, shift); - output[base + 4] = descale_and_clamp(tmp13 - out0 + rounding, shift); + const int shift = CONST_BITS + PASS1_BITS + 3; + const int rounding = 1 << (shift - 1); + left = uchar4( + descale_and_clamp(tmp10 + out3 + rounding, shift), + descale_and_clamp(tmp11 + out2 + rounding, shift), + descale_and_clamp(tmp12 + out1 + rounding, shift), + descale_and_clamp(tmp13 + out0 + rounding, shift) + ); + right = uchar4( + descale_and_clamp(tmp13 - out0 + rounding, shift), + descale_and_clamp(tmp12 - out1 + rounding, shift), + descale_and_clamp(tmp11 - out2 + rounding, shift), + descale_and_clamp(tmp10 - out3 + rounding, shift) + ); } inline void idct_islow( thread const short input[64], thread uchar output[64] ) { - thread int work[64]; - bool upper_half_zero = true; - for (uint i = 32; i < 64; ++i) { - if (input[i] != 0) { - upper_half_zero = false; - break; + thread const short4 *rows = reinterpret_cast(input); + int4 left_cols[8]; + int4 right_cols[8]; + idct_islow_columns_x4( + int4(rows[0]), int4(rows[2]), int4(rows[4]), int4(rows[6]), + int4(rows[8]), int4(rows[10]), int4(rows[12]), int4(rows[14]), + left_cols + ); + idct_islow_columns_x4( + int4(rows[1]), int4(rows[3]), int4(rows[5]), int4(rows[7]), + int4(rows[9]), int4(rows[11]), int4(rows[13]), int4(rows[15]), + right_cols + ); + for (uint row = 0; row < 8u; ++row) { + uchar4 left; + uchar4 right; + idct_islow_row(left_cols[row], right_cols[row], left, right); + thread uchar *out_row = output + row * 8u; + for (uint i = 0; i < 4u; ++i) { + out_row[i] = left[i]; + out_row[i + 4u] = right[i]; } } - for (uint col = 0; col < 8; ++col) { - if (upper_half_zero) { - idct_1d_column_bottom_half_zero(input, work, col); - } else { - idct_1d_column(input, work, col); - } - } - for (uint row = 0; row < 8; ++row) { - idct_1d_row(work, output, row); - } } inline void idct_islow_dc_only( @@ -904,6 +799,92 @@ inline void deposit_block( } } +// Stores one output row with deposit_block's clipping and store widths. +inline void deposit_block_row( + device uchar *plane, + uint stride, + uint x, + uint row_y, + uint copy_width, + bool vector_store, + uchar4 left, + uchar4 right +) { + device uchar *dst = plane + row_y * stride + x; + if (vector_store) { + *(device uchar4 *)(dst) = left; + *(device uchar4 *)(dst + 4u) = right; + return; + } + for (uint bx = 0; bx < copy_width; ++bx) { + dst[bx] = bx < 4u ? left[bx] : right[bx - 4u]; + } +} + +// idct_islow followed by deposit_block, with the block kept in registers and +// each row stored as soon as it is computed. +inline void idct_islow_deposit( + thread const short coeffs[64], + device uchar *plane, + uint stride, + uint width, + uint height, + uint x, + uint y +) { + if (x >= width || y >= height) { + return; + } + thread const short4 *rows = reinterpret_cast(coeffs); + int4 left_cols[8]; + int4 right_cols[8]; + idct_islow_columns_x4( + int4(rows[0]), int4(rows[2]), int4(rows[4]), int4(rows[6]), + int4(rows[8]), int4(rows[10]), int4(rows[12]), int4(rows[14]), + left_cols + ); + idct_islow_columns_x4( + int4(rows[1]), int4(rows[3]), int4(rows[5]), int4(rows[7]), + int4(rows[9]), int4(rows[11]), int4(rows[13]), int4(rows[15]), + right_cols + ); + + const uint copy_width = min(8u, width - x); + const uint copy_height = min(8u, height - y); + const bool vector_store = copy_width == 8u && copy_height == 8u && (stride & 3u) == 0u; + for (uint row = 0; row < 8u; ++row) { + if (row >= copy_height) { + break; + } + uchar4 left; + uchar4 right; + idct_islow_row(left_cols[row], right_cols[row], left, right); + deposit_block_row(plane, stride, x, y + row, copy_width, vector_store, left, right); + } +} + +// idct_islow_dc_only followed by deposit_block. +inline void deposit_dc_block( + short dc_coeff, + device uchar *plane, + uint stride, + uint width, + uint height, + uint x, + uint y +) { + if (x >= width || y >= height) { + return; + } + const uchar4 pixel = uchar4(clamp_u8(((int(dc_coeff) + 4) >> 3) + 128)); + const uint copy_width = min(8u, width - x); + const uint copy_height = min(8u, height - y); + const bool vector_store = copy_width == 8u && copy_height == 8u && (stride & 3u) == 0u; + for (uint row = 0; row < copy_height; ++row) { + deposit_block_row(plane, stride, x, y + row, copy_width, vector_store, pixel, pixel); + } +} + inline bool decode_idct_deposit_block( thread BitReader &br, device const uchar *bytes, @@ -919,15 +900,17 @@ inline bool decode_idct_deposit_block( uint height, uint x, uint y, - thread short coeffs[64], - thread uchar pixels[64] + thread short coeffs[64] ) { bool dc_only = false; if (!decode_block(br, bytes, len, dc_table, ac_table, quant, prev_dc, status, coeffs, dc_only)) { return false; } - idct_block(coeffs, dc_only, pixels); - deposit_block(plane, stride, width, height, x, y, pixels); + if (dc_only) { + deposit_dc_block(coeffs[0], plane, stride, width, height, x, y); + } else { + idct_islow_deposit(coeffs, plane, stride, width, height, x, y); + } return true; } diff --git a/crates/j2k-jpeg-metal/src/shaders_shared.metal b/crates/j2k-jpeg-metal/src/shaders_shared.metal index 1b4202a07..25b230dd6 100644 --- a/crates/j2k-jpeg-metal/src/shaders_shared.metal +++ b/crates/j2k-jpeg-metal/src/shaders_shared.metal @@ -257,20 +257,16 @@ struct JpegEntropyCheckpoint { uint reserved_tail; }; -struct JpegHuffmanTable { - uchar bits[16]; - ushort values_len; - ushort reserved; - uchar values[256]; -}; - struct PreparedHuffman { int min_code[17]; int max_code[17]; int val_offset[17]; uchar values[256]; - uchar fast_symbol[512]; - uchar fast_len[512]; + // (code length << 8) | symbol for codes of at most 9 bits; 0 otherwise. + ushort fast[512]; + // (value << 8) | (run << 4) | total length when an AC code and its extra + // bits fit in the 9-bit lookahead; 0 otherwise. + short fast_ac[512]; ushort values_len; }; diff --git a/crates/j2k-jpeg-metal/src/surface/batch_texture.rs b/crates/j2k-jpeg-metal/src/surface/batch_texture.rs index b88b22a12..e66f0898f 100644 --- a/crates/j2k-jpeg-metal/src/surface/batch_texture.rs +++ b/crates/j2k-jpeg-metal/src/surface/batch_texture.rs @@ -270,6 +270,14 @@ impl MetalBatchTextureOutput { }) } + pub(crate) fn account_texture_handle_capacity( + &self, + budget: &mut crate::batch_allocation::BatchMetadataBudget, + ) -> Result<(), Error> { + budget.account_capacity::(self.set.textures.capacity())?; + Ok(()) + } + #[cfg(test)] pub(crate) fn shares_allocation_set_with(&self, other: &Self) -> bool { Arc::ptr_eq(&self.set, &other.set) diff --git a/crates/j2k-jpeg-metal/src/tests.rs b/crates/j2k-jpeg-metal/src/tests.rs index 26bd54157..cf26c1905 100644 --- a/crates/j2k-jpeg-metal/src/tests.rs +++ b/crates/j2k-jpeg-metal/src/tests.rs @@ -311,24 +311,46 @@ fn assert_prepared_huffman_matches_shared(label: &str, table: &JpegHuffmanTable) "{label} values" ); - let mut fast_symbol = [0u8; 512]; - let mut fast_len = [0u8; 512]; - for idx in 0..canonical.huffsize_len { - let len = usize::from(canonical.huffsize[idx]); - if len == 0 || len > 9 { - continue; - } - let code = usize::from(canonical.huffcode[idx]); - let prefix = code << (9 - len); - let fill = 1usize << (9 - len); - for suffix in 0..fill { - fast_symbol[prefix | suffix] = table.values[idx]; - fast_len[prefix | suffix] = canonical.huffsize[idx]; - } - } + // Independent reference: walk every 9-bit lookahead through the + // canonical codes, as the shader's slow path would. + for lookahead in 0..512usize { + let hit = (0..canonical.huffsize_len).find(|&idx| { + let len = usize::from(canonical.huffsize[idx]); + len != 0 && len <= 9 && lookahead >> (9 - len) == usize::from(canonical.huffcode[idx]) + }); + let expected_fast = hit.map_or(0, |idx| { + (u16::from(canonical.huffsize[idx]) << 8) | u16::from(table.values[idx]) + }); + assert_eq!( + prepared.fast[lookahead], expected_fast, + "{label} fast[{lookahead:#05x}]" + ); - assert_eq!(prepared.fast_symbol, fast_symbol, "{label} fast_symbol"); - assert_eq!(prepared.fast_len, fast_len, "{label} fast_len"); + let expected_fast_ac = hit.and_then(|idx| { + let code_len = usize::from(canonical.huffsize[idx]); + let symbol = table.values[idx]; + let ssss = usize::from(symbol & 0x0f); + if ssss == 0 || code_len + ssss > 9 { + return None; + } + let extra = (lookahead >> (9 - code_len - ssss)) & ((1 << ssss) - 1); + let extra = i32::try_from(extra).expect("extra bits fit in i32"); + let value = if extra < 1 << (ssss - 1) { + extra + (-1 << ssss) + 1 + } else { + extra + }; + let value = i8::try_from(value).ok()?; + let run = i16::from(symbol >> 4); + let total = i16::try_from(code_len + ssss).expect("length fits in i16"); + Some(i16::from(value) * 256 + run * 16 + total) + }); + assert_eq!( + prepared.fast_ac[lookahead], + expected_fast_ac.unwrap_or(0), + "{label} fast_ac[{lookahead:#05x}]" + ); + } } #[cfg(target_os = "macos")] @@ -919,6 +941,144 @@ fn rgb8_fast444_batch_decode_can_write_into_reusable_metal_output_buffer() { } } +#[cfg(target_os = "macos")] +#[test] +fn rgb8_four_table_groups_preserve_buffer_order_and_reuse() { + if !should_run_metal_runtime() { + return; + } + + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (64, 64); + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for subsampling in [ + JpegSubsampling::Ybr420, + JpegSubsampling::Ybr422, + JpegSubsampling::Ybr444, + ] { + let jpegs = [95, 85, 70, 50].map(|quality| { + encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality, + subsampling, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode table group") + }); + let expected: Vec<_> = jpegs + .iter() + .map(|jpeg| { + CpuDecoder::new(&jpeg.data) + .expect("CPU decoder") + .decode_request(DecodeRequest::full(PixelFormat::Rgb8)) + .expect("CPU decode") + .0 + }) + .collect(); + let order = [0, 1, 2, 3, 2, 0, 3, 1]; + let inputs = order.map(|index| jpegs[index].data.as_slice()); + let output = MetalBatchOutputBuffer::new_rgb8_tiles(&session, dimensions, inputs.len()) + .expect("buffer output"); + for _ in 0..2 { + let surfaces = decode_rgb8_buffer_batch_with_session( + Rgb8MetalBatchSource::Bytes(&inputs), + Rgb8MetalBatchOp::Full, + MetalBufferBatchTarget::Reusable(&output), + &session, + ) + .expect("decode interleaved table groups"); + for (index, surface) in surfaces.into_iter().enumerate() { + let surface = surface.expect("surface"); + let (buffer, offset) = surface.metal_buffer_trusted().expect("metal buffer"); + assert!(std::ptr::eq(buffer, output.buffer_trusted())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + expected[order[index]].as_slice() + ); + } + } + } +} + +#[cfg(target_os = "macos")] +#[test] +fn concurrent_four_group_buffer_batches_share_session_without_stalling() { + if !should_run_metal_runtime() { + return; + } + + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let start = std::sync::Barrier::new(2); + std::thread::scope(|scope| { + for variant in [0u8, 79] { + let session = &session; + let start = &start; + scope.spawn(move || { + let dimensions = (64, 64); + let mut rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for pixel in rgb.chunks_exact_mut(3) { + pixel[0] = pixel[0].wrapping_add(variant); + } + let jpegs = [95, 85, 70, 50].map(|quality| { + encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality, + subsampling: JpegSubsampling::Ybr420, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode concurrent table group") + }); + let expected: Vec<_> = jpegs + .iter() + .map(|jpeg| { + CpuDecoder::new(&jpeg.data) + .expect("CPU decoder") + .decode_request(DecodeRequest::full(PixelFormat::Rgb8)) + .expect("CPU decode") + .0 + }) + .collect(); + let inputs = jpegs.each_ref().map(|jpeg| jpeg.data.as_slice()); + let output = + MetalBatchOutputBuffer::new_rgb8_tiles(session, dimensions, inputs.len()) + .expect("buffer output"); + + start.wait(); + let surfaces = decode_rgb8_buffer_batch_with_session( + Rgb8MetalBatchSource::Bytes(&inputs), + Rgb8MetalBatchOp::Full, + MetalBufferBatchTarget::Reusable(&output), + session, + ) + .expect("decode concurrent table groups"); + assert_eq!(surfaces.len(), inputs.len()); + for (index, surface) in surfaces.into_iter().enumerate() { + let surface = surface.expect("surface"); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + expected[index].as_slice() + ); + } + }); + } + }); +} + #[cfg(target_os = "macos")] #[expect( clippy::too_many_lines, diff --git a/crates/j2k-jpeg-metal/src/tests/textures.rs b/crates/j2k-jpeg-metal/src/tests/textures.rs index 7b93d6e31..1e55d2fc2 100644 --- a/crates/j2k-jpeg-metal/src/tests/textures.rs +++ b/crates/j2k-jpeg-metal/src/tests/textures.rs @@ -9,6 +9,68 @@ fn metal_session() -> Option { .then(|| MetalBackendSession::system_default().expect("Metal backend session")) } +#[test] +fn rgb8_four_table_groups_preserve_texture_order_and_reuse() { + let Some(session) = metal_session() else { + return; + }; + let dimensions = (64, 64); + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for subsampling in [ + JpegSubsampling::Ybr420, + JpegSubsampling::Ybr422, + JpegSubsampling::Ybr444, + ] { + let jpegs = [95, 85, 70, 50].map(|quality| { + encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality, + subsampling, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode table group") + }); + let expected: Vec<_> = jpegs + .iter() + .map(|jpeg| { + let (rgb, _) = CpuDecoder::new(&jpeg.data) + .expect("CPU decoder") + .decode_request(DecodeRequest::full(PixelFormat::Rgb8)) + .expect("CPU decode"); + rgb_to_rgba_opaque(&rgb) + }) + .collect(); + let order = [0, 1, 2, 3, 2, 0, 3, 1]; + let inputs = order.map(|index| jpegs[index].data.as_slice()); + let expected_tiles = order.map(|index| expected[index].as_slice()); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, inputs.len()) + .expect("texture output"); + for _ in 0..2 { + let tiles = decode_rgb8_texture_batch_with_session( + Rgb8MetalBatchSource::Bytes(&inputs), + Rgb8MetalBatchOp::Full, + MetalTextureBatchTarget::Reusable(&output), + &session, + ) + .expect("decode interleaved table groups"); + assert_reusable_rgba_texture_tiles( + &session, + &output, + tiles, + dimensions, + &expected_tiles, + ); + } + } +} + #[test] fn rgb8_fast444_batch_decode_can_write_into_reusable_metal_textures() { let Some(session) = metal_session() else { @@ -1409,10 +1471,11 @@ fn rgb8_texture_batch_decode_avoids_private_rgba_staging_buffers() { return; } + // The three private allocations are the Y/Cb/Cr component planes. let cases = [ (BASELINE_420, (16, 16), 3), (BASELINE_422, (16, 8), 3), - (BASELINE_444, (8, 8), 0), + (BASELINE_444, (8, 8), 3), ]; for (input, dimensions, expected_private_allocations) in cases { @@ -1447,7 +1510,7 @@ fn rgb8_texture_batch_decode_avoids_private_rgba_staging_buffers() { #[cfg(target_os = "macos")] #[test] -fn rgb8_fast444_texture_batch_decode_fuses_directly_into_reusable_metal_textures() { +fn rgb8_fast444_texture_batch_decode_uses_reusable_component_planes() { let Some(session) = metal_session() else { return; }; @@ -1473,8 +1536,8 @@ fn rgb8_fast444_texture_batch_decode_fuses_directly_into_reusable_metal_textures assert_reusable_rgba_texture_tiles(&session, &output, tiles, (8, 8), &expected_tiles); assert_eq!( compute::jpeg_private_buffer_allocations_for_test(), - 0, - "fused 4:4:4 texture batch decode should not allocate private Y/Cb/Cr staging planes" + 3, + "4:4:4 texture batch decode should stage one set of private Y/Cb/Cr planes" ); } @@ -1614,10 +1677,10 @@ fn rgb8_table_mixed_fast444_texture_batch_groups_resident_dispatches() { assert_eq!(actual_rgba.as_slice(), expected_tiles[index].as_slice()); } assert_eq!( - compute::jpeg_private_buffer_allocations_for_test(), - 0, - "table-mixed resident 4:4:4 texture dispatches should not allocate private Y/Cb/Cr staging planes" - ); + compute::jpeg_private_buffer_allocations_for_test(), + 6, + "table-mixed resident 4:4:4 texture groups should stage Y/Cb/Cr planes once per in-flight group" + ); } #[cfg(target_os = "macos")] @@ -1841,8 +1904,8 @@ fn rgb8_table_mixed_fast422_texture_batch_groups_resident_dispatches() { } assert_eq!( compute::jpeg_private_buffer_allocations_for_test(), - 3, - "subsampled texture decode allocates only three reusable component planes" + 6, + "two in-flight table groups each retain three reusable component planes" ); } @@ -2369,7 +2432,7 @@ fn rgb8_table_mixed_restart_fast420_texture_batch_groups_resident_dispatches() { } assert_eq!( compute::jpeg_private_buffer_allocations_for_test(), - 3, - "subsampled texture decode allocates only three reusable component planes" + 6, + "two in-flight table groups each retain three reusable component planes" ); } diff --git a/crates/j2k-jpeg-metal/tests/batch.rs b/crates/j2k-jpeg-metal/tests/batch.rs index ee6f30781..e1d8443a1 100644 --- a/crates/j2k-jpeg-metal/tests/batch.rs +++ b/crates/j2k-jpeg-metal/tests/batch.rs @@ -253,6 +253,72 @@ fn auto_small_restart_tile_batch_stays_cpu_surface() { assert_eq!(session.submissions().expect("session submissions"), 1); } +#[cfg(target_os = "macos")] +#[test] +fn auto_compatible_256_tile_batches_use_metal_and_preserve_pixels() { + if !should_run_metal_runtime() { + return; + } + for sampling in [ + jpeg_encoder::SamplingFactor::F_2_2, + jpeg_encoder::SamplingFactor::F_2_1, + ] { + let inputs = (0_u8..16) + .map(|index| { + let mut rgb = j2k_test_support::patterned_rgb8(256, 256); + for pixel in rgb.chunks_exact_mut(3) { + pixel[0] = pixel[0].wrapping_add(index.wrapping_mul(17)); + pixel[2] ^= index.wrapping_mul(29); + } + let mut input = Vec::new(); + let mut encoder = jpeg_encoder::Encoder::new(&mut input, 90); + encoder.set_sampling_factor(sampling); + encoder + .encode(&rgb, 256, 256, jpeg_encoder::ColorType::Rgb) + .expect("JPEG encode"); + input + }) + .collect::>(); + let expected = inputs + .iter() + .map(|input| { + CpuDecoder::new(input) + .expect("CPU decoder") + .decode_request(DecodeRequest::full(PixelFormat::Rgb8)) + .expect("CPU decode") + .0 + }) + .collect::>(); + for (count, distinct) in [(64, 1), (16, 16)] { + let mut context = JpegDecoderContext::default(); + let mut pool = ScratchPool::new(); + let mut session = MetalSession::default(); + let submissions = (0..count) + .map(|index| { + ::submit_tile_to_device( + &mut context, + &mut session, + &mut pool, + &inputs[index % distinct], + PixelFormat::Rgb8, + BackendRequest::Auto, + ) + .expect("Auto submit") + }) + .collect::>(); + for (index, submission) in submissions.into_iter().enumerate() { + let surface = submission.wait().expect("Auto result"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!( + surface.as_bytes().expect("pixels"), + expected[index % distinct].as_slice() + ); + } + assert_eq!(session.submissions().expect("submissions"), 1); + } + } +} + #[cfg(target_os = "macos")] #[test] fn auto_restart_wsi_tile_batch_stays_on_cpu_without_promotion_evidence() { diff --git a/crates/j2k-jpeg/benches/decode_cpu.rs b/crates/j2k-jpeg/benches/decode_cpu.rs index 1659d2183..e5603888c 100644 --- a/crates/j2k-jpeg/benches/decode_cpu.rs +++ b/crates/j2k-jpeg/benches/decode_cpu.rs @@ -13,9 +13,6 @@ use j2k_jpeg::{ }; use j2k_test_support::{patterned_gray8, patterned_rgb8}; -const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; -const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; - fn zeroed_bytes(len: usize) -> Vec { let mut bytes = Vec::new(); bytes @@ -47,7 +44,6 @@ struct DecodeCase { mode: DecodeMode, fast_packet: FastPacketKind, expected_output: Option>, - expected_checksum: u64, } impl DecodeCase { @@ -63,13 +59,14 @@ impl DecodeCase { assert_eq!(decoder.info().dimensions, (width, height)); let expected_output = match mode { DecodeMode::Buffer(format) => Some(decode_buffer_output(&decoder, format)), - DecodeMode::Rows => None, + DecodeMode::Rows => { + assert_eq!( + decode_rows_output(&decoder), + decode_buffer_output(&decoder, PixelFormat::Rgb8), + ); + None + } }; - let expected_checksum = expected_output.as_ref().map_or_else( - || decode_rows_checksum(&decoder), - |output| fnv1a_update(FNV_OFFSET_BASIS, output), - ); - assert_ne!(expected_checksum, FNV_OFFSET_BASIS); Self { name, width, @@ -78,7 +75,6 @@ impl DecodeCase { mode, fast_packet, expected_output, - expected_checksum, } } @@ -87,18 +83,12 @@ impl DecodeCase { } } -fn fnv1a_update(mut hash: u64, bytes: &[u8]) -> u64 { - for byte in bytes { - hash = (hash ^ u64::from(*byte)).wrapping_mul(FNV_PRIME); - } - hash -} - fn bytes_per_pixel(format: PixelFormat) -> usize { match format { PixelFormat::Gray8 => 1, PixelFormat::Rgb8 => 3, - _ => panic!("decode benchmark only supports Gray8 and Rgb8"), + PixelFormat::Rgba8 => 4, + _ => panic!("decode benchmark only supports eight-bit output"), } } @@ -113,43 +103,54 @@ fn decode_buffer_output(decoder: &Decoder<'_>, format: PixelFormat) -> Vec { output } -struct ChecksumSink { - hash: u64, +struct ValidationSink { + pixels: Vec, expected_y: u32, row_bytes: usize, } -impl ChecksumSink { +impl ValidationSink { fn new(row_bytes: usize) -> Self { Self { - hash: FNV_OFFSET_BASIS, + pixels: Vec::new(), expected_y: 0, row_bytes, } } } -impl RowSink for ChecksumSink { +impl RowSink for ValidationSink { type Error = JpegError; fn write_row(&mut self, y: u32, row: &[u8]) -> Result<(), Self::Error> { assert_eq!(y, self.expected_y); assert_eq!(row.len(), self.row_bytes); - self.hash = fnv1a_update(self.hash, row); + self.pixels.extend_from_slice(row); self.expected_y += 1; Ok(()) } } -fn decode_rows_checksum(decoder: &Decoder<'_>) -> u64 { +fn decode_rows_output(decoder: &Decoder<'_>) -> Vec { let (width, height) = decoder.info().dimensions; - let mut sink = ChecksumSink::new(width as usize * 3); + let mut sink = ValidationSink::new(width as usize * 3); let outcome = decoder .decode_rows(&mut sink) .expect("generated benchmark JPEG row decode must succeed"); assert_eq!((outcome.decoded.w, outcome.decoded.h), (width, height)); assert_eq!(sink.expected_y, height); - sink.hash + sink.pixels +} + +struct BenchSink; + +impl RowSink for BenchSink { + type Error = JpegError; + + fn write_row(&mut self, _y: u32, row: &[u8]) -> Result<(), Self::Error> { + std::hint::black_box(row); + Ok(()) + } } fn encode_gray(width: u32, height: u32) -> Vec { @@ -225,7 +226,7 @@ fn decode_cases() -> Vec { let rgb_420 = encode_rgb(512, 512, JpegSubsampling::Ybr420); let mut cases = Vec::new(); cases - .try_reserve_exact(7) + .try_reserve_exact(8) .expect("reserve deterministic benchmark cases"); cases.push(DecodeCase::new( "gray8_512", @@ -259,6 +260,14 @@ fn decode_cases() -> Vec { DecodeMode::Buffer(PixelFormat::Rgb8), FastPacketKind::Ybr420, )); + cases.push(DecodeCase::new( + "rgba8_512_420", + 512, + 512, + rgb_420.clone(), + DecodeMode::Buffer(PixelFormat::Rgba8), + FastPacketKind::Ybr420, + )); cases.push(DecodeCase::new( "rgb8_512_420_restart7", 512, @@ -490,7 +499,6 @@ fn bench_decode_cpu(c: &mut Criterion) { let mut group = c.benchmark_group("jpeg_cpu_decode_runtime"); for case in &cases { let decoder = Decoder::new(&case.bytes).expect("benchmark JPEG must parse"); - let expected_checksum = case.expected_checksum; group.throughput(Throughput::Elements(case.pixels())); match case.mode { DecodeMode::Buffer(format) => { @@ -501,22 +509,18 @@ fn bench_decode_cpu(c: &mut Criterion) { let outcome = decoder .decode_into(&mut output, stride, format) .expect("benchmark decode must succeed"); - std::hint::black_box(outcome); - let checksum = fnv1a_update(FNV_OFFSET_BASIS, &output); - debug_assert_eq!(checksum, expected_checksum); - std::hint::black_box(checksum); + std::hint::black_box((&output, outcome)); }); }); } DecodeMode::Rows => { group.bench_function(case.name, |b| { b.iter(|| { - let mut sink = ChecksumSink::new(case.width as usize * 3); + let mut sink = BenchSink; let outcome = decoder .decode_rows(&mut sink) .expect("benchmark row decode must succeed"); - debug_assert_eq!(sink.hash, expected_checksum); - std::hint::black_box((outcome, sink.hash)); + std::hint::black_box(outcome); }); }); } diff --git a/crates/j2k-jpeg/fuzz/Cargo.lock b/crates/j2k-jpeg/fuzz/Cargo.lock index b47de3c57..87f4a9415 100644 --- a/crates/j2k-jpeg/fuzz/Cargo.lock +++ b/crates/j2k-jpeg/fuzz/Cargo.lock @@ -59,9 +59,9 @@ checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "fearless_simd" -version = "0.7.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" +checksum = "f3772b63c40606beea8fe1f7b06a60de27ceb9bfc2d9dad3427368ca41dc7e62" [[package]] name = "find-msvc-tools" diff --git a/crates/j2k-jpeg/src/adapter/baseline_encode/tests.rs b/crates/j2k-jpeg/src/adapter/baseline_encode/tests.rs index e15260a59..2293c297c 100644 --- a/crates/j2k-jpeg/src/adapter/baseline_encode/tests.rs +++ b/crates/j2k-jpeg/src/adapter/baseline_encode/tests.rs @@ -14,108 +14,6 @@ use super::{encode_jpeg_baseline_gpu_batch, encode_jpeg_baseline_gpu_tile}; use crate::encoder::{JpegBackend, JpegEncodeError, JpegEncodeOptions, JpegSubsampling}; use crate::PixelFormat; -#[test] -fn baseline_encode_modules_stay_focused_and_fragment_free() { - const ROOT: &str = include_str!("../baseline_encode.rs"); - const ALLOCATION: &str = include_str!("allocation.rs"); - const FRAME: &str = include_str!("frame.rs"); - const ORCHESTRATE: &str = include_str!("orchestrate.rs"); - const ORCHESTRATE_BATCH: &str = include_str!("orchestrate/batch.rs"); - const ORCHESTRATE_BATCH_GROUP: &str = include_str!("orchestrate/batch/group.rs"); - const PLANNING: &str = include_str!("planning.rs"); - const PLANNING_BATCH: &str = include_str!("planning/batch.rs"); - const TABLES: &str = include_str!("tables.rs"); - const TYPES: &str = include_str!("types.rs"); - const VALIDATION: &str = include_str!("validation.rs"); - - let modules = [ - ("baseline_encode.rs", ROOT, 45usize), - ("baseline_encode/allocation.rs", ALLOCATION, 300), - ("baseline_encode/frame.rs", FRAME, 220), - ("baseline_encode/orchestrate.rs", ORCHESTRATE, 190), - ( - "baseline_encode/orchestrate/batch.rs", - ORCHESTRATE_BATCH, - 190, - ), - ( - "baseline_encode/orchestrate/batch/group.rs", - ORCHESTRATE_BATCH_GROUP, - 130, - ), - ("baseline_encode/planning.rs", PLANNING, 240), - ("baseline_encode/planning/batch.rs", PLANNING_BATCH, 120), - ("baseline_encode/tables.rs", TABLES, 260), - ("baseline_encode/types.rs", TYPES, 260), - ("baseline_encode/validation.rs", VALIDATION, 190), - ]; - for (path, source, max_lines) in modules { - let line_count = source.lines().count(); - assert!( - line_count <= max_lines, - "{path} grew to {line_count} lines; split it before exceeding {max_lines}" - ); - assert!( - !source.contains("include!(") && !source.contains("#[path"), - "{path} must remain a real Rust module, not a textual source fragment" - ); - assert!( - !source.contains("pub use ") || !source.contains("::*"), - "{path} must not hide its API behind a wildcard re-export" - ); - } - - for declaration in [ - "mod allocation;", - "mod frame;", - "mod orchestrate;", - "mod planning;", - "mod tables;", - "mod types;", - "mod validation;", - ] { - assert!( - ROOT.contains(declaration), - "baseline_encode facade lost required module boundary {declaration}" - ); - } - assert!( - ORCHESTRATE.contains("mod batch;"), - "GPU baseline orchestration lost its batch execution boundary" - ); -} - -#[test] -fn baseline_encode_error_and_marker_literals_remain_path_owned() { - const FRAME: &str = include_str!("frame.rs"); - const ORCHESTRATE: &str = include_str!("orchestrate.rs"); - const ORCHESTRATE_BATCH: &str = include_str!("orchestrate/batch.rs"); - const ORCHESTRATE_BATCH_GROUP: &str = include_str!("orchestrate/batch/group.rs"); - const PLANNING: &str = include_str!("planning.rs"); - let orchestration = [ORCHESTRATE, ORCHESTRATE_BATCH, ORCHESTRATE_BATCH_GROUP].concat(); - - for marker in ["DQT", "DRI", "SOF0", "DHT", "SOS"] { - assert_eq!( - FRAME.matches(&format!("\"{marker}\"")).count(), - 1, - "{marker} segment name moved or duplicated" - ); - } - for message in [ - "GPU JPEG baseline batch returned the wrong number of entropy chunks", - "GPU JPEG baseline entropy output exceeded its planned capacity", - ] { - assert_eq!(orchestration.matches(message).count(), 1); - } - for message in [ - "JPEG MCU count overflow", - "JPEG entropy capacity overflow", - "JPEG entropy capacity exceeds usize", - ] { - assert_eq!(PLANNING.matches(message).count(), 1); - } -} - fn rgb_tile() -> JpegBaselineGpuEncodeTile { JpegBaselineGpuEncodeTile { byte_offset: 32, diff --git a/crates/j2k-jpeg/src/adapter/device_plan.rs b/crates/j2k-jpeg/src/adapter/device_plan.rs index 05a7471ae..dcb185204 100644 --- a/crates/j2k-jpeg/src/adapter/device_plan.rs +++ b/crates/j2k-jpeg/src/adapter/device_plan.rs @@ -4,6 +4,7 @@ use crate::allocation::{ checked_add_allocation_bytes, checked_allocation_bytes, ensure_allocation_bytes, try_reserve_for_len_with_live_budget, }; +use crate::context::MAX_DECODER_CONTEXT_ALLOCATION_BYTES; use crate::decoder::Decoder; use crate::error::JpegError; use crate::info::{ColorSpace, SofKind}; @@ -108,7 +109,12 @@ pub fn build_device_plan<'a>( cadence_mcus.max(1), restart_interval.map(u32::from), ); - let retained_decoder_bytes = retained_decoder_allocation_bytes(decoder)?; + // The shared decoder context stays live while the plan is built; charge it + // once beside this decoder's own retained bytes. + let retained_decoder_bytes = checked_add_allocation_bytes( + MAX_DECODER_CONTEXT_ALLOCATION_BYTES, + retained_decoder_allocation_bytes(decoder)?, + )?; let output_bytes = device_plan_output_allocation_bytes( expected_checkpoint_count, warning_count, diff --git a/crates/j2k-jpeg/src/adapter/mod.rs b/crates/j2k-jpeg/src/adapter/mod.rs index 5ab3bb045..b41dc5caf 100644 --- a/crates/j2k-jpeg/src/adapter/mod.rs +++ b/crates/j2k-jpeg/src/adapter/mod.rs @@ -48,6 +48,11 @@ pub fn decoder_bytes<'a>(decoder: &'a Decoder<'a>) -> &'a [u8] { /// Return allocator-reported retained bytes for one prepared decoder graph. /// +/// This counts only what the decoder owns. The shared +/// [`DecoderContext`](crate::DecoderContext) that many decoders build through +/// is not included, so summing this over a batch counts each decoder once and +/// the context not at all; decode workspace planning charges the context. +/// /// # Errors /// /// Returns a typed JPEG invariant or allocation-size error. diff --git a/crates/j2k-jpeg/src/baseline_entropy.rs b/crates/j2k-jpeg/src/baseline_entropy.rs index a693e3106..459453717 100644 --- a/crates/j2k-jpeg/src/baseline_entropy.rs +++ b/crates/j2k-jpeg/src/baseline_entropy.rs @@ -10,7 +10,7 @@ use crate::encoded_output::CappedBytes; pub(crate) struct BitWriter { bytes: CappedBytes, - current: u8, + current: u64, used: u8, } @@ -24,27 +24,43 @@ impl BitWriter { } fn write_bits(&mut self, code: u32, len: u8) -> Result<(), JpegEncodeError> { - for bit_idx in (0..len).rev() { - let bit = u8::from(((code >> bit_idx) & 1) != 0); - self.current = (self.current << 1) | bit; - self.used += 1; - if self.used == 8 { - self.push_byte(self.current)?; - self.current = 0; - self.used = 0; - } + debug_assert!(len <= 32); + if self.used + len > 64 { + self.flush_bytes()?; } + self.current = (self.current << len) | (u64::from(code) & ((1u64 << len) - 1)); + self.used += len; Ok(()) } + fn flush_bytes(&mut self) -> Result<(), JpegEncodeError> { + // Eight data bytes plus, at worst, eight JPEG stuffing bytes. Reserve + // output once per word rather than once for every emitted byte. + let mut bytes = [0u8; 16]; + let mut len = 0; + while self.used >= 8 { + self.used -= 8; + let byte = ((self.current >> self.used) & 0xff) as u8; + bytes[len] = byte; + len += 1; + if byte == 0xff { + bytes[len] = 0; + len += 1; + } + } + self.current &= (1u64 << self.used) - 1; + self.bytes.extend_from_slice(&bytes[..len]) + } + fn align_with_ones(&mut self) -> Result<(), JpegEncodeError> { + self.flush_bytes()?; if self.used == 0 { return Ok(()); } let remaining = 8 - self.used; self.current <<= remaining; - self.current |= (1u8 << remaining) - 1; - self.push_byte(self.current)?; + self.current |= (1u64 << remaining) - 1; + self.push_byte(self.current.to_le_bytes()[0])?; self.current = 0; self.used = 0; Ok(()) diff --git a/crates/j2k-jpeg/src/batch_session.rs b/crates/j2k-jpeg/src/batch_session.rs index 62f648362..d9f1fdcfe 100644 --- a/crates/j2k-jpeg/src/batch_session.rs +++ b/crates/j2k-jpeg/src/batch_session.rs @@ -38,10 +38,9 @@ type BatchResultSlot = j2k_core::BatchResultSlot; /// Reusable JPEG tile-batch runtime for WSI viewport loops. /// /// The session keeps one decoder context and scratch pool per active worker -/// during a batch and retains worker slots across calls. Before planning the -/// next batch it may preserve one bounded decoder context, while evicting -/// stale scratch and other contexts so the planning decoder retains the full -/// codec memory allowance. Callers continue to own compressed inputs and +/// across calls. Planning counts retained worker storage against the codec +/// memory allowance and evicts it only when a new request would exceed that +/// allowance. Callers continue to own compressed inputs and /// decoded output buffers. #[derive(Debug)] pub struct JpegBatchSession { @@ -144,17 +143,22 @@ impl JpegBatchSession { } let job_count = jobs.len(); let planning_metadata = self.prepare_job_planning(job_count)?; - let planning_context = self.planning_context()?; - let plans = plan_regular_jobs(jobs, planning_metadata, planning_context, |job, ctx| { - planned_jpeg_tile_decode_live_bytes( - job.input, - ctx, - fmt, - None, - j2k_core::Downscale::None, - decode_options, - ) - })?; + let plans = plan_regular_jobs( + jobs, + planning_metadata, + &mut self.workers, + |job, ctx, external_live_bytes| { + planned_jpeg_tile_decode_live_bytes( + job.input, + ctx, + fmt, + None, + j2k_core::Downscale::None, + decode_options, + external_live_bytes, + ) + }, + )?; let batch = self.prepare_batch::( &plans, vec_capacity_bytes(&plans)?, @@ -202,17 +206,22 @@ impl JpegBatchSession { } let job_count = jobs.len(); let planning_metadata = self.prepare_job_planning(job_count)?; - let planning_context = self.planning_context()?; - let plans = plan_per_tile_jobs(jobs, planning_metadata, planning_context, |job, ctx| { - planned_jpeg_tile_decode_live_bytes( - job.input.as_bytes(), - ctx, - PixelFormat::Rgb8, - None, - j2k_core::Downscale::None, - job.options, - ) - })?; + let plans = plan_per_tile_jobs( + jobs, + planning_metadata, + &mut self.workers, + |job, ctx, external_live_bytes| { + planned_jpeg_tile_decode_live_bytes( + job.input.as_bytes(), + ctx, + PixelFormat::Rgb8, + None, + j2k_core::Downscale::None, + job.options, + external_live_bytes, + ) + }, + )?; let batch = self.prepare_batch::>( &plans, vec_capacity_bytes(&plans)?, @@ -271,17 +280,22 @@ impl JpegBatchSession { } let job_count = jobs.len(); let planning_metadata = self.prepare_job_planning(job_count)?; - let planning_context = self.planning_context()?; - let plans = plan_regular_jobs(jobs, planning_metadata, planning_context, |job, ctx| { - planned_jpeg_tile_decode_live_bytes( - job.input, - ctx, - fmt, - None, - job.scale, - decode_options, - ) - })?; + let plans = plan_regular_jobs( + jobs, + planning_metadata, + &mut self.workers, + |job, ctx, external_live_bytes| { + planned_jpeg_tile_decode_live_bytes( + job.input, + ctx, + fmt, + None, + job.scale, + decode_options, + external_live_bytes, + ) + }, + )?; let batch = self.prepare_batch::( &plans, vec_capacity_bytes(&plans)?, @@ -346,17 +360,22 @@ impl JpegBatchSession { } let job_count = jobs.len(); let planning_metadata = self.prepare_job_planning(job_count)?; - let planning_context = self.planning_context()?; - let plans = plan_regular_jobs(jobs, planning_metadata, planning_context, |job, ctx| { - planned_jpeg_tile_decode_live_bytes( - job.input, - ctx, - fmt, - Some(job.roi.into()), - job.scale, - decode_options, - ) - })?; + let plans = plan_regular_jobs( + jobs, + planning_metadata, + &mut self.workers, + |job, ctx, external_live_bytes| { + planned_jpeg_tile_decode_live_bytes( + job.input, + ctx, + fmt, + Some(job.roi.into()), + job.scale, + decode_options, + external_live_bytes, + ) + }, + )?; let batch = self.prepare_batch::( &plans, vec_capacity_bytes(&plans)?, diff --git a/crates/j2k-jpeg/src/batch_session/planning.rs b/crates/j2k-jpeg/src/batch_session/planning.rs index 6f33be16f..11247f853 100644 --- a/crates/j2k-jpeg/src/batch_session/planning.rs +++ b/crates/j2k-jpeg/src/batch_session/planning.rs @@ -1,15 +1,18 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use alloc::vec::Vec; +use std::sync::Mutex; use j2k_core::{ - BatchDecodeError, BatchInfrastructureError, TileBatchError as IndexedTileBatchError, + BatchDecodeError, BatchInfrastructureError, CodecContext, + TileBatchError as IndexedTileBatchError, }; use super::allocation::{ ensure_metadata_bytes, ensure_planning_phase, try_vec_with_retained_metadata, vec_capacity_bytes, PlannedJob, }; +use super::worker::WorkerSlot; use crate::context::DecoderContext; use crate::decoder::{ PlannedJpegTileDecode, PreparedJpegTileJob, TileBatchError, TileDecodeJob, @@ -17,6 +20,52 @@ use crate::decoder::{ }; use crate::error::JpegError; +fn plan_job( + job: &T, + workers: &mut [Mutex], + owned_context: &mut DecoderContext, + planner: &mut impl FnMut(&T, &mut DecoderContext, usize) -> Result, +) -> Result, BatchInfrastructureError> { + let mut retained = owned_context.retained_allocation_bytes(); + for slot in &mut *workers { + retained = retained.saturating_add( + slot.get_mut() + .map_err(|_| BatchInfrastructureError::SchedulerPoisoned)? + .retained_bytes(), + ); + } + let context = match workers.first_mut() { + Some(slot) => slot + .get_mut() + .map_err(|_| BatchInfrastructureError::SchedulerPoisoned)? + .planning_context(), + None => &mut *owned_context, + }; + let external_live_bytes = retained - context.retained_allocation_bytes(); + let result = planner(job, context, external_live_bytes); + if retained != 0 && matches!(result, Err(JpegError::MemoryCapExceeded { .. })) { + // Parsing and construction charge all warm storage to the same codec + // cap. Retry without it only when that retained storage prevents a fit. + for slot in &mut *workers { + slot.get_mut() + .map_err(|_| BatchInfrastructureError::SchedulerPoisoned)? + .release_allocations(); + } + owned_context.clear(); + return Ok(match workers.first_mut() { + Some(slot) => planner( + job, + slot.get_mut() + .map_err(|_| BatchInfrastructureError::SchedulerPoisoned)? + .planning_context(), + 0, + ), + None => planner(job, owned_context, 0), + }); + } + Ok(result) +} + pub(super) trait BatchJobOutput { fn out_len(&self) -> usize; } @@ -65,8 +114,8 @@ pub(super) fn planned_job_chunk( pub(super) fn plan_regular_jobs( jobs: &[T], retained_metadata_bytes: usize, - context: Option<&mut DecoderContext>, - mut planner: impl FnMut(&T, &mut DecoderContext) -> Result, + workers: &mut [Mutex], + mut planner: impl FnMut(&T, &mut DecoderContext, usize) -> Result, ) -> Result, TileBatchError> { let mut plans = try_vec_with_retained_metadata( jobs.len(), @@ -80,9 +129,8 @@ pub(super) fn plan_regular_jobs( )?; ensure_planning_phase(planning_metadata)?; let mut owned_context = DecoderContext::new(); - let context = context.unwrap_or(&mut owned_context); for (index, job) in jobs.iter().enumerate() { - match planner(job, context) { + match plan_job(job, workers, &mut owned_context, &mut planner)? { Ok(plan) => plans.push(planned_job(plan)), Err(source) => { return Err(BatchDecodeError::Tile(IndexedTileBatchError { @@ -98,8 +146,8 @@ pub(super) fn plan_regular_jobs( pub(super) fn plan_per_tile_jobs( jobs: &[T], retained_metadata_bytes: usize, - context: Option<&mut DecoderContext>, - mut planner: impl FnMut(&T, &mut DecoderContext) -> Result, + workers: &mut [Mutex], + mut planner: impl FnMut(&T, &mut DecoderContext, usize) -> Result, ) -> Result, BatchInfrastructureError> { let mut plans = try_vec_with_retained_metadata( jobs.len(), @@ -113,9 +161,8 @@ pub(super) fn plan_per_tile_jobs( )?; ensure_planning_phase(planning_metadata)?; let mut owned_context = DecoderContext::new(); - let context = context.unwrap_or(&mut owned_context); for job in jobs { - match planner(job, context) { + match plan_job(job, workers, &mut owned_context, &mut planner)? { Ok(plan) => plans.push(planned_job(plan)), Err(error) => plans.push(PlannedJob::Reject(error)), } diff --git a/crates/j2k-jpeg/src/batch_session/runtime.rs b/crates/j2k-jpeg/src/batch_session/runtime.rs index fa07b8428..dd7a91a25 100644 --- a/crates/j2k-jpeg/src/batch_session/runtime.rs +++ b/crates/j2k-jpeg/src/batch_session/runtime.rs @@ -16,7 +16,6 @@ use super::{ available_tile_batch_workers, BatchResultSlot, JpegBatchSession, SMALL_OUTPUT_BYTES, SMALL_OUTPUT_DEFAULT_WORKER_CAP, }; -use crate::context::DecoderContext; impl JpegBatchSession { pub(super) fn prepare_job_planning( @@ -31,21 +30,6 @@ impl JpegBatchSession { }, )?; - // One existing context may participate in the planning decoder's - // authoritative codec claim. All scratch and every other context are - // evicted before parsing so no stale codec owner sits beside that full - // claim. Worker-slot descriptors remain metadata and are charged below. - for (index, slot) in self.workers.iter_mut().enumerate() { - let worker = slot - .get_mut() - .map_err(|_| BatchInfrastructureError::SchedulerPoisoned)?; - if index == 0 { - worker.prepare_for_planning(); - } else { - worker.release_allocations(); - } - } - let mut worker_metadata_bytes = vec_capacity_bytes(&self.workers)?; let combined = match ensure_metadata_bytes( worker_metadata_bytes, @@ -68,19 +52,6 @@ impl JpegBatchSession { Ok(worker_metadata_bytes) } - pub(super) fn planning_context( - &mut self, - ) -> Result, BatchInfrastructureError> { - self.workers - .first_mut() - .map(|slot| { - slot.get_mut() - .map(WorkerSlot::planning_context) - .map_err(|_| BatchInfrastructureError::SchedulerPoisoned) - }) - .transpose() - } - pub(super) fn prepare_batch( &mut self, plans: &[PlannedJob], diff --git a/crates/j2k-jpeg/src/batch_session/tests.rs b/crates/j2k-jpeg/src/batch_session/tests.rs index 25b950fee..5fc406aa6 100644 --- a/crates/j2k-jpeg/src/batch_session/tests.rs +++ b/crates/j2k-jpeg/src/batch_session/tests.rs @@ -4,6 +4,45 @@ use super::*; use crate::decoder::Decoder; use j2k_test_support::JPEG_BASELINE_420_16X16; +#[test] +fn repeated_batches_reuse_each_workers_decode_cache() { + use j2k_core::CodecContext; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + let mut outputs = [vec![0u8; 16 * 16 * 3], vec![0u8; 16 * 16 * 3]]; + let mut previous_hits = [0; 2]; + for iteration in 0..3 { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: JPEG_BASELINE_420_16X16, + out, + stride: 16 * 3, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .unwrap(); + assert_eq!(outputs[0], outputs[1]); + for (slot, previous) in session.workers.iter_mut().zip(&mut previous_hits) { + let hits = slot + .get_mut() + .unwrap() + .planning_context() + .cache_stats() + .hits; + if iteration != 0 { + assert!( + hits > *previous, + "a warm worker must reuse its cached tables" + ); + } + *previous = hits; + } + } +} + #[test] fn one_shot_session_caps_default_workers_for_small_outputs() { const JOBS: usize = 64; diff --git a/crates/j2k-jpeg/src/batch_session/worker.rs b/crates/j2k-jpeg/src/batch_session/worker.rs index dae042cb1..81c74fe9d 100644 --- a/crates/j2k-jpeg/src/batch_session/worker.rs +++ b/crates/j2k-jpeg/src/batch_session/worker.rs @@ -164,12 +164,6 @@ impl WorkerSlot { self.pool = ScratchPool::default(); } - pub(super) fn prepare_for_planning(&mut self) { - // Planning reuses one bounded decoder context, but no stale decode - // scratch may coexist with the planning decoder's full codec claim. - self.pool = ScratchPool::default(); - } - pub(super) fn planning_context(&mut self) -> &mut DecoderContext { &mut self.ctx } diff --git a/crates/j2k-jpeg/src/color/upsample.rs b/crates/j2k-jpeg/src/color/upsample.rs index c0e6feeb7..3641a7396 100644 --- a/crates/j2k-jpeg/src/color/upsample.rs +++ b/crates/j2k-jpeg/src/color/upsample.rs @@ -135,6 +135,33 @@ pub(crate) fn upsample_h2v2_fancy_rows( emit_h2v2_row(next, curr, output_width, out_bot); } +/// Emit one visible output row of a 4:4:0 (h1v2) component, as libjpeg-turbo's +/// `h1v2_fancy_upsample` does: three quarters of the nearest input row plus +/// one quarter of the next nearest, which is the row above for the top output +/// row of a pair (rounding bias 1) and the row below for the bottom (bias 2). +pub(crate) fn upsample_h1v2_fancy_row( + prev: &[u8], + curr: &[u8], + next: &[u8], + output_width: usize, + output_is_bottom: bool, + out: &mut [u8], +) { + let (far, bias) = if output_is_bottom { + (next, 2) + } else { + (prev, 1) + }; + for ((slot, &near), &far) in out[..output_width] + .iter_mut() + .zip(&curr[..output_width]) + .zip(&far[..output_width]) + { + let sum = 3 * u16::from(near) + u16::from(far) + bias; + *slot = u8::try_from(sum >> 2).expect("weighted mean of u8 samples fits u8"); + } +} + /// Emit one visible output row from a 4:2:0 fancy-upsampled chroma triple. pub(crate) fn upsample_h2v2_fancy_row( prev: &[u8], diff --git a/crates/j2k-jpeg/src/color/ycbcr.rs b/crates/j2k-jpeg/src/color/ycbcr.rs index bf03f8de4..d35ff1656 100644 --- a/crates/j2k-jpeg/src/color/ycbcr.rs +++ b/crates/j2k-jpeg/src/color/ycbcr.rs @@ -79,12 +79,16 @@ pub(crate) fn ycbcr_to_rgb(y: u8, cb: u8, cr: u8) -> (u8, u8, u8) { /// /// Returned values are clamped to the native 12-bit range `[0, 4095]`, not /// scaled to the full `u16` range. +/// +/// libjpeg-turbo has no SIMD converter for 12-bit samples, so this follows the +/// C tables in `jdcolor.c`: green adds the rounded *negated* chroma terms, +/// which differs from `ycbcr_to_rgb`'s SIMD-matching form on exact ties. pub(crate) fn ycbcr12_to_rgb16(y: u16, cb: u16, cr: u16) -> (u16, u16, u16) { let y = i32::from(y); let cb_centered = i32::from(cb) - 2048; let cr_centered = i32::from(cr) - 2048; let r = y + ((FIX_1_40200 * cr_centered + ROUND) >> 16); - let g = y - ((FIX_0_34414 * cb_centered + FIX_0_71414 * cr_centered + ROUND) >> 16); + let g = y + ((-FIX_0_34414 * cb_centered - FIX_0_71414 * cr_centered + ROUND) >> 16); let b = y + ((FIX_1_77200 * cb_centered + ROUND) >> 16); (clamp_to_12bit(r), clamp_to_12bit(g), clamp_to_12bit(b)) diff --git a/crates/j2k-jpeg/src/context.rs b/crates/j2k-jpeg/src/context.rs index a9bb4b8f2..8b19a9801 100644 --- a/crates/j2k-jpeg/src/context.rs +++ b/crates/j2k-jpeg/src/context.rs @@ -877,12 +877,13 @@ mod tests { #[test] fn decode_plan_cache_entry_boundary_bypasses_oversized_keys() { let plan = empty_plan(0); + let max_key_bytes = MAX_DECODE_PLAN_CACHE_BYTES - plan.retained_allocation_bytes().unwrap(); assert_eq!( - decode_plan_entry_bytes(MAX_DECODE_PLAN_CACHE_BYTES, &plan).unwrap(), + decode_plan_entry_bytes(max_key_bytes, &plan).unwrap(), MAX_DECODE_PLAN_CACHE_BYTES ); assert!( - decode_plan_entry_bytes(MAX_DECODE_PLAN_CACHE_BYTES + 1, &plan).unwrap() + decode_plan_entry_bytes(max_key_bytes + 1, &plan).unwrap() > MAX_DECODE_PLAN_CACHE_BYTES ); } diff --git a/crates/j2k-jpeg/src/decoder.rs b/crates/j2k-jpeg/src/decoder.rs index f2707198c..db6ccb655 100644 --- a/crates/j2k-jpeg/src/decoder.rs +++ b/crates/j2k-jpeg/src/decoder.rs @@ -5,7 +5,7 @@ use crate::backend::Backend; use crate::context::DecoderContext; use crate::entropy::block::{decode_block_with_activity, BlockActivity, CoefficientBlock}; -use crate::entropy::huffman::{HuffmanTable, PreparedHuffmanTableId, PreparedHuffmanTables}; +use crate::entropy::huffman::{PreparedHuffmanTableId, PreparedHuffmanTables}; use crate::entropy::progressive::{ decode_progressive, decode_progressive_dct_blocks, PreparedProgressiveComponentPlan, PreparedProgressivePlan, PreparedProgressiveScan, PreparedProgressiveScanComponent, @@ -447,6 +447,15 @@ impl<'a> Decoder<'a> { view: JpegView<'a>, ctx: &mut DecoderContext, operation: impl FnOnce(&Self) -> Result, + ) -> Result { + Self::with_view_in_context_with_external_live(view, ctx, 0, operation) + } + + pub(super) fn with_view_in_context_with_external_live( + view: JpegView<'a>, + ctx: &mut DecoderContext, + external_live_bytes: usize, + operation: impl FnOnce(&Self) -> Result, ) -> Result { let cache_prefix = if view.options == DecodeOptions::default() && matches!( @@ -460,14 +469,16 @@ impl<'a> Decoder<'a> { None }; let Some(cache_prefix) = cache_prefix else { - let decoder = Self::from_view_in_context(view, ctx)?; + let decoder = + Self::from_view_in_context_with_external_live(view, ctx, external_live_bytes)?; return operation(&decoder); }; let Some(cached_plan) = ctx.cached_decode_plan(cache_prefix) else { - let decoder = Self::from_view_in_context(view, ctx)?; + let decoder = + Self::from_view_in_context_with_external_live(view, ctx, external_live_bytes)?; return operation(&decoder); }; - Self::validate_cached_plan_lease(&view.header, ctx, cached_plan, 0)?; + Self::validate_cached_plan_lease(&view.header, ctx, cached_plan, external_live_bytes)?; let Some((lease, plan)) = ctx.lease_cached_decode_plan(cache_prefix)? else { return Err(JpegError::InternalInvariant { reason: "validated cached decode plan disappeared before execution", diff --git a/crates/j2k-jpeg/src/decoder/allocation.rs b/crates/j2k-jpeg/src/decoder/allocation.rs index 9de79b22c..33e9076bf 100644 --- a/crates/j2k-jpeg/src/decoder/allocation.rs +++ b/crates/j2k-jpeg/src/decoder/allocation.rs @@ -12,20 +12,27 @@ use crate::error::Warning; use super::warning_ownership::warning_merge_peak_bytes; use super::{ - checked_usize_product, Decoder, HuffmanTable, JpegError, ParsedHeader, PreparedDecodePlan, - PreparedProgressiveComponentPlan, PreparedProgressiveScan, PreparedProgressiveScanComponent, - SofKind, COMPONENT_IMAGE_METADATA_BYTES, DEFAULT_MAX_DECODE_BYTES, + checked_usize_product, Decoder, JpegError, ParsedHeader, PreparedDecodePlan, + PreparedHuffmanTables, PreparedProgressiveComponentPlan, PreparedProgressiveScan, + PreparedProgressiveScanComponent, SofKind, COMPONENT_IMAGE_METADATA_BYTES, + DEFAULT_MAX_DECODE_BYTES, }; impl Decoder<'_> { - /// Exact retained host bytes that remain live while the CPU checkpoint - /// cache grows. The cache itself is deliberately excluded so replacement - /// growth can count the old and new cache capacities exactly once. + /// Exact retained host bytes this decoder owns that remain live while the + /// CPU checkpoint cache grows. The cache itself is deliberately excluded so + /// replacement growth can count the old and new cache capacities exactly + /// once. + /// + /// The shared [`DecoderContext`](crate::DecoderContext) reserve, + /// `MAX_DECODER_CONTEXT_ALLOCATION_BYTES`, is not owned by any one decoder: + /// many decoders share one context. Each budget with a live context + /// charges that reserve once (decode workspace planning, checkpoint + /// growth, device planning), so summing decoders never multiplies it. pub(crate) fn retained_allocation_bytes_excluding_cpu_checkpoint_cache( &self, ) -> Result { - let mut total = MAX_DECODER_CONTEXT_ALLOCATION_BYTES; - total = checked_add_allocation_bytes(total, self.plan.retained_allocation_bytes()?)?; + let mut total = self.plan.retained_allocation_bytes()?; if let Some(progressive) = &self.progressive_plan { total = checked_add_allocation_bytes(total, progressive.retained_allocation_bytes()?)?; } @@ -90,7 +97,7 @@ pub(super) fn progressive_prepared_allocation_bytes( )?; total = checked_add_allocation_bytes( total, - checked_allocation_bytes::(huffman_table_count)?, + PreparedHuffmanTables::allocation_bytes_for_capacity(huffman_table_count)?, )?; total = checked_add_allocation_bytes( total, @@ -241,7 +248,9 @@ mod tests { let huffman_tables = 7usize; let expected = components * size_of::() + scan_components * size_of::() - + huffman_tables * size_of::() + + huffman_tables * size_of::() + + 2 * (size_of::>() + + 2 * size_of::()) + scans * size_of::() + components * size_of::(); assert_eq!( @@ -257,8 +266,11 @@ mod tests { } #[test] - fn prepared_decode_formula_counts_one_inline_arena() { - let expected = size_of::() + 2 * size_of::(); + fn prepared_decode_formula_counts_shared_arena_and_owner() { + let expected = size_of::() + + 2 * size_of::() + + size_of::>() + + 2 * size_of::(); assert_eq!( prepared_decode_plan_allocation_bytes(1, 2).unwrap(), expected @@ -271,6 +283,8 @@ mod tests { let before = decoder .retained_allocation_bytes_excluding_cpu_checkpoint_cache() .expect("bounded decoder baseline"); + let owned_before = crate::adapter::decoder_retained_allocation_bytes(&decoder) + .expect("empty-cache decoder ownership"); let workspace_before = decoder .decode_workspace_cap() .expect("empty-cache workspace cap"); @@ -290,8 +304,11 @@ mod tests { let after = decoder .retained_allocation_bytes_excluding_cpu_checkpoint_cache() .expect("bounded decoder baseline"); + let owned_after = crate::adapter::decoder_retained_allocation_bytes(&decoder) + .expect("populated-cache decoder ownership"); assert_eq!(before, after); + assert_eq!(owned_after, owned_before + retained_checkpoint_bytes); assert_eq!( decoder .decode_workspace_cap() diff --git a/crates/j2k-jpeg/src/decoder/extended12/tests.rs b/crates/j2k-jpeg/src/decoder/extended12/tests.rs index a3e2f8933..7be5e8772 100644 --- a/crates/j2k-jpeg/src/decoder/extended12/tests.rs +++ b/crates/j2k-jpeg/src/decoder/extended12/tests.rs @@ -2,133 +2,6 @@ use super::{upsample_h2v1_sample_at, upsample_h2v2_rows_at}; -#[test] -fn extended12_decode_modules_stay_focused_and_fragment_free() { - const ROOT: &str = include_str!("../extended12.rs"); - const PLANES: &str = include_str!("planes.rs"); - const PLANE_ALLOCATION: &str = include_str!("planes/allocation.rs"); - const PROGRESSIVE: &str = include_str!("progressive.rs"); - const PROGRESSIVE_444: &str = include_str!("progressive/color444.rs"); - const PROGRESSIVE_SUBSAMPLED: &str = include_str!("progressive/subsampled.rs"); - const PROGRESSIVE_FOUR: &str = include_str!("progressive/four_component.rs"); - const RGBA: &str = include_str!("rgba.rs"); - const SAMPLING: &str = include_str!("sampling.rs"); - const SEQUENTIAL: &str = include_str!("sequential.rs"); - const SEQUENTIAL_444: &str = include_str!("sequential/color444.rs"); - const SEQUENTIAL_SUBSAMPLED: &str = include_str!("sequential/subsampled.rs"); - const SEQUENTIAL_FOUR: &str = include_str!("sequential/four_component.rs"); - const STATE: &str = include_str!("state.rs"); - const UPSAMPLE: &str = include_str!("upsample.rs"); - const WRITERS: &str = include_str!("writers.rs"); - - let modules = [ - ("extended12.rs", ROOT, 40usize), - ("extended12/planes.rs", PLANES, 400), - ("extended12/planes/allocation.rs", PLANE_ALLOCATION, 320), - ("extended12/progressive.rs", PROGRESSIVE, 210), - ("extended12/progressive/color444.rs", PROGRESSIVE_444, 120), - ( - "extended12/progressive/subsampled.rs", - PROGRESSIVE_SUBSAMPLED, - 100, - ), - ( - "extended12/progressive/four_component.rs", - PROGRESSIVE_FOUR, - 80, - ), - ("extended12/rgba.rs", RGBA, 70), - ("extended12/sampling.rs", SAMPLING, 280), - ("extended12/sequential.rs", SEQUENTIAL, 260), - ("extended12/sequential/color444.rs", SEQUENTIAL_444, 120), - ( - "extended12/sequential/subsampled.rs", - SEQUENTIAL_SUBSAMPLED, - 100, - ), - ( - "extended12/sequential/four_component.rs", - SEQUENTIAL_FOUR, - 180, - ), - ("extended12/state.rs", STATE, 70), - ("extended12/upsample.rs", UPSAMPLE, 170), - ("extended12/writers.rs", WRITERS, 380), - ]; - - for (path, source, max_lines) in modules { - let line_count = source.lines().count(); - assert!( - line_count <= max_lines, - "{path} grew to {line_count} lines; split it before exceeding {max_lines}" - ); - assert!( - !source.contains("include!(") && !source.contains("#[path"), - "{path} must remain a real Rust module, not a textual source fragment" - ); - } - - for declaration in [ - "mod planes;", - "mod progressive;", - "mod rgba;", - "mod sampling;", - "mod sequential;", - "mod state;", - "mod upsample;", - "mod writers;", - ] { - assert!( - ROOT.contains(declaration), - "extended12 facade lost required module boundary {declaration}" - ); - } - for route in [PROGRESSIVE, SEQUENTIAL] { - for declaration in ["mod color444;", "mod four_component;", "mod subsampled;"] { - assert!( - route.contains(declaration), - "extended12 route lost required path boundary {declaration}" - ); - } - } - assert!( - PLANES.contains("mod allocation;"), - "extended12 plane orchestration lost its allocation boundary" - ); -} - -#[test] -fn extended12_diagnostic_literals_survive_module_moves_exactly_once_per_path() { - const PROGRESSIVE_SUBSAMPLED: &str = include_str!("progressive/subsampled.rs"); - const SEQUENTIAL_SUBSAMPLED: &str = include_str!("sequential/subsampled.rs"); - const WRITERS: &str = include_str!("writers.rs"); - - assert_eq!( - PROGRESSIVE_SUBSAMPLED - .matches("4:4:4 path is handled directly") - .count(), - 1 - ); - assert_eq!( - SEQUENTIAL_SUBSAMPLED - .matches("4:4:4 path is handled directly") - .count(), - 1 - ); - assert_eq!( - WRITERS - .matches("12-bit four-component path only accepts CMYK/YCCK") - .count(), - 1 - ); - assert_eq!( - WRITERS - .matches("12-bit four-component plane path only accepts CMYK/YCCK") - .count(), - 1 - ); -} - #[test] fn extended12_generic_upsampling_keeps_edge_and_rounding_goldens() { let row_u8 = [0u8, 1, 255]; @@ -150,3 +23,97 @@ fn extended12_generic_upsampling_keeps_edge_and_rounding_goldens() { .collect::>(); assert_eq!(actual_h2v2, [150, 200, 300, 475, 850]); } + +#[test] +fn extended12_420_writers_replicate_last_real_chroma_row_instead_of_padding() { + use super::planes::Extended12Plane; + use super::sampling::Extended12ColorSampling; + use super::writers::{ + write_extended12_color420_planes_region, write_extended12_four_component_planes_region, + Extended12Output, Extended12RgbProjection, Extended12WriteRegion, + }; + use crate::info::{ColorSpace, DownscaleFactor, Rect}; + + // 5x6 image in one 16x16 MCU: chroma rows 0..3 are real, 3..8 padding. + let (width, height) = (5u32, 6u32); + let real_chroma_rows = 3; + let plane = |seed: u16, luma: bool, padding: Option| { + let (stride, rows) = if luma { (16, 16) } else { (8, 8) }; + let mut pixels = vec![0u16; stride * rows]; + for (offset, sample) in pixels.iter_mut().enumerate() { + let (row, col) = (offset / stride, offset % stride); + let row = match padding { + Some(value) if !luma && row >= real_chroma_rows => { + *sample = value; + continue; + } + None if !luma => row.min(real_chroma_rows - 1), + _ => row, + }; + let (row, col) = (u16::try_from(row).unwrap(), u16::try_from(col).unwrap()); + *sample = (seed + row * 397 + col * 211) % 4096; + } + Extended12Plane { + pixels, + stride, + width: if luma { 5 } else { 3 }, + } + }; + let region = Extended12WriteRegion { + output_rect: Rect { + x: 0, + y: 0, + w: width, + h: height, + }, + dimensions: (width, height), + downscale: DownscaleFactor::Full, + output: Extended12Output::Rgb16, + }; + let stride = width as usize * 6; + let color = |padding: Option| { + let planes = [ + plane(100, true, None), + plane(1_900, false, padding), + plane(2_300, false, padding), + ]; + let mut out = vec![0u8; stride * height as usize]; + write_extended12_color420_planes_region( + &mut out, + stride, + region, + Extended12RgbProjection::YCbCr, + &planes, + ); + out + }; + let four_component = |padding: Option| { + let planes = [ + plane(100, true, None), + plane(1_900, false, padding), + plane(2_300, false, padding), + plane(700, false, padding), + ]; + let mut out = vec![0u8; stride * height as usize]; + write_extended12_four_component_planes_region( + &mut out, + stride, + region, + ColorSpace::Ycck, + Extended12ColorSampling::S420, + &planes, + ); + out + }; + + let color_replicated = color(None); + let four_replicated = four_component(None); + for padding in [0u16, 4095, 1234] { + assert_eq!(color(Some(padding)), color_replicated, "padding {padding}"); + assert_eq!( + four_component(Some(padding)), + four_replicated, + "padding {padding}" + ); + } +} diff --git a/crates/j2k-jpeg/src/decoder/extended12/upsample.rs b/crates/j2k-jpeg/src/decoder/extended12/upsample.rs index 178599a6b..f20ea26e6 100644 --- a/crates/j2k-jpeg/src/decoder/extended12/upsample.rs +++ b/crates/j2k-jpeg/src/decoder/extended12/upsample.rs @@ -2,7 +2,6 @@ //! Point sampling and fancy chroma upsampling for extended-precision planes. -use super::super::lossless_helpers::upsample_h2v1_u16_at; use super::planes::Extended12Plane; pub(super) fn sample_extended12_plane_at( @@ -23,18 +22,46 @@ pub(super) fn upsample_extended12_plane_h2v1_at( ) -> u16 { let height = plane.pixels.len() / plane.stride; let y = source_y.min(height - 1); - upsample_h2v1_u16_at(extended12_plane_row(plane, y), source_x) + upsample_extended12_h2v1_at(extended12_plane_row(plane, y), source_x) } +/// Fancy 4:2:2 sample of a DCT-coded 12-bit row, rounded as libjpeg-turbo's +/// `h2v1_fancy_upsample`: +1 toward the left neighbour, +2 toward the right. +/// Lossless decoding keeps `upsample_h2v1_u16_at`. +pub(super) fn upsample_extended12_h2v1_at(row: &[u16], output_x: usize) -> u16 { + debug_assert!(!row.is_empty()); + let last = row.len() - 1; + let sample = (output_x / 2).min(last); + let near = 3 * u32::from(row[sample]); + let value = if output_x.is_multiple_of(2) { + if sample == 0 { + return row[0]; + } + (near + u32::from(row[sample - 1]) + 1) >> 2 + } else { + if sample == last { + return row[last]; + } + (near + u32::from(row[sample + 1]) + 2) >> 2 + }; + u16::try_from(value).expect("weighted mean of 12-bit samples fits u16") +} + +/// Fancy 4:2:0 sample at full-resolution (`source_x`, `source_y`). +/// +/// `chroma_height` is the plane's real sample height, `ceil(image_height / 2)`. +/// The plane itself is padded to whole MCUs; like libjpeg-turbo, the last real +/// row is replicated instead of blending in that padding. pub(super) fn upsample_extended12_plane_h2v2_at( plane: &Extended12Plane, + chroma_height: usize, source_x: usize, source_y: usize, ) -> u16 { - let height = plane.pixels.len() / plane.stride; - let chroma_y = (source_y / 2).min(height - 1); + debug_assert!(chroma_height > 0 && chroma_height <= plane.pixels.len() / plane.stride); + let chroma_y = (source_y / 2).min(chroma_height - 1); let prev_y = chroma_y.saturating_sub(1); - let next_y = (chroma_y + 1).min(height - 1); + let next_y = (chroma_y + 1).min(chroma_height - 1); upsample_h2v2_u16_rows_at( extended12_plane_row(plane, prev_y), extended12_plane_row(plane, chroma_y), diff --git a/crates/j2k-jpeg/src/decoder/extended12/writers.rs b/crates/j2k-jpeg/src/decoder/extended12/writers.rs index 4a63f2f71..b9dd0160c 100644 --- a/crates/j2k-jpeg/src/decoder/extended12/writers.rs +++ b/crates/j2k-jpeg/src/decoder/extended12/writers.rs @@ -2,13 +2,13 @@ //! Scaling- and ROI-aware extended-precision output writers. -use super::super::lossless_helpers::upsample_h2v1_u16_at; use super::super::{ColorSpace, DownscaleFactor, Rect}; use super::planes::Extended12Plane; use super::sampling::Extended12ColorSampling; use super::upsample::{ - extended12_plane_row, sample_extended12_plane_at, upsample_extended12_plane_h2v1_at, - upsample_extended12_plane_h2v2_at, upsample_h2v2_u16_rows_at, + extended12_plane_row, sample_extended12_plane_at, upsample_extended12_h2v1_at, + upsample_extended12_plane_h2v1_at, upsample_extended12_plane_h2v2_at, + upsample_h2v2_u16_rows_at, }; #[derive(Debug, Clone, Copy)] @@ -155,8 +155,8 @@ pub(super) fn write_extended12_color422_planes_region( [chroma_y * planes[1].stride..chroma_y * planes[1].stride + planes[1].width]; let cr_row = &planes[2].pixels [chroma_y * planes[2].stride..chroma_y * planes[2].stride + planes[2].width]; - let c1 = upsample_h2v1_u16_at(cb_row, source_x); - let c2 = upsample_h2v1_u16_at(cr_row, source_x); + let c1 = upsample_extended12_h2v1_at(cb_row, source_x); + let c2 = upsample_extended12_h2v1_at(cr_row, source_x); let (r, g, b) = match projection { Extended12RgbProjection::Identity => (y, c1, c2), Extended12RgbProjection::YCbCr => crate::color::ycbcr::ycbcr12_to_rgb16(y, c1, c2), @@ -181,13 +181,14 @@ pub(super) fn write_extended12_color420_planes_region( let (width, height) = region.dimensions; let denom = region.downscale.denominator(); let output_rect = region.output_rect; + // Real chroma rows; the planes below them hold MCU padding. + let chroma_height = (height as usize).div_ceil(2); for output_y in output_rect.y..output_rect.y + output_rect.h { let source_y = output_y.saturating_mul(denom).min(height - 1) as usize; let dst_row = (output_y - output_rect.y) as usize; for output_x in output_rect.x..output_rect.x + output_rect.w { let source_x = output_x.saturating_mul(denom).min(width - 1) as usize; let y = planes[0].pixels[source_y * planes[0].stride + source_x]; - let chroma_height = planes[1].pixels.len() / planes[1].stride; let chroma_y = (source_y / 2).min(chroma_height - 1); let prev_y = chroma_y.saturating_sub(1); let next_y = (chroma_y + 1).min(chroma_height - 1); @@ -230,6 +231,8 @@ pub(super) fn write_extended12_four_component_planes_region( let (width, height) = region.dimensions; let denom = region.downscale.denominator(); let output_rect = region.output_rect; + // Real 4:2:0 chroma rows; the planes below them hold MCU padding. + let chroma_height = (height as usize).div_ceil(2); for output_y in output_rect.y..output_rect.y + output_rect.h { let source_y = output_y.saturating_mul(denom).min(height - 1) as usize; let dst_row = (output_y - output_rect.y) as usize; @@ -248,9 +251,24 @@ pub(super) fn write_extended12_four_component_planes_region( upsample_extended12_plane_h2v1_at(&planes[3], source_x, source_y), ), Extended12ColorSampling::S420 => ( - upsample_extended12_plane_h2v2_at(&planes[1], source_x, source_y), - upsample_extended12_plane_h2v2_at(&planes[2], source_x, source_y), - upsample_extended12_plane_h2v2_at(&planes[3], source_x, source_y), + upsample_extended12_plane_h2v2_at( + &planes[1], + chroma_height, + source_x, + source_y, + ), + upsample_extended12_plane_h2v2_at( + &planes[2], + chroma_height, + source_x, + source_y, + ), + upsample_extended12_plane_h2v2_at( + &planes[3], + chroma_height, + source_x, + source_y, + ), ), }; let (r, g, b) = match color_space { diff --git a/crates/j2k-jpeg/src/decoder/plan.rs b/crates/j2k-jpeg/src/decoder/plan.rs index cef7cd11b..f6bb2c7a9 100644 --- a/crates/j2k-jpeg/src/decoder/plan.rs +++ b/crates/j2k-jpeg/src/decoder/plan.rs @@ -79,10 +79,14 @@ impl Decoder<'_> { .ok_or(JpegError::InternalInvariant { reason: "parsed SOS offset is outside the JPEG input", })?; - let plan = - ctx.resolve_decode_plan(header_prefix, retained_parsed_bytes, |ctx| { - Self::build_prepared_plan(&header, &info, ctx, &mut construction) - })?; + let plan = ctx.resolve_decode_plan( + header_prefix, + crate::allocation::checked_add_allocation_bytes( + external_live_bytes, + retained_parsed_bytes, + )?, + |ctx| Self::build_prepared_plan(&header, &info, ctx, &mut construction), + )?; construction.rebase_after_plan_cache( ctx.retained_allocation_bytes(), plan.retained_allocation_bytes()?, diff --git a/crates/j2k-jpeg/src/decoder/routing/dispatch.rs b/crates/j2k-jpeg/src/decoder/routing/dispatch.rs index ee2e13c99..616fd0942 100644 --- a/crates/j2k-jpeg/src/decoder/routing/dispatch.rs +++ b/crates/j2k-jpeg/src/decoder/routing/dispatch.rs @@ -45,7 +45,7 @@ impl Decoder<'_> { OutputFormat::Rgba8 { alpha } | OutputFormat::Rgba8Scaled { alpha, .. } => { let mut writer = Rgba8Writer::new_with_backend(out, stride, output_rect.w, alpha, self.backend); - self.decode_with_writer(pool, &mut writer, downscale, source_roi) + self.decode_rgb_with_writer(pool, &mut writer, downscale, source_roi) } OutputFormat::Gray8 | OutputFormat::Gray8Scaled { .. } => { let mut writer = Gray8Writer::new(out, stride, output_rect.w); diff --git a/crates/j2k-jpeg/src/decoder/sequential.rs b/crates/j2k-jpeg/src/decoder/sequential.rs index ede0b848a..a9ed2f27a 100644 --- a/crates/j2k-jpeg/src/decoder/sequential.rs +++ b/crates/j2k-jpeg/src/decoder/sequential.rs @@ -3,6 +3,7 @@ //! Sequential and progressive writer-based decode paths. use crate::allocation::checked_add_allocation_bytes; +use crate::context::MAX_DECODER_CONTEXT_ALLOCATION_BYTES; use super::{ checkpoint_before_mcu, decode_progressive, decode_scan_baseline, decode_scan_baseline_rgb, @@ -93,8 +94,13 @@ impl Decoder<'_> { return Ok(None); } + // The shared decoder context stays live beside the decode, charged + // once as in `decode_workspace_cap`. let retained_decoder_baseline_bytes = checked_add_allocation_bytes( - self.retained_allocation_bytes_excluding_cpu_checkpoint_cache()?, + checked_add_allocation_bytes( + MAX_DECODER_CONTEXT_ALLOCATION_BYTES, + self.retained_allocation_bytes_excluding_cpu_checkpoint_cache()?, + )?, external_decode_phase_bytes, )?; let mut cache = diff --git a/crates/j2k-jpeg/src/decoder/tests.rs b/crates/j2k-jpeg/src/decoder/tests.rs index b05201c68..2491ee6a8 100644 --- a/crates/j2k-jpeg/src/decoder/tests.rs +++ b/crates/j2k-jpeg/src/decoder/tests.rs @@ -445,6 +445,51 @@ fn large_fast_420_region_decode_populates_cpu_entropy_checkpoints() { .any(|checkpoint| checkpoint.mcu_index >= CPU_ROI_CHECKPOINT_MIN_TARGET_MCUS)); } +#[test] +fn decoder_retained_bytes_are_owned_metadata_without_the_context_reserve() { + use crate::context::MAX_DECODER_CONTEXT_ALLOCATION_BYTES; + + for bytes in [dc_only_420_jpeg(16, 16), dc_only_420_jpeg(1024, 2048)] { + let dec = Decoder::new(&bytes).expect("decoder"); + let expected = dec.plan.retained_allocation_bytes().expect("plan bytes") + + dec.warnings.capacity() * core::mem::size_of::(); + let retained = dec + .retained_allocation_bytes_excluding_cpu_checkpoint_cache() + .expect("retained bytes"); + assert_eq!(retained, expected); + assert!(retained < MAX_DECODER_CONTEXT_ALLOCATION_BYTES / 64); + } +} + +#[test] +fn cpu_checkpoint_budget_charges_the_shared_context_reserve_once() { + use crate::context::MAX_DECODER_CONTEXT_ALLOCATION_BYTES; + use j2k_core::DEFAULT_MAX_HOST_ALLOCATION_BYTES as CAP; + + let bytes = dc_only_420_jpeg(1024, 2048); + let target_mcu = CPU_ROI_CHECKPOINT_MIN_TARGET_MCUS + 5; + let headroom = 64 * 1024; + let decode_phase_bytes = |reserve_room: usize| { + let dec = Decoder::new(&bytes).expect("decoder"); + let retained = dec + .retained_allocation_bytes_excluding_cpu_checkpoint_cache() + .expect("retained bytes"); + let external = CAP - reserve_room - retained - headroom; + let scan_bytes = &bytes[dec.plan.scan_offset..]; + dec.checkpoint_for_mcu(scan_bytes, target_mcu, external) + }; + + // Room for one reserve beside the decode phase: the cache may grow. + assert!(decode_phase_bytes(MAX_DECODER_CONTEXT_ALLOCATION_BYTES) + .expect("one context reserve fits") + .is_some()); + // No room for the reserve: growth must fail with a typed cap error. + assert!(matches!( + decode_phase_bytes(0), + Err(JpegError::MemoryCapExceeded { cap: CAP, .. }) + )); +} + #[derive(Default)] struct GrayRows { rows: Vec<(u32, Vec)>, diff --git a/crates/j2k-jpeg/src/decoder/tile.rs b/crates/j2k-jpeg/src/decoder/tile.rs index d6c806db1..ad3c99f68 100644 --- a/crates/j2k-jpeg/src/decoder/tile.rs +++ b/crates/j2k-jpeg/src/decoder/tile.rs @@ -128,9 +128,12 @@ pub(crate) fn planned_jpeg_tile_decode_live_bytes( roi: Option, scale: Downscale, options: DecodeOptions, + external_live_bytes: usize, ) -> Result { - let view = JpegView::parse_with_options(input, options)?; - Decoder::with_view_in_context(view, ctx, |decoder| { + let parsing_live_bytes = + checked_add_allocation_bytes(external_live_bytes, ctx.retained_allocation_bytes())?; + let view = JpegView::parse_with_options_and_external_live(input, options, parsing_live_bytes)?; + Decoder::with_view_in_context_with_external_live(view, ctx, external_live_bytes, |decoder| { let source_rect = roi.unwrap_or_else(|| Rect::full(decoder.info.dimensions)); if !source_rect.is_within(decoder.info.dimensions) { return Err(JpegError::RectOutOfBounds { diff --git a/crates/j2k-jpeg/src/decoder/tile/tests.rs b/crates/j2k-jpeg/src/decoder/tile/tests.rs index 78f274ec0..eb3730a30 100644 --- a/crates/j2k-jpeg/src/decoder/tile/tests.rs +++ b/crates/j2k-jpeg/src/decoder/tile/tests.rs @@ -308,6 +308,7 @@ fn warmed_batch_planning_reuses_the_cached_plan_without_cloning() { None, Downscale::None, DecodeOptions::default(), + 0, ) .expect("first batch plan"); let clones = context.decode_plan_clone_count(); @@ -319,6 +320,7 @@ fn warmed_batch_planning_reuses_the_cached_plan_without_cloning() { None, Downscale::None, DecodeOptions::default(), + 0, ) .expect("warmed batch plan"); @@ -701,6 +703,7 @@ fn tile_planning_accounts_full_roi_and_rejects_out_of_bounds_roi() { None, Downscale::None, DecodeOptions::default(), + 0, ) .expect("full tile plan"); let roi = Rect { @@ -716,6 +719,7 @@ fn tile_planning_accounts_full_roi_and_rejects_out_of_bounds_roi() { Some(roi), Downscale::Quarter, DecodeOptions::default(), + 0, ) .expect("region tile plan"); let error = planned_jpeg_tile_decode_live_bytes( @@ -730,6 +734,7 @@ fn tile_planning_accounts_full_roi_and_rejects_out_of_bounds_roi() { }), Downscale::None, DecodeOptions::default(), + 0, ) .expect_err("out-of-bounds region must fail planning"); @@ -776,3 +781,18 @@ fn malformed_one_shot_input_does_not_mutate_output() { assert_eq!(output, [0xa5; 16]); assert!(!error.is_api_misuse()); } +#[test] +fn planning_charges_retained_workers_before_parsing_or_preparing() { + let mut context = crate::DecoderContext::new(); + let error = planned_jpeg_tile_decode_live_bytes( + j2k_test_support::JPEG_BASELINE_420_16X16, + &mut context, + PixelFormat::Rgb8, + None, + Downscale::None, + DecodeOptions::default(), + crate::decoder::DEFAULT_MAX_DECODE_BYTES, + ) + .unwrap_err(); + assert!(matches!(error, JpegError::MemoryCapExceeded { requested, cap } if requested > cap)); +} diff --git a/crates/j2k-jpeg/src/decoder/view.rs b/crates/j2k-jpeg/src/decoder/view.rs index 7cf6b1bd1..70511e045 100644 --- a/crates/j2k-jpeg/src/decoder/view.rs +++ b/crates/j2k-jpeg/src/decoder/view.rs @@ -54,7 +54,7 @@ impl<'a> JpegView<'a> { Ok(Self::from_header(input, header, options)) } - fn parse_with_options_and_external_live( + pub(super) fn parse_with_options_and_external_live( input: &'a [u8], options: DecodeOptions, external_live_bytes: usize, diff --git a/crates/j2k-jpeg/src/encoded_output/tests.rs b/crates/j2k-jpeg/src/encoded_output/tests.rs index cd1d9443e..bfcbcd4b2 100644 --- a/crates/j2k-jpeg/src/encoded_output/tests.rs +++ b/crates/j2k-jpeg/src/encoded_output/tests.rs @@ -47,12 +47,3 @@ fn allocator_reported_capacity_is_checked_after_reservation() { }) )); } - -#[test] -fn encoded_output_module_stays_focused() { - const SOURCE: &str = include_str!("../encoded_output.rs"); - assert!( - SOURCE.lines().count() <= 160, - "encoded output storage should be split before it exceeds 160 lines" - ); -} diff --git a/crates/j2k-jpeg/src/encoder/entropy.rs b/crates/j2k-jpeg/src/encoder/entropy.rs index 179fa1b6f..ddf6f1d56 100644 --- a/crates/j2k-jpeg/src/encoder/entropy.rs +++ b/crates/j2k-jpeg/src/encoder/entropy.rs @@ -222,15 +222,3 @@ where } Ok(()) } - -#[cfg(test)] -mod tests { - #[test] - fn entropy_orchestration_module_stays_focused() { - const SOURCE: &str = include_str!("entropy.rs"); - assert!( - SOURCE.lines().count() <= 430, - "entropy orchestration should be split before it exceeds 430 lines" - ); - } -} diff --git a/crates/j2k-jpeg/src/encoder/entropy/restart.rs b/crates/j2k-jpeg/src/encoder/entropy/restart.rs index a16e2fa9a..8a26e736c 100644 --- a/crates/j2k-jpeg/src/encoder/entropy/restart.rs +++ b/crates/j2k-jpeg/src/encoder/entropy/restart.rs @@ -396,10 +396,4 @@ mod tests { && cap == DEFAULT_MAX_HOST_ALLOCATION_BYTES )); } - - #[test] - fn restart_entropy_module_stays_focused() { - const SOURCE: &str = include_str!("restart.rs"); - assert!(SOURCE.lines().count() <= 420); - } } diff --git a/crates/j2k-jpeg/src/encoder/entropy/workspace.rs b/crates/j2k-jpeg/src/encoder/entropy/workspace.rs index 07aaef383..fc0fac952 100644 --- a/crates/j2k-jpeg/src/encoder/entropy/workspace.rs +++ b/crates/j2k-jpeg/src/encoder/entropy/workspace.rs @@ -99,12 +99,3 @@ fn cap_overflow() -> JpegEncodeError { cap: DEFAULT_MAX_HOST_ALLOCATION_BYTES, } } - -#[cfg(test)] -mod tests { - #[test] - fn workspace_module_stays_focused() { - const SOURCE: &str = include_str!("workspace.rs"); - assert!(SOURCE.lines().count() <= 130); - } -} diff --git a/crates/j2k-jpeg/src/encoder/transform.rs b/crates/j2k-jpeg/src/encoder/transform.rs index 04eb304a2..da37e1ac1 100644 --- a/crates/j2k-jpeg/src/encoder/transform.rs +++ b/crates/j2k-jpeg/src/encoder/transform.rs @@ -11,34 +11,89 @@ pub(super) fn fdct_quantize( quant: &[u8; 64], cosine: &[[f64; 8]; 8], ) -> [i32; 64] { - let mut coeffs = [0i32; 64]; - for v in 0..8 { + let mut columns = [[0.0; 8]; 8]; + for y in 0..8 { + let row = fdct_8(core::array::from_fn(|x| { + f64::from(block[y * 8 + x]) - 128.0 + })); for u in 0..8 { - let mut sum = 0.0; - for y in 0..8 { - for x in 0..8 { - let sample = f64::from(block[y * 8 + x]) - 128.0; - sum += sample * cosine[u][x] * cosine[v][y]; + columns[u][y] = row[u]; + } + } + let mut coeffs = [0i32; 64]; + for (u, column) in columns.into_iter().enumerate() { + for (v, value) in fdct_8(column).into_iter().enumerate() { + let natural = v * 8 + u; + let mut quantized = + value / (8.0 * AAN_SCALE[u] * AAN_SCALE[v] * f64::from(quant[natural])); + // Reassociation can move an exact half-integer across the rounding + // boundary. Preserve the original codestream on those rare ties. + if (quantized.abs().fract() - 0.5).abs() < 1e-9 { + let mut direct = 0.0; + for y in 0..8 { + for x in 0..8 { + direct += + (f64::from(block[y * 8 + x]) - 128.0) * cosine[u][x] * cosine[v][y]; + } } + let cu = if u == 0 { + core::f64::consts::FRAC_1_SQRT_2 + } else { + 1.0 + }; + let cv = if v == 0 { + core::f64::consts::FRAC_1_SQRT_2 + } else { + 1.0 + }; + quantized = 0.25 * cu * cv * direct / f64::from(quant[natural]); } - let cu = if u == 0 { - core::f64::consts::FRAC_1_SQRT_2 - } else { - 1.0 - }; - let cv = if v == 0 { - core::f64::consts::FRAC_1_SQRT_2 - } else { - 1.0 - }; - let natural = v * 8 + u; - let transformed = 0.25 * cu * cv * sum; - coeffs[natural] = (transformed / f64::from(quant[natural])).round() as i32; + coeffs[natural] = quantized.round() as i32; } } coeffs } +// Arai–Agui–Nakajima scaling: the five-multiply 1-D transform defers +// normalization to quantization. F64 plus the tie fallback above preserves +// the existing encoder's rounding, including its byte-level golden fixtures. +const AAN_SCALE: [f64; 8] = [ + 1.0, + 1.387_039_845_322_147_5, + 1.306_562_964_876_376_6, + 1.175_875_602_419_358_8, + 1.0, + 0.785_694_958_387_102_2, + 0.541_196_100_146_197_1, + 0.275_899_379_282_943_1, +]; + +#[inline] +fn fdct_8(input: [f64; 8]) -> [f64; 8] { + let sum: [f64; 4] = core::array::from_fn(|i| input[i] + input[7 - i]); + let diff: [f64; 4] = core::array::from_fn(|i| input[i] - input[7 - i]); + let outer = sum[0] + sum[3]; + let inner = sum[1] + sum[2]; + let even = sum[0] - sum[3]; + let rotation = (even + sum[1] - sum[2]) * core::f64::consts::FRAC_1_SQRT_2; + let left = diff[3] + diff[2]; + let right = diff[1] + diff[0]; + let cross = (left - right) * 0.382_683_432_365_089_8; + let low = left * 0.541_196_100_146_197 + cross; + let high = right * 1.306_562_964_876_376_6 + cross; + let middle = (diff[2] + diff[1]) * core::f64::consts::FRAC_1_SQRT_2; + [ + outer + inner, + diff[0] + middle + high, + even + rotation, + diff[0] - middle - low, + outer - inner, + diff[0] - middle + low, + even - rotation, + diff[0] + middle - high, + ] +} + pub(super) fn cosine_table() -> [[f64; 8]; 8] { let mut table = [[0.0; 8]; 8]; for (u, row) in (0u32..8).zip(&mut table) { @@ -48,3 +103,66 @@ pub(super) fn cosine_table() -> [[f64; 8]; 8] { } table } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[expect( + clippy::cast_possible_truncation, + reason = "the reference transform covers bounded eight-bit samples and quantized coefficients" + )] + fn separable_dct_preserves_quantization_including_rounding_ties() { + let cosine = cosine_table(); + let mut seed = 17u32; + for case in 0..128 { + let block = core::array::from_fn(|i| { + seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + match case { + 0..=3 => [0, 127, 128, 255][case], + 4 => { + if i == 0 { + 255 + } else { + 0 + } + } + 5 => { + if i % 2 == 0 { + 255 + } else { + 0 + } + } + _ => (seed >> 24) as u8, + } + }); + for q in [1u8, 3, 16, 255] { + let actual = fdct_quantize(&block, &[q; 64], &cosine); + let reference = core::array::from_fn(|k| { + let (v, u) = (k / 8, k % 8); + let mut sum = 0.0; + for y in 0..8 { + for x in 0..8 { + sum += + (f64::from(block[y * 8 + x]) - 128.0) * cosine[u][x] * cosine[v][y]; + } + } + let cu = if u == 0 { + core::f64::consts::FRAC_1_SQRT_2 + } else { + 1.0 + }; + let cv = if v == 0 { + core::f64::consts::FRAC_1_SQRT_2 + } else { + 1.0 + }; + (0.25 * cu * cv * sum / f64::from(q)).round() as i32 + }); + assert_eq!(actual, reference, "case {case}, quantizer {q}"); + } + } + } +} diff --git a/crates/j2k-jpeg/src/entropy/huffman.rs b/crates/j2k-jpeg/src/entropy/huffman.rs index c4961a6c7..7479a7048 100644 --- a/crates/j2k-jpeg/src/entropy/huffman.rs +++ b/crates/j2k-jpeg/src/entropy/huffman.rs @@ -14,6 +14,7 @@ use crate::error::{HuffmanFailure, JpegError}; use crate::internal::bit_reader::BitReader; use crate::parse::tables::{HuffmanTableRole, HuffmanValues, RawHuffmanTable}; +use alloc::sync::Arc; use alloc::vec::Vec; use core::num::NonZeroU32; @@ -62,13 +63,12 @@ pub(crate) struct AcHuffmanTable<'a>(&'a HuffmanTable); #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct PreparedHuffmanTableId(NonZeroU32); -/// Heap owner for every compiled Huffman table referenced by one decoder. -/// -/// Table values are fully inline, so this vector is the only Huffman-table -/// allocation retained by prepared decode metadata. +/// Compiled tables are immutable after construction and shared by cached plans. +/// The size-dependent vector is allocated fallibly; the fixed Arc owner is +/// charged separately, as it is for the shared packet-cache owners. #[derive(Debug)] pub(crate) struct PreparedHuffmanTables { - entries: Vec, + entries: Arc>, } pub(crate) type CanonicalHuffmanDerivation = j2k_codec_math::jpeg::CanonicalHuffmanDerivation; @@ -411,6 +411,16 @@ impl PreparedHuffmanTableId { } impl PreparedHuffmanTables { + const OWNER_BYTES: usize = + core::mem::size_of::>() + 2 * core::mem::size_of::(); + + pub(crate) fn allocation_bytes_for_capacity(capacity: usize) -> Result { + crate::allocation::checked_add_allocation_bytes( + Self::OWNER_BYTES, + crate::allocation::checked_allocation_bytes::(capacity)?, + ) + } + #[cfg(test)] pub(crate) fn try_with_capacity(capacity: usize) -> Result { let mut live_bytes = 0; @@ -426,6 +436,11 @@ impl PreparedHuffmanTables { live_bytes: &mut usize, cap: usize, ) -> Result { + let requested = live_bytes.saturating_add(Self::OWNER_BYTES); + if requested > cap { + return Err(JpegError::MemoryCapExceeded { requested, cap }); + } + *live_bytes = requested; let mut entries = Vec::new(); crate::allocation::try_reserve_for_len_with_live_budget( &mut entries, @@ -433,7 +448,9 @@ impl PreparedHuffmanTables { live_bytes, cap, )?; - Ok(Self { entries }) + Ok(Self { + entries: Arc::new(entries), + }) } pub(crate) fn push( @@ -446,7 +463,11 @@ impl PreparedHuffmanTables { reason: "prepared Huffman arena exceeded its reserved capacity", }); } - self.entries.push(table); + Arc::get_mut(&mut self.entries) + .ok_or(JpegError::InternalInvariant { + reason: "prepared Huffman arena was shared before construction finished", + })? + .push(table); Ok(id) } @@ -484,7 +505,7 @@ impl PreparedHuffmanTables { } pub(crate) fn retained_allocation_bytes(&self) -> Result { - crate::allocation::checked_allocation_bytes::(self.entries.capacity()) + Self::allocation_bytes_for_capacity(self.entries.capacity()) } pub(crate) fn try_clone_with_live_budget( @@ -492,15 +513,16 @@ impl PreparedHuffmanTables { live_bytes: &mut usize, cap: usize, ) -> Result { - let mut entries = Vec::new(); - crate::allocation::try_reserve_for_len_with_live_budget( - &mut entries, - self.entries.len(), - live_bytes, - cap, - )?; - entries.extend(self.entries.iter().cloned()); - Ok(Self { entries }) + // Charge the full retained arena to each owner conservatively, even + // though cloning only increments the reference count. + let requested = live_bytes.saturating_add(self.retained_allocation_bytes()?); + if requested > cap { + return Err(JpegError::MemoryCapExceeded { requested, cap }); + } + *live_bytes = requested; + Ok(Self { + entries: Arc::clone(&self.entries), + }) } pub(crate) fn len(&self) -> usize { @@ -818,6 +840,8 @@ mod tests { assert_eq!( arena.retained_allocation_bytes().unwrap(), arena.capacity() * core::mem::size_of::() + + core::mem::size_of::>() + + 2 * core::mem::size_of::() ); } diff --git a/crates/j2k-jpeg/src/entropy/progressive/render.rs b/crates/j2k-jpeg/src/entropy/progressive/render.rs index f4b91430b..cce31a1e0 100644 --- a/crates/j2k-jpeg/src/entropy/progressive/render.rs +++ b/crates/j2k-jpeg/src/entropy/progressive/render.rs @@ -6,7 +6,9 @@ use alloc::vec::Vec; use crate::allocation::{checked_allocation_len, try_reserve_for_len_with_live_budget}; use crate::backend::Backend; -use crate::color::upsample::{upsample_h2v1_fancy_row, upsample_h2v2_fancy_row}; +use crate::color::upsample::{ + upsample_h1v2_fancy_row, upsample_h2v1_fancy_row, upsample_h2v2_fancy_row, +}; use crate::entropy::block::clamp_i16; use crate::entropy::ZIGZAG; use crate::error::JpegError; @@ -167,6 +169,14 @@ fn upsample_component_row( let curr = component_row(component, image, sample_y); let next = component_row(component, image, next_y); upsample_h2v2_fancy_row(prev, curr, next, out.len(), y % 2 == 1, out); + } else if h_ratio == 1 && v_ratio == 2 { + let sample_y = ((y / 2) as usize).min(component.sample_height.saturating_sub(1) as usize); + let prev_y = sample_y.saturating_sub(1); + let next_y = (sample_y + 1).min(component.sample_height.saturating_sub(1) as usize); + let prev = component_row(component, image, prev_y); + let curr = component_row(component, image, sample_y); + let next = component_row(component, image, next_y); + upsample_h1v2_fancy_row(prev, curr, next, out.len(), y % 2 == 1, out); } else { upsample_nearest(plan, component, image, y, out); } diff --git a/crates/j2k-jpeg/src/entropy/sequential/emit.rs b/crates/j2k-jpeg/src/entropy/sequential/emit.rs index bed498f1e..9228fe9eb 100644 --- a/crates/j2k-jpeg/src/entropy/sequential/emit.rs +++ b/crates/j2k-jpeg/src/entropy/sequential/emit.rs @@ -5,8 +5,6 @@ mod output; mod region420; mod rgb; mod rgb444; -#[cfg(test)] -mod structure_tests; mod types; mod upsample; diff --git a/crates/j2k-jpeg/src/entropy/sequential/emit/four_component.rs b/crates/j2k-jpeg/src/entropy/sequential/emit/four_component.rs index 9de1b5e02..473af5bc1 100644 --- a/crates/j2k-jpeg/src/entropy/sequential/emit/four_component.rs +++ b/crates/j2k-jpeg/src/entropy/sequential/emit/four_component.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{ - super::{PreparedDecodePlan, StripeBuffer}, + super::PreparedDecodePlan, types::StripeNeighbors, upsample::{ upsample_component_row_stripe, StripeComponentUpsample, StripeComponentUpsampleSpec, @@ -14,15 +14,26 @@ use crate::{ internal::scratch::RgbGenericRows, }; +/// One output row of a stripe: its stripe-local index, the rows the stripe +/// emits, and the output width. +#[derive(Clone, Copy)] +pub(super) struct FourComponentRow { + pub(super) local_y: u32, + pub(super) stripe_rows: usize, + pub(super) width: usize, +} + pub(super) fn fill_four_component_rgb_row( plan: &PreparedDecodePlan, - prev: Option<&StripeBuffer>, - curr: &StripeBuffer, - next: Option<&StripeBuffer>, - local_y: u32, - width: usize, + neighbors: StripeNeighbors<'_>, + row: FourComponentRow, scratch: &mut RgbGenericRows, ) -> Result<(), JpegError> { + let FourComponentRow { + local_y, + stripe_rows, + width, + } = row; let (c0_h, c0_v) = plan .sampling .component(0) @@ -45,7 +56,6 @@ pub(super) fn fill_four_component_rgb_row( .ok_or(JpegError::UnsupportedComponentCount { count: 3 })?; let max_h = u32::from(plan.sampling.max_h); let max_v = u32::from(plan.sampling.max_v); - let neighbors = StripeNeighbors { prev, curr, next }; upsample_component_row_stripe(StripeComponentUpsample { neighbors, @@ -56,6 +66,7 @@ pub(super) fn fill_four_component_rgb_row( max_h, max_v, local_y_out: local_y, + stripe_rows, width, }, out: &mut scratch.r, @@ -69,6 +80,7 @@ pub(super) fn fill_four_component_rgb_row( max_h, max_v, local_y_out: local_y, + stripe_rows, width, }, out: &mut scratch.g, @@ -82,6 +94,7 @@ pub(super) fn fill_four_component_rgb_row( max_h, max_v, local_y_out: local_y, + stripe_rows, width, }, out: &mut scratch.b, @@ -95,6 +108,7 @@ pub(super) fn fill_four_component_rgb_row( max_h, max_v, local_y_out: local_y, + stripe_rows, width, }, out: &mut scratch.k, diff --git a/crates/j2k-jpeg/src/entropy/sequential/emit/output.rs b/crates/j2k-jpeg/src/entropy/sequential/emit/output.rs index e03ed871d..14191dd21 100644 --- a/crates/j2k-jpeg/src/entropy/sequential/emit/output.rs +++ b/crates/j2k-jpeg/src/entropy/sequential/emit/output.rs @@ -2,7 +2,7 @@ use super::{ super::{is_ycbcr_420, scaled_dimensions, OutputScratch, PreparedDecodePlan}, - four_component::fill_four_component_rgb_row, + four_component::{fill_four_component_rgb_row, FourComponentRow}, types::{StripeEmit, StripeNeighbors}, upsample::{ upsample_420_pair, upsample_component_row_stripe, Stripe420PairSpec, Stripe420PairUpsample, @@ -88,6 +88,7 @@ pub(in crate::entropy::sequential) fn emit_stripe( spec: Stripe420PairSpec { plane_idx: 1, local_y_out: local_y as u32, + stripe_rows, width, }, top: &mut scratch.cb_top, @@ -98,6 +99,7 @@ pub(in crate::entropy::sequential) fn emit_stripe( spec: Stripe420PairSpec { plane_idx: 2, local_y_out: local_y as u32, + stripe_rows, width, }, top: &mut scratch.cr_top, @@ -136,6 +138,7 @@ pub(in crate::entropy::sequential) fn emit_stripe( max_h, max_v, local_y_out: local_y as u32, + stripe_rows, width, }, out: &mut scratch.cb_up, @@ -149,6 +152,7 @@ pub(in crate::entropy::sequential) fn emit_stripe( max_h, max_v, local_y_out: local_y as u32, + stripe_rows, width, }, out: &mut scratch.cr_up, @@ -196,6 +200,7 @@ pub(in crate::entropy::sequential) fn emit_stripe( max_h, max_v, local_y_out: local_y as u32, + stripe_rows, width, }, out: &mut scratch.r, @@ -209,6 +214,7 @@ pub(in crate::entropy::sequential) fn emit_stripe( max_h, max_v, local_y_out: local_y as u32, + stripe_rows, width, }, out: &mut scratch.g, @@ -222,6 +228,7 @@ pub(in crate::entropy::sequential) fn emit_stripe( max_h, max_v, local_y_out: local_y as u32, + stripe_rows, width, }, out: &mut scratch.b, @@ -241,11 +248,12 @@ pub(in crate::entropy::sequential) fn emit_stripe( for local_y in 0..stripe_rows { fill_four_component_rgb_row( plan, - prev, - curr, - next, - local_y as u32, - width, + neighbors, + FourComponentRow { + local_y: local_y as u32, + stripe_rows, + width, + }, scratch, )?; writer.write_rgb_row( diff --git a/crates/j2k-jpeg/src/entropy/sequential/emit/region420.rs b/crates/j2k-jpeg/src/entropy/sequential/emit/region420.rs index 502adc536..c3308a9d5 100644 --- a/crates/j2k-jpeg/src/entropy/sequential/emit/region420.rs +++ b/crates/j2k-jpeg/src/entropy/sequential/emit/region420.rs @@ -3,7 +3,7 @@ use super::{ super::{scaled_dimensions, PreparedDecodePlan}, types::{Fast420RegionStripe, StripeNeighbors}, - upsample::component_row_triplet, + upsample::{component_row_triplet, valid_component_rows}, }; use crate::{ backend::{Backend, Rgb420ChromaRows, Rgb420Crop, Rgb420CroppedRowPair, Rgb420RowPair}, @@ -54,6 +54,7 @@ pub(in crate::entropy::sequential) fn emit_stripe_rgb_420_region< let crop_width = region_layout.crop_end - region_layout.crop_start; let crop_len = crop_width * 3; let use_direct_crop = should_use_direct_420_crop(backend, downscale, row_width, crop_width); + let chroma_valid_rows = valid_component_rows(stripe_rows, 2); let mut local_y = 0usize; while local_y < stripe_rows { let next_local_y = local_y + 1; @@ -75,12 +76,14 @@ pub(in crate::entropy::sequential) fn emit_stripe_rgb_420_region< curr.plane(1), next.map(|stripe| stripe.plane(1)), chroma_y, + chroma_valid_rows, ); let (prev_cr, curr_cr, next_cr) = component_row_triplet( prev.map(|stripe| stripe.plane(2)), curr.plane(2), next.map(|stripe| stripe.plane(2)), chroma_y, + chroma_valid_rows, ); let chroma = Rgb420ChromaRows::new( &prev_cb[..chroma_width], diff --git a/crates/j2k-jpeg/src/entropy/sequential/emit/rgb.rs b/crates/j2k-jpeg/src/entropy/sequential/emit/rgb.rs index 70f6cc088..ae0898315 100644 --- a/crates/j2k-jpeg/src/entropy/sequential/emit/rgb.rs +++ b/crates/j2k-jpeg/src/entropy/sequential/emit/rgb.rs @@ -2,11 +2,11 @@ use super::{ super::{is_ycbcr_420, scaled_dimensions, PreparedDecodePlan, RgbOutputScratch}, - four_component::fill_four_component_rgb_row, + four_component::{fill_four_component_rgb_row, FourComponentRow}, types::{StripeEmit, StripeNeighbors}, upsample::{ - component_row_triplet, upsample_component_row_stripe, StripeComponentUpsample, - StripeComponentUpsampleSpec, + component_row_triplet, upsample_component_row_stripe, valid_component_rows, + StripeComponentUpsample, StripeComponentUpsampleSpec, }, }; use crate::{ @@ -66,6 +66,7 @@ pub(in crate::entropy::sequential) fn emit_stripe_rgb String { - fs::read_to_string(Path::new(env!("CARGO_MANIFEST_DIR")).join("src").join(path)) - .unwrap_or_else(|error| panic!("read {path}: {error}")) -} - -#[test] -fn sequential_emit_uses_focused_real_modules() { - let allow_attribute = ["#[", "allow"].concat(); - let include_macro = ["include", "!("].concat(); - let wildcard_import = ["use super::", "*"].concat(); - let wildcard_reexport = ["pub(super) use self::", "*"].concat(); - let root = source("entropy/sequential/emit.rs"); - - assert!( - root.lines().count() < 50, - "sequential/emit.rs must remain a focused module shell" - ); - for declaration in [ - "mod four_component;", - "mod output;", - "mod region420;", - "mod rgb;", - "mod rgb444;", - "mod types;", - "mod upsample;", - ] { - assert!( - root.contains(declaration), - "sequential/emit.rs must contain {declaration}" - ); - } - for required_item in [ - "emit_stripe", - "emit_stripe_rgb", - "emit_stripe_rgb_420_region", - "emit_stripe_rgb_444", - "Fast420RegionStripe", - "StripeEmit", - "StripeNeighbors", - ] { - assert!( - root.contains(required_item), - "sequential/emit.rs must reexport {required_item}" - ); - } - assert!(!root.contains(&allow_attribute)); - assert!(!root.contains(&include_macro)); - assert!(!root.contains(&wildcard_reexport)); - - for (path, max_lines) in MODULES { - let module = source(path); - assert!( - module.lines().count() < *max_lines, - "{path} must stay below its focused-module line-count ratchet" - ); - assert!( - !module.contains(&allow_attribute), - "{path} must avoid broad allows" - ); - assert!( - !module.contains(&include_macro), - "{path} must be a real module" - ); - assert!( - !module.contains(&wildcard_import), - "{path} must use explicit imports" - ); - } -} diff --git a/crates/j2k-jpeg/src/entropy/sequential/emit/upsample.rs b/crates/j2k-jpeg/src/entropy/sequential/emit/upsample.rs index 37e3ce2cf..ce0b545b0 100644 --- a/crates/j2k-jpeg/src/entropy/sequential/emit/upsample.rs +++ b/crates/j2k-jpeg/src/entropy/sequential/emit/upsample.rs @@ -2,7 +2,8 @@ use super::{super::StripePlane, types::StripeNeighbors}; use crate::color::upsample::{ - upsample_1x1, upsample_h2v1_fancy_row, upsample_h2v2_fancy_row, upsample_h2v2_fancy_rows, + upsample_1x1, upsample_h1v2_fancy_row, upsample_h2v1_fancy_row, upsample_h2v2_fancy_row, + upsample_h2v2_fancy_rows, }; #[derive(Clone, Copy)] @@ -13,6 +14,7 @@ pub(super) struct StripeComponentUpsampleSpec { pub(super) max_h: u32, pub(super) max_v: u32, pub(super) local_y_out: u32, + pub(super) stripe_rows: usize, pub(super) width: usize, } @@ -26,6 +28,7 @@ pub(super) struct StripeComponentUpsample<'a, 'b> { pub(super) struct Stripe420PairSpec { pub(super) plane_idx: usize, pub(super) local_y_out: u32, + pub(super) stripe_rows: usize, pub(super) width: usize, } @@ -36,18 +39,40 @@ pub(super) struct Stripe420PairUpsample<'a, 'b> { pub(super) bot: &'b mut [u8], } +/// Rows of a component plane that hold image samples when its stripe emits +/// `stripe_rows` output rows. Only the image's final stripe is short; the rest +/// of that stripe's plane is MCU padding. +pub(super) fn valid_component_rows(stripe_rows: usize, v_ratio: usize) -> usize { + stripe_rows.div_ceil(v_ratio) +} + +/// Returns the rows above, at, and below `local_row` for vertical upsampling. +/// +/// `valid_rows` counts the rows of `curr` that hold image samples. Below the +/// last of them the current row is replicated, as libjpeg-turbo does +/// (`set_bottom_pointers` in `jdmainct.c`), so decoded MCU padding never +/// reaches the output. pub(in crate::entropy::sequential) fn component_row_triplet<'a>( prev: Option>, curr: StripePlane<'a>, next: Option>, local_row: usize, + valid_rows: usize, ) -> (&'a [u8], &'a [u8], &'a [u8]) { fn plane_row(plane: StripePlane<'_>, row: usize) -> &[u8] { let start = row * plane.stride; &plane.data[start..start + plane.stride] } - let curr_rows = curr.rows; + let curr_rows = curr.rows.min(valid_rows); + debug_assert!( + local_row < curr_rows, + "row {local_row} is outside the image" + ); + debug_assert!( + next.is_none() || curr_rows == curr.rows, + "only the final stripe may end before its padded height" + ); let prev_row = if local_row == 0 { match prev { Some(plane) => plane_row(plane, plane.rows - 1), @@ -86,6 +111,7 @@ pub(super) fn upsample_component_row_stripe(request: StripeComponentUpsample<'_, max_h, max_v, local_y_out, + stripe_rows, width, } = spec; let v_ratio = max_v / comp_v; @@ -98,6 +124,7 @@ pub(super) fn upsample_component_row_stripe(request: StripeComponentUpsample<'_, curr_plane, next.map(|stripe| stripe.plane(plane_idx)), chroma_y as usize, + valid_component_rows(stripe_rows, v_ratio as usize), ); match (h_ratio, v_ratio) { @@ -108,6 +135,16 @@ pub(super) fn upsample_component_row_stripe(request: StripeComponentUpsample<'_, let chroma_cols = width.div_ceil(2); upsample_h2v1_fancy_row(&curr_row[..chroma_cols], width, out); } + (1, 2) => { + upsample_h1v2_fancy_row( + &prev_row[..width], + &curr_row[..width], + &next_row[..width], + width, + !local_y_out.is_multiple_of(2), + out, + ); + } (2, 2) => { let chroma_cols = width.div_ceil(2); upsample_h2v2_fancy_row( @@ -143,6 +180,7 @@ pub(super) fn upsample_420_pair(request: Stripe420PairUpsample<'_, '_>) { let Stripe420PairSpec { plane_idx, local_y_out, + stripe_rows, width, } = spec; let curr_plane = curr.plane(plane_idx); @@ -153,6 +191,7 @@ pub(super) fn upsample_420_pair(request: Stripe420PairUpsample<'_, '_>) { curr_plane, next.map(|stripe| stripe.plane(plane_idx)), chroma_y as usize, + valid_component_rows(stripe_rows, 2), ); upsample_h2v2_fancy_rows(prev_row, curr_row, next_row, width, top, bot); diff --git a/crates/j2k-jpeg/src/entropy/sequential/generic/row.rs b/crates/j2k-jpeg/src/entropy/sequential/generic/row.rs index 6e83ed865..576980465 100644 --- a/crates/j2k-jpeg/src/entropy/sequential/generic/row.rs +++ b/crates/j2k-jpeg/src/entropy/sequential/generic/row.rs @@ -64,6 +64,17 @@ pub(super) fn decode_mcu_row( } let mut pixels_4x4 = [0u8; 16]; let mut pixels_2x2 = [0u8; 4]; + // Resolve immutable Huffman handles outside the MCU loop, including the + // generic grayscale, restart, and subsampled paths. + let mut resolved = [context.plan.resolved_component(0)?; 4]; + if context.plan.components.len() > resolved.len() { + return Err(JpegError::InternalInvariant { + reason: "sequential decode supports at most four components", + }); + } + for (tables, comp) in resolved.iter_mut().zip(&context.plan.components) { + *tables = context.plan.resolve_component(comp)?; + } for mx in 0..context.mcus_per_row { if consume_restart_marker_if_due( state.br, @@ -79,10 +90,10 @@ pub(super) fn decode_mcu_row( *state.mcus_since_restart = 0; } - for comp in &context.plan.components { + for (comp, tables) in context.plan.components.iter().zip(&resolved) { let plane_idx = comp.output_index; - let dc_table = context.plan.dc_table(comp)?; - let ac_table = context.plan.ac_table(comp)?; + let dc_table = tables.dc_table; + let ac_table = tables.ac_table; let in_region = mx >= context.stripe_mcu_start && mx < stripe_mcu_end; let local_mcu_x0_px = mx.saturating_sub(context.stripe_mcu_start) * u32::from(comp.h) * block_size; diff --git a/crates/j2k-jpeg/src/entropy/sequential/plan.rs b/crates/j2k-jpeg/src/entropy/sequential/plan.rs index 1704038df..fbd179d8a 100644 --- a/crates/j2k-jpeg/src/entropy/sequential/plan.rs +++ b/crates/j2k-jpeg/src/entropy/sequential/plan.rs @@ -5,7 +5,7 @@ use alloc::vec::Vec; use crate::allocation::{checked_add_allocation_bytes, checked_allocation_bytes}; -use crate::entropy::huffman::{HuffmanTable, PreparedHuffmanTableId, PreparedHuffmanTables}; +use crate::entropy::huffman::{PreparedHuffmanTableId, PreparedHuffmanTables}; use crate::error::JpegError; use crate::info::{ColorSpace, SamplingFactors}; @@ -46,7 +46,7 @@ impl PreparedDecodePlan { let mut total = checked_allocation_bytes::(component_count)?; total = checked_add_allocation_bytes( total, - checked_allocation_bytes::(huffman_table_count)?, + PreparedHuffmanTables::allocation_bytes_for_capacity(huffman_table_count)?, )?; Ok(total) } diff --git a/crates/j2k-jpeg/src/entropy/sequential/tests.rs b/crates/j2k-jpeg/src/entropy/sequential/tests.rs index 8fb3d4598..49c2a53fd 100644 --- a/crates/j2k-jpeg/src/entropy/sequential/tests.rs +++ b/crates/j2k-jpeg/src/entropy/sequential/tests.rs @@ -9,89 +9,6 @@ use crate::Decoder; use alloc::vec; use j2k_test_support::JPEG_BASELINE_420_16X16; -#[test] -fn sequential_decode_modules_stay_focused_and_fragment_free() { - const ROOT: &str = include_str!("../sequential.rs"); - const GENERIC: &str = include_str!("generic.rs"); - const GENERIC_DRIVER: &str = include_str!("generic/driver.rs"); - const GENERIC_ROW: &str = include_str!("generic/row.rs"); - const DCT: &str = include_str!("dct.rs"); - const DCT_ALLOCATION: &str = include_str!("dct/allocation.rs"); - const OUTPUT_SCRATCH: &str = include_str!("output_scratch.rs"); - const PLAN: &str = include_str!("plan.rs"); - const PLAN_RESOLVED: &str = include_str!("plan/resolved.rs"); - const RGB444: &str = include_str!("rgb444.rs"); - const STRIPE: &str = include_str!("stripe.rs"); - const FAST420: &str = include_str!("fast420/mod.rs"); - const FAST420_ROWS: &str = include_str!("fast420/rows.rs"); - - let modules = [ - ("sequential.rs", ROOT, 240usize), - ("sequential/generic.rs", GENERIC, 180), - ("sequential/generic/driver.rs", GENERIC_DRIVER, 260), - ("sequential/generic/row.rs", GENERIC_ROW, 230), - ("sequential/dct.rs", DCT, 200), - ("sequential/dct/allocation.rs", DCT_ALLOCATION, 320), - ("sequential/output_scratch.rs", OUTPUT_SCRATCH, 60), - ("sequential/plan.rs", PLAN, 120), - ("sequential/plan/resolved.rs", PLAN_RESOLVED, 80), - ("sequential/rgb444.rs", RGB444, 300), - ("sequential/stripe.rs", STRIPE, 240), - ("sequential/fast420/mod.rs", FAST420, 650), - ("sequential/fast420/rows.rs", FAST420_ROWS, 700), - ]; - - for (path, source, max_lines) in modules { - let line_count = source.lines().count(); - assert!( - line_count <= max_lines, - "{path} grew to {line_count} lines; split it before exceeding {max_lines}" - ); - assert!( - !source.contains("include!(") && !source.contains("#[path"), - "{path} must remain a real Rust module, not a textual source fragment" - ); - } - - for declaration in [ - "mod dct;", - "mod fast420;", - "mod generic;", - "mod output_scratch;", - "mod plan;", - "mod rgb444;", - "mod stripe;", - ] { - assert!( - ROOT.contains(declaration), - "sequential facade lost required module boundary {declaration}" - ); - } - assert!( - DCT.contains("mod allocation;"), - "sequential DCT execution lost its allocation boundary" - ); - for declaration in ["mod driver;", "mod row;"] { - assert!( - GENERIC.contains(declaration), - "generic sequential owner lost required boundary {declaration}" - ); - } - for shared_owner in [ - "struct ScanSetup", - "struct ScanBuffers", - "trait StripeEmitter", - "fn decode_scan_rows", - ] { - assert!( - GENERIC_DRIVER.contains(shared_owner), - "generic scan driver lost typed shared owner {shared_owner}" - ); - } - assert!(!GENERIC.contains("clippy::too_many_lines")); - assert!(!GENERIC_DRIVER.contains("macro_rules!")); -} - #[test] fn fast_tile_rgb_matches_generic_baseline_decode() { let dec = Decoder::new(JPEG_BASELINE_420_16X16).expect("fixture must parse"); @@ -397,7 +314,7 @@ fn deposit_dc_block_writes_uniform_rows_without_temp_block() { } #[test] -fn component_row_triplet_uses_neighbor_stripes_and_clamps_edges() { +fn component_row_triplet_uses_neighbor_stripes_and_clamps_to_real_rows() { let prev = StripeBuffer { planes: vec![vec![], vec![10, 11, 12, 13, 14, 15], vec![]], plane_strides: vec![0, 2, 0], @@ -419,32 +336,194 @@ fn component_row_triplet_uses_neighbor_stripes_and_clamps_edges() { let next_plane = Some(next.plane(1)); let (top_prev, top_curr, top_next) = - component_row_triplet(prev_plane, curr_plane, next_plane, 0); + component_row_triplet(prev_plane, curr_plane, next_plane, 0, 3); assert_eq!(top_prev, &[14, 15]); assert_eq!(top_curr, &[20, 21]); assert_eq!(top_next, &[22, 23]); let (mid_prev, mid_curr, mid_next) = - component_row_triplet(prev_plane, curr_plane, next_plane, 1); + component_row_triplet(prev_plane, curr_plane, next_plane, 1, 3); assert_eq!(mid_prev, &[20, 21]); assert_eq!(mid_curr, &[22, 23]); assert_eq!(mid_next, &[24, 25]); let (bot_prev, bot_curr, bot_next) = - component_row_triplet(prev_plane, curr_plane, next_plane, 2); + component_row_triplet(prev_plane, curr_plane, next_plane, 2, 3); assert_eq!(bot_prev, &[22, 23]); assert_eq!(bot_curr, &[24, 25]); assert_eq!(bot_next, &[30, 31]); - let (clamp_prev, clamp_curr, clamp_next) = component_row_triplet(None, curr_plane, None, 0); + let (clamp_prev, clamp_curr, clamp_next) = component_row_triplet(None, curr_plane, None, 0, 3); assert_eq!(clamp_prev, &[20, 21]); assert_eq!(clamp_curr, &[20, 21]); assert_eq!(clamp_next, &[22, 23]); - let (tail_prev, tail_curr, tail_next) = component_row_triplet(None, curr_plane, None, 2); + let (tail_prev, tail_curr, tail_next) = component_row_triplet(None, curr_plane, None, 2, 3); assert_eq!(tail_prev, &[22, 23]); assert_eq!(tail_curr, &[24, 25]); assert_eq!(tail_next, &[24, 25]); + + // A final stripe cut short by the image height ends at its last real row, + // as libjpeg-turbo does; row 2 is MCU padding and must not be blended in. + let (short_prev, short_curr, short_next) = + component_row_triplet(Some(prev.plane(1)), curr_plane, None, 1, 2); + assert_eq!(short_prev, &[20, 21]); + assert_eq!(short_curr, &[22, 23]); + assert_eq!(short_next, &[22, 23]); + + let (single_prev, single_curr, single_next) = + component_row_triplet(Some(prev.plane(1)), curr_plane, None, 0, 1); + assert_eq!(single_prev, &[14, 15]); + assert_eq!(single_curr, &[20, 21]); + assert_eq!(single_next, &[20, 21]); +} + +#[test] +fn final_420_stripe_replicates_last_real_chroma_row_instead_of_padding() { + // (color space, downscale, image height). Each height leaves the final + // stripe with an even number of output rows, so its last row blends the + // last real chroma row with the row below it. + let cases = [ + (ColorSpace::YCbCr, DownscaleFactor::Full, 18u32), + (ColorSpace::YCbCr, DownscaleFactor::Full, 28), + (ColorSpace::YCbCr, DownscaleFactor::Half, 20), + (ColorSpace::Rgb, DownscaleFactor::Full, 18), + (ColorSpace::Rgb, DownscaleFactor::Full, 20), + ]; + for (color_space, downscale, height) in cases { + // Interleaved-RGB emitter, then the generic component-writer emitter. + for rgb_emitter in [true, false] { + let emit = |padding| { + emit_final_420_stripe(color_space, downscale, height, padding, rgb_emitter) + }; + let replicated = emit(None); + for padding in [0u8, 255, 77] { + assert_eq!( + emit(Some(padding)), + replicated, + "{color_space:?} {downscale:?} height {height} rgb_emitter {rgb_emitter}: \ + chroma padding {padding} leaked into the output" + ); + } + } + } +} + +/// Emits the final stripe of a two-stripe 4:2:0 image. Chroma rows below the +/// image's last real chroma row hold `padding`, or replicate that last real +/// row when `padding` is `None`. +fn emit_final_420_stripe( + color_space: ColorSpace, + downscale: DownscaleFactor, + height: u32, + padding: Option, + rgb_emitter: bool, +) -> Vec { + let block = downscale.output_block_size() as usize; + let (scaled_width, scaled_height) = scaled_dimensions((13, height), downscale); + let width = scaled_width as usize; + let real_chroma_rows = (scaled_height as usize - 2 * block).div_ceil(2); + let prev = synthetic_420_stripe(block, 5, None); + let curr = synthetic_420_stripe(block, 40, Some((real_chroma_rows, padding))); + let plan = PreparedDecodePlan { + components: vec![], + huffman_tables: PreparedHuffmanTables::try_with_capacity(0).expect("empty test arena"), + sampling: SamplingFactors::from_validated_components(&[(2, 2), (1, 1), (1, 1)]), + color_space, + restart_interval: None, + dimensions: (13, height), + scan_offset: 0, + scratch_bytes: 0, + }; + let mut rgb_rows = crate::internal::scratch::RgbGenericRows { + r: vec![0; width], + g: vec![0; width], + b: vec![0; width], + k: vec![0; width], + }; + let mut ycbcr420_rows = crate::internal::scratch::YCbCr420Rows { + cb_top: vec![0; width], + cb_bot: vec![0; width], + cr_top: vec![0; width], + cr_bot: vec![0; width], + }; + let mut out = vec![0u8; width * scaled_height as usize * 3]; + let mut writer = Rgb8Writer::new(&mut out, width * 3, scaled_width); + let emit = super::emit::StripeEmit { + prev: Some(&prev), + curr: &curr, + next: None, + stripe_index: 1, + source_width: width, + downscale, + }; + let result = match (rgb_emitter, color_space) { + (true, ColorSpace::YCbCr) => super::emit::emit_stripe_rgb( + &plan, + Backend::detect(), + &mut writer, + &mut RgbOutputScratch::YCbCr420, + emit, + ), + (true, _) => super::emit::emit_stripe_rgb( + &plan, + Backend::detect(), + &mut writer, + &mut RgbOutputScratch::RgbGeneric(&mut rgb_rows), + emit, + ), + (false, ColorSpace::YCbCr) => super::emit::emit_stripe( + &plan, + &mut writer, + &mut OutputScratch::YCbCr420(&mut ycbcr420_rows), + emit, + ), + (false, _) => super::emit::emit_stripe( + &plan, + &mut writer, + &mut OutputScratch::RgbGeneric(&mut rgb_rows), + emit, + ), + }; + result.expect("emit final stripe"); + out.split_off(width * 2 * block * 3) +} + +/// One-MCU-wide 4:2:0 stripe of patterned samples. With +/// `Some((real_rows, padding))`, chroma rows from `real_rows` on hold +/// `padding`, or repeat the last real row when `padding` is `None`. +fn synthetic_420_stripe( + block: usize, + seed: usize, + chroma_padding: Option<(usize, Option)>, +) -> StripeBuffer { + let strides = [2 * block, block, block]; + let rows = [2 * block, block, block]; + let planes = (0..3) + .map(|index| { + (0..strides[index] * rows[index]) + .map(|offset| { + let (row, col) = (offset / strides[index], offset % strides[index]); + let row = match chroma_padding { + Some((real_rows, padding)) if index > 0 && row >= real_rows => { + if let Some(value) = padding { + return value; + } + real_rows - 1 + } + _ => row, + }; + u8::try_from((seed + index * 71 + row * 29 + col * 13) & 0xFF) + .expect("fixture is byte-masked") + }) + .collect() + }) + .collect(); + StripeBuffer { + planes, + plane_strides: strides.to_vec(), + plane_rows: rows.to_vec(), + } } #[test] diff --git a/crates/j2k-jpeg/src/idct/scalar.rs b/crates/j2k-jpeg/src/idct/scalar.rs index 4d738c767..bd224a65a 100644 --- a/crates/j2k-jpeg/src/idct/scalar.rs +++ b/crates/j2k-jpeg/src/idct/scalar.rs @@ -71,23 +71,77 @@ pub(crate) fn idct_islow_dc_only_pixel(dc_coeff: i16) -> u8 { idct_islow_dc_only_sample::(dc_coeff) } +/// Fractional bits kept between the passes of the 12-bit IDCT. libjpeg-turbo's +/// `jidctint.c` keeps one instead of the 8-bit path's two, to leave headroom. +const PASS1_BITS_12: usize = 1; + /// Inverse DCT of a 12-bit JPEG block, returning native 0..4095 sample values. +/// +/// Mirrors `jidctint.c` built with `BITS_IN_JSAMPLE == 12`: `PASS1_BITS = 1` +/// and `JLONG` (64-bit on LP64 targets) intermediates, so nothing wraps. The +/// zero-AC shortcuts of the C code give the same result as the full +/// computation here and are omitted. pub(crate) fn idct_islow_12bit(input: &[i16; 64], output: &mut [u16; 64]) { - let mut work = [Wrapping(0i32); 64]; - if input[32..].iter().all(|&coeff| coeff == 0) { - for col in 0..8 { - idct_1d_column_bottom_half_zero(input, &mut work, col); - } - } else { - for col in 0..8 { - idct_1d_column(input, &mut work, col); + let mut work = [0i64; 64]; + let column_shift = CONST_BITS - PASS1_BITS_12; + for col in 0..8 { + let samples = core::array::from_fn(|row| i64::from(input[row * 8 + col])); + let outputs = islow_1d_i64(samples); + for (row, value) in outputs.into_iter().enumerate() { + work[row * 8 + col] = descale_i64(value, column_shift); } } + let row_shift = CONST_BITS + PASS1_BITS_12 + 3; for row in 0..8 { - idct_1d_row::(&work, output, row); + let samples = core::array::from_fn(|col| work[row * 8 + col]); + let outputs = islow_1d_i64(samples); + for (col, value) in outputs.into_iter().enumerate() { + let sample = (descale_i64(value, row_shift) + 2048).clamp(0, 4095); + output[row * 8 + col] = u16::try_from(sample).expect("clamped 12-bit sample"); + } } } +/// One ISLOW 1-D pass in 64-bit arithmetic; outputs are scaled by +/// `2^CONST_BITS` and not yet descaled. +fn islow_1d_i64(p: [i64; 8]) -> [i64; 8] { + let fix = i64::from; + let z1 = (p[2] + p[6]) * fix(idct::FIX_0_541196100); + let tmp2 = z1 - p[6] * fix(idct::FIX_1_847759065); + let tmp3 = z1 + p[2] * fix(idct::FIX_0_765366865); + let tmp0 = (p[0] + p[4]) << CONST_BITS; + let tmp1 = (p[0] - p[4]) << CONST_BITS; + let (tmp10, tmp13) = (tmp0 + tmp3, tmp0 - tmp3); + let (tmp11, tmp12) = (tmp1 + tmp2, tmp1 - tmp2); + + let (t0, t1, t2, t3) = (p[7], p[5], p[3], p[1]); + let z5 = (t0 + t2 + t1 + t3) * fix(idct::FIX_1_175875602); + let z1 = -(t0 + t3) * fix(idct::FIX_0_899976223); + let z2 = -(t1 + t2) * fix(idct::FIX_2_562915447); + let z3 = -(t0 + t2) * fix(idct::FIX_1_961570560) + z5; + let z4 = -(t1 + t3) * fix(idct::FIX_0_390180644) + z5; + let t0 = t0 * fix(idct::FIX_0_298631336) + z1 + z3; + let t1 = t1 * fix(idct::FIX_2_053119869) + z2 + z4; + let t2 = t2 * fix(idct::FIX_3_072711026) + z2 + z3; + let t3 = t3 * fix(idct::FIX_1_501321110) + z1 + z4; + + [ + tmp10 + t3, + tmp11 + t2, + tmp12 + t1, + tmp13 + t0, + tmp13 - t0, + tmp12 - t1, + tmp11 - t2, + tmp10 - t3, + ] +} + +/// `DESCALE` from `jdct.h`: round half up, then arithmetic shift. +const fn descale_i64(value: i64, shift: usize) -> i64 { + (value + (1 << (shift - 1))) >> shift +} + /// Return the uniform native 12-bit sample produced by the DC-only ISLOW path. pub(crate) fn idct_islow_12bit_dc_only_sample(dc_coeff: i16) -> u16 { idct_islow_dc_only_sample::(dc_coeff) diff --git a/crates/j2k-jpeg/src/output/rgba8.rs b/crates/j2k-jpeg/src/output/rgba8.rs index 77ed7b6f5..b29ced459 100644 --- a/crates/j2k-jpeg/src/output/rgba8.rs +++ b/crates/j2k-jpeg/src/output/rgba8.rs @@ -5,7 +5,7 @@ use crate::backend::Backend; use crate::error::JpegError; -use crate::output::OutputWriter; +use crate::output::{InterleavedRgbWriter, OutputWriter}; pub(crate) struct Rgba8Writer<'o> { out: &'o mut [u8], @@ -15,6 +15,41 @@ pub(crate) struct Rgba8Writer<'o> { backend: Backend, } +impl InterleavedRgbWriter for Rgba8Writer<'_> { + fn with_rgb_rows(&mut self, y: u32, row_count: usize, fill: F) -> Result + where + F: FnOnce(&mut [u8], Option<&mut [u8]>) -> Result, + { + let width = self.width as usize; + let start = y as usize * self.stride; + let (top, bottom) = match row_count { + 1 => (&mut self.out[start..start + width * 4], None), + 2 => { + let (head, tail) = self.out.split_at_mut(start + self.stride); + ( + &mut head[start..start + width * 4], + Some(&mut tail[..width * 4]), + ) + } + _ => unreachable!("Rgba8Writer only supports one or two rows"), + }; + let mut bottom = bottom; + let result = fill( + &mut top[..width * 3], + bottom.as_deref_mut().map(|row| &mut row[..width * 3]), + )?; + // The fused RGB kernels write into the caller's RGBA storage. Expanding + // backwards preserves unread RGB pixels and needs no intermediate rows. + for row in core::iter::once(top).chain(bottom) { + for x in (0..width).rev() { + let rgb = [row[x * 3], row[x * 3 + 1], row[x * 3 + 2]]; + row[x * 4..x * 4 + 4].copy_from_slice(&[rgb[0], rgb[1], rgb[2], self.alpha]); + } + } + Ok(result) + } +} + impl<'o> Rgba8Writer<'o> { #[cfg(test)] pub(crate) fn new(out: &'o mut [u8], stride: usize, width: u32, alpha: u8) -> Self { diff --git a/crates/j2k-jpeg/src/simd/x86.rs b/crates/j2k-jpeg/src/simd/x86.rs index 14e3cd12d..37572c571 100644 --- a/crates/j2k-jpeg/src/simd/x86.rs +++ b/crates/j2k-jpeg/src/simd/x86.rs @@ -7,7 +7,7 @@ use j2k_core::CpuFeatures; /// Proof that AVX2 and the operating-system AVX register state are available. /// /// This deliberately represents AVX2 alone, rather than -/// `fearless_simd::Avx2`, whose v0.7 contract is the wider x86-64-v3 feature +/// `fearless_simd::Avx2`, whose contract is the wider x86-64-v3 feature /// set. Keeping the token private prevents callers from forging capability /// state while preserving the decoder's existing AVX2 acceleration envelope. #[derive(Debug, Clone, Copy)] diff --git a/crates/j2k-jpeg/tests/libjpeg_turbo_compare.rs b/crates/j2k-jpeg/tests/libjpeg_turbo_compare.rs index de19a1b6c..ac69ad6be 100644 --- a/crates/j2k-jpeg/tests/libjpeg_turbo_compare.rs +++ b/crates/j2k-jpeg/tests/libjpeg_turbo_compare.rs @@ -13,15 +13,20 @@ use j2k_test_support::{ JPEG_BASELINE_420_16X16, JPEG_BASELINE_422_16X8, JPEG_BASELINE_422_16X8_RGB, JPEG_GRAYSCALE_8X8, }; -#[test] -fn turbojpeg_rgb_and_region_match_j2k_fixtures() { - let require_turbo = std::env::var_os("J2K_REQUIRE_LIBJPEG_TURBO").is_some(); - let turbo_available = libjpeg_turbo::is_available(); +/// Whether libjpeg-turbo comparisons can run. They skip when it is missing, +/// unless `J2K_REQUIRE_LIBJPEG_TURBO` makes that a failure. +fn turbo_available() -> bool { + let available = libjpeg_turbo::is_available(); assert!( - !require_turbo || turbo_available, + available || std::env::var_os("J2K_REQUIRE_LIBJPEG_TURBO").is_none(), "J2K_REQUIRE_LIBJPEG_TURBO is set but libjpeg-turbo is unavailable" ); - if !turbo_available { + available +} + +#[test] +fn turbojpeg_rgb_and_region_match_j2k_fixtures() { + if !turbo_available() { return; } @@ -179,3 +184,495 @@ fn crop_rgb(full: &[u8], width: usize, roi: Rect) -> Vec { } out } + +#[test] +fn rgba420_and_non_row_aligned_restarts_match_turbo_with_padded_output() { + use j2k_jpeg::{encode_jpeg_baseline, JpegEncodeOptions, JpegSamples, JpegSubsampling}; + if !turbo_available() { + return; + } + let (width, height) = (35, 33); + let pixels = j2k_test_support::patterned_rgb8(width, height); + let mut turbo = libjpeg_turbo::TurboJpegDecoder::new().unwrap(); + for restart_interval in [None, Some(1), Some(7)] { + let encoded = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &pixels, + width, + height, + }, + JpegEncodeOptions { + subsampling: JpegSubsampling::Ybr420, + restart_interval, + ..Default::default() + }, + ) + .unwrap(); + let reference = turbo.decode_rgb(&encoded.data).unwrap(); + let decoder = Decoder::new(&encoded.data).unwrap(); + let stride = width as usize * 4 + 5; + let mut output = vec![0xa5; stride * height as usize]; + decoder + .decode_into(&mut output, stride, PixelFormat::Rgba8) + .unwrap(); + for (row, rgb) in output + .chunks_exact(stride) + .zip(reference.chunks_exact(width as usize * 3)) + { + for (rgba, expected) in row[..width as usize * 4] + .chunks_exact(4) + .zip(rgb.chunks_exact(3)) + { + assert_eq!(&rgba[..3], expected, "restart {restart_interval:?}"); + assert_eq!(rgba[3], 255); + } + assert_eq!(&row[width as usize * 4..], &[0xa5; 5]); + } + let roi = Rect { + x: 17, + y: 15, + w: 13, + h: 17, + }; + let (region, _) = decoder + .decode_request(DecodeRequest::region(PixelFormat::Rgba8, roi)) + .unwrap(); + let expected = crop_rgb(&reference, width as usize, roi); + for (rgba, rgb) in region.chunks_exact(4).zip(expected.chunks_exact(3)) { + assert_eq!(&rgba[..3], rgb); + assert_eq!(rgba[3], 255); + } + } +} + +/// 4:2:0 frames whose height ends inside an MCU row. The encoded rows below +/// the frame height hold contrasting colours, so blending decoded MCU padding +/// into the last output row, instead of replicating the last real chroma row +/// as libjpeg-turbo does, shows up as a large difference. +#[test] +fn ybr420_bottom_rows_match_turbo_when_height_is_not_mcu_aligned() { + use j2k_jpeg::{encode_jpeg_baseline, JpegEncodeOptions, JpegSamples, JpegSubsampling}; + if !turbo_available() { + return; + } + let mut turbo = libjpeg_turbo::TurboJpegDecoder::new().unwrap(); + let width = 130u32; + // Heights = 2 (mod 4) and 4, 8, 12 (mod 16), with aligned and odd controls. + for height in [34u32, 64, 65, 66, 68, 72, 76] { + let encoded_height = height.div_ceil(16) * 16; + let pixels = rgb8_with_contrasting_rows_below(width, encoded_height, height); + for restart_interval in [None, Some(1), Some(7)] { + let mut jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &pixels, + width, + height: encoded_height, + }, + JpegEncodeOptions { + subsampling: JpegSubsampling::Ybr420, + restart_interval, + ..Default::default() + }, + ) + .unwrap() + .data; + set_baseline_frame_height(&mut jpeg, height); + let context = format!("height {height}, restart {restart_interval:?}"); + let reference = turbo.decode_rgb(&jpeg).unwrap(); + let decoder = Decoder::new(&jpeg).unwrap(); + + let (rgb, _) = decoder + .decode_request(DecodeRequest::full(PixelFormat::Rgb8)) + .unwrap(); + assert_rgb_rows_eq(&rgb, 3, &reference, width, &format!("{context}, RGB8")); + let (rgba, _) = decoder + .decode_request(DecodeRequest::full(PixelFormat::Rgba8)) + .unwrap(); + assert_rgb_rows_eq(&rgba, 4, &reference, width, &format!("{context}, RGBA8")); + + let roi = Rect { + x: 17, + y: height - 9, + w: 90, + h: 9, + }; + let expected = crop_rgb(&reference, width as usize, roi); + let (region, _) = decoder + .decode_request(DecodeRequest::region(PixelFormat::Rgb8, roi)) + .unwrap(); + assert_rgb_rows_eq(®ion, 3, &expected, roi.w, &format!("{context}, region")); + } + } +} + +fn rgb8_with_contrasting_rows_below(width: u32, height: u32, real_height: u32) -> Vec { + let mut pixels = j2k_test_support::patterned_rgb8(width, height); + let row_len = width as usize * 3; + for (x, pixel) in pixels[real_height as usize * row_len..] + .chunks_exact_mut(3) + .enumerate() + { + let magenta_or_blue = if (x % width as usize / 3).is_multiple_of(2) { + 255 + } else { + 0 + }; + pixel.copy_from_slice(&[magenta_or_blue, 0, 255]); + } + pixels +} + +/// Rewrites the SOF0 frame height, walking marker segments from SOI. +fn set_baseline_frame_height(jpeg: &mut [u8], height: u32) { + let mut offset = 2; + loop { + assert_eq!(jpeg[offset], 0xFF, "marker expected at byte {offset}"); + let marker = jpeg[offset + 1]; + if marker == 0xC0 { + let height = u16::try_from(height).unwrap().to_be_bytes(); + jpeg[offset + 5..offset + 7].copy_from_slice(&height); + return; + } + assert_ne!(marker, 0xDA, "no SOF0 before the first scan"); + let len = usize::from(u16::from_be_bytes([jpeg[offset + 2], jpeg[offset + 3]])); + offset += 2 + len; + } +} + +fn assert_rgb_rows_eq(actual: &[u8], channels: usize, expected_rgb: &[u8], width: u32, what: &str) { + let width = width as usize; + assert_eq!( + actual.len() / channels, + expected_rgb.len() / 3, + "{what}: size" + ); + for (row, (actual_row, expected_row)) in actual + .chunks_exact(width * channels) + .zip(expected_rgb.chunks_exact(width * 3)) + .enumerate() + { + let matches = actual_row + .chunks_exact(channels) + .zip(expected_row.chunks_exact(3)) + .all(|(actual, expected)| &actual[..3] == expected); + assert!(matches, "{what}: row {row} differs from libjpeg-turbo"); + } +} + +/// Compression and 12-bit decompression through `TurboJPEG` 3: fixtures and +/// oracles the shared benchmark adapter does not provide. +#[cfg(all(has_libjpeg_turbo, has_libjpeg_turbo_v3))] +mod turbo_codec { + use std::ffi::{c_char, c_int, c_void, CStr}; + + const TJINIT_COMPRESS: c_int = 0; + const TJINIT_DECOMPRESS: c_int = 1; + const TJPARAM_QUALITY: c_int = 3; + const TJPARAM_SUBSAMP: c_int = 4; + const TJPARAM_PROGRESSIVE: c_int = 12; + pub(crate) const TJPF_RGB: c_int = 0; + pub(crate) const TJPF_GRAY: c_int = 6; + pub(crate) const TJSAMP_444: c_int = 0; + pub(crate) const TJSAMP_422: c_int = 1; + pub(crate) const TJSAMP_420: c_int = 2; + pub(crate) const TJSAMP_GRAY: c_int = 3; + pub(crate) const TJSAMP_440: c_int = 4; + + unsafe extern "C" { + fn tj3Init(init_type: c_int) -> *mut c_void; + fn tj3Destroy(handle: *mut c_void); + fn tj3GetErrorStr(handle: *mut c_void) -> *mut c_char; + fn tj3Set(handle: *mut c_void, param: c_int, value: c_int) -> c_int; + fn tj3Free(buffer: *mut c_void); + fn tj3Compress8( + handle: *mut c_void, + src: *const u8, + width: c_int, + pitch: c_int, + height: c_int, + pixel_format: c_int, + jpeg: *mut *mut u8, + jpeg_size: *mut usize, + ) -> c_int; + fn tj3Compress12( + handle: *mut c_void, + src: *const i16, + width: c_int, + pitch: c_int, + height: c_int, + pixel_format: c_int, + jpeg: *mut *mut u8, + jpeg_size: *mut usize, + ) -> c_int; + fn tj3Decompress12( + handle: *mut c_void, + jpeg: *const u8, + jpeg_size: usize, + dst: *mut i16, + pitch: c_int, + pixel_format: c_int, + ) -> c_int; + } + + /// Encoding parameters for a generated fixture. + #[derive(Clone, Copy, Debug)] + pub(crate) struct Encode { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) pixel_format: c_int, + pub(crate) subsampling: c_int, + pub(crate) quality: c_int, + pub(crate) progressive: bool, + } + + struct Handle(*mut c_void); + + impl Handle { + fn new(init_type: c_int) -> Self { + // SAFETY: tj3Init has no preconditions; a null result is checked. + let handle = unsafe { tj3Init(init_type) }; + assert!(!handle.is_null(), "tj3Init returned null"); + Self(handle) + } + + fn check(&self, rc: c_int, what: &str) { + if rc == 0 { + return; + } + // SAFETY: the handle is live; TurboJPEG returns a static C string. + let message = unsafe { CStr::from_ptr(tj3GetErrorStr(self.0)) }; + panic!("{what}: {}", message.to_string_lossy()); + } + + fn configure(&self, spec: Encode) { + for (param, value) in [ + (TJPARAM_QUALITY, spec.quality), + (TJPARAM_SUBSAMP, spec.subsampling), + (TJPARAM_PROGRESSIVE, c_int::from(spec.progressive)), + ] { + // SAFETY: the handle is live and the parameter ids are TurboJPEG 3's. + self.check(unsafe { tj3Set(self.0, param, value) }, "tj3Set"); + } + } + } + + impl Drop for Handle { + fn drop(&mut self) { + // SAFETY: the handle came from tj3Init and is destroyed once. + unsafe { tj3Destroy(self.0) }; + } + } + + fn channels(pixel_format: c_int) -> usize { + if pixel_format == TJPF_GRAY { + 1 + } else { + 3 + } + } + + fn c(value: usize) -> c_int { + c_int::try_from(value).expect("dimension fits c_int") + } + + /// Copies a TurboJPEG-allocated JPEG buffer out and frees it. + fn take_jpeg(jpeg: *mut u8, size: usize) -> Vec { + // SAFETY: TurboJPEG allocated `size` bytes at `jpeg` for this call. + let bytes = unsafe { std::slice::from_raw_parts(jpeg, size) }.to_vec(); + // SAFETY: the buffer came from TurboJPEG and is freed once. + unsafe { tj3Free(jpeg.cast()) }; + bytes + } + + pub(crate) fn compress8(pixels: &[u8], spec: Encode) -> Vec { + let handle = Handle::new(TJINIT_COMPRESS); + handle.configure(spec); + let pitch = spec.width as usize * channels(spec.pixel_format); + assert_eq!(pixels.len(), pitch * spec.height as usize); + let (mut jpeg, mut size) = (std::ptr::null_mut(), 0usize); + // SAFETY: `pixels` holds `height` rows of `pitch` samples; TurboJPEG + // allocates the output buffer, which take_jpeg frees. + let rc = unsafe { + tj3Compress8( + handle.0, + pixels.as_ptr(), + c(spec.width as usize), + c(pitch), + c(spec.height as usize), + spec.pixel_format, + &raw mut jpeg, + &raw mut size, + ) + }; + handle.check(rc, "tj3Compress8"); + take_jpeg(jpeg, size) + } + + pub(crate) fn compress12(samples: &[i16], spec: Encode) -> Vec { + let handle = Handle::new(TJINIT_COMPRESS); + handle.configure(spec); + let pitch = spec.width as usize * channels(spec.pixel_format); + assert_eq!(samples.len(), pitch * spec.height as usize); + let (mut jpeg, mut size) = (std::ptr::null_mut(), 0usize); + // SAFETY: as for compress8, with 12-bit samples in `i16`. + let rc = unsafe { + tj3Compress12( + handle.0, + samples.as_ptr(), + c(spec.width as usize), + c(pitch), + c(spec.height as usize), + spec.pixel_format, + &raw mut jpeg, + &raw mut size, + ) + }; + handle.check(rc, "tj3Compress12"); + take_jpeg(jpeg, size) + } + + /// Full-frame 12-bit decode with libjpeg-turbo's defaults (ISLOW IDCT, + /// fancy upsampling). + pub(crate) fn decompress12( + jpeg: &[u8], + width: u32, + height: u32, + pixel_format: c_int, + ) -> Vec { + let handle = Handle::new(TJINIT_DECOMPRESS); + let pitch = width as usize * channels(pixel_format); + let mut out = vec![0i16; pitch * height as usize]; + // SAFETY: `out` holds `height` rows of `pitch` samples. + let rc = unsafe { + tj3Decompress12( + handle.0, + jpeg.as_ptr(), + jpeg.len(), + out.as_mut_ptr(), + c(pitch), + pixel_format, + ) + }; + handle.check(rc, "tj3Decompress12"); + out + } +} + +/// Smooth gradients plus deterministic noise, so blocks carry many AC terms. +#[cfg(all(has_libjpeg_turbo, has_libjpeg_turbo_v3))] +fn textured_samples(width: u32, height: u32, channels: u32, max: u32) -> Vec { + let mut state = 0x9e37_79b9_u32; + let span = width + height; + let mut samples = Vec::with_capacity((width * height * channels) as usize); + for y in 0..height { + for x in 0..width { + for channel in 0..channels { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + let base = ((x + y) * max / span + channel * max / 3 + (x * y) % 97) % (max + 1); + let noise = state % (max / 8 + 1); + samples.push((base + noise).min(max)); + } + } + } + samples +} + +#[cfg(all(has_libjpeg_turbo, has_libjpeg_turbo_v3))] +#[test] +fn ycbcr440_matches_turbo_h1v2_fancy_upsampling() { + use turbo_codec::{Encode, TJPF_RGB, TJSAMP_440}; + + if !turbo_available() { + return; + } + let mut turbo = libjpeg_turbo::TurboJpegDecoder::new().expect("turbojpeg decoder"); + // Odd, even, and MCU-aligned heights; baseline and progressive. + for (width, height) in [(64, 64), (67, 45), (40, 34), (33, 17)] { + for (quality, progressive) in [(75, false), (95, false), (90, true)] { + let rgb = textured_samples(width, height, 3, 255) + .into_iter() + .map(|sample| u8::try_from(sample).expect("8-bit sample")) + .collect::>(); + let spec = Encode { + width, + height, + pixel_format: TJPF_RGB, + subsampling: TJSAMP_440, + quality, + progressive, + }; + let jpeg = turbo_codec::compress8(&rgb, spec); + let expected = turbo.decode_rgb(&jpeg).expect("turbojpeg 4:4:0 decode"); + let (actual, _) = Decoder::new(&jpeg) + .expect("4:4:0 decoder") + .decode_request(DecodeRequest::full(PixelFormat::Rgb8)) + .expect("4:4:0 decode"); + assert_rgb_rows_eq(&actual, 3, &expected, width, &format!("{spec:?}")); + } + } +} + +#[cfg(all(has_libjpeg_turbo, has_libjpeg_turbo_v3))] +#[test] +fn extended12_decodes_match_turbo() { + use turbo_codec::{ + Encode, TJPF_GRAY, TJPF_RGB, TJSAMP_420, TJSAMP_422, TJSAMP_444, TJSAMP_GRAY, + }; + + if !turbo_available() { + return; + } + for (pixel_format, subsampling) in [ + (TJPF_GRAY, TJSAMP_GRAY), + (TJPF_RGB, TJSAMP_444), + (TJPF_RGB, TJSAMP_422), + (TJPF_RGB, TJSAMP_420), + ] { + let (channels, format) = if pixel_format == TJPF_GRAY { + (1, PixelFormat::Gray16) + } else { + (3, PixelFormat::Rgb16) + }; + for (width, height) in [(64, 48), (67, 45)] { + for quality in [75, 95, 100] { + let samples = textured_samples(width, height, channels, 4095) + .into_iter() + .map(|sample| i16::try_from(sample).expect("12-bit sample")) + .collect::>(); + let spec = Encode { + width, + height, + pixel_format, + subsampling, + quality, + progressive: false, + }; + let jpeg = turbo_codec::compress12(&samples, spec); + let expected = turbo_codec::decompress12(&jpeg, width, height, pixel_format); + let (actual, _) = Decoder::new(&jpeg) + .expect("12-bit decoder") + .decode_request(DecodeRequest::full(format)) + .expect("12-bit decode"); + let actual = actual + .chunks_exact(2) + .map(|pair| i16::from_ne_bytes([pair[0], pair[1]])) + .collect::>(); + assert_eq!(actual.len(), expected.len(), "{spec:?}: size"); + if let Some(index) = actual.iter().zip(&expected).position(|(a, e)| a != e) { + let max_delta = actual + .iter() + .zip(&expected) + .map(|(a, e)| (i32::from(*a) - i32::from(*e)).abs()) + .max() + .unwrap_or(0); + panic!( + "{spec:?}: sample {index} is {} (turbo {}); max delta {max_delta}", + actual[index], expected[index] + ); + } + } + } + } +} diff --git a/crates/j2k-metal/src/batch_decoder/encoder_count_tests.rs b/crates/j2k-metal/src/batch_decoder/encoder_count_tests.rs index be2c7295d..04d92bce2 100644 --- a/crates/j2k-metal/src/batch_decoder/encoder_count_tests.rs +++ b/crates/j2k-metal/src/batch_decoder/encoder_count_tests.rs @@ -277,13 +277,11 @@ fn irreversible_groups_batch_transform_stages_without_changing_pixels() { } #[test] -fn large_irreversible_groups_bound_the_working_set_without_changing_pixels() { - // P27 retained per-image reconstruction above 20 MiB after chunking regressed. - for (dimensions, count, expected_sequences) in [ - ((1024, 1024), 6, 7), - ((1024, 1024), 9, 10), - ((1025, 513), 11, 12), - ] { +fn large_irreversible_groups_batch_every_level_without_changing_pixels() { + // P36: with P31's fused lifting, batches reconstruct every level in one + // batched sequence per component, including stages above 20 MiB (P20/P27 + // had fallen back to per-image reconstruction there). + for (dimensions, count) in [((1024, 1024), 6), ((1024, 1024), 9), ((1025, 513), 11)] { if assert_external_group_uses_one_command_buffer_and_encoder( FixtureRoute::Ht97, FixtureColor::Gray, @@ -296,8 +294,8 @@ fn large_irreversible_groups_bound_the_working_set_without_changing_pixels() { } assert_eq!( crate::engine::idwt97_stage_sequences_for_test(), - expected_sequences, - "{dimensions:?} batch {count}: one small level plus bounded final-level chunks", + FixtureColor::Gray.channels() * 2, + "{dimensions:?} batch {count}: IDWT stage sequences must not grow with batch size", ); } } diff --git a/crates/j2k-metal/src/buffer_pool/state.rs b/crates/j2k-metal/src/buffer_pool/state.rs index e1ee5f4a3..fff5cafb0 100644 --- a/crates/j2k-metal/src/buffer_pool/state.rs +++ b/crates/j2k-metal/src/buffer_pool/state.rs @@ -9,7 +9,15 @@ use crate::metal_types::{Buffer, Device}; use super::MetalBufferPoolDiagnostics; -const DEFAULT_RETAINED_BYTES_PER_POOL: usize = 256 * 1024 * 1024; +/// Retained-byte bounds for one pool. A pool keeps completed buffers cached for +/// exact-size reuse; decodes still allocate what they need beyond it. Each +/// runtime owns a private and a shared pool, so one session can retain up to +/// twice the per-pool cap until it is dropped. +const MIN_RETAINED_BYTES_PER_POOL: usize = 256 * 1024 * 1024; +const MAX_RETAINED_BYTES_PER_POOL: usize = 1024 * 1024 * 1024; +/// Share of the device's recommended working set one pool may retain: about +/// 680 MB on an 8 GB Mac, the 1 GiB ceiling from 16 GB up. +const RETAINED_WORKING_SET_DIVISOR: u64 = 8; const DEFAULT_PRIVATE_RETAINED_BUFFERS_PER_POOL: usize = crate::resident_limits::RESIDENT_PRIVATE_POOL_BUFFER_LIMIT; const DEFAULT_SHARED_RETAINED_BUFFERS_PER_POOL: usize = 64; @@ -30,13 +38,20 @@ impl PoolLimits { } fn for_device(device: &Device, retained_buffers: usize) -> Self { - let device_limit = device.maxBufferLength(); Self { - retained_bytes: device_limit.min(DEFAULT_RETAINED_BYTES_PER_POOL), + retained_bytes: retained_bytes_for_device( + device.maxBufferLength(), + device.recommendedMaxWorkingSetSize(), + ), retained_buffers, } } + #[cfg(test)] + pub(super) const fn retained_bytes_for_test(self) -> usize { + self.retained_bytes + } + #[cfg(test)] pub(super) const fn new(retained_bytes: usize, retained_buffers: usize) -> Self { Self { @@ -46,6 +61,19 @@ impl PoolLimits { } } +/// Per-pool retained-byte cap: an eighth of the recommended working set, +/// clamped to [256 MiB, 1 GiB] and never above the largest single buffer. +pub(super) fn retained_bytes_for_device( + max_buffer_length: usize, + recommended_working_set_bytes: u64, +) -> usize { + let scaled = usize::try_from(recommended_working_set_bytes / RETAINED_WORKING_SET_DIVISOR) + .unwrap_or(usize::MAX); + scaled + .clamp(MIN_RETAINED_BYTES_PER_POOL, MAX_RETAINED_BYTES_PER_POOL) + .min(max_buffer_length) +} + #[derive(Default)] struct PoolCounters { peak_cached_bytes: usize, diff --git a/crates/j2k-metal/src/buffer_pool/tests/production_limits.rs b/crates/j2k-metal/src/buffer_pool/tests/production_limits.rs index 7f43df0a6..c64e4dc4a 100644 --- a/crates/j2k-metal/src/buffer_pool/tests/production_limits.rs +++ b/crates/j2k-metal/src/buffer_pool/tests/production_limits.rs @@ -1,8 +1,63 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::{device, MetalBufferPools}; +#[cfg(target_os = "macos")] +use crate::metal_types::prelude::*; + +use super::super::state::retained_bytes_for_device; +use super::{device, MetalBufferPools, PoolLimits}; use j2k_metal_support::{checked_private_buffer, checked_shared_buffer}; +const MIB: usize = 1024 * 1024; +const GIB: u64 = 1024 * 1024 * 1024; + +fn eighth(working_set: u64) -> usize { + usize::try_from(working_set / 8).expect("test working set fits usize") +} + +#[test] +fn retained_cap_is_an_eighth_of_the_working_set_between_256_mib_and_1_gib() { + let any_buffer = usize::MAX; + // An 8 GB Mac reports roughly two thirds of RAM as its working set. + let eight_gb_class = 16 * GIB / 3; + assert_eq!( + retained_bytes_for_device(any_buffer, eight_gb_class), + eighth(eight_gb_class) + ); + // 16 GB and larger devices reach the 1 GiB ceiling. + assert_eq!( + retained_bytes_for_device(any_buffer, 32 * GIB / 3), + 1024 * MIB + ); + assert_eq!(retained_bytes_for_device(any_buffer, 96 * GIB), 1024 * MIB); + // Small or unreported working sets keep the previous 256 MiB cap. + assert_eq!(retained_bytes_for_device(any_buffer, GIB), 256 * MIB); + assert_eq!(retained_bytes_for_device(any_buffer, 0), 256 * MIB); +} + +#[test] +fn retained_cap_never_exceeds_the_largest_device_buffer() { + assert_eq!(retained_bytes_for_device(300 * MIB, 64 * GIB), 300 * MIB); + assert_eq!(retained_bytes_for_device(100 * MIB, 0), 100 * MIB); +} + +#[test] +fn production_retained_cap_matches_the_device_formula_for_both_pools() { + let device = device(); + let expected = retained_bytes_for_device( + device.maxBufferLength(), + device.recommendedMaxWorkingSetSize(), + ); + assert!(expected <= 1024 * MIB); + assert_eq!( + PoolLimits::private_for_device(&device).retained_bytes_for_test(), + expected + ); + assert_eq!( + PoolLimits::shared_for_device(&device).retained_bytes_for_test(), + expected + ); +} + #[test] fn production_private_and_shared_record_limits_are_independent() { const LEGACY_SHARED_RECORD_LIMIT: usize = 64; diff --git a/crates/j2k-metal/src/decoder/routes.rs b/crates/j2k-metal/src/decoder/routes.rs index ee14b8e59..22e92dbff 100644 --- a/crates/j2k-metal/src/decoder/routes.rs +++ b/crates/j2k-metal/src/decoder/routes.rs @@ -115,6 +115,22 @@ impl J2kDecoder<'_> { #[cfg(target_os = "macos")] let selected = match request.op { + MetalDecodeOp::Full + if request.backend == BackendRequest::Auto + && j2k::J2kDecoder::inspect_support(self.bytes) + .ok() + .is_some_and(|support| { + routing::auto_full_decode_uses_metal( + plan.output_dims(), + support.component_count(), + request.fmt, + support.transfer_syntax, + support.payload_kind, + ) + }) => + { + BackendRequest::Metal + } MetalDecodeOp::Scaled(scale) if request.backend == BackendRequest::Auto && j2k::J2kDecoder::inspect_support(self.bytes) diff --git a/crates/j2k-metal/src/encode/tests/dwt_parity.rs b/crates/j2k-metal/src/encode/tests/dwt_parity.rs index 06b81a677..a56600616 100644 --- a/crates/j2k-metal/src/encode/tests/dwt_parity.rs +++ b/crates/j2k-metal/src/encode/tests/dwt_parity.rs @@ -55,51 +55,71 @@ fn metal_forward_dwt53_matches_reference_for_fractional_stage_samples() { return; } - let width = 8; - let height = 8; - let samples = (0..width * height) - .map(|idx| f32::from(u16::try_from(idx).expect("test index fits u16")) * 0.5 - 15.25) - .collect::>(); - let expected = - forward_dwt53_reference(&samples, width, height, 1).expect("native 5/3 reference DWT"); - let mut accelerator = MetalEncodeStageAccelerator::default(); + for (width, height, num_levels) in [ + (8, 8, 1), + (31, 7, 1), + (32, 7, 1), + (33, 7, 1), + (63, 7, 1), + (64, 7, 1), + (65, 35, 2), + ] { + let samples = (0..width * height) + .map(|idx| { + if width == 8 && height == 8 { + return f32::from(u16::try_from(idx).expect("test index fits u16")) * 0.5 + - 15.25; + } + let x = idx % width; + let y = idx / width; + let mixed = (x * 37 + y * 53 + x * y * 11) % 257; + let checker = if (x + y) & 1 == 0 { 0.125 } else { -0.25 }; + f32::from(u16::try_from(mixed).expect("bounded test sample fits u16")) * 0.375 + - 48.625 + + checker + }) + .collect::>(); + let expected = forward_dwt53_reference(&samples, width, height, num_levels) + .expect("native 5/3 reference DWT"); + let mut accelerator = MetalEncodeStageAccelerator::default(); - let actual = accelerator - .encode_forward_dwt53(J2kForwardDwt53Job { - samples: &samples, - width, - height, - num_levels: 1, - }) - .expect("metal DWT 5/3 stage") - .expect("metal DWT 5/3 dispatch"); + let actual = accelerator + .encode_forward_dwt53(J2kForwardDwt53Job { + samples: &samples, + width, + height, + num_levels, + }) + .expect("metal DWT 5/3 stage") + .expect("metal DWT 5/3 dispatch"); - assert_eq!(actual.ll_width, expected.ll_width); - assert_eq!(actual.ll_height, expected.ll_height); - assert_slice_near(&actual.ll, &expected.ll, "LL"); - assert_eq!(actual.levels.len(), expected.levels.len()); - for (index, (actual, expected)) in actual.levels.iter().zip(&expected.levels).enumerate() { - assert_eq!(actual.width, expected.width, "level {index} width"); - assert_eq!(actual.height, expected.height, "level {index} height"); - assert_eq!( - actual.low_width, expected.low_width, - "level {index} low width" - ); - assert_eq!( - actual.low_height, expected.low_height, - "level {index} low height" - ); - assert_eq!( - actual.high_width, expected.high_width, - "level {index} high width" - ); - assert_eq!( - actual.high_height, expected.high_height, - "level {index} high height" - ); - assert_slice_near(&actual.hl, &expected.hl, "HL"); - assert_slice_near(&actual.lh, &expected.lh, "LH"); - assert_slice_near(&actual.hh, &expected.hh, "HH"); + assert_eq!(actual.ll_width, expected.ll_width); + assert_eq!(actual.ll_height, expected.ll_height); + assert_slice_near(&actual.ll, &expected.ll, "LL"); + assert_eq!(actual.levels.len(), expected.levels.len()); + for (index, (actual, expected)) in actual.levels.iter().zip(&expected.levels).enumerate() { + assert_eq!(actual.width, expected.width, "level {index} width"); + assert_eq!(actual.height, expected.height, "level {index} height"); + assert_eq!( + actual.low_width, expected.low_width, + "level {index} low width" + ); + assert_eq!( + actual.low_height, expected.low_height, + "level {index} low height" + ); + assert_eq!( + actual.high_width, expected.high_width, + "level {index} high width" + ); + assert_eq!( + actual.high_height, expected.high_height, + "level {index} high height" + ); + assert_slice_near(&actual.hl, &expected.hl, "HL"); + assert_slice_near(&actual.lh, &expected.lh, "LH"); + assert_slice_near(&actual.hh, &expected.hh, "HH"); + } } } diff --git a/crates/j2k-metal/src/encode/tests/resident_batches.rs b/crates/j2k-metal/src/encode/tests/resident_batches.rs index a0ce7f18e..5864a1ba0 100644 --- a/crates/j2k-metal/src/encode/tests/resident_batches.rs +++ b/crates/j2k-metal/src/encode/tests/resident_batches.rs @@ -254,74 +254,72 @@ fn metal_padded_private_batch_encode_to_metal_buffers_exposes_per_frame_bytes() #[cfg(target_os = "macos")] #[test] -#[expect( - clippy::cast_sign_loss, - reason = "bounded synthetic pixel expression is nonnegative" -)] fn metal_padded_private_batch_dwt_encode_to_metal_buffers_round_trips() { if !should_run_metal_runtime() { return; } - let first: Vec = (0..128 * 128 * 3) - .map(|i| ((i * 17 + i / 3) & 0xFF) as u8) - .collect(); - let second: Vec = (0..128 * 128 * 3) - .map(|i| 255u8.wrapping_sub(((i * 23 + i / 5) & 0xFF) as u8)) - .collect(); let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let first_buffer = crate::benchmark_private_buffer_with_bytes(&session, &first) - .expect("private benchmark input buffer"); - let second_buffer = crate::benchmark_private_buffer_with_bytes(&session, &second) - .expect("private benchmark input buffer"); - let tiles = [ - super::super::MetalLosslessEncodeTile { - buffer: &first_buffer, - byte_offset: 0, - width: 128, - height: 128, - pitch_bytes: 128 * 3, - output_width: 128, - output_height: 128, - format: PixelFormat::Rgb8, - }, - super::super::MetalLosslessEncodeTile { - buffer: &second_buffer, - byte_offset: 0, - width: 128, - height: 128, - pitch_bytes: 128 * 3, - output_width: 128, - output_height: 128, - format: PixelFormat::Rgb8, - }, - ]; + for (width, height) in [(128u32, 128u32), (65, 35)] { + let first: Vec = (0..width * height * 3) + .map(|i| ((i * 17 + i / 3) & 0xFF) as u8) + .collect(); + let second: Vec = (0..width * height * 3) + .map(|i| 255u8.wrapping_sub(((i * 23 + i / 5) & 0xFF) as u8)) + .collect(); + let first_buffer = crate::benchmark_private_buffer_with_bytes(&session, &first) + .expect("private benchmark input buffer"); + let second_buffer = crate::benchmark_private_buffer_with_bytes(&session, &second) + .expect("private benchmark input buffer"); + let tiles = [ + super::super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width, + height, + pitch_bytes: width as usize * 3, + output_width: width, + output_height: height, + format: PixelFormat::Rgb8, + }, + super::super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width, + height, + pitch_bytes: width as usize * 3, + output_width: width, + output_height: height, + format: PixelFormat::Rgb8, + }, + ]; - let encoded = super::super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( - &tiles, - &lossless_options! { - backend: EncodeBackendPreference::RequireDevice, - validation: J2kEncodeValidation::External, - }, - &session, - ) - .expect("Metal padded DWT buffer batch lossless encode to Metal buffers"); + let encoded = super::super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( + &tiles, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + validation: J2kEncodeValidation::External, + }, + &session, + ) + .expect("Metal padded DWT buffer batch lossless encode to Metal buffers"); - assert_eq!(encoded.len(), 2); - for (frame, expected) in encoded.iter().zip([first, second]) { - assert!(!frame.input_copy_used); - assert!(frame.resident.coefficient_prep_used); - assert!(frame.resident.packetization_used); - assert!(frame.resident.codestream_assembly_used); - let codestream = frame - .encoded - .codestream_bytes() - .expect("Metal codestream bytes are CPU-readable"); - let decoded = Image::new(&codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_decoded_bytes_match(&decoded.data, &expected); + assert_eq!(encoded.len(), 2); + for (frame, expected) in encoded.iter().zip([first, second]) { + assert!(!frame.input_copy_used); + assert!(frame.resident.coefficient_prep_used); + assert!(frame.resident.packetization_used); + assert!(frame.resident.codestream_assembly_used); + let codestream = frame + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_decoded_bytes_match(&decoded.data, &expected); + } } } diff --git a/crates/j2k-metal/src/engine/abi.rs b/crates/j2k-metal/src/engine/abi.rs index 49c24a2f9..61856a4f4 100644 --- a/crates/j2k-metal/src/engine/abi.rs +++ b/crates/j2k-metal/src/engine/abi.rs @@ -261,7 +261,9 @@ pub(crate) struct J2kRepeatedIdwtSingleDecompositionParams { pub(crate) batch_count: u32, } -#[cfg(target_os = "macos")] +/// One lifting step of the original per-step 9/7 kernels, now used only by +/// the full-grid reference oracle in the lifting parity tests. +#[cfg(all(test, target_os = "macos"))] #[repr(C)] #[derive(Clone, Copy)] pub(crate) struct J2kIdwt97StepParams { @@ -271,6 +273,20 @@ pub(crate) struct J2kIdwt97StepParams { pub(crate) _reserved1: u32, } +/// All four lifting steps of one 9/7 axis; mirrors `J2kIdwt97LiftSteps` in +/// `idwt.metal`. `first_parity` is the parity step 0 updates; later steps +/// alternate. `high_pass_bits` carries the vertical high-pass scale. +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kIdwt97LiftSteps { + pub(crate) coefficients: [f32; 4], + pub(crate) first_parity: u32, + pub(crate) high_pass_bits: u32, + pub(crate) _reserved0: u32, + pub(crate) _reserved1: u32, +} + #[cfg(target_os = "macos")] pub(crate) const J2K_MCT_STATUS_OK: u32 = 0; #[cfg(target_os = "macos")] @@ -285,6 +301,9 @@ pub(crate) struct J2kInverseMctParams { pub(crate) _addend0: f32, pub(crate) _addend1: f32, pub(crate) _addend2: f32, + /// Nonzero rounds irreversible samples ties-to-even before the addend, + /// as integer output requires; zero keeps float-plane semantics. + pub(crate) _round_centered: u32, } #[cfg(target_os = "macos")] @@ -381,6 +400,9 @@ pub(crate) struct J2kStoreParams { pub(crate) output_x: u32, pub(crate) output_y: u32, pub(crate) addend: f32, + /// Nonzero rounds the centered sample ties-to-even before the addend, as + /// the CPU does for irreversible integer output. + pub(crate) round_centered: u32, } #[cfg(target_os = "macos")] @@ -400,6 +422,8 @@ pub(crate) struct J2kRepeatedStoreParams { pub(crate) output_y: u32, pub(crate) addend: f32, pub(crate) batch_count: u32, + /// See [`J2kStoreParams::round_centered`]. + pub(crate) round_centered: u32, } #[cfg(target_os = "macos")] @@ -516,6 +540,16 @@ pub(crate) struct J2kHtRepeatedBatchParams { pub(crate) batch_count: u32, } +/// Blocks per SIMD group for the cooperative HT VLC kernels; mirrors +/// `J2kHtVlcDispatchParams` in `ht_cleanup_simd.metal`. +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kHtVlcDispatchParams { + pub(crate) job_count: u32, + pub(crate) lanes_per_simd: u32, +} + #[cfg(target_os = "macos")] #[repr(C)] #[derive(Clone, Copy)] @@ -1048,7 +1082,7 @@ impl_gpu_readback_abi!( J2kClassicStatus, J2kIdwtSingleDecompositionParams, J2kRepeatedIdwtSingleDecompositionParams, - J2kIdwt97StepParams, + J2kIdwt97LiftSteps, J2kInverseMctParams, J2kForwardRctParams, J2kForwardIctParams, @@ -1065,6 +1099,7 @@ impl_gpu_readback_abi!( J2kHtCleanupParams, J2kHtCleanupBatchJob, J2kHtRepeatedBatchParams, + J2kHtVlcDispatchParams, J2kClassicRepeatedBatchParams, J2kHtStatus, J2kClassicEncodeParams, @@ -1094,6 +1129,9 @@ impl_gpu_readback_abi!( J2kCodestreamAssemblyStatus, ); +#[cfg(all(test, target_os = "macos"))] +impl_gpu_readback_abi!(J2kIdwt97StepParams); + #[cfg(all(test, target_os = "macos"))] mod gpu_readback_abi_tests { use core::mem::{align_of, offset_of, size_of}; diff --git a/crates/j2k-metal/src/engine/decode_dispatch.rs b/crates/j2k-metal/src/engine/decode_dispatch.rs index 43d68c40e..a206fa7c3 100644 --- a/crates/j2k-metal/src/engine/decode_dispatch.rs +++ b/crates/j2k-metal/src/engine/decode_dispatch.rs @@ -12,7 +12,7 @@ use crate::profile_env::{ use super::abi::{ J2kClassicCleanupBatchJob, J2kClassicRepeatedBatchParams, J2kClassicSegment, J2kClassicStatus, J2kGrayStoreParams, J2kHtCleanupBatchJob, J2kHtRepeatedBatchParams, J2kHtStatus, - J2kIdwt97StepParams, J2kIdwtSingleDecompositionParams, J2kInverseMctParams, + J2kIdwt97LiftSteps, J2kIdwtSingleDecompositionParams, J2kInverseMctParams, J2kRepeatedGrayStoreParams, J2kRepeatedIdwtSingleDecompositionParams, J2kRepeatedStoreParams, J2kStoreParams, J2K_CLASSIC_MAX_HEIGHT, J2K_CLASSIC_MAX_WIDTH, J2K_CLASSIC_STATUS_OK, }; diff --git a/crates/j2k-metal/src/engine/decode_dispatch/classic_subband.rs b/crates/j2k-metal/src/engine/decode_dispatch/classic_subband.rs index f224968de..1f96f598c 100644 --- a/crates/j2k-metal/src/engine/decode_dispatch/classic_subband.rs +++ b/crates/j2k-metal/src/engine/decode_dispatch/classic_subband.rs @@ -9,7 +9,7 @@ use super::{ dispatch_classic_cleanup_plain_dev_repeated_batched_in_command_buffer, dispatch_classic_cleanup_repeated_batched_in_command_buffer, dispatch_classic_store_repeated_batched_in_command_buffer, dispatch_zero_u32_buffer_in_encoder, - new_shared_buffer, take_classic_coefficients_scratch_buffer, + new_compute_command_encoder, new_shared_buffer, take_classic_coefficients_scratch_buffer, take_classic_states_scratch_buffer, Buffer, ClassicCleanupBatchDispatch, ClassicPlainDevRepeatedCleanupDispatch, ClassicRepeatedCleanupDispatch, ClassicRepeatedStoreDispatch, CommandBufferRef, ComputeCommandEncoderRef, DirectScratchBuffer, @@ -26,6 +26,7 @@ struct ClassicBatchView<'a> { jobs: &'a [J2kClassicCleanupBatchJob], segments: &'a [J2kClassicSegment], output_plane_len: usize, + zero_fill: bool, repeated_overflow_message: &'static str, } @@ -39,6 +40,7 @@ impl<'a> ClassicBatchView<'a> { jobs: &job.jobs, segments: &job.segments, output_plane_len: job.width as usize * job.height as usize, + zero_fill: job.zero_fill, repeated_overflow_message: "classic J2K MetalDirect repeated job count overflow", } } @@ -51,6 +53,7 @@ impl<'a> ClassicBatchView<'a> { jobs: &group.jobs, segments: &group.segments, output_plane_len: group.total_coefficients, + zero_fill: group.zero_fill, repeated_overflow_message: "classic J2K MetalDirect repeated grouped job count overflow", } @@ -153,6 +156,19 @@ fn dispatch_repeated_classic_cleanup( } } +#[cfg(target_os = "macos")] +fn zero_repeated_classic_output( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + output: &Buffer, + word_count: usize, +) -> Result<(), Error> { + let encoder = new_compute_command_encoder(command_buffer)?; + let result = dispatch_zero_u32_buffer_in_encoder(runtime, &encoder, output, word_count); + encoder.endEncoding(); + result +} + #[cfg(target_os = "macos")] fn empty_repeated_classic_execution( runtime: &MetalRuntime, @@ -180,6 +196,19 @@ fn encode_repeated_classic_batch_to_buffer_in_command_buffer( output: &Buffer, scratch_buffers: &mut Vec, ) -> Result<(Vec, DirectStatusCheck), Error> { + // The output is a recycled scratch buffer that may hold an earlier + // decode's coefficients. Samples no code-block job writes (empty blocks, + // or a job-less batch) must read as zero, as on the non-repeated paths. + if count != 0 && (batch.zero_fill || batch.jobs.is_empty()) { + let word_count = + batch + .output_plane_len + .checked_mul(count) + .ok_or_else(|| Error::MetalKernel { + message: batch.repeated_overflow_message.to_string(), + })?; + zero_repeated_classic_output(runtime, command_buffer, output, word_count)?; + } if count == 0 || batch.jobs.is_empty() { return empty_repeated_classic_execution(runtime); } diff --git a/crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/execution.rs b/crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/execution.rs index b5764376a..81e3e1344 100644 --- a/crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/execution.rs +++ b/crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/execution.rs @@ -101,7 +101,7 @@ pub(in crate::engine) fn encode_metal_ht_batches_in_encoder( status_buffer: &status_buffer, status_offset_bytes: status_offset, }, - ); + )?; let chunk_status_bytes = chunk .job_count() .checked_mul(abi_size_of::()) @@ -289,7 +289,7 @@ impl RepeatedHtChunkEncoder<'_, '_> { status_buffer: self.status_buffer, status_offset_bytes: status_offset, }, - ); + )?; let chunk_status_count = chunk .job_count() diff --git a/crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/tests.rs b/crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/tests.rs index 9756be363..637288825 100644 --- a/crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/tests.rs +++ b/crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/tests.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 mod cache; +mod cooperative; mod parity; mod planning; mod status; diff --git a/crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/tests/cooperative.rs b/crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/tests/cooperative.rs new file mode 100644 index 000000000..ec83cad51 --- /dev/null +++ b/crates/j2k-metal/src/engine/decode_dispatch/ht_chunks/tests/cooperative.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! The cooperative (one SIMD group per code block) cleanup decoder must match +//! the CPU oracle and the legacy one-thread-per-block kernel bit for bit. + +#[cfg(target_os = "macos")] +use crate::metal_types::prelude::*; + +use core::mem::size_of; + +use j2k_core::HtGpuJobPassBucket; +use j2k_native::{DecodeSettings, DecoderContext, EncodeOptions, Image}; + +use super::super::{ + default_metal_ht_chunk_limits, plan_metal_ht_chunks, HtBatchInput, J2kHtCleanupBatchJob, + MetalHtPipelineKind, +}; +use crate::engine::abi::{J2kHtStatus, J2K_HT_STATUS_OK}; +use crate::engine::resident_codestream::{ + dispatch_ht_cleanup_batched_in_encoder_with_status_offset, HtCleanupBatchDispatch, +}; +use crate::engine::{ + checked_buffer_slice, commit_and_wait_metal, copied_slice_buffer, + decode_prepared_ht_sub_band_group_on_cpu_profile, new_command_buffer, + new_compute_command_encoder, prepare_direct_grayscale_plan, with_runtime, zeroed_shared_buffer, + MetalRuntime, +}; + +struct Fixture { + name: &'static str, + width: u32, + height: u32, + bit_depth: u8, + samples: Vec, + options: EncodeOptions, +} + +/// Deterministic xorshift noise: high-entropy `MagSgn` streams produce many +/// 0xFF bytes, which exercises bit-unstuffing inside the cooperative window. +fn noise(len: usize, seed: u32) -> Vec { + let mut state = seed | 1; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state.to_le_bytes()[1] + }) + .collect() +} + +fn gradient(width: u32, height: u32) -> Vec { + (0..height) + .flat_map(|y| (0..width).map(move |x| ((x * 3 + y * 5 + (x * y) / 7) & 0xFF) as u8)) + .collect() +} + +fn fixtures() -> Vec { + let lossless = |levels, cbw, cbh| EncodeOptions { + reversible: true, + num_decomposition_levels: levels, + code_block_width_exp: cbw, + code_block_height_exp: cbh, + ..EncodeOptions::default() + }; + let lossy = |levels| EncodeOptions { + reversible: false, + num_decomposition_levels: levels, + ..EncodeOptions::default() + }; + vec![ + Fixture { + name: "gradient_256_64x64", + width: 256, + height: 256, + bit_depth: 8, + samples: gradient(256, 256), + options: lossless(3, 4, 4), + }, + Fixture { + name: "noise_256_64x64", + width: 256, + height: 256, + bit_depth: 8, + samples: noise(256 * 256, 0x1234_5678), + options: lossless(2, 4, 4), + }, + Fixture { + name: "noise16_128_64x64", + width: 128, + height: 128, + bit_depth: 16, + samples: noise(128 * 128 * 2, 0x0bad_cafe), + options: lossless(2, 4, 4), + }, + Fixture { + name: "noise_odd_131x67_32x32", + width: 131, + height: 67, + bit_depth: 8, + samples: noise(131 * 67, 0x2468_ace0), + options: lossless(2, 3, 3), + }, + Fixture { + name: "noise_512x32_wide_256x16", + width: 512, + height: 32, + bit_depth: 8, + samples: noise(512 * 32, 0x1357_9bdf), + options: lossless(1, 6, 2), + }, + Fixture { + name: "noise_256x66_wide_128x32", + width: 256, + height: 66, + bit_depth: 8, + samples: noise(256 * 66, 0x0f0f_0f0f), + options: lossless(1, 5, 3), + }, + Fixture { + name: "noise_lossy97_192", + width: 192, + height: 192, + bit_depth: 8, + samples: noise(192 * 192, 0x7777_1111), + options: lossy(3), + }, + Fixture { + name: "gradient_lossy97_odd_97x151", + width: 97, + height: 151, + bit_depth: 8, + samples: gradient(97, 151), + options: lossy(2), + }, + ] +} + +struct CleanupWorkload { + name: &'static str, + coded_data: Vec, + jobs: Vec, + output_words: usize, + expected: Vec, +} + +fn cleanup_workloads() -> Vec { + let mut workloads = Vec::new(); + for fixture in fixtures() { + let bytes = j2k_native::encode_htj2k( + &fixture.samples, + fixture.width, + fixture.height, + 1, + fixture.bit_depth, + false, + &fixture.options, + ) + .unwrap_or_else(|error| panic!("encode {}: {error:?}", fixture.name)); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("fixture image"); + let mut context = DecoderContext::default(); + let direct = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct fixture plan"); + let prepared = prepare_direct_grayscale_plan(&direct).expect("prepared fixture plan"); + for group in &prepared.ht_groups { + let expected = decode_prepared_ht_sub_band_group_on_cpu_profile(group, None) + .expect("CPU coefficient oracle"); + let input = HtBatchInput { + source_index: 0, + payload: group.payload_source.as_ht_payload_source(), + jobs: &group.jobs, + output_base: 0, + execution_owner: &group.execution_owner, + }; + let plan = plan_metal_ht_chunks(&[input], default_metal_ht_chunk_limits()) + .expect("chunk plan"); + assert_eq!( + plan.chunk_count(), + 1, + "{}: fixture fits one chunk", + fixture.name + ); + let chunk = plan.pack_chunk(0).expect("packed chunk"); + assert_eq!( + chunk.bucket, + HtGpuJobPassBucket::CleanupOnly, + "{}: native encoder emits cleanup-only blocks", + fixture.name + ); + workloads.push(CleanupWorkload { + name: fixture.name, + coded_data: chunk.coded_data, + jobs: chunk.jobs, + output_words: group.total_coefficients, + expected, + }); + } + } + workloads +} + +#[derive(Clone, Copy)] +enum CleanupRoute { + /// One thread per block (`j2k_decode_ht_cleanup_batched_cleanup_only`). + Legacy, + /// The production cleanup-only dispatcher (cooperative VLC + `MagSgn`). + Production, +} + +/// Decodes `replicas` copies of `workload` (each into its own output span) +/// through `route`, returning the output words and the GPU time in seconds. +fn decode_with( + runtime: &MetalRuntime, + route: CleanupRoute, + workload: &CleanupWorkload, + replicas: usize, +) -> (Vec, f64) { + let mut jobs = Vec::with_capacity(workload.jobs.len() * replicas); + for replica in 0..replicas { + let shift = u32::try_from(replica * workload.output_words).expect("replica offset"); + jobs.extend(workload.jobs.iter().map(|job| J2kHtCleanupBatchJob { + output_offset: job.output_offset + shift, + ..*job + })); + } + let output_words = workload.output_words * replicas; + let coded = copied_slice_buffer(&runtime.device, &workload.coded_data).expect("coded buffer"); + let jobs_buffer = copied_slice_buffer(&runtime.device, &jobs).expect("jobs buffer"); + let decoded = zeroed_shared_buffer(&runtime.device, output_words * size_of::()) + .expect("decoded buffer"); + let status = zeroed_shared_buffer(&runtime.device, jobs.len() * size_of::()) + .expect("status buffer"); + let kernels = runtime.decode().expect("decode kernels"); + let command_buffer = new_command_buffer(&runtime.queue).expect("command buffer"); + let encoder = new_compute_command_encoder(&command_buffer).expect("encoder"); + match route { + CleanupRoute::Legacy => { + encoder.setComputePipelineState(&kernels.ht_cleanup_batched_cleanup_only); + encoder.set_buffer(0, Some(&coded), 0); + encoder.set_buffer(1, Some(&decoded), 0); + encoder.set_buffer(2, Some(&jobs_buffer), 0); + encoder.set_buffer(3, Some(&kernels.ht_vlc_table0), 0); + encoder.set_buffer(4, Some(&kernels.ht_vlc_table1), 0); + encoder.set_buffer(5, Some(&kernels.ht_uvlc_table0), 0); + encoder.set_buffer(6, Some(&kernels.ht_uvlc_table1), 0); + encoder.set_buffer(7, Some(&status), 0); + encoder.dispatchThreads_threadsPerThreadgroup( + j2k_metal_support::mtl_size(jobs.len() as u64, 1, 1), + j2k_metal_support::mtl_size(32.min(jobs.len()) as u64, 1, 1), + ); + } + CleanupRoute::Production => { + dispatch_ht_cleanup_batched_in_encoder_with_status_offset( + kernels, + &encoder, + MetalHtPipelineKind::CleanupOnly, + HtCleanupBatchDispatch { + coded_data: &coded, + jobs: &jobs_buffer, + job_count: jobs.len(), + decoded: &decoded, + status_buffer: &status, + status_offset_bytes: 0, + }, + ) + .expect("cooperative dispatch"); + } + } + encoder.endEncoding(); + commit_and_wait_metal(&command_buffer).expect("cleanup decode"); + let statuses = checked_buffer_slice::(&status, jobs.len(), "statuses") + .expect("status readback"); + if let Some(status) = statuses + .iter() + .find(|status| status.code != J2K_HT_STATUS_OK) + { + panic!( + "{}: cleanup decode reported code {} detail {}", + workload.name, status.code, status.detail + ); + } + let words = checked_buffer_slice::(&decoded, output_words, "coefficients") + .expect("coefficient readback"); + ( + words, + command_buffer.GPUEndTime() - command_buffer.GPUStartTime(), + ) +} + +#[test] +fn cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit() { + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + let workloads = cleanup_workloads(); + assert!( + workloads.len() >= fixtures().len(), + "every fixture must contribute cleanup-only work" + ); + with_runtime(|runtime| { + assert_eq!( + runtime + .decode()? + .ht_cleanup_magsgn_batched + .threadExecutionWidth(), + 32, + "the cooperative decoder assumes 32-lane SIMD groups" + ); + for workload in &workloads { + let expected: Vec = workload + .expected + .iter() + .map(|value| value.to_bits()) + .collect(); + let (legacy, _) = decode_with(runtime, CleanupRoute::Legacy, workload, 1); + assert_eq!(legacy, expected, "{}: legacy vs CPU", workload.name); + // One replica runs one block per SIMD group; larger batches pack + // 8 and then 32 blocks into each VLC SIMD group. + for replicas in [1, 12, 96] { + let (cooperative, _) = + decode_with(runtime, CleanupRoute::Production, workload, replicas); + for (replica, words) in cooperative.chunks(workload.output_words).enumerate() { + assert!( + words == expected.as_slice(), + "{}: cooperative replica {replica} of {replicas} differs from CPU", + workload.name + ); + } + } + } + Ok(()) + }) + .expect("cooperative cleanup parity"); +} diff --git a/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible.rs b/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible.rs index bf4b7cf7d..b304549f9 100644 --- a/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible.rs +++ b/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible.rs @@ -5,20 +5,18 @@ use crate::metal_types::prelude::*; #[cfg(test)] use super::super::checked_buffer_slice; +#[cfg(test)] +use super::super::dispatch_3d_pipeline; use super::super::{ checked_buffer_copy_into, commit_and_wait_metal, copied_slice_buffer, dispatch_2d_pipeline, - dispatch_3d_pipeline, hybrid_stage_signpost, label_compute_encoder, new_command_buffer, - new_compute_command_encoder, new_shared_buffer, with_runtime, Buffer, CommandBufferRef, - ComputeCommandEncoderRef, Error, J2kIdwt97StepParams, J2kIdwtSingleDecompositionParams, - J2kSingleDecompositionIdwtJob, SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE, + hybrid_stage_signpost, label_compute_encoder, new_command_buffer, new_compute_command_encoder, + new_shared_buffer, with_runtime, Buffer, CommandBufferRef, ComputeCommandEncoderRef, Error, + J2kIdwt97LiftSteps, J2kIdwtSingleDecompositionParams, J2kSingleDecompositionIdwtJob, + SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE, }; use super::{checked_host_output_layout, IdwtSubBandBuffers, SingleIdwtDispatch}; use j2k_codec_math::dwt; -const fn parity_axis_len(length: u32, odd: bool) -> u32 { - length / 2 + if odd { 0 } else { length % 2 } -} - pub(crate) fn decode_irreversible97_single_decomposition_idwt( job: J2kSingleDecompositionIdwtJob<'_>, output: &mut [f32], @@ -254,6 +252,23 @@ pub(super) fn dispatch_irreversible97_horizontal_scale( encoder.memory_barrier_with_resources(&[decoded]); } +/// Threadgroup geometry of the fused lifting kernels; must match +/// `J2K_IDWT97_*` in `idwt.metal`. Each horizontal group owns whole rows and +/// each vertical group a whole strip of columns. +const IDWT97_ROWS_PER_GROUP: u32 = 4; +const IDWT97_ROW_THREADS: u32 = 64; +const IDWT97_COL_TILE: u32 = 32; +const IDWT97_COL_ROW_THREADS: u32 = 8; + +const IDWT97_LIFT_COEFFICIENTS: [f32; 4] = [ + dwt::IDWT97_NEG_DELTA_F32, + dwt::IDWT97_NEG_GAMMA_F32, + dwt::IDWT97_NEG_BETA_F32, + dwt::IDWT97_NEG_ALPHA_F32, +]; + +/// Encodes the horizontal lifting steps, vertical scale, and vertical lifting +/// steps as two fused tile dispatches. pub(super) fn dispatch_irreversible97_stages_after_horizontal_scale( encoder: &ComputeCommandEncoderRef, kernels: &crate::engine::runtime::DecodeKernels, @@ -266,86 +281,78 @@ pub(super) fn dispatch_irreversible97_stages_after_horizontal_scale( #[cfg(test)] crate::engine::test_counters::record_idwt97_stage_sequence(); - let horizontal_even_is_odd = ((params.x0 + params.output_x) & 1) != 0; - encoder.setComputePipelineState(&kernels.idwt_irreversible97_horizontal_step); + if params.width == 0 || params.height == 0 || batch_count == 0 { + return; + } + let size = j2k_metal_support::mtl_size; encoder.set_buffer(0, Some(decoded), decoded_offset as u64); encoder.set_bytes::(1, ¶ms); - for (coefficient, odd) in [ - (dwt::IDWT97_NEG_DELTA_F32, horizontal_even_is_odd), - (dwt::IDWT97_NEG_GAMMA_F32, !horizontal_even_is_odd), - (dwt::IDWT97_NEG_BETA_F32, horizontal_even_is_odd), - (dwt::IDWT97_NEG_ALPHA_F32, !horizontal_even_is_odd), - ] { - let step = J2kIdwt97StepParams { - coefficient, - parity: u32::from(odd), + + if params.width > 1 { + let horizontal = J2kIdwt97LiftSteps { + coefficients: IDWT97_LIFT_COEFFICIENTS, + first_parity: (params.x0 + params.output_x) & 1, + high_pass_bits: high_pass.to_bits(), _reserved0: 0, _reserved1: 0, }; - encoder.set_bytes::(2, &step); - let horizontal_step_grid = ( - parity_axis_len(params.width, odd), + let groups = ( + 1_u32, + params.height.div_ceil(IDWT97_ROWS_PER_GROUP), + batch_count, + ); + #[cfg(test)] + crate::engine::test_counters::record_idwt97_logical_dispatch(( + params.width, params.height, batch_count, + )); + encoder.setComputePipelineState(&kernels.idwt_irreversible97_horizontal_lift_fused); + encoder.set_bytes::(2, &horizontal); + encoder.dispatchThreadgroups_threadsPerThreadgroup( + size( + u64::from(groups.0), + u64::from(groups.1), + u64::from(groups.2), + ), + size( + u64::from(IDWT97_ROW_THREADS), + u64::from(IDWT97_ROWS_PER_GROUP), + 1, + ), ); - if horizontal_step_grid.0 != 0 { - #[cfg(test)] - crate::engine::test_counters::record_idwt97_logical_dispatch(horizontal_step_grid); - dispatch_3d_pipeline( - encoder, - &kernels.idwt_irreversible97_horizontal_step, - horizontal_step_grid, - ); - } encoder.memory_barrier_with_resources(&[decoded]); } - encoder.setComputePipelineState(&kernels.idwt_irreversible97_vertical_scale); - encoder.set_buffer(0, Some(decoded), decoded_offset as u64); - encoder.set_bytes::(1, ¶ms); - encoder.set_bytes::(2, &high_pass); - let vertical_scale_grid = (params.width, params.height, batch_count); + + let vertical = J2kIdwt97LiftSteps { + coefficients: IDWT97_LIFT_COEFFICIENTS, + first_parity: (params.y0 + params.output_y) & 1, + high_pass_bits: high_pass.to_bits(), + _reserved0: 0, + _reserved1: 0, + }; + let groups = (params.width.div_ceil(IDWT97_COL_TILE), 1_u32, batch_count); #[cfg(test)] - crate::engine::test_counters::record_idwt97_logical_dispatch(vertical_scale_grid); - dispatch_3d_pipeline( - encoder, - &kernels.idwt_irreversible97_vertical_scale, - vertical_scale_grid, + crate::engine::test_counters::record_idwt97_logical_dispatch(( + params.width, + params.height, + batch_count, + )); + encoder.setComputePipelineState(&kernels.idwt_irreversible97_vertical_fused); + encoder.set_bytes::(2, &vertical); + encoder.dispatchThreadgroups_threadsPerThreadgroup( + size( + u64::from(groups.0), + u64::from(groups.1), + u64::from(groups.2), + ), + size( + u64::from(IDWT97_COL_TILE), + u64::from(IDWT97_COL_ROW_THREADS), + 1, + ), ); encoder.memory_barrier_with_resources(&[decoded]); - - let vertical_even_is_odd = ((params.y0 + params.output_y) & 1) != 0; - encoder.setComputePipelineState(&kernels.idwt_irreversible97_vertical_step); - encoder.set_buffer(0, Some(decoded), decoded_offset as u64); - encoder.set_bytes::(1, ¶ms); - for (coefficient, odd) in [ - (dwt::IDWT97_NEG_DELTA_F32, vertical_even_is_odd), - (dwt::IDWT97_NEG_GAMMA_F32, !vertical_even_is_odd), - (dwt::IDWT97_NEG_BETA_F32, vertical_even_is_odd), - (dwt::IDWT97_NEG_ALPHA_F32, !vertical_even_is_odd), - ] { - let step = J2kIdwt97StepParams { - coefficient, - parity: u32::from(odd), - _reserved0: 0, - _reserved1: 0, - }; - encoder.set_bytes::(2, &step); - let vertical_step_grid = ( - params.width, - parity_axis_len(params.height, odd), - batch_count, - ); - if vertical_step_grid.1 != 0 { - #[cfg(test)] - crate::engine::test_counters::record_idwt97_logical_dispatch(vertical_step_grid); - dispatch_3d_pipeline( - encoder, - &kernels.idwt_irreversible97_vertical_step, - vertical_step_grid, - ); - } - encoder.memory_barrier_with_resources(&[decoded]); - } } #[cfg(test)] diff --git a/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/interleave_tests.rs b/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/interleave_tests.rs index 289960cd4..01a0fdc0e 100644 --- a/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/interleave_tests.rs +++ b/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/interleave_tests.rs @@ -285,11 +285,12 @@ fn irreversible97_production_decomposition_dispatch_count() { encoder.endEncoding(); commit_and_wait_metal(&command).unwrap(); let (positions, dispatches) = idwt97_logical_dispatches_for_test(); - assert_eq!(dispatches, 10, "actual production launches, batch {batch}"); + // Interleave with horizontal scale, then one fused pass per axis. + assert_eq!(dispatches, 3, "actual production launches, batch {batch}"); assert_eq!( positions, - 6 * count, - "actual requested production positions" + 3 * count, + "each production pass covers every sample once" ); } Ok(()) diff --git a/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/parity_tests.rs b/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/parity_tests.rs index 0534b9bb0..b5263c1dd 100644 --- a/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/parity_tests.rs +++ b/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/parity_tests.rs @@ -1,15 +1,23 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::*; +use crate::engine::abi::J2kIdwt97StepParams; use crate::engine::runtime::MetalRuntime; use crate::metal_types::ComputePipelineState; use j2k_metal_support::MetalPipelineLoader; // Original full-grid lifting kernels, kept only as an independent test oracle. -const REFERENCE_STEPS: &str = r"kernel void audit_idwt97_horizontal_reference( +const REFERENCE_STEPS: &str = r"struct J2kIdwt97ReferenceStep { + float coefficient; + uint parity; + uint reserved0; + uint reserved1; +}; + +kernel void audit_idwt97_horizontal_reference( device float *out [[buffer(0)]], constant J2kIdwtSingleDecompositionParams ¶ms [[buffer(1)]], - constant J2kIdwt97StepParams &step [[buffer(2)]], + constant J2kIdwt97ReferenceStep &step [[buffer(2)]], uint3 gid [[thread_position_in_grid]] ) { if (gid.x >= params.width || gid.y >= params.height || params.width <= 1u @@ -26,10 +34,36 @@ const REFERENCE_STEPS: &str = r"kernel void audit_idwt97_horizontal_reference( out[idx]); } +kernel void audit_idwt97_vertical_scale_reference( + device float *out [[buffer(0)]], + constant J2kIdwtSingleDecompositionParams ¶ms [[buffer(1)]], + constant float &high_pass [[buffer(2)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + out += ulong(gid.z) * params.width * params.height; + const float KAPPA = CODEC_MATH_DWT97_KAPPA; + float sample = out[gid.y * params.width + gid.x]; + + if (params.height == 1u) { + if (((params.y0 + params.output_y) & 1u) != 0u) { + sample *= 0.5f; + } + } else { + const uint first_even_y = (params.y0 + params.output_y) & 1u; + sample *= (gid.y & 1u) == first_even_y ? KAPPA : high_pass; + } + + out[gid.y * params.width + gid.x] = sample; +} + kernel void audit_idwt97_vertical_reference( device float *out [[buffer(0)]], constant J2kIdwtSingleDecompositionParams ¶ms [[buffer(1)]], - constant J2kIdwt97StepParams &step [[buffer(2)]], + constant J2kIdwt97ReferenceStep &step [[buffer(2)]], uint3 gid [[thread_position_in_grid]] ) { if (gid.x >= params.width || gid.y >= params.height || params.height <= 1u @@ -47,47 +81,10 @@ kernel void audit_idwt97_vertical_reference( } "; -#[derive(Clone, Copy)] -enum Stage { - Scale(f32), - Lift(J2kIdwt97StepParams), -} - -fn compare_stage( - runtime: &MetalRuntime, - buffers: &[Buffer; 2], - pipelines: [&ComputePipelineState; 2], - params: &J2kIdwtSingleDecompositionParams, - batch: u32, - stage: Stage, -) { - const PREFIX: usize = 4; - let command = new_command_buffer(&runtime.queue).expect("comparison command"); - let encoder = new_compute_command_encoder(&command).expect("comparison encoder"); - for (buffer, pipeline) in buffers.iter().zip(pipelines) { - encoder.setComputePipelineState(pipeline); - encoder.set_buffer(0, Some(buffer), (PREFIX * size_of::()) as u64); - encoder.set_bytes::(1, params); - match stage { - Stage::Scale(high_pass) => encoder.set_bytes::(2, &high_pass), - Stage::Lift(step) => encoder.set_bytes::(2, &step), - } - // Full original bounds remain a legal upper bound for a compact - // kernel's guards. Production grid coverage is checked separately. - dispatch_3d_pipeline(&encoder, pipeline, (params.width, params.height, batch)); - } - encoder.endEncoding(); - commit_and_wait_metal(&command).expect("completed comparison"); - let len = PREFIX + params.width as usize * params.height as usize * batch as usize + PREFIX; - let expected = checked_buffer_slice::(&buffers[0], len, "reference stages").unwrap(); - let actual = checked_buffer_slice::(&buffers[1], len, "production stages").unwrap(); - for (index, (actual, expected)) in actual.iter().zip(&expected).enumerate() { - assert!(actual.is_finite()); - assert_eq!(actual.to_bits(), expected.to_bits(), "coefficient {index}"); - } - for value in actual[..PREFIX].iter().chain(&actual[len - PREFIX..]) { - assert_eq!(value.to_bits(), 7.0_f32.to_bits(), "offset guard changed"); - } +struct References { + horizontal: ComputePipelineState, + vertical_scale: ComputePipelineState, + vertical: ComputePipelineState, } fn geometry_params(geometry: (u32, u32, u32, u32, u32, u32)) -> J2kIdwtSingleDecompositionParams { @@ -118,75 +115,109 @@ fn geometry_params(geometry: (u32, u32, u32, u32, u32, u32)) -> J2kIdwtSingleDec } } +/// Runs the original one-pass-per-step lifting sequence on `buffer`. +fn encode_reference_stages( + encoder: &ComputeCommandEncoderRef, + references: &References, + buffer: &Buffer, + offset: u64, + params: &J2kIdwtSingleDecompositionParams, + batch: u32, + high_pass: f32, +) { + let grid = (params.width, params.height, batch); + let lifts = |origin: u32, pipeline: &ComputePipelineState| { + let even = origin & 1; + for (coefficient, parity) in [ + (dwt::IDWT97_NEG_DELTA_F32, even), + (dwt::IDWT97_NEG_GAMMA_F32, 1 - even), + (dwt::IDWT97_NEG_BETA_F32, even), + (dwt::IDWT97_NEG_ALPHA_F32, 1 - even), + ] { + let step = J2kIdwt97StepParams { + coefficient, + parity, + _reserved0: 0, + _reserved1: 0, + }; + encoder.setComputePipelineState(pipeline); + encoder.set_buffer(0, Some(buffer), offset); + encoder.set_bytes::(1, params); + encoder.set_bytes::(2, &step); + dispatch_3d_pipeline(encoder, pipeline, grid); + encoder.memory_barrier_with_resources(&[buffer]); + } + }; + lifts(params.x0 + params.output_x, &references.horizontal); + encoder.setComputePipelineState(&references.vertical_scale); + encoder.set_buffer(0, Some(buffer), offset); + encoder.set_bytes::(1, params); + encoder.set_bytes::(2, &high_pass); + dispatch_3d_pipeline(encoder, &references.vertical_scale, grid); + encoder.memory_barrier_with_resources(&[buffer]); + lifts(params.y0 + params.output_y, &references.vertical); +} + fn compare_geometry( runtime: &MetalRuntime, - references: &[ComputePipelineState; 2], + references: &References, geometry: (u32, u32, u32, u32, u32, u32), batch: u32, high_pass: f32, ) { - let (width, height, x0, y0, output_x, output_y) = geometry; + const PREFIX: usize = 4; + let (width, height, ..) = geometry; let params = geometry_params(geometry); let count = width as usize * height as usize * batch as usize; - let mut seed = vec![7.0; count + 8]; - for (index, value) in seed[4..count + 4].iter_mut().enumerate() { + let mut seed = vec![7.0; count + 2 * PREFIX]; + for (index, value) in seed[PREFIX..count + PREFIX].iter_mut().enumerate() { *value = f32::from(i16::try_from(index % 257).unwrap() - 128) * 0.03125; } let buffers = [ copied_slice_buffer(&runtime.device, &seed).unwrap(), copied_slice_buffer(&runtime.device, &seed).unwrap(), ]; - let kernels = runtime.decode().expect("production kernels"); - for (axis, origin, offset, scale, lift) in [ - ( - 0, - x0, - output_x, - &kernels.idwt_irreversible97_horizontal_scale, - &kernels.idwt_irreversible97_horizontal_step, - ), - ( - 1, - y0, - output_y, - &kernels.idwt_irreversible97_vertical_scale, - &kernels.idwt_irreversible97_vertical_step, - ), - ] { - compare_stage( - runtime, - &buffers, - [scale, scale], - ¶ms, - batch, - Stage::Scale(high_pass), + let offset = (PREFIX * size_of::()) as u64; + let command = new_command_buffer(&runtime.queue).expect("comparison command"); + let encoder = new_compute_command_encoder(&command).expect("comparison encoder"); + encode_reference_stages( + &encoder, + references, + &buffers[0], + offset, + ¶ms, + batch, + high_pass, + ); + dispatch_irreversible97_stages_after_horizontal_scale( + &encoder, + runtime.decode().expect("production kernels"), + &buffers[1], + PREFIX * size_of::(), + params, + high_pass, + batch, + ); + encoder.endEncoding(); + commit_and_wait_metal(&command).expect("completed comparison"); + let len = count + 2 * PREFIX; + let expected = checked_buffer_slice::(&buffers[0], len, "reference stages").unwrap(); + let actual = checked_buffer_slice::(&buffers[1], len, "fused stages").unwrap(); + for (index, (actual, expected)) in actual.iter().zip(&expected).enumerate() { + assert!(actual.is_finite()); + assert_eq!( + actual.to_bits(), + expected.to_bits(), + "geometry {geometry:?} batch {batch}: coefficient {index}" ); - let even = (origin + offset) & 1; - for (coefficient, parity) in [ - (dwt::IDWT97_NEG_DELTA_F32, even), - (dwt::IDWT97_NEG_GAMMA_F32, 1 - even), - (dwt::IDWT97_NEG_BETA_F32, even), - (dwt::IDWT97_NEG_ALPHA_F32, 1 - even), - ] { - compare_stage( - runtime, - &buffers, - [&references[axis], lift], - ¶ms, - batch, - Stage::Lift(J2kIdwt97StepParams { - coefficient, - parity, - _reserved0: 0, - _reserved1: 0, - }), - ); - } + } + for value in actual[..PREFIX].iter().chain(&actual[len - PREFIX..]) { + assert_eq!(value.to_bits(), 7.0_f32.to_bits(), "offset guard changed"); } } #[test] -fn irreversible97_lifting_intermediates_match_full_grid_reference_bits() { +fn irreversible97_fused_lifting_matches_full_grid_reference_bits() { if !j2k_test_support::metal_runtime_gate(module_path!()) { return; } @@ -196,12 +227,17 @@ fn irreversible97_lifting_intermediates_match_full_grid_reference_bits() { crate::engine::shader_source::decode_shader_source() ); let loader = MetalPipelineLoader::new(&runtime.device, &source).expect("reference library"); - let references = [ - loader + let references = References { + horizontal: loader .pipeline("audit_idwt97_horizontal_reference") .unwrap(), - loader.pipeline("audit_idwt97_vertical_reference").unwrap(), - ]; + vertical_scale: loader + .pipeline("audit_idwt97_vertical_scale_reference") + .unwrap(), + vertical: loader.pipeline("audit_idwt97_vertical_reference").unwrap(), + }; + // Small edge cases, then shapes spanning several row tiles (128) and + // column tiles (64), with partial final tiles and odd origins. for geometry in [ (1, 1, 0, 0, 0, 0), (1, 1, 1, 1, 0, 0), @@ -212,6 +248,12 @@ fn irreversible97_lifting_intermediates_match_full_grid_reference_bits() { (5, 7, 0, 0, 1, 1), (18, 31, 1, 1, 2, 3), (129, 7, 0, 1, 3, 2), + (320, 240, 0, 0, 0, 0), + (257, 131, 1, 0, 0, 1), + (131, 257, 0, 1, 1, 0), + (640, 65, 1, 1, 0, 0), + (2, 200, 0, 0, 0, 1), + (200, 2, 1, 0, 0, 0), ] { for batch in [1, 3, 16] { for high_pass in [ @@ -224,11 +266,11 @@ fn irreversible97_lifting_intermediates_match_full_grid_reference_bits() { } Ok(()) }) - .expect("intermediate bitwise comparison"); + .expect("fused lifting bitwise comparison"); } #[test] -fn irreversible97_stage_grids_remove_inactive_parity_positions() { +fn irreversible97_fused_stages_encode_one_pass_per_axis() { if !j2k_test_support::metal_runtime_gate(module_path!()) { return; } @@ -253,44 +295,21 @@ fn irreversible97_stage_grids_remove_inactive_parity_positions() { commit_and_wait_metal(&command)?; let (positions, dispatches) = crate::engine::test_counters::idwt97_logical_dispatches_for_test(); + let horizontal_lift = usize::from(width > 1); assert_eq!( positions, - 6 * samples.len(), - "two scales plus eight half-parity lifts" + (2 + horizontal_lift) * samples.len(), + "horizontal scale, horizontal lifts, and vertical scale plus lifts each cover every sample once" ); assert_eq!( dispatches, - 2 + if width == 1 { 2 } else { 4 } + if height == 1 { 2 } else { 4 }, - "zero-count parity dispatches must be skipped" + 2 + horizontal_lift, + "single-column planes skip the horizontal lifting pass" ); } Ok(()) }) - .expect("production parity grids"); -} - -#[test] -fn compact_parity_axis_enumerates_original_active_coordinates() { - for length in 0..=257 { - for odd in [false, true] { - let parity = u32::from(odd); - let original = (0..length) - .filter(|index| index & 1 == parity) - .collect::>(); - let compact = (0..parity_axis_len(length, odd)) - .map(|index| index * 2 + parity) - .collect::>(); - assert_eq!(compact, original, "length={length}, odd={odd}"); - } - } - assert_eq!(parity_axis_len(u32::MAX, false), 1_u32 << 31); - assert_eq!(parity_axis_len(u32::MAX, true), (1_u32 << 31) - 1); - for (odd, expected_last) in [(false, u32::MAX - 1), (true, u32::MAX - 2)] { - let last = (parity_axis_len(u32::MAX, odd) - 1) - .checked_mul(2) - .and_then(|index| index.checked_add(u32::from(odd))); - assert_eq!(last, Some(expected_last)); - } + .expect("production fused grids"); } #[path = "interleave_tests.rs"] diff --git a/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/performance.rs b/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/performance.rs index e1d9ae170..06dc2452e 100644 --- a/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/performance.rs +++ b/crates/j2k-metal/src/engine/decode_dispatch/idwt/irreversible/performance.rs @@ -396,7 +396,8 @@ fn metal_irreversible97_stage_gpu_timing() { ); let (probe_logical_positions, probe_dispatches) = crate::engine::test_counters::idwt97_logical_dispatches_for_test(); - assert_eq!(probe_dispatches, 10); + // Horizontal scale, then one fused pass per axis. + assert_eq!(probe_dispatches, 3); let warm_started = Instant::now(); while warm_started.elapsed() < WARM_DURATION { diff --git a/crates/j2k-metal/src/engine/decode_dispatch/mct.rs b/crates/j2k-metal/src/engine/decode_dispatch/mct.rs index b4bc9e63c..96abf05b8 100644 --- a/crates/j2k-metal/src/engine/decode_dispatch/mct.rs +++ b/crates/j2k-metal/src/engine/decode_dispatch/mct.rs @@ -44,6 +44,9 @@ pub(crate) fn decode_inverse_mct(job: J2kInverseMctJob<'_>) -> Result encoder.memory_barrier_with_resources(&[r, g, b]), [r, g, b, a] => encoder.memory_barrier_with_resources(&[r, g, b, a]), diff --git a/crates/j2k-metal/src/engine/direct_grayscale_execute/component_plane/execution.rs b/crates/j2k-metal/src/engine/direct_grayscale_execute/component_plane/execution.rs index 8ca7601ca..5df731b0d 100644 --- a/crates/j2k-metal/src/engine/direct_grayscale_execute/component_plane/execution.rs +++ b/crates/j2k-metal/src/engine/direct_grayscale_execute/component_plane/execution.rs @@ -64,6 +64,7 @@ struct ComponentPlaneExecution<'a> { scratch_buffers: &'a mut Vec, bands: Vec, final_plane: FinalComponentPlane, + round_centered_store: bool, } impl ComponentPlaneExecution<'_> { @@ -343,6 +344,7 @@ impl ComponentPlaneExecution<'_> { output_x: store.output_x, output_y: store.output_y, addend: store.addend, + round_centered: u32::from(self.round_centered_store), }, ); if let Some(started) = encode_started { @@ -400,6 +402,7 @@ pub(in crate::engine) fn encode_prepared_direct_component_plane_in_encoder( scratch_buffers, bands, final_plane: FinalComponentPlane::empty(), + round_centered_store: plan.round_centered_store, }; let mut step_idx = 0; while step_idx < plan.steps.len() { diff --git a/crates/j2k-metal/src/engine/direct_grayscale_execute/destination/submission.rs b/crates/j2k-metal/src/engine/direct_grayscale_execute/destination/submission.rs index 57bda8508..3c50ad6eb 100644 --- a/crates/j2k-metal/src/engine/direct_grayscale_execute/destination/submission.rs +++ b/crates/j2k-metal/src/engine/direct_grayscale_execute/destination/submission.rs @@ -186,6 +186,8 @@ pub(in crate::engine::direct_grayscale_execute) fn commit_direct_destination( metadata: DirectExecutionMetadata, consumer_ordering: DirectDestinationConsumerOrdering, ) -> Result { + #[cfg(test)] + crate::engine::test_counters::record_direct_destination_command_buffer(&command_buffer); let mut consumer_waits = Vec::new(); #[cfg(test)] let mut known_consumer_event_ptr = None; diff --git a/crates/j2k-metal/src/engine/direct_grayscale_execute/single.rs b/crates/j2k-metal/src/engine/direct_grayscale_execute/single.rs index a1d1029e5..a8887447e 100644 --- a/crates/j2k-metal/src/engine/direct_grayscale_execute/single.rs +++ b/crates/j2k-metal/src/engine/direct_grayscale_execute/single.rs @@ -152,6 +152,7 @@ fn encode_prepared_direct_grayscale_plan_in_command_buffer_inner( destination, destination_item_index, destination_written: false, + round_centered_store: plan.round_centered_store, }; let result = (|| { let mut step_idx = 0; diff --git a/crates/j2k-metal/src/engine/direct_grayscale_execute/single/execution.rs b/crates/j2k-metal/src/engine/direct_grayscale_execute/single/execution.rs index 39def9ef5..9a0f2103c 100644 --- a/crates/j2k-metal/src/engine/direct_grayscale_execute/single/execution.rs +++ b/crates/j2k-metal/src/engine/direct_grayscale_execute/single/execution.rs @@ -43,6 +43,7 @@ pub(super) struct SingleGrayscaleExecution<'a> { pub(super) destination: Option<&'a MetalImageDestination>, pub(super) destination_item_index: usize, pub(super) destination_written: bool, + pub(super) round_centered_store: bool, } impl SingleGrayscaleExecution<'_> { @@ -285,48 +286,60 @@ impl SingleGrayscaleExecution<'_> { )?); } } else { - let output_span = checked_f32_span( - store.output_width as usize, - store.output_height as usize, - "J2K MetalDirect single stored component plane", - )?; - let output = take_f32_scratch_buffer(self.runtime, output_span.elements)?; - let params = J2kStoreParams { - input_width: store.input_rect.width(), - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - }; - dispatch_store_component_buffer_in_encoder_with_offsets( - self.runtime.decode()?, - self.encoder, - &input, - input_offset, - &output.buffer, - 0, - params, - ); - self.encoder - .memory_barrier_with_resources(&[&output.buffer]); - self.retained_buffers.push(output.buffer.clone()); - self.final_surface = Some(encode_gray_plane_to_surface_in_encoder( - self.runtime, - self.encoder, - &output.buffer, - self.dimensions, - self.bit_depth, - self.fmt, - )?); - self.scratch_buffers.push(output); + self.encode_store_through_plane(store, &input, input_offset)?; } Ok(()) } + /// Stores into an f32 plane, then packs it to a non-gray output format. + fn encode_store_through_plane( + &mut self, + store: &J2kDirectStoreStep, + input: &Buffer, + input_offset: usize, + ) -> Result<(), Error> { + let output_span = checked_f32_span( + store.output_width as usize, + store.output_height as usize, + "J2K MetalDirect single stored component plane", + )?; + let output = take_f32_scratch_buffer(self.runtime, output_span.elements)?; + let params = J2kStoreParams { + input_width: store.input_rect.width(), + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + round_centered: u32::from(self.round_centered_store), + }; + dispatch_store_component_buffer_in_encoder_with_offsets( + self.runtime.decode()?, + self.encoder, + input, + input_offset, + &output.buffer, + 0, + params, + ); + self.encoder + .memory_barrier_with_resources(&[&output.buffer]); + self.retained_buffers.push(output.buffer.clone()); + self.final_surface = Some(encode_gray_plane_to_surface_in_encoder( + self.runtime, + self.encoder, + &output.buffer, + self.dimensions, + self.bit_depth, + self.fmt, + )?); + self.scratch_buffers.push(output); + Ok(()) + } + pub(super) fn encode_step(&mut self, step: &PreparedDirectGrayscaleStep) -> Result<(), Error> { match step { PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { diff --git a/crates/j2k-metal/src/engine/direct_plan_types.rs b/crates/j2k-metal/src/engine/direct_plan_types.rs index 115e54f32..96d7ecfa3 100644 --- a/crates/j2k-metal/src/engine/direct_plan_types.rs +++ b/crates/j2k-metal/src/engine/direct_plan_types.rs @@ -17,6 +17,11 @@ mod allocation; pub(crate) struct PreparedDirectGrayscalePlan { pub(super) dimensions: (u32, u32), pub(super) bit_depth: u8, + /// Whether the final store rounds centered samples ties-to-even before + /// the level shift, as the CPU does for integer output. False only for + /// the first three components of an MCT color plan, which the inverse + /// transform rounds instead. + pub(super) round_centered_store: bool, pub(super) tier1_prepare_mode: DirectTier1Mode, pub(super) steps: Vec, pub(super) classic_groups: Vec, @@ -49,6 +54,19 @@ unsafe impl Send for PreparedDirectColorPlan {} // readers cannot race host or GPU writes through this owner. unsafe impl Sync for PreparedDirectColorPlan {} +/// Applies the CPU's color rounding split to freshly prepared component plans: +/// with MCT, components 0-2 are rounded after the inverse transform and any +/// alpha component at its own store. +pub(super) fn with_color_store_rounding( + mut component_plans: Vec, + mct: bool, +) -> Vec { + for (index, plan) in component_plans.iter_mut().enumerate() { + plan.round_centered_store = !(mct && index < 3); + } + component_plans +} + pub(super) enum PreparedDirectGrayscaleStep { ClassicSubBand(PreparedClassicSubBand), HtSubBand(PreparedHtSubBand), diff --git a/crates/j2k-metal/src/engine/direct_prepare/color.rs b/crates/j2k-metal/src/engine/direct_prepare/color.rs index 5f85ca525..16f7cf75b 100644 --- a/crates/j2k-metal/src/engine/direct_prepare/color.rs +++ b/crates/j2k-metal/src/engine/direct_prepare/color.rs @@ -9,6 +9,7 @@ use super::{ J2kReferencedHtj2kPlan, NativeGrayscalePlan, PreparedDirectColorPlan, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, ReferencedClassicPayloadCursor, }; +use crate::engine::direct_plan_types::with_color_store_rounding; use std::sync::Arc; #[cfg(target_os = "macos")] @@ -34,7 +35,7 @@ pub(crate) fn prepare_referenced_classic_color_plan( signed, mct: prepared.mct, transform: prepared.transform, - component_plans: prepared.component_plans, + component_plans: with_color_store_rounding(prepared.component_plans, prepared.mct), }) } @@ -61,7 +62,7 @@ pub(crate) fn prepare_referenced_classic_rgba_plan( signed, mct: prepared.mct, transform: prepared.transform, - component_plans: prepared.component_plans, + component_plans: with_color_store_rounding(prepared.component_plans, prepared.mct), }) } @@ -252,7 +253,7 @@ pub(crate) fn prepare_referenced_htj2k_color_plan( signed, mct: prepared.mct, transform: prepared.transform, - component_plans: prepared.component_plans, + component_plans: with_color_store_rounding(prepared.component_plans, prepared.mct), }) } @@ -279,7 +280,7 @@ pub(crate) fn prepare_referenced_htj2k_rgba_plan( signed, mct: prepared.mct, transform: prepared.transform, - component_plans: prepared.component_plans, + component_plans: with_color_store_rounding(prepared.component_plans, prepared.mct), }) } diff --git a/crates/j2k-metal/src/engine/direct_prepare/grayscale.rs b/crates/j2k-metal/src/engine/direct_prepare/grayscale.rs index b5baf3ca6..6d21469df 100644 --- a/crates/j2k-metal/src/engine/direct_prepare/grayscale.rs +++ b/crates/j2k-metal/src/engine/direct_prepare/grayscale.rs @@ -205,6 +205,7 @@ pub(super) fn prepare_direct_grayscale_plan_with_tier1_mode( Ok(PreparedDirectGrayscalePlan { dimensions: plan.dimensions, bit_depth: plan.bit_depth, + round_centered_store: true, tier1_prepare_mode, steps, classic_groups, diff --git a/crates/j2k-metal/src/engine/direct_prepare/referenced.rs b/crates/j2k-metal/src/engine/direct_prepare/referenced.rs index 420c40c80..f6213ce32 100644 --- a/crates/j2k-metal/src/engine/direct_prepare/referenced.rs +++ b/crates/j2k-metal/src/engine/direct_prepare/referenced.rs @@ -99,6 +99,7 @@ pub(super) fn finish_referenced_component_plan( Ok(PreparedDirectGrayscalePlan { dimensions, bit_depth, + round_centered_store: true, tier1_prepare_mode, steps, classic_groups, diff --git a/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission.rs b/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission.rs index b6a8857d9..af6ad3134 100644 --- a/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission.rs +++ b/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission.rs @@ -33,6 +33,7 @@ struct SubmissionContext<'a, 'p, 'r> { count: usize, broadcast_tier1_inputs: bool, profile_stages: bool, + round_centered_store: bool, resources: &'r mut StackedComponentResources, } @@ -97,6 +98,7 @@ pub(super) fn submit_stacked_component_commands<'p>( count: plan.count, broadcast_tier1_inputs: plan.broadcast_tier1_inputs, profile_stages, + round_centered_store: plan.first.round_centered_store, resources, }; diff --git a/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission/final_store.rs b/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission/final_store.rs index c217f7590..1243d0b63 100644 --- a/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission/final_store.rs +++ b/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission/final_store.rs @@ -77,6 +77,7 @@ impl SubmissionContext<'_, '_, '_> { batch_count: u32::try_from(self.count).map_err(|_| Error::MetalKernel { message: "J2K MetalDirect color store batch count exceeds u32".to_string(), })?, + round_centered: u32::from(self.round_centered_store), }; if let Some(encoder) = self.compute_encoder { dispatch_store_component_repeated_in_encoder( diff --git a/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission/reconstruction.rs b/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission/reconstruction.rs index ba18f5c87..62262dde5 100644 --- a/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission/reconstruction.rs +++ b/crates/j2k-metal/src/engine/direct_stacked_batch/command_submission/reconstruction.rs @@ -37,6 +37,12 @@ impl SubmissionContext<'_, '_, '_> { idwt: &PreparedDirectIdwt, output: &crate::metal_types::Buffer, ) -> Result<(), Error> { + #[cfg(test)] + if !crate::engine::test_counters::decode_stage_enabled( + crate::engine::test_counters::DecodeStageLimit::Idwt, + ) { + return Ok(()); + } let (ll, low_low_stride) = lookup_repeated_direct_band_layout_entry( &self.resources.band_sets, idwt.step.ll_band_id, @@ -113,6 +119,12 @@ impl SubmissionContext<'_, '_, '_> { output: &crate::metal_types::Buffer, span: &CheckedF32BatchSpan, ) -> Result<(), Error> { + #[cfg(test)] + if !crate::engine::test_counters::decode_stage_enabled( + crate::engine::test_counters::DecodeStageLimit::Idwt, + ) { + return Ok(()); + } for (instance_idx, bands) in self.resources.band_sets.iter().enumerate() { let PreparedDirectGrayscaleStep::Idwt(step) = &self.plans[instance_idx].steps[step_idx] else { @@ -173,9 +185,13 @@ impl SubmissionContext<'_, '_, '_> { )?; let output = take_f32_scratch_buffer(self.runtime, span.total_elements)?; let encode_started = self.profile_stages.then(Instant::now); - // Bound the simultaneously reconstructed planes: the measured 16-image - // 640x480 stage wins, while a 64 MiB 1024x1024 stage regresses (P20). - let use_single = self.count == 1 || span.total_bytes > 20 * 1024 * 1024; + // Batches reconstruct every instance in one dispatch per level. P20 + // bounded this at 20 MiB when the per-step kernels regressed at + // 16 x 1024x1024; with the fused lifting (P31) the batched route is + // 11% faster there in GPU time (P36). + let use_single = self.count == 1; + #[cfg(test)] + let use_single = use_single || crate::engine::test_counters::per_image_idwt_forced(); if idwt.step.transform == J2kWaveletTransform::Irreversible97 && use_single { self.encode_distinct_irreversible97_idwt(step_idx, &output.buffer, &span)?; } else { diff --git a/crates/j2k-metal/src/engine/direct_stacked_batch/repeated_grayscale/execution.rs b/crates/j2k-metal/src/engine/direct_stacked_batch/repeated_grayscale/execution.rs index e2ce0c370..17a622b50 100644 --- a/crates/j2k-metal/src/engine/direct_stacked_batch/repeated_grayscale/execution.rs +++ b/crates/j2k-metal/src/engine/direct_stacked_batch/repeated_grayscale/execution.rs @@ -24,6 +24,7 @@ struct RepeatedGrayscaleExecution<'a> { band_sets: Vec>, surfaces: Vec, stacked_outputs: bool, + round_centered_store: bool, } impl RepeatedGrayscaleExecution<'_> { @@ -98,6 +99,7 @@ pub(in crate::engine) fn encode_repeated_direct_grayscale_plan_in_command_buffer band_sets, surfaces, stacked_outputs: true, + round_centered_store: plan.round_centered_store, }; let mut step_idx = 0; while step_idx < plan.steps.len() { diff --git a/crates/j2k-metal/src/engine/direct_stacked_batch/repeated_grayscale/execution/final_store.rs b/crates/j2k-metal/src/engine/direct_stacked_batch/repeated_grayscale/execution/final_store.rs index 368fbaabf..fe8ece3f7 100644 --- a/crates/j2k-metal/src/engine/direct_stacked_batch/repeated_grayscale/execution/final_store.rs +++ b/crates/j2k-metal/src/engine/direct_stacked_batch/repeated_grayscale/execution/final_store.rs @@ -92,6 +92,7 @@ impl RepeatedGrayscaleExecution<'_> { output_y: store.output_y, addend: store.addend, batch_count, + round_centered: u32::from(self.round_centered_store), }, )?; self.retained_buffers.push(output.buffer.clone()); @@ -131,6 +132,7 @@ impl RepeatedGrayscaleExecution<'_> { output_x: store.output_x, output_y: store.output_y, addend: store.addend, + round_centered: u32::from(self.round_centered_store), }; dispatch_store_component_buffer_in_command_buffer_with_offsets( self.runtime, diff --git a/crates/j2k-metal/src/engine/direct_stacked_batch/validation.rs b/crates/j2k-metal/src/engine/direct_stacked_batch/validation.rs index 0c988168e..75d176a69 100644 --- a/crates/j2k-metal/src/engine/direct_stacked_batch/validation.rs +++ b/crates/j2k-metal/src/engine/direct_stacked_batch/validation.rs @@ -140,6 +140,7 @@ pub(in super::super) fn supports_stacked_direct_component_plane_batch( if plans.iter().any(|plan| { plan.dimensions != first.dimensions || plan.bit_depth != first.bit_depth + || plan.round_centered_store != first.round_centered_store || plan.steps.len() != first.steps.len() }) { return false; diff --git a/crates/j2k-metal/src/engine/forward_transform.rs b/crates/j2k-metal/src/engine/forward_transform.rs index ce66c2991..7721bded1 100644 --- a/crates/j2k-metal/src/engine/forward_transform.rs +++ b/crates/j2k-metal/src/engine/forward_transform.rs @@ -165,6 +165,7 @@ pub(crate) fn encode_forward_dwt53( input, output, params, + params.current_width, "J2K forward DWT 5/3 vertical", )?; active_is_a = !active_is_a; @@ -178,6 +179,7 @@ pub(crate) fn encode_forward_dwt53( input, output, params, + params.low_width, "J2K forward DWT 5/3 horizontal", )?; active_is_a = !active_is_a; @@ -541,6 +543,7 @@ pub(super) fn dispatch_forward_dwt53_pass( input: &Buffer, output: &Buffer, params: J2kForwardDwt53Params, + dispatch_width: u32, label: &str, ) -> Result<(), Error> { let encoder = new_compute_command_encoder(command_buffer)?; @@ -549,11 +552,7 @@ pub(super) fn dispatch_forward_dwt53_pass( encoder.set_buffer(0, Some(input), 0); encoder.set_buffer(1, Some(output), 0); encoder.set_bytes::(2, ¶ms); - dispatch_2d_pipeline( - &encoder, - pipeline, - (params.current_width, params.current_height), - ); + dispatch_2d_pipeline(&encoder, pipeline, (dispatch_width, params.current_height)); encoder.endEncoding(); Ok(()) } @@ -565,6 +564,7 @@ pub(super) fn dispatch_forward_dwt53_batched_pass( inputs: &[Buffer], outputs: &[Buffer], params: J2kForwardDwt53BatchedParams, + dispatch_width: u32, label: &str, ) -> Result<(), Error> { debug_assert!(!inputs.is_empty()); @@ -591,7 +591,7 @@ pub(super) fn dispatch_forward_dwt53_batched_pass( &encoder, pipeline, ( - params.current_width, + dispatch_width, params.current_height, params.component_count, ), diff --git a/crates/j2k-metal/src/engine/lossless_prepare/commands.rs b/crates/j2k-metal/src/engine/lossless_prepare/commands.rs index ea21f3dd4..93028c572 100644 --- a/crates/j2k-metal/src/engine/lossless_prepare/commands.rs +++ b/crates/j2k-metal/src/engine/lossless_prepare/commands.rs @@ -260,6 +260,7 @@ pub(in crate::engine) fn dispatch_forward_dwt53_on_buffers( src, dst, params, + params.current_width, "J2K coefficient prep DWT 5/3 vertical", )?; active_is_input = !active_is_input; @@ -272,6 +273,7 @@ pub(in crate::engine) fn dispatch_forward_dwt53_on_buffers( src, dst, params, + params.low_width, "J2K coefficient prep DWT 5/3 horizontal", )?; active_is_input = !active_is_input; @@ -348,6 +350,7 @@ pub(in crate::engine) fn dispatch_forward_dwt53_components_on_buffers( inputs, outputs, params, + params.current_width, "J2K coefficient prep DWT 5/3 vertical", )?; active_is_input = !active_is_input; @@ -364,6 +367,7 @@ pub(in crate::engine) fn dispatch_forward_dwt53_components_on_buffers( inputs, outputs, params, + params.low_width, "J2K coefficient prep DWT 5/3 horizontal", )?; active_is_input = !active_is_input; @@ -424,6 +428,7 @@ pub(in crate::engine) fn dispatch_forward_dwt53_on_buffers_split_profile( src, dst, params, + params.current_width, "J2K coefficient prep DWT 5/3 vertical", )?; command_buffer.commit(); @@ -442,6 +447,7 @@ pub(in crate::engine) fn dispatch_forward_dwt53_on_buffers_split_profile( src, dst, params, + params.low_width, "J2K coefficient prep DWT 5/3 horizontal", )?; command_buffer.commit(); @@ -514,6 +520,7 @@ pub(in crate::engine) fn dispatch_forward_dwt53_components_split_profile( inputs, outputs, params, + params.current_width, "J2K coefficient prep DWT 5/3 vertical", )?; command_buffer.commit(); @@ -536,6 +543,7 @@ pub(in crate::engine) fn dispatch_forward_dwt53_components_split_profile( inputs, outputs, params, + params.low_width, "J2K coefficient prep DWT 5/3 horizontal", )?; command_buffer.commit(); diff --git a/crates/j2k-metal/src/engine/resident_codestream/ht_cleanup.rs b/crates/j2k-metal/src/engine/resident_codestream/ht_cleanup.rs index a671e06b0..a8231c176 100644 --- a/crates/j2k-metal/src/engine/resident_codestream/ht_cleanup.rs +++ b/crates/j2k-metal/src/engine/resident_codestream/ht_cleanup.rs @@ -13,6 +13,8 @@ use super::{ MetalRuntime, J2K_HT_STATUS_OK, }; #[cfg(target_os = "macos")] +use crate::engine::abi::J2kHtVlcDispatchParams; +#[cfg(target_os = "macos")] use crate::engine::decode_dispatch::MetalHtPipelineKind; #[cfg(target_os = "macos")] @@ -38,6 +40,117 @@ pub(in crate::engine) struct HtCleanupRepeatedBatchDispatch<'a> { pub(in crate::engine) status_offset_bytes: u64, } +/// SIMD width, code blocks per `MagSgn` threadgroup, and VLC threads per +/// threadgroup of the two-kernel cleanup decode in `ht_cleanup_simd.metal`; +/// all three must match the shader constants. +#[cfg(target_os = "macos")] +const HT_SIMD_LANES: usize = 32; +#[cfg(target_os = "macos")] +const HT_SIMD_BLOCKS_PER_GROUP: usize = 4; +#[cfg(target_os = "macos")] +const HT_VLC_THREADS_PER_GROUP: usize = 32; +/// SIMD groups the VLC kernel aims to keep in flight. Each block's MEL/VLC +/// walk is one long serial chain: packing many blocks into a SIMD group adds +/// divergence, while spreading them over many groups makes the walk +/// issue-bound. On a 16-core M4 Pro, 40-150 groups measured best for 64 to +/// 3,072 blocks. +#[cfg(target_os = "macos")] +const HT_VLC_TARGET_SIMD_GROUPS: usize = 96; + +#[cfg(target_os = "macos")] +fn ht_vlc_lanes_per_simd(total_jobs: usize) -> usize { + total_jobs + .div_ceil(HT_VLC_TARGET_SIMD_GROUPS) + .next_power_of_two() + .clamp(1, HT_SIMD_LANES) +} + +/// Cleanup-only decode split into a thread-per-block MEL/VLC kernel and a +/// SIMD-group-per-block `MagSgn` kernel. +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +struct CooperativeHtCleanup<'a> { + vlc: &'a crate::metal_types::ComputePipelineState, + magsgn: &'a crate::metal_types::ComputePipelineState, + magsgn_threads_per_group: usize, +} + +#[cfg(target_os = "macos")] +impl<'a> CooperativeHtCleanup<'a> { + /// `None` when the device's SIMD width is not the one the kernels assume. + fn new( + vlc: &'a crate::metal_types::ComputePipelineState, + magsgn: &'a crate::metal_types::ComputePipelineState, + job_count: usize, + ) -> Option { + if magsgn.threadExecutionWidth() != HT_SIMD_LANES + || vlc.maxTotalThreadsPerThreadgroup() < HT_VLC_THREADS_PER_GROUP + { + return None; + } + let blocks = HT_SIMD_BLOCKS_PER_GROUP + .min(magsgn.maxTotalThreadsPerThreadgroup() / HT_SIMD_LANES) + .min(job_count); + (blocks > 0).then_some(Self { + vlc, + magsgn, + magsgn_threads_per_group: blocks * HT_SIMD_LANES, + }) + } + + /// Encodes both kernels over buffers the caller has already bound. The + /// `MagSgn` kernel reads the per-quad VLC words the first kernel leaves in + /// `decoded`, and may overwrite `status` with a `MagSgn` failure. + fn encode( + self, + encoder: &ComputeCommandEncoderRef, + job_count: usize, + batch_count: u32, + decoded: &Buffer, + status: &Buffer, + ) -> Result<(), Error> { + let total_jobs = + job_count + .checked_mul(batch_count as usize) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal cooperative cleanup job count overflow".to_string(), + })?; + let lanes_per_simd = ht_vlc_lanes_per_simd(total_jobs); + let dispatch = J2kHtVlcDispatchParams { + job_count: u32::try_from(job_count).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal cooperative cleanup job count exceeds u32".to_string(), + })?, + lanes_per_simd: u32::try_from(lanes_per_simd).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal cooperative cleanup lane count exceeds u32".to_string(), + })?, + }; + let vlc_threads = job_count.div_ceil(lanes_per_simd) * HT_SIMD_LANES; + encoder.set_bytes::(9, &dispatch); + encoder.setComputePipelineState(self.vlc); + encoder.dispatchThreads_threadsPerThreadgroup( + j2k_metal_support::mtl_size(vlc_threads as u64, u64::from(batch_count), 1), + j2k_metal_support::mtl_size(HT_VLC_THREADS_PER_GROUP as u64, 1, 1), + ); + encoder.memory_barrier_with_resources(&[decoded, status]); + #[cfg(test)] + if !crate::engine::test_counters::decode_stage_enabled( + crate::engine::test_counters::DecodeStageLimit::Tier1, + ) { + return Ok(()); + } + encoder.setComputePipelineState(self.magsgn); + encoder.dispatchThreads_threadsPerThreadgroup( + j2k_metal_support::mtl_size( + (job_count * HT_SIMD_LANES) as u64, + u64::from(batch_count), + 1, + ), + j2k_metal_support::mtl_size(self.magsgn_threads_per_group as u64, 1, 1), + ); + Ok(()) + } +} + #[cfg(target_os = "macos")] pub(in crate::engine) fn dispatch_ht_cleanup( runtime: &MetalRuntime, @@ -62,7 +175,13 @@ pub(in crate::engine) fn dispatch_ht_cleanup( )?, )?; encoder.memory_barrier_with_resources(&[decoded]); - encoder.setComputePipelineState(&runtime.decode()?.ht_cleanup); + let kernels = runtime.decode()?; + let pipeline = if params.number_of_coding_passes == 1 { + &kernels.ht_cleanup_cleanup_only + } else { + &kernels.ht_cleanup + }; + encoder.setComputePipelineState(pipeline); encoder.set_buffer(0, Some(&input), 0); encoder.set_buffer(1, Some(decoded), 0); encoder.set_bytes::(2, ¶ms); @@ -106,25 +225,40 @@ pub(in crate::engine) fn dispatch_ht_cleanup_batched( ht_batch_output_word_count(jobs)?, )?; encoder.memory_barrier_with_resources(&[decoded]); - encoder.setComputePipelineState(&runtime.decode()?.ht_cleanup_batched); + let kernels = runtime.decode()?; + let cleanup_only = jobs.iter().all(|job| job.number_of_coding_passes == 1); + let cooperative = cleanup_only + .then(|| { + CooperativeHtCleanup::new( + &kernels.ht_cleanup_vlc_batched, + &kernels.ht_cleanup_magsgn_batched, + jobs.len(), + ) + }) + .flatten(); + let pipeline = match (cooperative, cleanup_only) { + (Some(cooperative), _) => cooperative.vlc, + (None, true) => &kernels.ht_cleanup_batched_cleanup_only, + (None, false) => &kernels.ht_cleanup_batched, + }; + encoder.setComputePipelineState(pipeline); encoder.set_buffer(0, Some(&input), 0); encoder.set_buffer(1, Some(decoded), 0); encoder.set_buffer(2, Some(&jobs_buffer), 0); - encoder.set_buffer(3, Some(&runtime.decode()?.ht_vlc_table0), 0); - encoder.set_buffer(4, Some(&runtime.decode()?.ht_vlc_table1), 0); - encoder.set_buffer(5, Some(&runtime.decode()?.ht_uvlc_table0), 0); - encoder.set_buffer(6, Some(&runtime.decode()?.ht_uvlc_table1), 0); + encoder.set_buffer(3, Some(&kernels.ht_vlc_table0), 0); + encoder.set_buffer(4, Some(&kernels.ht_vlc_table1), 0); + encoder.set_buffer(5, Some(&kernels.ht_uvlc_table0), 0); + encoder.set_buffer(6, Some(&kernels.ht_uvlc_table1), 0); encoder.set_buffer(7, Some(&status_buffer), 0); - let width = runtime - .decode()? - .ht_cleanup_batched - .threadExecutionWidth() - .max(1) - .min(jobs.len()); - encoder.dispatchThreads_threadsPerThreadgroup( - j2k_metal_support::mtl_size(jobs.len() as u64, 1, 1), - j2k_metal_support::mtl_size(width as u64, 1, 1), - ); + if let Some(cooperative) = cooperative { + cooperative.encode(&encoder, jobs.len(), 1, decoded, &status_buffer)?; + } else { + let width = pipeline.threadExecutionWidth().max(1).min(jobs.len()); + encoder.dispatchThreads_threadsPerThreadgroup( + j2k_metal_support::mtl_size(jobs.len() as u64, 1, 1), + j2k_metal_support::mtl_size(width as u64, 1, 1), + ); + } encoder.endEncoding(); commit_and_wait_metal(&command_buffer)?; @@ -150,11 +284,21 @@ pub(in crate::engine) fn dispatch_ht_cleanup_batched_in_encoder_with_status_offs encoder: &ComputeCommandEncoderRef, pipeline_kind: MetalHtPipelineKind, dispatch: HtCleanupBatchDispatch<'_>, -) { - let pipeline = match pipeline_kind { - MetalHtPipelineKind::CleanupOnly => &kernels.ht_cleanup_batched_cleanup_only, - MetalHtPipelineKind::SigProp => &kernels.ht_cleanup_batched_sigprop, - MetalHtPipelineKind::MagRef => &kernels.ht_cleanup_batched_magref, +) -> Result<(), Error> { + let cooperative = (pipeline_kind == MetalHtPipelineKind::CleanupOnly) + .then(|| { + CooperativeHtCleanup::new( + &kernels.ht_cleanup_vlc_batched, + &kernels.ht_cleanup_magsgn_batched, + dispatch.job_count, + ) + }) + .flatten(); + let pipeline = match (pipeline_kind, cooperative) { + (MetalHtPipelineKind::CleanupOnly, Some(cooperative)) => cooperative.vlc, + (MetalHtPipelineKind::CleanupOnly, None) => &kernels.ht_cleanup_batched_cleanup_only, + (MetalHtPipelineKind::SigProp, _) => &kernels.ht_cleanup_batched_sigprop, + (MetalHtPipelineKind::MagRef, _) => &kernels.ht_cleanup_batched_magref, }; encoder.setComputePipelineState(pipeline); encoder.set_buffer(0, Some(dispatch.coded_data), 0); @@ -169,6 +313,15 @@ pub(in crate::engine) fn dispatch_ht_cleanup_batched_in_encoder_with_status_offs Some(dispatch.status_buffer), dispatch.status_offset_bytes, ); + if let Some(cooperative) = cooperative { + return cooperative.encode( + encoder, + dispatch.job_count, + 1, + dispatch.decoded, + dispatch.status_buffer, + ); + } let width = pipeline .threadExecutionWidth() .max(1) @@ -177,6 +330,7 @@ pub(in crate::engine) fn dispatch_ht_cleanup_batched_in_encoder_with_status_offs j2k_metal_support::mtl_size(dispatch.job_count as u64, 1, 1), j2k_metal_support::mtl_size(width as u64, 1, 1), ); + Ok(()) } #[cfg(target_os = "macos")] @@ -185,11 +339,23 @@ pub(in crate::engine) fn dispatch_ht_cleanup_repeated_batched_in_encoder_with_st encoder: &ComputeCommandEncoderRef, pipeline_kind: MetalHtPipelineKind, dispatch: HtCleanupRepeatedBatchDispatch<'_>, -) { - let pipeline = match pipeline_kind { - MetalHtPipelineKind::CleanupOnly => &kernels.ht_cleanup_repeated_batched_cleanup_only, - MetalHtPipelineKind::SigProp => &kernels.ht_cleanup_repeated_batched_sigprop, - MetalHtPipelineKind::MagRef => &kernels.ht_cleanup_repeated_batched_magref, +) -> Result<(), Error> { + let cooperative = (pipeline_kind == MetalHtPipelineKind::CleanupOnly) + .then(|| { + CooperativeHtCleanup::new( + &kernels.ht_cleanup_vlc_repeated_batched, + &kernels.ht_cleanup_magsgn_repeated_batched, + dispatch.base_job_count, + ) + }) + .flatten(); + let pipeline = match (pipeline_kind, cooperative) { + (MetalHtPipelineKind::CleanupOnly, Some(cooperative)) => cooperative.vlc, + (MetalHtPipelineKind::CleanupOnly, None) => { + &kernels.ht_cleanup_repeated_batched_cleanup_only + } + (MetalHtPipelineKind::SigProp, _) => &kernels.ht_cleanup_repeated_batched_sigprop, + (MetalHtPipelineKind::MagRef, _) => &kernels.ht_cleanup_repeated_batched_magref, }; encoder.setComputePipelineState(pipeline); encoder.set_buffer(0, Some(dispatch.coded_data), 0); @@ -205,6 +371,15 @@ pub(in crate::engine) fn dispatch_ht_cleanup_repeated_batched_in_encoder_with_st Some(dispatch.status_buffer), dispatch.status_offset_bytes, ); + if let Some(cooperative) = cooperative { + return cooperative.encode( + encoder, + dispatch.base_job_count, + dispatch.repeated.batch_count, + dispatch.decoded, + dispatch.status_buffer, + ); + } let width = pipeline .threadExecutionWidth() .max(1) @@ -217,4 +392,5 @@ pub(in crate::engine) fn dispatch_ht_cleanup_repeated_batched_in_encoder_with_st ), j2k_metal_support::mtl_size(width as u64, 1, 1), ); + Ok(()) } diff --git a/crates/j2k-metal/src/engine/runtime/decode.rs b/crates/j2k-metal/src/engine/runtime/decode.rs index 35fb6eead..93db56943 100644 --- a/crates/j2k-metal/src/engine/runtime/decode.rs +++ b/crates/j2k-metal/src/engine/runtime/decode.rs @@ -33,9 +33,8 @@ pub(in crate::engine) struct DecodeKernels { pub(in crate::engine) idwt_reversible53_vertical_batched: ComputePipelineState, #[cfg(test)] pub(in crate::engine) idwt_irreversible97_horizontal_scale: ComputePipelineState, - pub(in crate::engine) idwt_irreversible97_vertical_scale: ComputePipelineState, - pub(in crate::engine) idwt_irreversible97_horizontal_step: ComputePipelineState, - pub(in crate::engine) idwt_irreversible97_vertical_step: ComputePipelineState, + pub(in crate::engine) idwt_irreversible97_horizontal_lift_fused: ComputePipelineState, + pub(in crate::engine) idwt_irreversible97_vertical_fused: ComputePipelineState, pub(in crate::engine) inverse_mct: ComputePipelineState, pub(in crate::engine) store_component: ComputePipelineState, pub(in crate::engine) store_component_repeated: ComputePipelineState, @@ -54,6 +53,7 @@ pub(in crate::engine) struct DecodeKernels { pub(in crate::engine) store_native_rgba_batch_u16: ComputePipelineState, pub(in crate::engine) store_native_rgba_batch_i16: ComputePipelineState, pub(in crate::engine) ht_cleanup: ComputePipelineState, + pub(in crate::engine) ht_cleanup_cleanup_only: ComputePipelineState, pub(in crate::engine) ht_cleanup_batched: ComputePipelineState, pub(in crate::engine) ht_cleanup_batched_cleanup_only: ComputePipelineState, pub(in crate::engine) ht_cleanup_batched_sigprop: ComputePipelineState, @@ -61,6 +61,10 @@ pub(in crate::engine) struct DecodeKernels { pub(in crate::engine) ht_cleanup_repeated_batched_cleanup_only: ComputePipelineState, pub(in crate::engine) ht_cleanup_repeated_batched_sigprop: ComputePipelineState, pub(in crate::engine) ht_cleanup_repeated_batched_magref: ComputePipelineState, + pub(in crate::engine) ht_cleanup_vlc_batched: ComputePipelineState, + pub(in crate::engine) ht_cleanup_vlc_repeated_batched: ComputePipelineState, + pub(in crate::engine) ht_cleanup_magsgn_batched: ComputePipelineState, + pub(in crate::engine) ht_cleanup_magsgn_repeated_batched: ComputePipelineState, pub(in crate::engine) ht_vlc_table0: Buffer, pub(in crate::engine) ht_vlc_table1: Buffer, pub(in crate::engine) ht_uvlc_table0: Buffer, @@ -110,12 +114,10 @@ impl DecodeKernels { #[cfg(test)] idwt_irreversible97_horizontal_scale: loader .pipeline("j2k_idwt_irreversible97_horizontal_scale")?, - idwt_irreversible97_vertical_scale: loader - .pipeline("j2k_idwt_irreversible97_vertical_scale")?, - idwt_irreversible97_horizontal_step: loader - .pipeline("j2k_idwt_irreversible97_horizontal_step")?, - idwt_irreversible97_vertical_step: loader - .pipeline("j2k_idwt_irreversible97_vertical_step")?, + idwt_irreversible97_horizontal_lift_fused: loader + .pipeline("j2k_idwt_irreversible97_horizontal_lift_fused")?, + idwt_irreversible97_vertical_fused: loader + .pipeline("j2k_idwt_irreversible97_vertical_fused")?, inverse_mct: loader.pipeline("j2k_inverse_mct")?, store_component: loader.pipeline("j2k_store_component")?, store_component_repeated: loader.pipeline("j2k_store_component_repeated")?, @@ -139,6 +141,7 @@ impl DecodeKernels { store_native_rgba_batch_u16: loader.pipeline("j2k_store_native_rgba_batch_u16")?, store_native_rgba_batch_i16: loader.pipeline("j2k_store_native_rgba_batch_i16")?, ht_cleanup: loader.pipeline("j2k_decode_ht_cleanup")?, + ht_cleanup_cleanup_only: loader.pipeline("j2k_decode_ht_cleanup_cleanup_only")?, ht_cleanup_batched: loader.pipeline("j2k_decode_ht_cleanup_batched")?, ht_cleanup_batched_cleanup_only: loader .pipeline("j2k_decode_ht_cleanup_batched_cleanup_only")?, @@ -150,6 +153,12 @@ impl DecodeKernels { .pipeline("j2k_decode_ht_cleanup_repeated_batched_sigprop")?, ht_cleanup_repeated_batched_magref: loader .pipeline("j2k_decode_ht_cleanup_repeated_batched_magref")?, + ht_cleanup_vlc_batched: loader.pipeline("j2k_decode_ht_cleanup_vlc_batched")?, + ht_cleanup_vlc_repeated_batched: loader + .pipeline("j2k_decode_ht_cleanup_vlc_repeated_batched")?, + ht_cleanup_magsgn_batched: loader.pipeline("j2k_decode_ht_cleanup_magsgn_batched")?, + ht_cleanup_magsgn_repeated_batched: loader + .pipeline("j2k_decode_ht_cleanup_magsgn_repeated_batched")?, ht_vlc_table0: checked_shared_buffer_with_slice(device, ht_vlc_table0())?, ht_vlc_table1: checked_shared_buffer_with_slice(device, ht_vlc_table1())?, ht_uvlc_table0: checked_shared_buffer_with_slice(device, ht_uvlc_table0())?, diff --git a/crates/j2k-metal/src/engine/shader_source.rs b/crates/j2k-metal/src/engine/shader_source.rs index 7fd153ad5..6deb414fa 100644 --- a/crates/j2k-metal/src/engine/shader_source.rs +++ b/crates/j2k-metal/src/engine/shader_source.rs @@ -25,6 +25,7 @@ pub(super) fn decode_shader_source() -> String { include_str!("../sampled_plane.metal"), include_str!("../store_native_color_batch.metal"), include_str!("../ht_cleanup.metal"), + include_str!("../ht_cleanup_simd.metal"), ] .join("\n") } diff --git a/crates/j2k-metal/src/engine/surface_decode.rs b/crates/j2k-metal/src/engine/surface_decode.rs index 70f171e85..eff346e6e 100644 --- a/crates/j2k-metal/src/engine/surface_decode.rs +++ b/crates/j2k-metal/src/engine/surface_decode.rs @@ -23,8 +23,10 @@ pub(crate) fn decode_image_to_surface<'a>( ) -> Result { with_runtime(|runtime| { let mut code_block_decoder = MetalCodeBlockDecoder::default(); + // The planes are packed to integer surfaces, so irreversible samples + // must be rounded before the level shift, as the CPU decode does. let decoded = image - .decode_components_with_ht_decoder(context, &mut code_block_decoder) + .decode_components_for_integer_output_with_ht_decoder(context, &mut code_block_decoder) .map_err(native_decode_error)?; let stage = select_plane_stage(runtime, image, &decoded, &mut code_block_decoder)?; stage.finish_with_runtime(runtime, fmt) @@ -51,7 +53,7 @@ pub(crate) fn decode_image_region_to_surface<'a>( with_runtime(|runtime| { let mut code_block_decoder = MetalCodeBlockDecoder::default(); let decoded = image - .decode_region_components_with_ht_decoder( + .decode_region_components_for_integer_output_with_ht_decoder( context, (roi.x, roi.y, roi.w, roi.h), &mut code_block_decoder, diff --git a/crates/j2k-metal/src/engine/test_counters.rs b/crates/j2k-metal/src/engine/test_counters.rs index 766567e8a..7e36c80f3 100644 --- a/crates/j2k-metal/src/engine/test_counters.rs +++ b/crates/j2k-metal/src/engine/test_counters.rs @@ -343,3 +343,69 @@ pub(crate) fn record_idwt97_logical_dispatch(grid: (u32, u32, u32)) { ); IDWT97_STAGE_DISPATCHES.set(IDWT97_STAGE_DISPATCHES.get().saturating_add(1)); } + +/// Last decode stage whose GPU dispatches are encoded, for attributing GPU time +/// by truncation (`decode_stage_profile`). Stages keep their buffers and host +/// bookkeeping; only the dispatches of later stages are skipped, so outputs +/// are meaningless below `Full`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum DecodeStageLimit { + /// Zero-fill and the HT cleanup MEL/VLC kernel (it writes job status). + HtVlc, + /// Adds the HT cleanup `MagSgn` kernel. + Tier1, + /// Adds the inverse wavelet transform. + Idwt, + /// Adds the inverse color transform and final store. + Full, +} + +static DECODE_STAGE_LIMIT: std::sync::atomic::AtomicU8 = + std::sync::atomic::AtomicU8::new(DecodeStageLimit::Full as u8); + +pub(crate) fn set_decode_stage_limit_for_test(limit: DecodeStageLimit) { + DECODE_STAGE_LIMIT.store(limit as u8, Ordering::Relaxed); +} + +pub(crate) fn decode_stage_enabled(stage: DecodeStageLimit) -> bool { + DECODE_STAGE_LIMIT.load(Ordering::Relaxed) >= stage as u8 +} + +std::thread_local! { + /// `Some` while `decode_stage_profile` captures direct-destination command + /// buffers; capture is off otherwise so no test retains their resources. + static DIRECT_DESTINATION_COMMAND_BUFFERS: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +pub(crate) fn record_direct_destination_command_buffer( + command_buffer: &crate::metal_types::CommandBuffer, +) { + DIRECT_DESTINATION_COMMAND_BUFFERS.with(|buffers| { + if let Some(buffers) = buffers.borrow_mut().as_mut() { + buffers.push(command_buffer.clone()); + } + }); +} + +pub(crate) fn begin_direct_destination_capture_for_test() { + DIRECT_DESTINATION_COMMAND_BUFFERS.with(|buffers| *buffers.borrow_mut() = Some(Vec::new())); +} + +pub(crate) fn end_direct_destination_capture_for_test() -> Vec { + DIRECT_DESTINATION_COMMAND_BUFFERS + .with(|buffers| buffers.borrow_mut().take().unwrap_or_default()) +} + +/// Forces the per-image 9/7 IDWT route for batches, the pre-P36 behaviour +/// above 20 MiB, for same-process A/B probes. +static PER_IMAGE_IDWT_FORCED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +pub(crate) fn force_per_image_idwt_for_test(forced: bool) { + PER_IMAGE_IDWT_FORCED.store(forced, Ordering::Relaxed); +} + +pub(crate) fn per_image_idwt_forced() -> bool { + PER_IMAGE_IDWT_FORCED.load(Ordering::Relaxed) +} diff --git a/crates/j2k-metal/src/engine/tests.rs b/crates/j2k-metal/src/engine/tests.rs index f948cf89f..df3b61b74 100644 --- a/crates/j2k-metal/src/engine/tests.rs +++ b/crates/j2k-metal/src/engine/tests.rs @@ -2,6 +2,7 @@ mod capacity; mod classic; +mod decode_stage_profile; mod grouping; mod hybrid; mod hybrid_support; diff --git a/crates/j2k-metal/src/engine/tests/decode_stage_profile.rs b/crates/j2k-metal/src/engine/tests/decode_stage_profile.rs new file mode 100644 index 000000000..6c286939e --- /dev/null +++ b/crates/j2k-metal/src/engine/tests/decode_stage_profile.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! GPU time by stage for the resident color batch route, by truncation: each +//! variant encodes the dispatches up to one `DecodeStageLimit`, and successive +//! differences of the median GPU times attribute the cost. Variants are sampled +//! round-robin after a warm-up so clock ramping affects each alike. +//! +//! ```sh +//! J2K_REQUIRE_METAL_RUNTIME=1 cargo test --profile gpu-quick -p j2k-metal --lib -- \ +//! decode_stage_profile --include-ignored --nocapture --test-threads=1 +//! ``` + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use j2k::{BatchDecodeOptions, BatchLayout, EncodedImage, PreparedBatch}; +use j2k_native::{encode_htj2k, EncodeOptions}; + +use crate::engine::test_counters::{ + begin_direct_destination_capture_for_test, end_direct_destination_capture_for_test, + force_per_image_idwt_for_test, set_decode_stage_limit_for_test, DecodeStageLimit, +}; +use crate::metal_types::prelude::*; + +const LIMITS: [DecodeStageLimit; 4] = [ + DecodeStageLimit::HtVlc, + DecodeStageLimit::Tier1, + DecodeStageLimit::Idwt, + DecodeStageLimit::Full, +]; +const SAMPLES: usize = 21; +const WARM_UP: Duration = Duration::from_secs(2); + +/// Restores full decoding even if a probe panics. +struct FullStagesOnDrop; + +impl Drop for FullStagesOnDrop { + fn drop(&mut self) { + set_decode_stage_limit_for_test(DecodeStageLimit::Full); + force_per_image_idwt_for_test(false); + } +} + +/// Distinct 9/7 HT RGB codestreams, as `metal_idwt97_geometry_distinct` builds them. +fn distinct_batch(width: u32, height: u32, count: u8) -> Vec { + let options = EncodeOptions { + reversible: false, + num_decomposition_levels: 3, + guard_bits: 2, + ..EncodeOptions::default() + }; + (0..count) + .map(|index| { + let mut pixels = j2k_test_support::patterned_rgb8(width, height); + pixels[0] = pixels[0].wrapping_add(index); + let bytes = encode_htj2k(&pixels, width, height, 3, 8, false, &options) + .expect("encode stage-profile fixture"); + EncodedImage::full(Arc::from(bytes)) + }) + .collect() +} + +/// GPU seconds of one resident decode truncated at `limit`. +fn decode_gpu_seconds( + decoder: &mut crate::MetalBatchDecoder, + prepared: &PreparedBatch, + limit: DecodeStageLimit, +) -> f64 { + set_decode_stage_limit_for_test(limit); + begin_direct_destination_capture_for_test(); + let result = decoder + .decode_prepared(prepared) + .expect("stage-profile decode"); + let command_buffers = end_direct_destination_capture_for_test(); + assert!( + result.errors().is_empty(), + "stage-profile decode at {limit:?}: {:?}", + result.errors() + ); + assert!( + !command_buffers.is_empty(), + "the batch did not take the direct color destination route" + ); + command_buffers + .iter() + .map(|command_buffer| { + command_buffer.waitUntilCompleted(); + command_buffer.GPUEndTime() - command_buffer.GPUStartTime() + }) + .sum() +} + +fn median_ms(samples: &mut [f64]) -> f64 { + samples.sort_by(f64::total_cmp); + samples[samples.len() / 2] * 1e3 +} + +#[test] +#[ignore = "GPU stage attribution; run explicitly with --include-ignored --nocapture"] +fn decode_stage_profile() { + if !super::runtime::should_run_metal_runtime() { + return; + } + let _restore = FullStagesOnDrop; + for (width, height, count) in [(1024, 1024, 16), (640, 480, 16)] { + let mut decoder = + crate::MetalBatchDecoder::system_default_with_options(BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }) + .expect("Metal batch decoder"); + let prepared = decoder + .prepare(distinct_batch(width, height, count)) + .expect("prepare stage-profile batch"); + + let warm_until = Instant::now() + WARM_UP; + while Instant::now() < warm_until { + for limit in LIMITS { + decode_gpu_seconds(&mut decoder, &prepared, limit); + } + } + let mut samples = [(); 4].map(|()| Vec::with_capacity(SAMPLES)); + for _ in 0..SAMPLES { + for (slot, limit) in samples.iter_mut().zip(LIMITS) { + slot.push(decode_gpu_seconds(&mut decoder, &prepared, limit)); + } + } + let [vlc, tier1, idwt, full] = samples.map(|mut slot| median_ms(&mut slot)); + println!( + "{width}x{height} x{count} GPU ms (median of {SAMPLES}): full {full:.3} | \ + zero-fill+VLC {vlc:.3}, MagSgn {:.3}, IDWT {:.3}, ICT+store {:.3}", + tier1 - vlc, + idwt - tier1, + full - idwt, + ); + } +} + +/// IDWT and full GPU time with the per-image 9/7 IDWT route forced (the +/// pre-P36 behaviour above 20 MiB) and with the batched route, round-robin. +#[test] +#[ignore = "GPU A/B probe; run explicitly with --include-ignored --nocapture"] +fn batched_idwt_route_profile() { + if !super::runtime::should_run_metal_runtime() { + return; + } + let _restore = FullStagesOnDrop; + let routes = [true, false]; + for (width, height, count) in [(1024, 1024, 16), (640, 480, 16)] { + let mut decoder = + crate::MetalBatchDecoder::system_default_with_options(BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }) + .expect("Metal batch decoder"); + let prepared = decoder + .prepare(distinct_batch(width, height, count)) + .expect("prepare IDWT probe batch"); + let mut sample = |per_image: bool, stage: DecodeStageLimit| { + force_per_image_idwt_for_test(per_image); + decode_gpu_seconds(&mut decoder, &prepared, stage) + }; + let warm_until = Instant::now() + WARM_UP; + while Instant::now() < warm_until { + for per_image in routes { + sample(per_image, DecodeStageLimit::Full); + } + } + let mut samples = [(); 6].map(|()| Vec::with_capacity(SAMPLES)); + for _ in 0..SAMPLES { + for (index, per_image) in routes.into_iter().enumerate() { + samples[index * 3].push(sample(per_image, DecodeStageLimit::Tier1)); + samples[index * 3 + 1].push(sample(per_image, DecodeStageLimit::Idwt)); + samples[index * 3 + 2].push(sample(per_image, DecodeStageLimit::Full)); + } + } + let [t1_a, idwt_a, full_a, t1_b, idwt_b, full_b] = + samples.map(|mut slot| median_ms(&mut slot)); + println!( + "{width}x{height} x{count} GPU ms: per-image IDWT {:.3} full {full_a:.3} | \ + batched IDWT {:.3} full {full_b:.3}", + idwt_a - t1_a, + idwt_b - t1_b, + ); + } +} diff --git a/crates/j2k-metal/src/fdwt.metal b/crates/j2k-metal/src/fdwt.metal index f31d1c592..14d48da11 100644 --- a/crates/j2k-metal/src/fdwt.metal +++ b/crates/j2k-metal/src/fdwt.metal @@ -66,29 +66,44 @@ inline void j2k_fdwt53_horizontal_step( uint full_width, uint current_width, uint low_width, - uint2 gid + uint2 gid, + uint local_x, + uint simd_lane ) { + const bool active = gid.x < low_width; + const uint high_width = current_width - low_width; const uint row_base = gid.y * full_width; - if (gid.x < low_width) { - const uint even = gid.x * 2u; - const float left = gid.x > 0u - ? j2k_fdwt53_predict_row(src, row_base, current_width, gid.x - 1u) - : j2k_fdwt53_predict_row(src, row_base, current_width, 0u); - const float right = even + 1u < current_width - ? j2k_fdwt53_predict_row(src, row_base, current_width, gid.x) - : left; - dst[row_base + gid.x] = - src[row_base + even] + floor((left + right) * 0.25f + 0.5f); + float right = 0.0f; + if (active && gid.x < high_width) { + right = j2k_fdwt53_predict_row(src, row_base, current_width, gid.x); + } + // Each lane emits a low/high pair and shares its high prediction with the + // next lane. Nonuniform threadgroups can make SIMD groups cross rows; + // row and SIMD boundaries need an explicit one-sample halo. + const float shuffled_left = simd_shuffle_up(right, 1u); + + if (!active) { return; } - const uint high_index = gid.x - low_width; - dst[row_base + gid.x] = j2k_fdwt53_predict_row( - src, - row_base, - current_width, - high_index - ); + float left; + if (gid.x == 0u) { + left = right; + } else if (local_x == 0u || simd_lane == 0u) { + left = j2k_fdwt53_predict_row(src, row_base, current_width, gid.x - 1u); + } else { + left = shuffled_left; + } + if (gid.x >= high_width) { + right = left; + } + + const uint even = gid.x * 2u; + dst[row_base + gid.x] = + src[row_base + even] + floor((left + right) * 0.25f + 0.5f); + if (gid.x < high_width) { + dst[row_base + low_width + gid.x] = right; + } } inline void j2k_fdwt53_vertical_step( @@ -126,7 +141,9 @@ kernel void j2k_forward_dwt53_horizontal( device const float *src [[buffer(0)]], device float *dst [[buffer(1)]], constant J2kForwardDwt53Params ¶ms [[buffer(2)]], - uint2 gid [[thread_position_in_grid]] + uint2 gid [[thread_position_in_grid]], + uint2 local_id [[thread_position_in_threadgroup]], + uint simd_lane [[thread_index_in_simdgroup]] ) { if (gid.x >= params.current_width || gid.y >= params.current_height) { return; @@ -138,7 +155,9 @@ kernel void j2k_forward_dwt53_horizontal( params.full_width, params.current_width, params.low_width, - gid + gid, + local_id.x, + simd_lane ); } @@ -170,7 +189,9 @@ kernel void j2k_forward_dwt53_horizontal_batched( device float *dst1 [[buffer(4)]], device float *dst2 [[buffer(5)]], constant J2kForwardDwt53BatchedParams ¶ms [[buffer(6)]], - uint3 gid [[thread_position_in_grid]] + uint3 gid [[thread_position_in_grid]], + uint3 local_id [[thread_position_in_threadgroup]], + uint simd_lane [[thread_index_in_simdgroup]] ) { if ( gid.x >= params.current_width || @@ -188,7 +209,9 @@ kernel void j2k_forward_dwt53_horizontal_batched( params.full_width, params.current_width, params.low_width, - gid.xy + gid.xy, + local_id.x, + simd_lane ); } diff --git a/crates/j2k-metal/src/generated/promotion.rs b/crates/j2k-metal/src/generated/promotion.rs index ee682ff90..7f68c3842 100644 --- a/crates/j2k-metal/src/generated/promotion.rs +++ b/crates/j2k-metal/src/generated/promotion.rs @@ -10,8 +10,15 @@ use crate::routing::promotion::{PromotionCell, PromotionOperation as Operation}; const METAL_PART1: &str = "162a47f7a96b2be88abebc100aab672513af04895532863fa1a293660546f879"; const METAL_PART15: &str = "cfa66686d053bb3e2d4c8756abaf84aab65d8505a635795cefd38de53573c1f5"; const METAL_HOST_OUTPUT: &str = "c98f11c0b2a2a96853953ceee7ea672e0e5044bdb8abbd397c8c36eb82fe53b8"; +const METAL_LOCAL_COMBINED: &str = + "66f9d83f932efb7df6cab2de0048849f09343da00915e22794b0056b29f2aa5f"; -pub(crate) const SOURCE_EVIDENCE: &[&str] = &[METAL_PART1, METAL_PART15, METAL_HOST_OUTPUT]; +pub(crate) const SOURCE_EVIDENCE: &[&str] = &[ + METAL_PART1, + METAL_PART15, + METAL_HOST_OUTPUT, + METAL_LOCAL_COMBINED, +]; #[cfg(any(test, target_os = "macos"))] const fn decode_cell( @@ -38,52 +45,116 @@ const fn decode_cell( #[cfg(any(test, target_os = "macos"))] pub(crate) const PROMOTION_CELLS: &[PromotionCell] = &[ decode_cell( - PixelFormat::Gray8, - Syntax::Jpeg2000Lossy, + PixelFormat::Rgb8, + Syntax::HtJpeg2000Lossless, Payload::Jpeg2000Codestream, Operation::Repeated, - (0, 0, 2_960_793, 16), - METAL_PART1, + (256, 256, 65_536, 16), + METAL_LOCAL_COMBINED, ), decode_cell( PixelFormat::Rgb8, - Syntax::Jpeg2000Lossy, + Syntax::HtJpeg2000Lossless, + Payload::JphFile, + Operation::Repeated, + (640, 480, 307_200, 16), + METAL_LOCAL_COMBINED, + ), + decode_cell( + PixelFormat::Gray8, + Syntax::HtJpeg2000Lossless, Payload::Jpeg2000Codestream, Operation::Repeated, - (0, 0, 307_200, 16), - METAL_PART1, + (640, 480, 307_200, 16), + METAL_LOCAL_COMBINED, ), decode_cell( PixelFormat::Rgb8, Syntax::Jpeg2000Lossless, Payload::Jpeg2000Codestream, Operation::Repeated, - (0, 0, 5_038_848, 16), - METAL_PART1, + (640, 480, 307_200, 16), + METAL_LOCAL_COMBINED, ), decode_cell( - PixelFormat::Rgb8, - Syntax::HtJpeg2000Lossy, + PixelFormat::Gray8, + Syntax::Jpeg2000Lossless, Payload::Jpeg2000Codestream, Operation::Repeated, - (640, 480, 0, 16), - METAL_PART15, + (2048, 2048, 4_194_304, 16), + METAL_LOCAL_COMBINED, + ), + decode_cell( + PixelFormat::Rgb8, + Syntax::HtJpeg2000Lossless, + Payload::Jpeg2000Codestream, + Operation::Full, + (640, 480, 307_200, 1), + METAL_LOCAL_COMBINED, ), decode_cell( PixelFormat::Rgb8, Syntax::HtJpeg2000Lossless, Payload::JphFile, + Operation::Full, + (640, 480, 307_200, 1), + METAL_LOCAL_COMBINED, + ), + decode_cell( + PixelFormat::Gray8, + Syntax::HtJpeg2000Lossless, + Payload::Jpeg2000Codestream, + Operation::Full, + (640, 480, 307_200, 1), + METAL_LOCAL_COMBINED, + ), + decode_cell( + PixelFormat::Rgb8, + Syntax::HtJpeg2000Lossy, + Payload::Jpeg2000Codestream, Operation::Repeated, - (768, 512, 0, 16), - METAL_PART15, + (256, 256, 65_536, 16), + METAL_LOCAL_COMBINED, ), decode_cell( PixelFormat::Rgb8, + Syntax::Jpeg2000Lossy, + Payload::Jpeg2000Codestream, + Operation::Repeated, + (1024, 1024, 1_048_576, 16), + METAL_LOCAL_COMBINED, + ), + decode_cell( + PixelFormat::Gray8, + Syntax::HtJpeg2000Lossy, + Payload::Jpeg2000Codestream, + Operation::Repeated, + (2048, 2048, 4_194_304, 16), + METAL_LOCAL_COMBINED, + ), + decode_cell( + PixelFormat::Gray8, + Syntax::Jpeg2000Lossy, + Payload::Jpeg2000Codestream, + Operation::Repeated, + (2048, 2048, 4_194_304, 16), + METAL_LOCAL_COMBINED, + ), + decode_cell( + PixelFormat::Rgb8, + Syntax::HtJpeg2000Lossy, + Payload::Jpeg2000Codestream, + Operation::Full, + (640, 480, 307_200, 1), + METAL_LOCAL_COMBINED, + ), + decode_cell( + PixelFormat::Gray8, Syntax::HtJpeg2000Lossy, Payload::Jpeg2000Codestream, - Operation::ScaledHalf, - (320, 240, 0, 1), - METAL_PART15, + Operation::Full, + (640, 480, 307_200, 1), + METAL_LOCAL_COMBINED, ), ]; @@ -105,5 +176,5 @@ pub(crate) fn auto_host_output_encode_qualifies(components: u16, width: u32, hei } pub(crate) fn auto_lossy_rgb8_encode_qualifies(pixels: usize) -> bool { - pixels >= 5_038_848 && SOURCE_EVIDENCE.contains(&METAL_PART1) + pixels >= 4_194_304 && SOURCE_EVIDENCE.contains(&METAL_LOCAL_COMBINED) } diff --git a/crates/j2k-metal/src/ht.rs b/crates/j2k-metal/src/ht.rs index 409d1048d..8217cd5f9 100644 --- a/crates/j2k-metal/src/ht.rs +++ b/crates/j2k-metal/src/ht.rs @@ -214,7 +214,10 @@ mod tests { #[cfg(target_os = "macos")] use crate::engine as compute; #[cfg(target_os = "macos")] - use j2k_native::{decode_ht_code_block_scalar, HtCodeBlockDecodeJob, HtCodeBlockDecoder}; + use j2k_native::{ + decode_ht_code_block_scalar, HtCodeBlockBatchJob, HtCodeBlockDecodeJob, HtCodeBlockDecoder, + HtSubBandDecodeJob, + }; use j2k_native::{ encode_htj2k, ColorSpace, DecodeSettings, DecoderContext, EncodeOptions, Image, }; @@ -551,4 +554,65 @@ mod tests { assert_eq!(actual, expected); } + + #[cfg(target_os = "macos")] + #[test] + fn metal_ht_subband_mixed_cleanup_and_refinement_matches_scalar() { + if !should_run_metal_runtime() { + return; + } + + let refinement = synthetic_refinement_job(3); + let mut cleanup = refinement.clone(); + cleanup.data.truncate(cleanup.cleanup_length as usize); + cleanup.refinement_length = 0; + cleanup.missing_bit_planes = cleanup + .missing_bit_planes + .checked_add(1) + .expect("cleanup missing-bitplane count fits u8"); + cleanup.number_of_coding_passes = 1; + + let width = cleanup.width + refinement.width; + let height = cleanup.height.max(refinement.height); + let output_len = width as usize * height as usize; + let cleanup_job = HtCodeBlockDecodeJob { + output_stride: width as usize, + ..cleanup.as_job() + }; + let refinement_job = HtCodeBlockDecodeJob { + output_stride: width as usize, + ..refinement.as_job() + }; + let jobs = [ + HtCodeBlockBatchJob { + output_x: 0, + output_y: 0, + code_block: cleanup_job, + }, + HtCodeBlockBatchJob { + output_x: cleanup.width, + output_y: 0, + code_block: refinement_job, + }, + ]; + + let mut expected = vec![0.0f32; output_len]; + decode_ht_code_block_scalar(cleanup_job, &mut expected).expect("scalar cleanup decode"); + decode_ht_code_block_scalar(refinement_job, &mut expected[cleanup.width as usize..]) + .expect("scalar refinement decode"); + + let mut actual = vec![13_579.0f32; output_len]; + compute::decode_ht_cleanup_sub_band( + HtSubBandDecodeJob { + width, + height, + jobs: &jobs, + }, + &mut actual, + false, + ) + .expect("mixed Metal sub-band decode"); + + assert_eq!(actual, expected); + } } diff --git a/crates/j2k-metal/src/ht_cleanup.metal b/crates/j2k-metal/src/ht_cleanup.metal index 3e81335e5..cc075f853 100644 --- a/crates/j2k-metal/src/ht_cleanup.metal +++ b/crates/j2k-metal/src/ht_cleanup.metal @@ -368,6 +368,18 @@ inline void decode_mag_sgn_sample_with_vn( value |= (v_n + 2u) << (p - 1u); } +template +inline uint ht_cleanup_output_bits(uint value, J2kHtCleanupParams params) { + if (ConvertOnStore) { + return coefficient_to_float_bits( + value, params.num_bitplanes + params.roi_shift, + params.dequantization_step, params.irreversible_midpoint, params.roi_shift + ); + } + return value; +} + +template inline void decode_ht_cleanup_common( device const uchar *coded_data, device uint *decoded_data, @@ -608,13 +620,13 @@ inline void decode_ht_cleanup_common( uint value0 = 0u; uint ignored_vn = 0u; decode_mag_sgn_sample_with_vn(magsgn, inf, 0u, uq, p, value0, ignored_vn); - decoded_data[dp] = value0; + decoded_data[dp] = ht_cleanup_output_bits(value0, params); uint value1 = 0u; uint v_n1 = 0u; decode_mag_sgn_sample_with_vn(magsgn, inf, 1u, uq, p, value1, v_n1); if (second_row_present) { - decoded_data[dp + stride] = value1; + decoded_data[dp + stride] = ht_cleanup_output_bits(value1, params); } v_n_scratch[vp] = prev_v_n | v_n1; prev_v_n = 0u; @@ -628,13 +640,13 @@ inline void decode_ht_cleanup_common( uint value2 = 0u; decode_mag_sgn_sample_with_vn(magsgn, inf, 2u, uq, p, value2, ignored_vn); - decoded_data[dp] = value2; + decoded_data[dp] = ht_cleanup_output_bits(value2, params); uint value3 = 0u; uint v_n3 = 0u; decode_mag_sgn_sample_with_vn(magsgn, inf, 3u, uq, p, value3, v_n3); if (second_row_present) { - decoded_data[dp + stride] = value3; + decoded_data[dp + stride] = ht_cleanup_output_bits(value3, params); } prev_v_n = v_n3; dp += 1u; @@ -670,13 +682,13 @@ inline void decode_ht_cleanup_common( uint value0 = 0u; uint ignored_vn = 0u; decode_mag_sgn_sample_with_vn(magsgn, inf, 0u, uq, p, value0, ignored_vn); - decoded_data[local_dp] = value0; + decoded_data[local_dp] = ht_cleanup_output_bits(value0, params); uint value1 = 0u; uint v_n1 = 0u; decode_mag_sgn_sample_with_vn(magsgn, inf, 1u, uq, p, value1, v_n1); if (local_second_row_present) { - decoded_data[local_dp + stride] = value1; + decoded_data[local_dp + stride] = ht_cleanup_output_bits(value1, params); } v_n_scratch[local_vp] = local_prev_v_n | v_n1; local_prev_v_n = 0u; @@ -690,13 +702,13 @@ inline void decode_ht_cleanup_common( uint value2 = 0u; decode_mag_sgn_sample_with_vn(magsgn, inf, 2u, uq, p, value2, ignored_vn); - decoded_data[local_dp] = value2; + decoded_data[local_dp] = ht_cleanup_output_bits(value2, params); uint value3 = 0u; uint v_n3 = 0u; decode_mag_sgn_sample_with_vn(magsgn, inf, 3u, uq, p, value3, v_n3); if (local_second_row_present) { - decoded_data[local_dp + stride] = value3; + decoded_data[local_dp + stride] = ht_cleanup_output_bits(value3, params); } local_prev_v_n = v_n3; local_dp += 1u; @@ -1051,14 +1063,12 @@ inline void decode_ht_cleanup_only_impl( ) { thread ushort scratch[J2K_HT_MAX_SCRATCH]; thread uint v_n_scratch[J2K_HT_MAX_VN]; - decode_ht_cleanup_common( + // Cleanup predictors use scratch state, so one-pass outputs can be final + // floats immediately. Refinement paths still need the packed integer words. + decode_ht_cleanup_common( coded_data, decoded_data, params, vlc_table0, vlc_table1, uvlc_table0, uvlc_table1, status, scratch, v_n_scratch ); - if (status->code != J2K_HT_STATUS_OK) { - return; - } - convert_ht_cleanup_coefficients(decoded_data, params); } inline void decode_ht_cleanup_impl( @@ -1073,7 +1083,7 @@ inline void decode_ht_cleanup_impl( ) { thread ushort scratch[J2K_HT_MAX_SCRATCH]; thread uint v_n_scratch[J2K_HT_MAX_VN]; - decode_ht_cleanup_common( + decode_ht_cleanup_common( coded_data, decoded_data, params, vlc_table0, vlc_table1, uvlc_table0, uvlc_table1, status, scratch, v_n_scratch ); @@ -1123,6 +1133,33 @@ kernel void j2k_decode_ht_cleanup( ); } +kernel void j2k_decode_ht_cleanup_cleanup_only( + device const uchar *coded_data [[buffer(0)]], + device uint *decoded_data [[buffer(1)]], + constant J2kHtCleanupParams ¶ms [[buffer(2)]], + constant ushort *vlc_table0 [[buffer(3)]], + constant ushort *vlc_table1 [[buffer(4)]], + constant ushort *uvlc_table0 [[buffer(5)]], + constant ushort *uvlc_table1 [[buffer(6)]], + device J2kHtStatus *status [[buffer(7)]], + uint gid [[thread_position_in_grid]] +) { + if (gid != 0u) { + return; + } + + decode_ht_cleanup_only_impl( + coded_data, + decoded_data, + params, + vlc_table0, + vlc_table1, + uvlc_table0, + uvlc_table1, + status + ); +} + kernel void j2k_decode_ht_cleanup_batched( device const uchar *coded_data [[buffer(0)]], device uint *decoded_data [[buffer(1)]], diff --git a/crates/j2k-metal/src/ht_cleanup_simd.metal b/crates/j2k-metal/src/ht_cleanup_simd.metal new file mode 100644 index 000000000..d7508acf7 --- /dev/null +++ b/crates/j2k-metal/src/ht_cleanup_simd.metal @@ -0,0 +1,906 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +// Two-kernel HT cleanup decode for cleanup-only code blocks. +// +// 1. `..._vlc_...`: one thread per code block walks the serial MEL/VLC +// streams. Thirty-two blocks share a SIMD group, so this latency-bound +// walk uses every lane. For quad (q, r) it stores `(u_q << 16) | inf` in +// the block's own output word at column 2q, row 2r, which the next kernel +// overwrites; no scratch allocation is needed. +// 2. `..._magsgn_...`: one SIMD group per code block decodes each quad row's +// MagSgn samples in parallel. Each lane owns one quad, derives its bit +// count from the stored VLC data and the previous row's exponents, and +// locates its bits with a SIMD prefix sum over a cooperatively destuffed +// threadgroup window. +// +// Arithmetic, bit consumption, validation and status codes mirror +// `decode_ht_cleanup_common`; only the schedule differs. + +constant uint J2K_HT_SIMD_LANES = 32u; +constant uint J2K_HT_SIMD_BLOCKS_PER_GROUP = 4u; +// 8192 destuffed bits. One 32-quad chunk consumes at most 3968 bits, and each +// refill appends at most 1024 bits, so live bits never wrap onto themselves. +constant uint J2K_HT_SIMD_WINDOW_WORDS = 256u; +constant uint J2K_HT_SIMD_WINDOW_MASK = J2K_HT_SIMD_WINDOW_WORDS - 1u; +constant uint J2K_HT_SIMD_REFILL_BYTES = J2K_HT_SIMD_LANES * 4u; +// Previous/current row significance context: two bits per quad, 16 quads per +// word, covering J2K_HT_MAX_WIDTH / 2 quads plus the zero quad read past the +// row end. The odd per-thread stride keeps lanes on distinct banks. +constant uint J2K_HT_VLC_CTX_WORDS = 9u; +constant uint J2K_HT_VLC_CTX_STRIDE = 2u * J2K_HT_VLC_CTX_WORDS + 1u; +constant uint J2K_HT_VLC_THREADS_PER_GROUP = 32u; + +// The VLC kernels run `lanes_per_simd` blocks per 32-thread SIMD group (the +// remaining lanes exit). Fewer blocks per group means less divergence between +// the serial walks; more means fewer groups to issue. The host picks the ratio +// from the job count. +struct J2kHtVlcDispatchParams { + uint job_count; + uint lanes_per_simd; +}; + +inline bool ht_vlc_job_index( + J2kHtVlcDispatchParams dispatch, + uint grid_x, + thread uint &job_index +) { + const uint lane = grid_x % J2K_HT_SIMD_LANES; + job_index = (grid_x / J2K_HT_SIMD_LANES) * dispatch.lanes_per_simd + lane; + return lane < dispatch.lanes_per_simd && job_index < dispatch.job_count; +} + +struct HtCleanupGeometry { + uint scup; + uint quads; + uint quad_rows; +}; + +// Validation shared by both kernels, in `decode_ht_cleanup_common` order. +// Returns the failing status code (and sets `detail`) or J2K_HT_STATUS_OK. +inline uint ht_simd_validate( + device const uchar *coded_data, + J2kHtCleanupParams params, + thread HtCleanupGeometry &geometry, + thread uint &detail +) { + detail = 0u; + geometry.scup = 0u; + geometry.quads = 0u; + geometry.quad_rows = 0u; + if (params.number_of_coding_passes > 3u) { + detail = 3u; + return J2K_HT_STATUS_FAIL; + } + // Empty blocks decode nothing and report OK; callers then see zero rows. + if (params.width == 0u || params.height == 0u) { + return J2K_HT_STATUS_OK; + } + if (params.width > J2K_HT_MAX_WIDTH || params.height > J2K_HT_MAX_HEIGHT || + params.width * params.height > J2K_HT_MAX_COEFFICIENTS) { + detail = 1u; + return J2K_HT_STATUS_UNSUPPORTED; + } + if (params.num_bitplanes == 0u || params.num_bitplanes > 31u || + params.roi_shift > 31u - params.num_bitplanes) { + detail = 2u; + return J2K_HT_STATUS_FAIL; + } + if (params.missing_msbs > 30u || params.missing_msbs == 30u) { + detail = 3u; + return J2K_HT_STATUS_FAIL; + } + const uint lcup = params.cleanup_length; + if (lcup < 2u || params.coded_len < lcup + params.refinement_length) { + detail = 4u; + return J2K_HT_STATUS_FAIL; + } + const uint scup = (uint(coded_data[lcup - 1u]) << 4u) + uint(coded_data[lcup - 2u] & uchar(0x0F)); + if (scup < 2u || scup > lcup || scup > 4079u) { + detail = 5u; + return J2K_HT_STATUS_FAIL; + } + const uint quad_rows = (params.height + 1u) / 2u; + const uint sstr = (params.width + 9u) & ~7u; + if (sstr > J2K_HT_MAX_SSTR || sstr * (quad_rows + 1u) > J2K_HT_MAX_SCRATCH) { + detail = 6u; + return J2K_HT_STATUS_UNSUPPORTED; + } + const uint quads = (params.width + 1u) / 2u; + if (quads + 2u > J2K_HT_MAX_VN) { + detail = 12u; + return J2K_HT_STATUS_UNSUPPORTED; + } + geometry.scup = scup; + geometry.quads = quads; + geometry.quad_rows = quad_rows; + return J2K_HT_STATUS_OK; +} + +// MEL decoder yielding the same run sequence as `mel_get_run`, one run on +// demand from a 64-bit MSB-first bit buffer instead of bit by bit. A byte +// following 0xFF contributes seven bits, and the final segment byte has its +// low nibble forced to ones. +struct HtSimdMel { + device const uchar *data; + uint pos; + uint remaining; + ulong acc; + uint acc_bits; + uint unstuff; + uint k; +}; + +inline HtSimdMel ht_simd_mel_new(device const uchar *data, uint lcup, uint scup) { + HtSimdMel mel; + mel.data = data; + mel.pos = lcup - scup; + mel.remaining = scup - 1u; + mel.acc = 0ul; + mel.acc_bits = 0u; + mel.unstuff = 0u; + mel.k = 0u; + return mel; +} + +inline int ht_simd_mel_get_run(thread HtSimdMel &mel) { + // A run consumes at most six bits; one byte always supplies at least seven. + if (mel.acc_bits < 6u) { + uint byte = 0xFFu; + if (mel.remaining > 0u) { + byte = uint(mel.data[mel.pos]); + mel.pos += 1u; + mel.remaining -= 1u; + } + if (mel.remaining == 0u) { + byte |= 0x0Fu; + } + const uint nbits = 8u - mel.unstuff; + const uint value = byte & ((1u << nbits) - 1u); + mel.acc |= ulong(value) << (64u - mel.acc_bits - nbits); + mel.acc_bits += nbits; + mel.unstuff = byte == 0xFFu ? 1u : 0u; + } + + // MEL_EXP = {0,0,0,1,1,1,2,2,2,3,3,4,5} + const uint eval = mel.k < 11u ? mel.k / 3u : mel.k - 7u; + if ((mel.acc >> 63) != 0ul) { + mel.k = min(mel.k + 1u, 12u); + mel.acc <<= 1; + mel.acc_bits -= 1u; + return int(((1u << eval) - 1u) << 1u); + } + mel.k = mel.k == 0u ? 0u : mel.k - 1u; + const uint bits = eval == 0u ? 0u : uint((mel.acc << 1) >> (64u - eval)); + mel.acc <<= 1u + eval; + mel.acc_bits -= 1u + eval; + return int((bits << 1u) | 1u); +} + +// The non-initial VLC context reads only inf bits 5 and 7 of the quads above +// (the bottom samples' significance), so each quad is recorded as two bits. +inline uint ht_simd_ctx_bits(uint inf) { + return ((inf >> 5u) & 1u) | ((inf >> 6u) & 2u); +} + +inline uint ht_simd_ctx_inf(uint bits) { + return ((bits & 1u) << 5u) | ((bits & 2u) << 6u); +} + +// Context for blocks at most 64 samples wide (32 quads): the previous row is +// a register consumed two quads per pair, and the current row is accumulated +// from the top four bits at a time, so no loop step shifts by a variable. +struct HtNarrowVlcContext { + ulong prev; + ulong cur; + uint pairs; + + uint above(uint offset) const { + return ht_simd_ctx_inf(uint(prev >> (2u * offset)) & 3u); + } + + void record_pair(uint inf0, uint inf1) { + const ulong bits = ulong(ht_simd_ctx_bits(inf0) | (ht_simd_ctx_bits(inf1) << 2u)); + cur = (cur >> 4u) | (bits << 60u); + prev >>= 4u; + pairs += 1u; + } + + void finish_row() { + prev = pairs < 16u ? cur >> (4u * (16u - pairs)) : cur; + cur = 0ul; + pairs = 0u; + } +}; + +inline HtNarrowVlcContext ht_narrow_vlc_context() { + HtNarrowVlcContext ctx; + ctx.prev = 0ul; + ctx.cur = 0ul; + ctx.pairs = 0u; + return ctx; +} + +// Context for wider blocks: two rows of 2-bit entries in this thread's slice +// of threadgroup memory, covering J2K_HT_MAX_WIDTH / 2 quads plus the zero +// quad read past the row end. +struct HtWideVlcContext { + threadgroup uint *prev; + threadgroup uint *cur; + uint quad; + + uint above(uint offset) const { + const uint q = quad + offset; + return ht_simd_ctx_inf((prev[q >> 4] >> ((q & 15u) * 2u)) & 3u); + } + + void record_pair(uint inf0, uint inf1) { + const uint bits = ht_simd_ctx_bits(inf0) | (ht_simd_ctx_bits(inf1) << 2u); + cur[quad >> 4] |= bits << ((quad & 15u) * 2u); + quad += 2u; + } + + void finish_row() { + threadgroup uint *done = cur; + cur = prev; + prev = done; + for (uint word = 0u; word < J2K_HT_VLC_CTX_WORDS; ++word) { + cur[word] = 0u; + } + quad = 0u; + } +}; + +inline HtWideVlcContext ht_wide_vlc_context(threadgroup uint *slice) { + HtWideVlcContext ctx; + ctx.prev = slice; + ctx.cur = slice + J2K_HT_VLC_CTX_WORDS; + ctx.quad = 0u; + for (uint word = 0u; word < 2u * J2K_HT_VLC_CTX_WORDS; ++word) { + slice[word] = 0u; + } + return ctx; +} + +// Reverse VLC reader for the thread-per-block kernel. It exposes the same bit +// sequence as `ReverseBitReader`, but refills four bytes at a time from a +// register that was loaded during the previous refill, so a refill never waits +// on memory. Buffering extra bits cannot change results: a quad pair reads at +// most 30 bits (two 7-bit codewords and a 16-bit UVLC prefix plus suffix) and +// every fetch leaves more than 32 valid bits. +struct HtVlcReader { + device const uchar *data; + int pos; + uint remaining; + ulong tmp; + uint bits; + uint unstuff; + uint ahead; +}; + +inline uint ht_vlc_load_ahead(device const uchar *data, int pos, uint remaining) { + uint word = 0u; + for (uint j = 0u; j < 4u; ++j) { + const uint byte = remaining > j ? uint(data[pos - int(j)]) : 0u; + word |= byte << (8u * j); + } + return word; +} + +inline HtVlcReader ht_vlc_reader_new(device const uchar *data, uint lcup, uint scup) { + const uchar d = data[lcup - 2u]; + const ulong tmp = ulong(d >> 4); + HtVlcReader reader; + reader.data = data; + reader.pos = int(lcup) - 3; + reader.remaining = scup - 2u; + reader.tmp = tmp; + reader.bits = 4u - uint((tmp & 0x7ul) == 0x7ul); + reader.unstuff = (d | uchar(0x0F)) > uchar(0x8F) ? 1u : 0u; + reader.ahead = ht_vlc_load_ahead(data, reader.pos, reader.remaining); + return reader; +} + +inline uint ht_vlc_fetch(thread HtVlcReader &reader) { + while (reader.bits <= 32u) { + const uint taken = min(reader.remaining, 4u); + const uint word = reader.ahead; + reader.pos -= int(taken); + reader.remaining -= taken; + reader.ahead = ht_vlc_load_ahead(reader.data, reader.pos, reader.remaining); + for (uint j = 0u; j < 4u; ++j) { + const uint raw = (word >> (8u * j)) & 0xFFu; + const bool stuffed = reader.unstuff != 0u && (raw & 0x7Fu) == 0x7Fu; + reader.tmp |= ulong(stuffed ? raw & 0x7Fu : raw) << reader.bits; + reader.bits += stuffed ? 7u : 8u; + reader.unstuff = raw > 0x8Fu ? 1u : 0u; + } + } + return uint(reader.tmp); +} + +inline void ht_simd_vlc_consume(thread HtVlcReader &vlc, uint count) { + vlc.tmp >>= count; + vlc.bits -= count; +} + +inline uint ht_simd_quad_word(uint inf, uint u_q) { + return (uint(ushort(u_q)) << 16u) | inf; +} + +// Each quad pair's VLC codewords, UVLC prefix and UVLC suffix are located in +// one fetched window, and the reader advances once per pair. Every field is +// taken from the same bits `reverse_reader_advance` would expose. +template +inline void ht_simd_vlc_initial_row( + thread HtSimdMel &mel, + thread HtVlcReader &vlc, + thread int &run, + thread Context &ctx, + device uint *quad_words, + uint width, + constant ushort *vlc_table0, + constant ushort *uvlc_table0 +) { + uint c_q = 0u; + uint quad = 0u; + uint x = 0u; + + while (x < width) { + const uint window = ht_vlc_fetch(vlc); + uint t0 = uint(vlc_table0[c_q + (window & 0x7Fu)]); + if (c_q == 0u) { + run -= 2; + t0 = run == -1 ? t0 : 0u; + if (run < 0) { + run = ht_simd_mel_get_run(mel); + } + } + x += 2u; + c_q = ((t0 & 0x10u) << 3u) | ((t0 & 0xE0u) << 2u); + uint used = t0 & 0x7u; + + const bool second = x < width; + uint t1 = uint(vlc_table0[c_q + ((window >> used) & 0x7Fu)]); + if (c_q == 0u && second) { + run -= 2; + t1 = run == -1 ? t1 : 0u; + if (run < 0) { + run = ht_simd_mel_get_run(mel); + } + } + if (!second) { + t1 = 0u; + } + x += 2u; + c_q = ((t1 & 0x10u) << 3u) | ((t1 & 0xE0u) << 2u); + used += t1 & 0x7u; + + uint uvlc_mode = ((t0 & 0x8u) << 3u) | ((t1 & 0x8u) << 4u); + if (uvlc_mode == 0xC0u) { + run -= 2; + if (run == -1) { + uvlc_mode += 0x40u; + } + if (run < 0) { + run = ht_simd_mel_get_run(mel); + } + } + + uint uvlc_entry = uint(uvlc_table0[uvlc_mode + ((window >> used) & 0x3Fu)]); + used += uvlc_entry & 0x7u; + uvlc_entry >>= 3u; + uint len = uvlc_entry & 0xFu; + const uint tmp = uint(vlc.tmp >> used) & ((1u << len) - 1u); + ht_simd_vlc_consume(vlc, used + len); + uvlc_entry >>= 4u; + len = uvlc_entry & 0x7u; + uvlc_entry >>= 3u; + + ctx.record_pair(t0, t1); + quad_words[quad * 2u] = + ht_simd_quad_word(t0, 1u + (uvlc_entry & 0x7u) + (tmp & ~(0xFFu << len))); + if (second) { + quad_words[quad * 2u + 2u] = + ht_simd_quad_word(t1, 1u + (uvlc_entry >> 3u) + (tmp >> len)); + } + quad += 2u; + } +} + +template +inline void ht_simd_vlc_non_initial_row( + thread HtSimdMel &mel, + thread HtVlcReader &vlc, + thread int &run, + thread Context &ctx, + device uint *quad_words, + uint width, + constant ushort *vlc_table1, + constant ushort *uvlc_table1 +) { + uint local_x = 0u; + uint local_c_q = 0u; + uint quad = 0u; + + while (local_x < width) { + const uint above0 = ctx.above(0u); + const uint above1 = ctx.above(1u); + const uint above2 = ctx.above(2u); + local_c_q |= (above0 & 0xA0u) << 2u; + local_c_q |= (above1 & 0x20u) << 4u; + + const uint window = ht_vlc_fetch(vlc); + uint t0 = uint(vlc_table1[local_c_q + (window & 0x7Fu)]); + if (local_c_q == 0u) { + run -= 2; + t0 = run == -1 ? t0 : 0u; + if (run < 0) { + run = ht_simd_mel_get_run(mel); + } + } + local_x += 2u; + + local_c_q = ((t0 & 0x40u) << 2u) | ((t0 & 0x80u) << 1u); + local_c_q |= above0 & 0x80u; + local_c_q |= (above1 & 0xA0u) << 2u; + local_c_q |= (above2 & 0x20u) << 4u; + uint used = t0 & 0x7u; + + const bool second = local_x < width; + uint t1 = uint(vlc_table1[local_c_q + ((window >> used) & 0x7Fu)]); + if (local_c_q == 0u && second) { + run -= 2; + t1 = run == -1 ? t1 : 0u; + if (run < 0) { + run = ht_simd_mel_get_run(mel); + } + } + if (!second) { + t1 = 0u; + } + local_x += 2u; + + local_c_q = ((t1 & 0x40u) << 2u) | ((t1 & 0x80u) << 1u); + local_c_q |= above1 & 0x80u; + used += t1 & 0x7u; + + const uint uvlc_mode = ((t0 & 0x8u) << 3u) | ((t1 & 0x8u) << 4u); + uint uvlc_entry = uint(uvlc_table1[uvlc_mode + ((window >> used) & 0x3Fu)]); + used += uvlc_entry & 0x7u; + uvlc_entry >>= 3u; + uint len = uvlc_entry & 0xFu; + const uint tmp = uint(vlc.tmp >> used) & ((1u << len) - 1u); + ht_simd_vlc_consume(vlc, used + len); + uvlc_entry >>= 4u; + len = uvlc_entry & 0x7u; + uvlc_entry >>= 3u; + + ctx.record_pair(t0, t1); + quad_words[quad * 2u] = + ht_simd_quad_word(t0, (uvlc_entry & 0x7u) + (tmp & ~(0xFFu << len))); + if (second) { + quad_words[quad * 2u + 2u] = + ht_simd_quad_word(t1, (uvlc_entry >> 3u) + (tmp >> len)); + } + quad += 2u; + } +} + +template +inline void ht_simd_vlc_rows( + thread HtSimdMel &mel, + thread HtVlcReader &vlc, + thread int &run, + thread Context &ctx, + device uint *decoded_data, + J2kHtCleanupParams params, + uint quad_rows, + constant ushort *vlc_table0, + constant ushort *vlc_table1, + constant ushort *uvlc_table0, + constant ushort *uvlc_table1 +) { + for (uint row = 0u; row < quad_rows; ++row) { + device uint *quad_words = + decoded_data + params.output_offset + row * 2u * params.output_stride; + if (row == 0u) { + ht_simd_vlc_initial_row( + mel, vlc, run, ctx, quad_words, params.width, vlc_table0, uvlc_table0 + ); + } else { + ht_simd_vlc_non_initial_row( + mel, vlc, run, ctx, quad_words, params.width, vlc_table1, uvlc_table1 + ); + } + ctx.finish_row(); + } +} + +inline void decode_ht_cleanup_vlc( + device const uchar *coded_data, + device uint *decoded_data, + J2kHtCleanupParams params, + constant ushort *vlc_table0, + constant ushort *vlc_table1, + constant ushort *uvlc_table0, + constant ushort *uvlc_table1, + device J2kHtStatus *status, + threadgroup uint *ctx_slice +) { + HtCleanupGeometry geometry; + uint detail = 0u; + const uint code = ht_simd_validate(coded_data, params, geometry, detail); + set_ht_status(status, code, detail); + if (code != J2K_HT_STATUS_OK || geometry.quad_rows == 0u) { + return; + } + + const uint lcup = params.cleanup_length; + HtSimdMel mel = ht_simd_mel_new(coded_data, lcup, geometry.scup); + HtVlcReader vlc = ht_vlc_reader_new(coded_data, lcup, geometry.scup); + int run = ht_simd_mel_get_run(mel); + + if (geometry.quads <= 32u) { + HtNarrowVlcContext ctx = ht_narrow_vlc_context(); + ht_simd_vlc_rows( + mel, vlc, run, ctx, decoded_data, params, geometry.quad_rows, + vlc_table0, vlc_table1, uvlc_table0, uvlc_table1 + ); + } else { + HtWideVlcContext ctx = ht_wide_vlc_context(ctx_slice); + ht_simd_vlc_rows( + mel, vlc, run, ctx, decoded_data, params, geometry.quad_rows, + vlc_table0, vlc_table1, uvlc_table0, uvlc_table1 + ); + } +} + +// Appends the next 128 raw MagSgn bytes, destuffed, to the window. Bytes past +// `len` read as 0xFF, and a byte following 0xFF contributes only its low seven +// bits, exactly as `forward_reader_fill` consumes them. +inline void ht_simd_window_refill( + device const uchar *data, + uint len, + threadgroup uint *window, + thread uint &raw_pos, + thread uint &window_end, + thread uint &prev_ff, + uint lane +) { + const uint first_empty = (window_end + 31u) >> 5; + window[(first_empty + lane) & J2K_HT_SIMD_WINDOW_MASK] = 0u; + + const uint base = raw_pos + lane * 4u; + uint raw[4]; + for (uint j = 0u; j < 4u; ++j) { + const uint k = base + j; + raw[j] = k < len ? uint(data[k]) : 0xFFu; + } + const uint last_ff = raw[3] == 0xFFu ? 1u : 0u; + const uint left_ff = simd_shuffle_up(last_ff, 1u); + uint stuffed = lane == 0u ? prev_ff : left_ff; + + uint bits = 0u; + uint count = 0u; + for (uint j = 0u; j < 4u; ++j) { + const uint value = stuffed != 0u ? (raw[j] & 0x7Fu) : raw[j]; + bits |= value << count; + count += 8u - stuffed; + stuffed = raw[j] == 0xFFu ? 1u : 0u; + } + + const uint offset = window_end + simd_prefix_exclusive_sum(count); + const uint total = simd_sum(count); + simdgroup_barrier(mem_flags::mem_threadgroup); + + const uint word = offset >> 5; + const uint shift = offset & 31u; + threadgroup atomic_uint *atomic_window = (threadgroup atomic_uint *)window; + atomic_fetch_or_explicit( + &atomic_window[word & J2K_HT_SIMD_WINDOW_MASK], bits << shift, memory_order_relaxed + ); + if (shift != 0u && shift + count > 32u) { + atomic_fetch_or_explicit( + &atomic_window[(word + 1u) & J2K_HT_SIMD_WINDOW_MASK], + bits >> (32u - shift), + memory_order_relaxed + ); + } + simdgroup_barrier(mem_flags::mem_threadgroup); + + raw_pos += J2K_HT_SIMD_REFILL_BYTES; + window_end += total; + prev_ff = simd_broadcast(last_ff, J2K_HT_SIMD_LANES - 1u); +} + +inline uint ht_simd_window_read32(threadgroup const uint *window, uint bit) { + const uint word = bit >> 5; + const uint shift = bit & 31u; + const uint lo = window[word & J2K_HT_SIMD_WINDOW_MASK]; + if (shift == 0u) { + return lo; + } + const uint hi = window[(word + 1u) & J2K_HT_SIMD_WINDOW_MASK]; + return (lo >> shift) | (hi << (32u - shift)); +} + +// Same result as `decode_mag_sgn_sample_with_vn` for a significant sample +// whose MagSgn bits start at `ms_val`. +inline uint ht_simd_mag_sgn_value( + uint ms_val, + uint m_n, + uint inf, + uint bit, + uint p, + thread uint &v_n +) { + uint value = ms_val << 31u; + const uint mask = m_n == 0u ? 0u : (1u << m_n) - 1u; + v_n = ms_val & mask; + v_n |= ((inf >> (8u + bit)) & 1u) << m_n; + v_n |= 1u; + value |= (v_n + 2u) << (p - 1u); + return value; +} + +inline uint ht_simd_output_bits(uint value, J2kHtCleanupParams params, uint k_max) { + return coefficient_to_float_bits( + value, k_max, params.dequantization_step, params.irreversible_midpoint, params.roi_shift + ); +} + +inline void decode_ht_cleanup_magsgn( + device const uchar *coded_data, + device uint *decoded_data, + J2kHtCleanupParams params, + device J2kHtStatus *status, + threadgroup uint *v_n, + threadgroup uint *window, + uint lane +) { + // The VLC kernel already reported validation failures; skip those blocks. + HtCleanupGeometry geometry; + uint detail = 0u; + if (ht_simd_validate(coded_data, params, geometry, detail) != J2K_HT_STATUS_OK) { + return; + } + + const uint width = params.width; + const uint height = params.height; + const uint stride = params.output_stride; + const uint quads = geometry.quads; + const uint p = 30u - params.missing_msbs; + const uint uq_limit = params.missing_msbs + 2u; + const uint k_max = params.num_bitplanes + params.roi_shift; + const uint magsgn_len = params.cleanup_length - geometry.scup; + + uint raw_pos = 0u; + uint window_end = 0u; + uint prev_ff = 0u; + uint bit_pos = 0u; + + for (uint row = 0u; row < geometry.quad_rows; ++row) { + const uint y = row * 2u; + const bool second_row_present = y + 1u < height; + device uint *row_out = decoded_data + params.output_offset + y * stride; + uint carry_v_n3 = 0u; + + for (uint chunk = 0u; chunk < quads; chunk += J2K_HT_SIMD_LANES) { + const uint q = chunk + lane; + const bool active = q < quads; + const bool has_right = active && q * 2u + 1u < width; + + uint inf = 0u; + uint uq = 0u; + if (active) { + const uint quad_word = row_out[q * 2u]; + inf = quad_word & 0xFFFFu; + const uint u_q = quad_word >> 16u; + if (row == 0u) { + uq = u_q; + } else { + uint gamma = inf & 0xF0u; + gamma &= gamma - 0x10u; + uint emax = v_n[q] | v_n[q + 1u]; + emax = 31u - clz(emax | 2u); + const uint kappa = gamma != 0u ? emax : 1u; + uq = u_q + kappa; + } + } + if (simd_any(active && uq > uq_limit)) { + if (lane == 0u) { + set_ht_status(status, J2K_HT_STATUS_FAIL, row == 0u ? 13u : 14u); + } + return; + } + + // Samples are consumed in column-major quad order: 0 and 1 always, + // 2 and 3 only when the quad's right column exists. + const uint m0 = (inf & sample_mask(0u)) != 0u ? uq - ((inf >> 12u) & 1u) : 0u; + const uint m1 = (inf & sample_mask(1u)) != 0u ? uq - ((inf >> 13u) & 1u) : 0u; + const uint m2 = has_right && (inf & sample_mask(2u)) != 0u + ? uq - ((inf >> 14u) & 1u) : 0u; + const uint m3 = has_right && (inf & sample_mask(3u)) != 0u + ? uq - ((inf >> 15u) & 1u) : 0u; + const uint quad_bits = m0 + m1 + m2 + m3; + const uint lane_bit = simd_prefix_exclusive_sum(quad_bits); + const uint chunk_bits = simd_sum(quad_bits); + + while (window_end < bit_pos + chunk_bits + 32u) { + ht_simd_window_refill( + coded_data, magsgn_len, window, raw_pos, window_end, prev_ff, lane + ); + } + + uint v_n1 = 0u; + uint v_n3 = 0u; + if (active) { + uint cursor = bit_pos + lane_bit; + device uint *out = row_out + q * 2u; + uint ignored_vn = 0u; + + uint value0 = 0u; + if ((inf & sample_mask(0u)) != 0u) { + value0 = ht_simd_mag_sgn_value( + ht_simd_window_read32(window, cursor), m0, inf, 0u, p, ignored_vn + ); + cursor += m0; + } + out[0] = ht_simd_output_bits(value0, params, k_max); + + uint value1 = 0u; + if ((inf & sample_mask(1u)) != 0u) { + value1 = ht_simd_mag_sgn_value( + ht_simd_window_read32(window, cursor), m1, inf, 1u, p, v_n1 + ); + cursor += m1; + } + if (second_row_present) { + out[stride] = ht_simd_output_bits(value1, params, k_max); + } + + if (has_right) { + uint value2 = 0u; + if ((inf & sample_mask(2u)) != 0u) { + value2 = ht_simd_mag_sgn_value( + ht_simd_window_read32(window, cursor), m2, inf, 2u, p, ignored_vn + ); + cursor += m2; + } + out[1] = ht_simd_output_bits(value2, params, k_max); + + uint value3 = 0u; + if ((inf & sample_mask(3u)) != 0u) { + value3 = ht_simd_mag_sgn_value( + ht_simd_window_read32(window, cursor), m3, inf, 3u, p, v_n3 + ); + } + if (second_row_present) { + out[stride + 1u] = ht_simd_output_bits(value3, params, k_max); + } + } + } + bit_pos += chunk_bits; + + // Every lane has read the previous row's exponents; publish this row's. + simdgroup_barrier(mem_flags::mem_threadgroup); + const uint left_v_n3 = simd_shuffle_up(v_n3, 1u); + if (active) { + v_n[q] = (lane == 0u ? carry_v_n3 : left_v_n3) | v_n1; + } + const uint last_lane = min(J2K_HT_SIMD_LANES - 1u, quads - 1u - chunk); + carry_v_n3 = simd_shuffle(v_n3, ushort(last_lane)); + } + if (lane == 0u) { + v_n[quads] = carry_v_n3; + } + simdgroup_barrier(mem_flags::mem_threadgroup); + } +} + +kernel void j2k_decode_ht_cleanup_vlc_batched( + device const uchar *coded_data [[buffer(0)]], + device uint *decoded_data [[buffer(1)]], + constant J2kHtCleanupBatchJob *jobs [[buffer(2)]], + constant ushort *vlc_table0 [[buffer(3)]], + constant ushort *vlc_table1 [[buffer(4)]], + constant ushort *uvlc_table0 [[buffer(5)]], + constant ushort *uvlc_table1 [[buffer(6)]], + device J2kHtStatus *status [[buffer(7)]], + constant J2kHtVlcDispatchParams &dispatch [[buffer(9)]], + uint gid [[thread_position_in_grid]], + uint tid [[thread_index_in_threadgroup]] +) { + threadgroup uint ctx[J2K_HT_VLC_THREADS_PER_GROUP * J2K_HT_VLC_CTX_STRIDE]; + uint job_index = 0u; + if (!ht_vlc_job_index(dispatch, gid, job_index)) { + return; + } + const constant J2kHtCleanupBatchJob &job = jobs[job_index]; + const J2kHtCleanupParams params = ht_cleanup_params_from_job(job, 1u, job.output_offset); + decode_ht_cleanup_vlc( + coded_data + job.coded_offset, decoded_data, params, + vlc_table0, vlc_table1, uvlc_table0, uvlc_table1, status + job_index, + ctx + tid * J2K_HT_VLC_CTX_STRIDE + ); +} + +kernel void j2k_decode_ht_cleanup_vlc_repeated_batched( + device const uchar *coded_data [[buffer(0)]], + device uint *decoded_data [[buffer(1)]], + constant J2kHtCleanupBatchJob *jobs [[buffer(2)]], + constant J2kHtRepeatedBatchParams &repeated [[buffer(3)]], + constant ushort *vlc_table0 [[buffer(4)]], + constant ushort *vlc_table1 [[buffer(5)]], + constant ushort *uvlc_table0 [[buffer(6)]], + constant ushort *uvlc_table1 [[buffer(7)]], + device J2kHtStatus *status [[buffer(8)]], + constant J2kHtVlcDispatchParams &dispatch [[buffer(9)]], + uint2 gid [[thread_position_in_grid]], + uint tid [[thread_index_in_threadgroup]] +) { + threadgroup uint ctx[J2K_HT_VLC_THREADS_PER_GROUP * J2K_HT_VLC_CTX_STRIDE]; + uint job_index = 0u; + if (!ht_vlc_job_index(dispatch, gid.x, job_index) || gid.y >= repeated.batch_count) { + return; + } + const constant J2kHtCleanupBatchJob &job = jobs[job_index]; + const J2kHtCleanupParams params = ht_cleanup_params_from_job( + job, 1u, job.output_offset + gid.y * repeated.output_plane_len + ); + decode_ht_cleanup_vlc( + coded_data + job.coded_offset, decoded_data, params, + vlc_table0, vlc_table1, uvlc_table0, uvlc_table1, + status + gid.y * repeated.job_count + job_index, + ctx + tid * J2K_HT_VLC_CTX_STRIDE + ); +} + +kernel void j2k_decode_ht_cleanup_magsgn_batched( + device const uchar *coded_data [[buffer(0)]], + device uint *decoded_data [[buffer(1)]], + constant J2kHtCleanupBatchJob *jobs [[buffer(2)]], + device J2kHtStatus *status [[buffer(7)]], + uint gid [[thread_position_in_grid]], + uint simd_index [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]] +) { + threadgroup uint v_n[J2K_HT_SIMD_BLOCKS_PER_GROUP][J2K_HT_MAX_VN]; + threadgroup uint window[J2K_HT_SIMD_BLOCKS_PER_GROUP][J2K_HT_SIMD_WINDOW_WORDS]; + + const uint job_index = gid / J2K_HT_SIMD_LANES; + const constant J2kHtCleanupBatchJob &job = jobs[job_index]; + const J2kHtCleanupParams params = ht_cleanup_params_from_job(job, 1u, job.output_offset); + decode_ht_cleanup_magsgn( + coded_data + job.coded_offset, decoded_data, params, status + job_index, + v_n[simd_index], window[simd_index], lane + ); +} + +kernel void j2k_decode_ht_cleanup_magsgn_repeated_batched( + device const uchar *coded_data [[buffer(0)]], + device uint *decoded_data [[buffer(1)]], + constant J2kHtCleanupBatchJob *jobs [[buffer(2)]], + constant J2kHtRepeatedBatchParams &repeated [[buffer(3)]], + device J2kHtStatus *status [[buffer(8)]], + uint2 gid [[thread_position_in_grid]], + uint simd_index [[simdgroup_index_in_threadgroup]], + uint lane [[thread_index_in_simdgroup]] +) { + threadgroup uint v_n[J2K_HT_SIMD_BLOCKS_PER_GROUP][J2K_HT_MAX_VN]; + threadgroup uint window[J2K_HT_SIMD_BLOCKS_PER_GROUP][J2K_HT_SIMD_WINDOW_WORDS]; + + const uint job_index = gid.x / J2K_HT_SIMD_LANES; + if (job_index >= repeated.job_count || gid.y >= repeated.batch_count) { + return; + } + const constant J2kHtCleanupBatchJob &job = jobs[job_index]; + const J2kHtCleanupParams params = ht_cleanup_params_from_job( + job, 1u, job.output_offset + gid.y * repeated.output_plane_len + ); + decode_ht_cleanup_magsgn( + coded_data + job.coded_offset, decoded_data, params, + status + gid.y * repeated.job_count + job_index, + v_n[simd_index], window[simd_index], lane + ); +} diff --git a/crates/j2k-metal/src/idwt.metal b/crates/j2k-metal/src/idwt.metal index e002895dd..6f2303512 100644 --- a/crates/j2k-metal/src/idwt.metal +++ b/crates/j2k-metal/src/idwt.metal @@ -575,68 +575,182 @@ kernel void j2k_idwt_irreversible97_horizontal_scale( out[gid.y * params.width + gid.x] = sample; } -kernel void j2k_idwt_irreversible97_vertical_scale( - device float *out [[buffer(0)]], - constant J2kIdwtSingleDecompositionParams ¶ms [[buffer(1)]], - constant float &high_pass [[buffer(2)]], - uint3 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.width || gid.y >= params.height) { - return; - } +// Fused 9/7 lifting. Each dispatch replaces four (horizontal) or five +// (vertical scale + four lifts) full-plane passes. A threadgroup owns whole +// rows (horizontal) or a strip of columns (vertical) and walks its tiles in +// order: a tile is loaded with a four-sample halo, every step is applied in +// threadgroup memory, and the tile is written back once. The halo before a +// tile comes from a carry of the previous tile's pre-lift samples (its device +// copy has already been overwritten); the halo after it is still unwritten. +// Step s only updates positions within 3 - s samples of the tile, so each +// neighbour it reads was produced by step s - 1 in the same buffer. Updates +// use the original one-pass-per-step kernels' exact expressions and edge +// mirroring (kept as the oracle in `irreversible/parity_tests.rs`), so results +// are bit-identical to running one full pass per step. + +constant uint J2K_IDWT97_HALO = 4u; +constant uint J2K_IDWT97_ROW_TILE = 128u; +constant uint J2K_IDWT97_ROWS_PER_GROUP = 4u; +constant uint J2K_IDWT97_ROW_THREADS = 64u; +constant uint J2K_IDWT97_COL_TILE = 32u; +constant uint J2K_IDWT97_COL_ROWS = 64u; +constant uint J2K_IDWT97_COL_ROW_THREADS = 8u; + +struct J2kIdwt97LiftSteps { + float4 coefficients; + uint first_parity; + uint high_pass_bits; + uint reserved0; + uint reserved1; +}; - out += ulong(gid.z) * params.width * params.height; - const float KAPPA = CODEC_MATH_DWT97_KAPPA; - float sample = out[gid.y * params.width + gid.x]; +inline uint idwt97_left(uint x) { + return periodic_symmetric_extension_left_u32(x, 1u); +} - if (params.height == 1u) { - if (((params.y0 + params.output_y) & 1u) != 0u) { - sample *= 0.5f; - } - } else { - const uint first_even_y = (params.y0 + params.output_y) & 1u; - sample *= (gid.y & 1u) == first_even_y ? KAPPA : high_pass; - } +inline uint idwt97_right(uint x, uint length) { + return periodic_symmetric_extension_right_u32(x, 1u, length); +} - out[gid.y * params.width + gid.x] = sample; +inline uint idwt97_first_of_parity(uint lo, uint parity) { + return lo + ((lo & 1u) != parity ? 1u : 0u); } -kernel void j2k_idwt_irreversible97_horizontal_step( +kernel void j2k_idwt_irreversible97_horizontal_lift_fused( device float *out [[buffer(0)]], constant J2kIdwtSingleDecompositionParams ¶ms [[buffer(1)]], - constant J2kIdwt97StepParams &step [[buffer(2)]], - uint3 gid [[thread_position_in_grid]] + constant J2kIdwt97LiftSteps &steps [[buffer(2)]], + uint3 group [[threadgroup_position_in_grid]], + uint3 local [[thread_position_in_threadgroup]] ) { - const uint x = 2u * gid.x + step.parity; - if (x >= params.width || gid.y >= params.height || params.width <= 1u) { - return; - } + threadgroup float rows[J2K_IDWT97_ROWS_PER_GROUP][J2K_IDWT97_ROW_TILE + 2u * J2K_IDWT97_HALO]; + threadgroup float carry[J2K_IDWT97_ROWS_PER_GROUP][J2K_IDWT97_HALO]; + const uint width = params.width; + const uint y = group.y * J2K_IDWT97_ROWS_PER_GROUP + local.y; + const bool row_active = y < params.height && width > 1u; + device float *row_ptr = out + ulong(group.z) * width * params.height + ulong(y) * width; + threadgroup float *row = rows[local.y]; + + for (uint tile_start = 0u; tile_start < width; tile_start += J2K_IDWT97_ROW_TILE) { + const uint tile_end = min(tile_start + J2K_IDWT97_ROW_TILE, width); + const uint load_start = tile_start > J2K_IDWT97_HALO ? tile_start - J2K_IDWT97_HALO : 0u; + const uint load_end = min(tile_end + J2K_IDWT97_HALO, width); + + threadgroup_barrier(mem_flags::mem_threadgroup); + if (row_active) { + for (uint x = tile_start + local.x; x < load_end; x += J2K_IDWT97_ROW_THREADS) { + row[x - load_start] = row_ptr[x]; + } + for (uint x = load_start + local.x; x < tile_start; x += J2K_IDWT97_ROW_THREADS) { + row[x - load_start] = carry[local.y][x - (tile_start - J2K_IDWT97_HALO)]; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (row_active && tile_end < width && local.x < J2K_IDWT97_HALO) { + carry[local.y][local.x] = row[tile_end - J2K_IDWT97_HALO + local.x - load_start]; + } - out += ulong(gid.z) * params.width * params.height; - const uint left = periodic_symmetric_extension_left_u32(x, 1u); - const uint right = periodic_symmetric_extension_right_u32(x, 1u, params.width); - const uint idx = gid.y * params.width + x; - out[idx] = fma(out[gid.y * params.width + left] + out[gid.y * params.width + right], - step.coefficient, - out[idx]); + for (uint step = 0u; step < 4u; ++step) { + threadgroup_barrier(mem_flags::mem_threadgroup); + if (!row_active) { + continue; + } + const uint reach = 3u - step; + const uint lo = tile_start > reach ? tile_start - reach : 0u; + const uint hi = min(tile_end + reach, width); + const uint first = idwt97_first_of_parity(lo, steps.first_parity ^ (step & 1u)); + const float coefficient = steps.coefficients[step]; + for (uint x = first + 2u * local.x; x < hi; x += 2u * J2K_IDWT97_ROW_THREADS) { + const uint i = x - load_start; + row[i] = fma( + row[idwt97_left(x) - load_start] + row[idwt97_right(x, width) - load_start], + coefficient, + row[i] + ); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (row_active) { + for (uint x = tile_start + local.x; x < tile_end; x += J2K_IDWT97_ROW_THREADS) { + row_ptr[x] = row[x - load_start]; + } + } + } } -kernel void j2k_idwt_irreversible97_vertical_step( +kernel void j2k_idwt_irreversible97_vertical_fused( device float *out [[buffer(0)]], constant J2kIdwtSingleDecompositionParams ¶ms [[buffer(1)]], - constant J2kIdwt97StepParams &step [[buffer(2)]], - uint3 gid [[thread_position_in_grid]] + constant J2kIdwt97LiftSteps &steps [[buffer(2)]], + uint3 group [[threadgroup_position_in_grid]], + uint3 local [[thread_position_in_threadgroup]] ) { - const uint y = 2u * gid.y + step.parity; - if (gid.x >= params.width || y >= params.height || params.height <= 1u) { - return; - } + threadgroup float tile[J2K_IDWT97_COL_ROWS + 2u * J2K_IDWT97_HALO][J2K_IDWT97_COL_TILE]; + threadgroup float carry[J2K_IDWT97_HALO][J2K_IDWT97_COL_TILE]; + const uint width = params.width; + const uint height = params.height; + const uint x = group.x * J2K_IDWT97_COL_TILE + local.x; + const bool column_active = x < width; + const bool lift = column_active && height > 1u; + device float *plane = out + ulong(group.z) * width * height; + + // Vertical scale, exactly as the original full-plane vertical scale pass. + const float KAPPA = CODEC_MATH_DWT97_KAPPA; + const float high_pass = as_type(steps.high_pass_bits); + const uint first_even_y = (params.y0 + params.output_y) & 1u; - out += ulong(gid.z) * params.width * params.height; - const uint above = periodic_symmetric_extension_left_u32(y, 1u); - const uint below = periodic_symmetric_extension_right_u32(y, 1u, params.height); - const uint idx = y * params.width + gid.x; - out[idx] = fma(out[above * params.width + gid.x] + out[below * params.width + gid.x], - step.coefficient, - out[idx]); + for (uint tile_start = 0u; tile_start < height; tile_start += J2K_IDWT97_COL_ROWS) { + const uint tile_end = min(tile_start + J2K_IDWT97_COL_ROWS, height); + const uint load_start = tile_start > J2K_IDWT97_HALO ? tile_start - J2K_IDWT97_HALO : 0u; + const uint load_end = min(tile_end + J2K_IDWT97_HALO, height); + + threadgroup_barrier(mem_flags::mem_threadgroup); + if (column_active) { + for (uint y = tile_start + local.y; y < load_end; y += J2K_IDWT97_COL_ROW_THREADS) { + float sample = plane[y * width + x]; + if (height == 1u) { + if (first_even_y != 0u) { + sample *= 0.5f; + } + } else { + sample *= (y & 1u) == first_even_y ? KAPPA : high_pass; + } + tile[y - load_start][local.x] = sample; + } + for (uint y = load_start + local.y; y < tile_start; y += J2K_IDWT97_COL_ROW_THREADS) { + tile[y - load_start][local.x] = carry[y - (tile_start - J2K_IDWT97_HALO)][local.x]; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (column_active && tile_end < height && local.y < J2K_IDWT97_HALO) { + carry[local.y][local.x] = tile[tile_end - J2K_IDWT97_HALO + local.y - load_start][local.x]; + } + + for (uint step = 0u; step < 4u; ++step) { + threadgroup_barrier(mem_flags::mem_threadgroup); + if (!lift) { + continue; + } + const uint reach = 3u - step; + const uint lo = tile_start > reach ? tile_start - reach : 0u; + const uint hi = min(tile_end + reach, height); + const uint first = idwt97_first_of_parity(lo, steps.first_parity ^ (step & 1u)); + const float coefficient = steps.coefficients[step]; + for (uint y = first + 2u * local.y; y < hi; y += 2u * J2K_IDWT97_COL_ROW_THREADS) { + const uint i = y - load_start; + tile[i][local.x] = fma( + tile[idwt97_left(y) - load_start][local.x] + + tile[idwt97_right(y, height) - load_start][local.x], + coefficient, + tile[i][local.x] + ); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + if (column_active) { + for (uint y = tile_start + local.y; y < tile_end; y += J2K_IDWT97_COL_ROW_THREADS) { + plane[y * width + x] = tile[y - load_start][local.x]; + } + } + } } diff --git a/crates/j2k-metal/src/mct.metal b/crates/j2k-metal/src/mct.metal index 6fd3bdfaf..c1360299c 100644 --- a/crates/j2k-metal/src/mct.metal +++ b/crates/j2k-metal/src/mct.metal @@ -24,9 +24,11 @@ kernel void j2k_inverse_mct( } if (params.transform == J2K_MCT_TRANSFORM_IRREVERSIBLE97) { - plane0[gid] = y2 * 1.402f + y0 + params.addend0; - plane1[gid] = y2 * -0.71414f + y1 * -0.34413f + y0 + params.addend1; - plane2[gid] = y1 * 1.772f + y0 + params.addend2; + const float3 centered = j2k_inverse_ict_centered(y0, y1, y2); + const bool round_centered = params.round_centered != 0u; + plane0[gid] = j2k_shift_centered_sample(centered[0], params.addend0, round_centered); + plane1[gid] = j2k_shift_centered_sample(centered[1], params.addend1, round_centered); + plane2[gid] = j2k_shift_centered_sample(centered[2], params.addend2, round_centered); return; } diff --git a/crates/j2k-metal/src/mct_abi.metal b/crates/j2k-metal/src/mct_abi.metal index bd6e4677a..3f086e905 100644 --- a/crates/j2k-metal/src/mct_abi.metal +++ b/crates/j2k-metal/src/mct_abi.metal @@ -9,6 +9,7 @@ struct J2kInverseMctParams { float addend0; float addend1; float addend2; + uint round_centered; }; struct J2kMctStatus { diff --git a/crates/j2k-metal/src/pack.metal b/crates/j2k-metal/src/pack.metal index 3d608c4cc..27b373fcd 100644 --- a/crates/j2k-metal/src/pack.metal +++ b/crates/j2k-metal/src/pack.metal @@ -34,6 +34,46 @@ struct J2kBatchedMctRgb8PackParams { float u8_scales[3]; }; +inline float j2k_round_ties_even_centered(float value) { + // Every finite f32 at or beyond 2^23 is already integral. Keeping those + // values out of the integer conversion also bounds the no-libdevice floor. + if (!isfinite(value) || abs(value) >= 8388608.0f) { + return value; + } + const float truncated = float(int(value)); + const float lower = truncated > value ? truncated - 1.0f : truncated; + const float upper = lower + 1.0f; + const float midpoint = lower + 0.5f; + if (value < midpoint) { + return lower; + } + if (value > midpoint || (int(lower) & 1) != 0) { + return upper; + } + return lower; +} + +// Integer output follows the CPU's `round_ties_even_then_add`: round the +// centered sample ties-to-even, then add the unsigned level shift. Adding the +// shift first loses one bit of precision, so `k + 0.5 - ulp` becomes a tie and +// rounds up. Reversible (5/3) samples are integral and pass through unchanged. +inline float j2k_shift_centered_sample(float centered, float addend, bool round_centered) { + return (round_centered ? j2k_round_ties_even_centered(centered) : centered) + addend; +} + +// The CPU inverse ICT's nested fused expressions (`j2c/mct.rs`, +// `direct_cpu/color.rs`). Contraction and reassociation are off so the +// compiler cannot re-fuse or reorder them. +inline float3 j2k_inverse_ict_centered(float y0, float y1, float y2) { +#pragma clang fp reassociate(off) +#pragma clang fp contract(off) + return float3( + fma(y2, 1.402f, y0), + fma(y2, -0.71414f, fma(y1, -0.34413f, y0)), + fma(y1, 1.772f, y0) + ); +} + inline uchar scale_to_u8(float sample, float max_value, float scale) { const float clamped = clamp(sample, 0.0f, max_value); return uchar(min(floor(clamped * scale + 0.5f), 255.0f)); @@ -108,9 +148,10 @@ kernel void j2k_pack_mct_rgb8( rgb1 = i1 + params.addends[1]; rgb2 = y1 + i1 + params.addends[2]; } else { - rgb0 = y2 * 1.402f + y0 + params.addends[0]; - rgb1 = y2 * -0.71414f + y1 * -0.34413f + y0 + params.addends[1]; - rgb2 = y1 * 1.772f + y0 + params.addends[2]; + const float3 centered = j2k_inverse_ict_centered(y0, y1, y2); + rgb0 = j2k_shift_centered_sample(centered[0], params.addends[0], true); + rgb1 = j2k_shift_centered_sample(centered[1], params.addends[1], true); + rgb2 = j2k_shift_centered_sample(centered[2], params.addends[2], true); } const uint out_idx = gid.y * params.out_stride + gid.x * 3u; @@ -146,9 +187,10 @@ kernel void j2k_pack_mct_rgb8_batched( rgb1 = i1 + params.addends[1]; rgb2 = y1 + i1 + params.addends[2]; } else { - rgb0 = y2 * 1.402f + y0 + params.addends[0]; - rgb1 = y2 * -0.71414f + y1 * -0.34413f + y0 + params.addends[1]; - rgb2 = y1 * 1.772f + y0 + params.addends[2]; + const float3 centered = j2k_inverse_ict_centered(y0, y1, y2); + rgb0 = j2k_shift_centered_sample(centered[0], params.addends[0], true); + rgb1 = j2k_shift_centered_sample(centered[1], params.addends[1], true); + rgb2 = j2k_shift_centered_sample(centered[2], params.addends[2], true); } const uint out_idx = gid.z * params.output_stride + gid.y * params.out_stride + gid.x * 3u; diff --git a/crates/j2k-metal/src/routing/mod.rs b/crates/j2k-metal/src/routing/mod.rs index f23d07798..a93615307 100644 --- a/crates/j2k-metal/src/routing/mod.rs +++ b/crates/j2k-metal/src/routing/mod.rs @@ -10,7 +10,9 @@ mod telemetry; pub(crate) use decision::{decide_route, decision_error, RouteDecision}; #[cfg(any(test, target_os = "macos"))] -pub(crate) use promotion::{auto_repeated_decode_uses_metal, auto_scaled_decode_uses_metal}; +pub(crate) use promotion::{ + auto_full_decode_uses_metal, auto_repeated_decode_uses_metal, auto_scaled_decode_uses_metal, +}; pub(crate) const AUTO_DECODE_CPU_FALLBACK_REASON: &str = "J2K Metal Auto decode stays on CPU until decode benchmark evidence justifies Metal routing"; diff --git a/crates/j2k-metal/src/routing/promotion.rs b/crates/j2k-metal/src/routing/promotion.rs index 752812195..c7ee683d6 100644 --- a/crates/j2k-metal/src/routing/promotion.rs +++ b/crates/j2k-metal/src/routing/promotion.rs @@ -4,6 +4,7 @@ use j2k_core::{CompressedPayloadKind, CompressedTransferSyntax, Downscale, Pixel #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum PromotionOperation { + Full, ScaledHalf, Repeated, } @@ -41,6 +42,29 @@ pub(crate) fn auto_scaled_decode_uses_metal( ) } +/// Single full-image decodes. Cells are measured on one-component sources +/// decoded to `Gray8` and three-component sources decoded to `Rgb8`; other +/// pairings stay on the CPU. +pub(crate) fn auto_full_decode_uses_metal( + dimensions: (u32, u32), + source_components: u16, + format: PixelFormat, + transfer_syntax: CompressedTransferSyntax, + payload_kind: CompressedPayloadKind, +) -> bool { + matches!( + (source_components, format), + (1, PixelFormat::Gray8) | (3, PixelFormat::Rgb8) + ) && qualifies( + dimensions, + format, + transfer_syntax, + payload_kind, + PromotionOperation::Full, + 1, + ) +} + pub(crate) fn auto_repeated_decode_uses_metal( dimensions: (u32, u32), format: PixelFormat, diff --git a/crates/j2k-metal/src/routing/tests.rs b/crates/j2k-metal/src/routing/tests.rs index f4b6061d7..0bb5c64fd 100644 --- a/crates/j2k-metal/src/routing/tests.rs +++ b/crates/j2k-metal/src/routing/tests.rs @@ -5,69 +5,45 @@ use j2k_core::{ use crate::Error; use super::{ - auto_repeated_decode_uses_metal, auto_scaled_decode_uses_metal, decide_route, decision_error, - rejection::ExplicitMetalRejection, RouteDecision, + auto_full_decode_uses_metal, auto_repeated_decode_uses_metal, auto_scaled_decode_uses_metal, + decide_route, decision_error, rejection::ExplicitMetalRejection, RouteDecision, }; #[test] -fn auto_repeated_decode_thresholds_match_verified_external_cells() { +fn auto_repeated_decode_thresholds_match_verified_cells() { use CompressedPayloadKind::{Jpeg2000Codestream as Raw, JphFile as Jph}; use CompressedTransferSyntax::{ HtJpeg2000Lossless as HtLossless, HtJpeg2000Lossy as HtLossy, Jpeg2000Lossless as ClassicLossless, Jpeg2000Lossy as ClassicLossy, }; + use PixelFormat::{Gray16, Gray8, Rgb8, Rgba8}; for (dimensions, format, batch, transfer_syntax, payload_kind, expected) in [ - ((512, 512), PixelFormat::Gray8, 16, ClassicLossy, Raw, false), - ((3323, 891), PixelFormat::Gray8, 16, ClassicLossy, Raw, true), - ( - (3323, 891), - PixelFormat::Gray8, - 16, - ClassicLossless, - Raw, - false, - ), - ( - (3323, 891), - PixelFormat::Gray16, - 16, - ClassicLossy, - Raw, - false, - ), - ((256, 149), PixelFormat::Rgb8, 16, ClassicLossy, Raw, false), - ((640, 480), PixelFormat::Rgb8, 16, ClassicLossy, Raw, true), - ( - (640, 480), - PixelFormat::Rgb8, - 16, - ClassicLossless, - Raw, - false, - ), - ( - (2592, 1944), - PixelFormat::Rgb8, - 16, - ClassicLossless, - Raw, - true, - ), - ((640, 480), PixelFormat::Rgba8, 16, ClassicLossy, Raw, false), - ( - (2592, 1944), - PixelFormat::Rgb8, - 15, - ClassicLossy, - Raw, - false, - ), - ((767, 512), PixelFormat::Rgb8, 16, HtLossless, Jph, false), - ((768, 512), PixelFormat::Rgb8, 16, HtLossless, Jph, true), - ((639, 480), PixelFormat::Rgb8, 16, HtLossy, Raw, false), - ((640, 480), PixelFormat::Rgb8, 16, HtLossy, Raw, true), - ((768, 512), PixelFormat::Gray8, 16, HtLossless, Jph, false), + ((255, 256), Rgb8, 16, HtLossless, Raw, false), + ((256, 256), Rgb8, 16, HtLossless, Raw, true), + ((639, 480), Rgb8, 16, HtLossless, Jph, false), + ((640, 480), Rgb8, 16, HtLossless, Jph, true), + ((640, 479), Gray8, 16, HtLossless, Raw, false), + ((640, 480), Gray8, 16, HtLossless, Raw, true), + ((639, 480), Rgb8, 16, ClassicLossless, Raw, false), + ((640, 480), Rgb8, 16, ClassicLossless, Raw, true), + ((2047, 2048), Gray8, 16, ClassicLossless, Raw, false), + ((2048, 2048), Gray8, 16, ClassicLossless, Raw, true), + ((640, 480), Rgb8, 15, ClassicLossless, Raw, false), + ((640, 480), Rgba8, 16, ClassicLossless, Raw, false), + ((2048, 2048), Gray16, 16, ClassicLossless, Raw, false), + ((255, 256), Rgb8, 16, HtLossy, Raw, false), + ((256, 256), Rgb8, 16, HtLossy, Raw, true), + // Part 1 lossy RGB8 measured slower at 640x480 and faster from 1024x1024. + ((640, 480), Rgb8, 16, ClassicLossy, Raw, false), + ((1023, 1024), Rgb8, 16, ClassicLossy, Raw, false), + ((1024, 1024), Rgb8, 16, ClassicLossy, Raw, true), + // Lossy Gray8 qualifies only from the measured 2048x2048 cells. + ((640, 480), Gray8, 16, HtLossy, Raw, false), + ((2047, 2048), Gray8, 16, HtLossy, Raw, false), + ((2048, 2048), Gray8, 16, HtLossy, Raw, true), + ((3323, 891), Gray8, 16, ClassicLossy, Raw, false), + ((2048, 2048), Gray8, 16, ClassicLossy, Raw, true), ] { assert_eq!( auto_repeated_decode_uses_metal( @@ -87,123 +63,89 @@ fn auto_repeated_decode_thresholds_match_verified_external_cells() { fn auto_repeated_decode_requires_the_measured_payload_kind() { use CompressedPayloadKind::{Jp2File as Jp2, Jpeg2000Codestream as Raw, JphFile as Jph}; use CompressedTransferSyntax::{ - HtJpeg2000Lossless as HtLossless, HtJpeg2000Lossy as HtLossy, Jpeg2000Lossy as ClassicLossy, + HtJpeg2000Lossless as HtLossless, Jpeg2000Lossless as ClassicLossless, }; - assert!(auto_repeated_decode_uses_metal( - (640, 480), - PixelFormat::Rgb8, - 16, - ClassicLossy, - Raw, - )); - assert!(!auto_repeated_decode_uses_metal( - (640, 480), - PixelFormat::Rgb8, - 16, - ClassicLossy, - Jp2, - )); - assert!(auto_repeated_decode_uses_metal( - (640, 480), - PixelFormat::Rgb8, - 16, - HtLossy, - Raw, - )); - assert!(!auto_repeated_decode_uses_metal( - (640, 480), - PixelFormat::Rgb8, - 16, - HtLossy, - Jph, - )); - assert!(auto_repeated_decode_uses_metal( - (768, 512), - PixelFormat::Rgb8, - 16, - HtLossless, - Jph, - )); - assert!(!auto_repeated_decode_uses_metal( - (768, 512), - PixelFormat::Rgb8, - 16, - HtLossless, - Raw, - )); + for (dimensions, transfer_syntax, payload_kind, expected) in [ + ((640, 480), ClassicLossless, Raw, true), + ((640, 480), ClassicLossless, Jp2, false), + ((256, 256), HtLossless, Raw, true), + ((256, 256), HtLossless, Jph, false), + ((640, 480), HtLossless, Jph, true), + ((640, 480), HtLossless, Jp2, false), + ] { + assert_eq!( + auto_repeated_decode_uses_metal( + dimensions, + PixelFormat::Rgb8, + 16, + transfer_syntax, + payload_kind, + ), + expected, + "{dimensions:?}/{transfer_syntax:?}/{payload_kind:?}" + ); + } } #[test] -fn auto_scaled_decode_threshold_matches_only_the_verified_ht_cell() { - use CompressedPayloadKind::{Jpeg2000Codestream as Raw, JphFile as Jph}; +fn auto_full_decode_thresholds_match_verified_ht_cells() { + use CompressedPayloadKind::{Jp2File as Jp2, Jpeg2000Codestream as Raw, JphFile as Jph}; + use CompressedTransferSyntax::{ + HtJpeg2000Lossless as HtLossless, HtJpeg2000Lossy as HtLossy, + Jpeg2000Lossless as ClassicLossless, Jpeg2000Lossy as ClassicLossy, + }; + use PixelFormat::{Gray8, Rgb8}; + + for (dimensions, components, format, transfer_syntax, payload_kind, expected) in [ + ((639, 480), 3, Rgb8, HtLossless, Raw, false), + ((640, 480), 3, Rgb8, HtLossless, Raw, true), + ((640, 480), 3, Rgb8, HtLossless, Jph, true), + ((640, 480), 3, Rgb8, HtLossless, Jp2, false), + ((640, 480), 1, Gray8, HtLossless, Raw, true), + ((640, 479), 1, Gray8, HtLossless, Raw, false), + // Only the measured source/output pairings qualify. + ((640, 480), 3, Gray8, HtLossless, Raw, false), + ((640, 480), 1, Rgb8, HtLossless, Raw, false), + ((639, 480), 3, Rgb8, HtLossy, Raw, false), + ((640, 480), 3, Rgb8, HtLossy, Raw, true), + ((640, 479), 1, Gray8, HtLossy, Raw, false), + ((640, 480), 1, Gray8, HtLossy, Raw, true), + // Part 1 full decodes measured slower on Metal, lossless and lossy. + ((2048, 2048), 3, Rgb8, ClassicLossless, Raw, false), + ((2048, 2048), 3, Rgb8, ClassicLossy, Raw, false), + ] { + assert_eq!( + auto_full_decode_uses_metal( + dimensions, + components, + format, + transfer_syntax, + payload_kind + ), + expected, + "{dimensions:?}/{components}/{format:?}/{transfer_syntax:?}/{payload_kind:?}" + ); + } +} + +#[test] +fn auto_scaled_decode_stays_on_cpu_without_a_verified_cell() { + use CompressedPayloadKind::Jpeg2000Codestream as Raw; use CompressedTransferSyntax::{HtJpeg2000Lossless as Lossless, HtJpeg2000Lossy as Lossy}; - assert!(auto_scaled_decode_uses_metal( - (320, 240), - 3, - PixelFormat::Rgb8, - Lossy, - Raw, - Downscale::Half, - )); - for (dimensions, components, fmt, transfer_syntax, payload_kind, scale) in [ - ( - (319, 240), - 3, - PixelFormat::Rgb8, - Lossy, - Raw, - Downscale::Half, - ), - ( - (320, 240), - 1, - PixelFormat::Rgb8, - Lossy, - Raw, - Downscale::Half, - ), - ( - (320, 240), - 3, - PixelFormat::Gray8, - Lossy, - Raw, - Downscale::Half, - ), - ( - (320, 240), - 3, - PixelFormat::Rgb8, - Lossless, - Raw, - Downscale::Half, - ), - ( - (320, 240), - 3, - PixelFormat::Rgb8, - Lossy, - Jph, - Downscale::Half, - ), - ( - (320, 240), - 3, - PixelFormat::Rgb8, - Lossy, - Raw, - Downscale::Quarter, - ), + for (dimensions, transfer_syntax) in [ + ((320, 240), Lossy), + ((1024, 1024), Lossy), + ((1024, 1024), Lossless), ] { assert!(!auto_scaled_decode_uses_metal( dimensions, - components, - fmt, + 3, + PixelFormat::Rgb8, transfer_syntax, - payload_kind, - scale, + Raw, + Downscale::Half, )); } } diff --git a/crates/j2k-metal/src/store.metal b/crates/j2k-metal/src/store.metal index 2e7a0953d..2e7491be9 100644 --- a/crates/j2k-metal/src/store.metal +++ b/crates/j2k-metal/src/store.metal @@ -13,6 +13,7 @@ struct J2kStoreParams { uint output_x; uint output_y; float addend; + uint round_centered; }; struct J2kRepeatedStoreParams { @@ -29,6 +30,7 @@ struct J2kRepeatedStoreParams { uint output_y; float addend; uint batch_count; + uint round_centered; }; struct J2kRepeatedGrayStoreParams { @@ -79,25 +81,6 @@ inline float j2k_unsigned_native_sample(float value, uint bit_depth) { return floor(clamp(value, 0.0f, max_value) + 0.5f); } -inline float j2k_round_ties_even_centered(float value) { - // Every finite f32 at or beyond 2^23 is already integral. Keeping those - // values out of the integer conversion also bounds the no-libdevice floor. - if (!isfinite(value) || abs(value) >= 8388608.0f) { - return value; - } - const float truncated = float(int(value)); - const float lower = truncated > value ? truncated - 1.0f : truncated; - const float upper = lower + 1.0f; - const float midpoint = lower + 0.5f; - if (value < midpoint) { - return lower; - } - if (value > midpoint || (int(lower) & 1) != 0) { - return upper; - } - return lower; -} - inline float3 j2k_native_color_samples( float value0, float value1, @@ -124,20 +107,12 @@ inline float3 j2k_native_color_samples( const float green = value0 - floor((value2 + value1) * 0.25f); return float3(value2 + green, green, value1 + green) + addends; } - { -#pragma clang fp reassociate(off) -#pragma clang fp contract(off) - const float3 centered = float3( - fma(value2, 1.402f, value0), - fma(value2, -0.71414f, fma(value1, -0.34413f, value0)), - fma(value1, 1.772f, value0) - ); - return float3( - j2k_round_ties_even_centered(centered[0]), - j2k_round_ties_even_centered(centered[1]), - j2k_round_ties_even_centered(centered[2]) - ) + addends; - } + const float3 centered = j2k_inverse_ict_centered(value0, value1, value2); + return float3( + j2k_round_ties_even_centered(centered[0]), + j2k_round_ties_even_centered(centered[1]), + j2k_round_ties_even_centered(centered[2]) + ) + addends; } inline uint j2k_native_color_output_index( @@ -210,7 +185,8 @@ kernel void j2k_store_component( 0u, 0u ); - output[indices.dst_idx] = input[indices.src_idx] + params.addend; + output[indices.dst_idx] = + j2k_shift_centered_sample(input[indices.src_idx], params.addend, params.round_centered != 0u); } kernel void j2k_store_component_repeated( @@ -235,7 +211,8 @@ kernel void j2k_store_component_repeated( gid.z * params.input_instance_stride, gid.z * output_plane_len ); - output[indices.dst_idx] = input[indices.src_idx] + params.addend; + output[indices.dst_idx] = + j2k_shift_centered_sample(input[indices.src_idx], params.addend, params.round_centered != 0u); } kernel void j2k_store_component_repeated_gray_u8( @@ -261,7 +238,7 @@ kernel void j2k_store_component_repeated_gray_u8( gid.z * input_plane_len, gid.z * output_plane_len ); - output[indices.dst_idx] = scale_to_u8(input[indices.src_idx] + params.addend, params.max_value, params.u8_scale); + output[indices.dst_idx] = scale_to_u8(j2k_shift_centered_sample(input[indices.src_idx], params.addend, true), params.max_value, params.u8_scale); } kernel void j2k_store_component_repeated_gray_u16( @@ -287,7 +264,7 @@ kernel void j2k_store_component_repeated_gray_u16( gid.z * input_plane_len, gid.z * output_plane_len ); - output[indices.dst_idx] = pack_to_u16(input[indices.src_idx] + params.addend, params.max_value, params.u16_scale); + output[indices.dst_idx] = pack_to_u16(j2k_shift_centered_sample(input[indices.src_idx], params.addend, true), params.max_value, params.u16_scale); } kernel void j2k_store_component_repeated_gray_i16( @@ -314,7 +291,7 @@ kernel void j2k_store_component_repeated_gray_i16( gid.z * output_plane_len ); output[indices.dst_idx] = j2k_pack_native_i16( - input[indices.src_idx] + params.addend, + j2k_shift_centered_sample(input[indices.src_idx], params.addend, true), params.max_value ); } @@ -331,7 +308,7 @@ kernel void j2k_store_component_repeated_gray_u8_contiguous( return; } - output[gid] = scale_to_u8(input[gid] + params.addend, params.max_value, params.u8_scale); + output[gid] = scale_to_u8(j2k_shift_centered_sample(input[gid], params.addend, true), params.max_value, params.u8_scale); } kernel void j2k_store_component_repeated_gray_u16_contiguous( @@ -346,7 +323,7 @@ kernel void j2k_store_component_repeated_gray_u16_contiguous( return; } - output[gid] = pack_to_u16(input[gid] + params.addend, params.max_value, params.u16_scale); + output[gid] = pack_to_u16(j2k_shift_centered_sample(input[gid], params.addend, true), params.max_value, params.u16_scale); } kernel void j2k_store_component_gray_u8( @@ -370,7 +347,7 @@ kernel void j2k_store_component_gray_u8( 0u, params.output_item_offset ); - output[indices.dst_idx] = scale_to_u8(input[indices.src_idx] + params.addend, params.max_value, params.u8_scale); + output[indices.dst_idx] = scale_to_u8(j2k_shift_centered_sample(input[indices.src_idx], params.addend, true), params.max_value, params.u8_scale); } kernel void j2k_store_component_gray_u16( @@ -394,7 +371,7 @@ kernel void j2k_store_component_gray_u16( 0u, params.output_item_offset ); - output[indices.dst_idx] = pack_to_u16(input[indices.src_idx] + params.addend, params.max_value, params.u16_scale); + output[indices.dst_idx] = pack_to_u16(j2k_shift_centered_sample(input[indices.src_idx], params.addend, true), params.max_value, params.u16_scale); } kernel void j2k_store_component_gray_i16( @@ -419,7 +396,7 @@ kernel void j2k_store_component_gray_i16( params.output_item_offset ); output[indices.dst_idx] = j2k_pack_native_i16( - input[indices.src_idx] + params.addend, + j2k_shift_centered_sample(input[indices.src_idx], params.addend, true), params.max_value ); } diff --git a/crates/j2k-metal/tests/auto_routing_parity.rs b/crates/j2k-metal/tests/auto_routing_parity.rs new file mode 100644 index 000000000..929bd393d --- /dev/null +++ b/crates/j2k-metal/tests/auto_routing_parity.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Route-parity pre-flight for an Auto-routing corpus. `benches/auto_routing` +//! stops at the first decode whose Metal route differs from the CPU route; +//! this check decodes every case and operation and reports every mismatch, so +//! a corpus can be vetted before the long timing run. +//! +//! ```sh +//! J2K_AUTO_ROUTING_MANIFEST=/path/manifest.json J2K_AUTO_ROUTING_ROOT=/path/corpus \ +//! cargo test --profile gpu-quick -p j2k-metal --test auto_routing_parity -- \ +//! --include-ignored --nocapture +//! ``` + +#![cfg(target_os = "macos")] + +use std::path::Path; + +use j2k_core::{BackendRequest, Downscale, PixelFormat, Rect}; +use j2k_metal::{J2kDecoder, MetalBackendSession, MetalDecodeRequest}; +use j2k_test_support::{ + load_auto_routing_manifest, AutoRoutingPixelFormat, AutoRoutingWorkloadKind, +}; + +fn decode(bytes: &[u8], request: MetalDecodeRequest, session: &MetalBackendSession) -> Vec { + J2kDecoder::new(bytes) + .expect("parity decoder") + .decode_request_to_device_with_session(request, session) + .expect("parity decode") + .as_bytes() + .expect("parity readback") + .into_owned() +} + +/// Operations the routing bench times, as `(label, request)` builders. +fn requests( + fmt: PixelFormat, + (width, height): (u32, u32), +) -> [(&'static str, impl Fn(BackendRequest) -> MetalDecodeRequest); 3] { + let roi = Rect { + x: width.saturating_sub((width / 2).max(1)) / 2, + y: height.saturating_sub((height / 2).max(1)) / 2, + w: (width / 2).max(1), + h: (height / 2).max(1), + }; + [ + ( + "full", + Box::new(move |backend| MetalDecodeRequest::full(fmt, backend)) + as Box MetalDecodeRequest>, + ), + ( + "roi", + Box::new(move |backend| MetalDecodeRequest::region(fmt, roi, backend)), + ), + ( + "half", + Box::new(move |backend| MetalDecodeRequest::scaled(fmt, Downscale::Half, backend)), + ), + ] +} + +#[test] +#[ignore = "corpus pre-flight; set J2K_AUTO_ROUTING_MANIFEST and J2K_AUTO_ROUTING_ROOT"] +fn auto_routing_corpus_routes_are_bit_identical() { + let manifest = std::env::var_os("J2K_AUTO_ROUTING_MANIFEST") + .expect("J2K_AUTO_ROUTING_MANIFEST must name the corpus manifest"); + let root = std::env::var_os("J2K_AUTO_ROUTING_ROOT") + .expect("J2K_AUTO_ROUTING_ROOT must name the corpus root"); + let set = load_auto_routing_manifest(Path::new(&manifest), Path::new(&root)) + .expect("load Auto-routing manifest"); + let session = MetalBackendSession::system_default().expect("Metal session"); + let mut mismatches = Vec::new(); + for workload in set + .workloads + .iter() + .filter(|workload| workload.kind == AutoRoutingWorkloadKind::Decode) + { + let fmt = match workload.pixel_format { + AutoRoutingPixelFormat::Gray8 => PixelFormat::Gray8, + AutoRoutingPixelFormat::Rgb8 => PixelFormat::Rgb8, + }; + let dimensions = j2k::J2kDecoder::inspect(&workload.bytes) + .expect("inspect parity case") + .dimensions; + for (label, request) in requests(fmt, dimensions) { + let cpu = decode(&workload.bytes, request(BackendRequest::Cpu), &session); + let metal = decode(&workload.bytes, request(BackendRequest::Metal), &session); + if cpu == metal { + continue; + } + let differing = cpu.iter().zip(&metal).filter(|(a, b)| a != b).count(); + let max_delta = cpu + .iter() + .zip(&metal) + .map(|(a, b)| a.abs_diff(*b)) + .max() + .unwrap_or(0); + mismatches.push(format!( + "{} {label}: {differing} of {} bytes differ, max delta {max_delta}, lengths {} / {}", + workload.id, + cpu.len(), + cpu.len(), + metal.len() + )); + } + } + for line in &mismatches { + println!("{line}"); + } + assert!( + mismatches.is_empty(), + "{} route mismatches", + mismatches.len() + ); +} diff --git a/crates/j2k-metal/tests/device.rs b/crates/j2k-metal/tests/device.rs index 2c2f580a6..62aaadd0c 100644 --- a/crates/j2k-metal/tests/device.rs +++ b/crates/j2k-metal/tests/device.rs @@ -491,6 +491,8 @@ mod external_batch; mod grayscale_external; #[path = "device/ht_roi.rs"] mod ht_roi; +#[path = "device/irreversible_output_rounding.rs"] +mod irreversible_output_rounding; #[path = "device/legacy_batch.rs"] mod legacy_batch; #[path = "device/multitile_color.rs"] diff --git a/crates/j2k-metal/tests/device/auto_single_routing.rs b/crates/j2k-metal/tests/device/auto_single_routing.rs index 00aa8e89f..d84b21006 100644 --- a/crates/j2k-metal/tests/device/auto_single_routing.rs +++ b/crates/j2k-metal/tests/device/auto_single_routing.rs @@ -2,59 +2,68 @@ use super::*; -#[test] -fn auto_promotes_only_the_qualified_scaled_htj2k_cell() { - if !should_run_metal_runtime() { - return; - } - - let width = 640; - let height = 480; - let pixels = j2k_test_support::gradient_u8(width, height, 3); - let codestream = encode_htj2k( - &pixels, - width, - height, - 3, - 8, - false, - &EncodeOptions { - reversible: false, - num_decomposition_levels: 6, - ..EncodeOptions::default() - }, - ) - .expect("encode qualified HTJ2K routing fixture"); - - let mut full = J2kDecoder::new(&codestream).expect("full decoder"); - let full = full - .decode_request_to_device_with_report(MetalDecodeRequest::full( - PixelFormat::Rgb8, - BackendRequest::Auto, - )) - .expect("Auto full decode"); - assert_eq!(full.report.selected_backend, BackendKind::Cpu); +fn auto_selected_backend(codestream: &[u8], request: MetalDecodeRequest) -> (BackendKind, Vec) { + let mut decoder = J2kDecoder::new(codestream).expect("Auto decoder"); + let routed = decoder + .decode_request_to_device_with_report(request) + .expect("Auto decode"); + let bytes = routed.surface.as_bytes().expect("Auto bytes").into_owned(); + (routed.report.selected_backend, bytes) +} - let request = - MetalDecodeRequest::scaled(PixelFormat::Rgb8, Downscale::Half, BackendRequest::Auto); - let mut cpu = J2kDecoder::new(&codestream).expect("CPU decoder"); - let expected = cpu +fn cpu_bytes(codestream: &[u8], request: MetalDecodeRequest) -> Vec { + J2kDecoder::new(codestream) + .expect("CPU decoder") .decode_request_to_device(MetalDecodeRequest { backend: BackendRequest::Cpu, ..request }) - .expect("CPU scaled decode") + .expect("CPU decode") .as_bytes() .expect("CPU bytes") - .into_owned(); - let mut auto = J2kDecoder::new(&codestream).expect("Auto decoder"); - let actual = auto - .decode_request_to_device_with_report(request) - .expect("Auto scaled decode"); + .into_owned() +} + +#[test] +fn auto_promotes_ht_full_decodes_and_keeps_half_scale_on_cpu() { + if !should_run_metal_runtime() { + return; + } + + let (width, height) = (640, 480); + let pixels = j2k_test_support::gradient_u8(width, height, 3); + let encode = |reversible| { + encode_htj2k( + &pixels, + width, + height, + 3, + 8, + false, + &EncodeOptions { + reversible, + num_decomposition_levels: 6, + ..EncodeOptions::default() + }, + ) + .expect("encode HTJ2K routing fixture") + }; + let full = MetalDecodeRequest::full(PixelFormat::Rgb8, BackendRequest::Auto); + let half = MetalDecodeRequest::scaled(PixelFormat::Rgb8, Downscale::Half, BackendRequest::Auto); - assert_eq!(actual.report.selected_backend, BackendKind::Metal); - assert_eq!( - actual.surface.as_bytes().expect("Metal bytes").as_ref(), - expected - ); + // Lossless full decodes at the measured 640x480 threshold run on Metal + // and match the CPU byte for byte. + let lossless = encode(true); + let (backend, actual) = auto_selected_backend(&lossless, full); + assert_eq!(backend, BackendKind::Metal); + assert_eq!(actual, cpu_bytes(&lossless, full)); + + // Lossy (9/7) full decodes qualify at the same measured threshold and are + // byte-identical; no half-scale cell qualified, so that stays on the CPU. + let lossy = encode(false); + for (request, expected) in [(full, BackendKind::Metal), (half, BackendKind::Cpu)] { + let (backend, actual) = auto_selected_backend(&lossy, request); + assert_eq!(backend, expected, "{request:?}"); + assert_eq!(actual, cpu_bytes(&lossy, request), "{request:?}"); + } } diff --git a/crates/j2k-metal/tests/device/auto_tile_batch.rs b/crates/j2k-metal/tests/device/auto_tile_batch.rs index 31a4fc35e..8e1170a34 100644 --- a/crates/j2k-metal/tests/device/auto_tile_batch.rs +++ b/crates/j2k-metal/tests/device/auto_tile_batch.rs @@ -43,7 +43,11 @@ fn submitted_auto_repeated_jph_lossless_rgb_768x512_batch16_uses_metal() { } #[test] -fn submitted_auto_repeated_raw_ht_lossless_rgb_768x512_batch16_stays_on_cpu() { +fn submitted_auto_repeated_raw_ht_lossless_rgb_768x512_batch16_uses_metal() { + if !should_run_metal_runtime() { + return; + } + let bytes = Arc::<[u8]>::from(fixture_ht_rgb_u8_sized(768, 512, 0)); let mut batch = MetalTileBatch::with_capacity(16); for _ in 0..16 { @@ -58,8 +62,19 @@ fn submitted_auto_repeated_raw_ht_lossless_rgb_768x512_batch16_stays_on_cpu() { let surfaces = batch.decode_all().expect("decode Auto raw HT RGB batch"); assert_eq!(surfaces.len(), 16); assert!(surfaces.iter().all(|surface| { - surface.backend_kind() == BackendKind::Cpu && surface.residency() == SurfaceResidency::Host + surface.backend_kind() == BackendKind::Metal + && surface.residency() == SurfaceResidency::MetalResidentDecode })); + let mut cpu = J2kDecoder::new(&bytes).expect("CPU HT RGB decoder"); + let mut expected = vec![0_u8; 768 * 512 * 3]; + cpu.decode_into(&mut expected, 768 * 3, PixelFormat::Rgb8) + .expect("CPU HT RGB oracle"); + for surface in surfaces { + assert_eq!( + surface.as_bytes().expect("Metal surface bytes").as_ref(), + expected + ); + } } #[test] diff --git a/crates/j2k-metal/tests/device/direct_repeated.rs b/crates/j2k-metal/tests/device/direct_repeated.rs index 41f29a3dd..4620d216d 100644 --- a/crates/j2k-metal/tests/device/direct_repeated.rs +++ b/crates/j2k-metal/tests/device/direct_repeated.rs @@ -267,3 +267,88 @@ fn explicit_metal_unsupported_rgba16_report_variants_are_rejected() { MetalDecodeRequest::region_scaled(PixelFormat::Rgba16, roi, scale, BackendRequest::Metal), )); } + +/// Gray Part 1 image of `size` pixels per side. Lossy fixtures are quantized +/// hard enough that high-frequency code blocks carry no coding passes, which +/// the Metal repeated classic path must zero-fill instead of decoding. +fn classic_gray_fixture(size: u32, reversible: bool, seed: u32) -> Vec { + let mut state = seed; + let pixels: Vec = (0..size * size) + .map(|index| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + let (x, y) = (index % size, index / size); + let base = (x + 2 * y) / 5 % 200 + 20; + u8::try_from(base + state % 29).expect("sample fits u8") + }) + .collect(); + let options = EncodeOptions { + reversible, + irreversible_quantization_scale: 64.0, + ..EncodeOptions::default() + }; + encode(&pixels, size, size, 1, 8, false, &options).expect("encode classic gray fixture") +} + +fn tile_batch_gray8(bytes: &Arc<[u8]>, backend: BackendRequest, count: usize) -> Vec> { + let mut batch = MetalTileBatch::with_capacity(count); + for _ in 0..count { + batch + .push_shared_tile_request( + Arc::clone(bytes), + MetalDecodeRequest::full(PixelFormat::Gray8, backend), + ) + .expect("queue repeated gray tile"); + } + batch + .decode_all() + .expect("repeated gray batch") + .iter() + .map(|surface| surface.as_bytes().expect("surface bytes").into_owned()) + .collect() +} + +#[test] +fn repeated_classic_batch_zero_fills_empty_code_blocks_in_recycled_buffers() { + const SIZE: u32 = 256; + const COUNT: usize = 16; + if !should_run_metal_runtime() { + return; + } + + let lossless: Arc<[u8]> = Arc::from(classic_gray_fixture(SIZE, true, 0x1357_9bdf)); + let lossy: Arc<[u8]> = Arc::from(classic_gray_fixture(SIZE, false, 0x2468_ace0)); + let image = j2k_native::Image::new(&lossy, &j2k_native::DecodeSettings::default()) + .expect("inspect lossy fixture"); + let plan = image + .build_direct_grayscale_plan_with_context(&mut j2k_native::DecoderContext::default()) + .expect("lossy direct plan"); + let empty_blocks = plan.steps.iter().any(|step| match step { + j2k_native::J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { + let covered: u64 = sub_band + .jobs + .iter() + .map(|job| u64::from(job.width) * u64::from(job.height)) + .sum(); + covered < u64::from(sub_band.width) * u64::from(sub_band.height) + || sub_band + .jobs + .iter() + .any(|job| job.number_of_coding_passes == 0) + } + _ => false, + }); + assert!(empty_blocks, "lossy fixture must contain empty code blocks"); + + let expected = tile_batch_gray8(&lossy, BackendRequest::Cpu, COUNT); + // Leave recycled scratch buffers of the same geometry holding nonzero + // coefficients, as a prior decode in a long-lived process does. + for round in 0..3 { + let _ = tile_batch_gray8(&lossless, BackendRequest::Metal, COUNT); + let actual = tile_batch_gray8(&lossy, BackendRequest::Metal, COUNT); + for (index, (actual, expected)) in actual.iter().zip(&expected).enumerate() { + assert_eq!(actual, expected, "round {round} item {index}"); + } + } +} diff --git a/crates/j2k-metal/tests/device/irreversible_output_rounding.rs b/crates/j2k-metal/tests/device/irreversible_output_rounding.rs new file mode 100644 index 000000000..da79df97f --- /dev/null +++ b/crates/j2k-metal/tests/device/irreversible_output_rounding.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Irreversible (9/7) integer output must match the CPU byte for byte. The CPU +//! rounds each centered sample ties-to-even and only then adds the unsigned +//! level shift (`round_ties_even_then_add`). Adding the shift first costs one +//! bit of f32 precision, so `k + 0.5 - ulp` becomes a tie and rounds up: Metal +//! was one code value high on roughly ten samples per million. + +use super::*; + +const SIZE: u32 = 512; +const ROI: Rect = Rect { + x: SIZE / 4, + y: SIZE / 4, + w: SIZE / 2, + h: SIZE / 2, +}; +const BATCH: usize = 3; + +#[derive(Clone, Copy, Debug)] +struct LossyCase { + components: u16, + ht: bool, + use_mct: bool, +} + +const CASES: [LossyCase; 5] = [ + LossyCase { + components: 3, + ht: true, + use_mct: true, + }, + LossyCase { + components: 3, + ht: false, + use_mct: true, + }, + LossyCase { + components: 3, + ht: true, + use_mct: false, + }, + LossyCase { + components: 1, + ht: true, + use_mct: false, + }, + LossyCase { + components: 1, + ht: false, + use_mct: false, + }, +]; + +fn lossy_fixture(case: LossyCase) -> Vec { + let components = u32::from(case.components); + let mut state = 0x1234_5678_u32; + let pixels: Vec = (0..SIZE * SIZE * components) + .map(|index| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + let (pixel, component) = (index / components, index % components); + let (x, y) = (pixel % SIZE, pixel / SIZE); + let base = (x * 3 + y * 5 + component * 40) / 9 % 200 + 20; + u8::try_from(base + state % 17).expect("sample fits u8") + }) + .collect(); + let options = EncodeOptions { + reversible: false, + use_mct: case.use_mct, + irreversible_quantization_scale: 4.0, + ..EncodeOptions::default() + }; + if case.ht { + encode_htj2k(&pixels, SIZE, SIZE, case.components, 8, false, &options) + } else { + encode(&pixels, SIZE, SIZE, case.components, 8, false, &options) + } + .expect("encode lossy 9/7 fixture") +} + +/// Samples whose unrounded CPU output lands exactly on `k + 0.5` after the +/// level shift: the inputs on which shift-then-round disagrees with the CPU. +#[expect( + clippy::float_cmp, + reason = "a tie is an exact property of the f32 value" +)] +fn shifted_tie_count(bytes: &[u8]) -> usize { + let image = j2k_native::Image::new(bytes, &j2k_native::DecodeSettings::default()) + .expect("inspect lossy fixture"); + let mut context = j2k_native::DecoderContext::default(); + let components = image + .decode_components_with_context(&mut context) + .expect("CPU float component decode"); + components + .planes() + .iter() + .flat_map(j2k_native::ComponentPlane::samples) + .filter(|sample| { + let doubled = **sample * 2.0; + doubled == doubled.floor() && doubled.rem_euclid(2.0) == 1.0 + }) + .count() +} + +type RequestBuilder = fn(PixelFormat, BackendRequest) -> MetalDecodeRequest; + +fn decode_single( + bytes: &[u8], + request: MetalDecodeRequest, + session: &MetalBackendSession, +) -> Vec { + J2kDecoder::new(bytes) + .expect("lossy decoder") + .decode_request_to_device_with_session(request, session) + .expect("lossy decode") + .as_bytes() + .expect("lossy readback") + .into_owned() +} + +fn decode_batch(bytes: &Arc<[u8]>, fmt: PixelFormat, backend: BackendRequest) -> Vec { + let mut batch = MetalTileBatch::with_capacity(BATCH); + for _ in 0..BATCH { + batch + .push_shared_tile_request(Arc::clone(bytes), MetalDecodeRequest::full(fmt, backend)) + .expect("queue lossy batch item"); + } + let surfaces = batch.decode_all().expect("lossy batch decode"); + assert_eq!(surfaces.len(), BATCH); + surfaces + .iter() + .flat_map(|surface| { + surface + .as_bytes() + .expect("lossy batch readback") + .into_owned() + }) + .collect() +} + +fn compare(label: &str, cpu: &[u8], metal: &[u8], failures: &mut Vec) { + if cpu.len() != metal.len() { + failures.push(format!("{label}: lengths {} / {}", cpu.len(), metal.len())); + return; + } + let mismatches: Vec = (0..cpu.len()).filter(|&i| cpu[i] != metal[i]).collect(); + if let Some(&first) = mismatches.first() { + failures.push(format!( + "{label}: {} of {} bytes differ; first at byte {first} (CPU {}, Metal {})", + mismatches.len(), + cpu.len(), + cpu[first], + metal[first], + )); + } +} + +#[test] +fn explicit_metal_irreversible_integer_output_matches_cpu_rounding() { + if !should_run_metal_runtime() { + return; + } + + let session = MetalBackendSession::system_default().expect("Metal session"); + let mut failures = Vec::new(); + for case in CASES { + let bytes = lossy_fixture(case); + assert!( + shifted_tie_count(&bytes) > 0, + "{case:?}: fixture has no shifted ties, so it cannot detect the rounding order" + ); + let shared: Arc<[u8]> = Arc::from(bytes.as_slice()); + let formats: &[PixelFormat] = if case.components == 1 { + &[PixelFormat::Gray8, PixelFormat::Gray16] + } else { + &[PixelFormat::Rgb8, PixelFormat::Rgba8, PixelFormat::Rgb16] + }; + for &fmt in formats { + let requests: [(&str, RequestBuilder); 3] = [ + ("full", MetalDecodeRequest::full), + ("roi", |fmt, backend| { + MetalDecodeRequest::region(fmt, ROI, backend) + }), + ("half", |fmt, backend| { + MetalDecodeRequest::scaled(fmt, Downscale::Half, backend) + }), + ]; + for (operation, request) in requests { + let cpu = decode_single(&bytes, request(fmt, BackendRequest::Cpu), &session); + let metal = decode_single(&bytes, request(fmt, BackendRequest::Metal), &session); + compare( + &format!("{case:?} {fmt:?} {operation}"), + &cpu, + &metal, + &mut failures, + ); + } + let cpu = decode_batch(&shared, fmt, BackendRequest::Cpu); + let metal = decode_batch(&shared, fmt, BackendRequest::Metal); + compare( + &format!("{case:?} {fmt:?} batch"), + &cpu, + &metal, + &mut failures, + ); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/crates/j2k-metal/tests/shader_integrity.rs b/crates/j2k-metal/tests/shader_integrity.rs index 27c993c2c..62b6f01fe 100644 --- a/crates/j2k-metal/tests/shader_integrity.rs +++ b/crates/j2k-metal/tests/shader_integrity.rs @@ -49,6 +49,7 @@ const SHADER_SOURCES: &[&str] = &[ PACKETIZATION_SOURCE, include_str!("../src/fdwt.metal"), include_str!("../src/ht_cleanup.metal"), + include_str!("../src/ht_cleanup_simd.metal"), include_str!("../src/idwt.metal"), include_str!("../src/mct.metal"), include_str!("../src/quantize.metal"), @@ -126,7 +127,7 @@ fn cleanup_only_ht_pipelines_cannot_instantiate_refinement_state() { assert!(!common.contains("J2K_HT_MAX_PREV_ROW_SIG")); let cleanup = metal_function_body(SOURCE, "inline void decode_ht_cleanup_only_impl("); - assert!(cleanup.contains("decode_ht_cleanup_common(")); + assert!(cleanup.contains("decode_ht_cleanup_common(")); assert!(!cleanup.contains("decode_ht_refinement_impl(")); assert!(!cleanup.contains("sigma")); assert!(!cleanup.contains("prev_row_sig")); @@ -136,6 +137,7 @@ fn cleanup_only_ht_pipelines_cannot_instantiate_refinement_state() { assert!(refinement.contains("thread ushort prev_row_sig[J2K_HT_MAX_PREV_ROW_SIG]")); for kernel in [ + "kernel void j2k_decode_ht_cleanup_cleanup_only(", "kernel void j2k_decode_ht_cleanup_batched_cleanup_only(", "kernel void j2k_decode_ht_cleanup_repeated_batched_cleanup_only(", ] { diff --git a/crates/j2k-native/src/direct_cpu.rs b/crates/j2k-native/src/direct_cpu.rs index 5e24250c8..a6371be60 100644 --- a/crates/j2k-native/src/direct_cpu.rs +++ b/crates/j2k-native/src/direct_cpu.rs @@ -23,7 +23,8 @@ pub use color::{execute_direct_color_plan_rgb8_into, execute_direct_color_plan_r mod component; use component::{ checked_area, checked_sub_band_job_output_range, execute_component_plan, execute_idwt_step, - prepare_sub_band_output, resize_and_zero, store_component, SubBandJobOutputRange, + prepare_sub_band_output, resize_and_zero, rounds_at_store, store_component, + SubBandJobOutputRange, }; mod referenced; pub use referenced::{ diff --git a/crates/j2k-native/src/direct_cpu/component.rs b/crates/j2k-native/src/direct_cpu/component.rs index 257d311d1..bf777980b 100644 --- a/crates/j2k-native/src/direct_cpu/component.rs +++ b/crates/j2k-native/src/direct_cpu/component.rs @@ -215,6 +215,13 @@ fn find_idwt_band(bands: &[DirectCpuBand], band_id: J2kDirectBandId) -> Result bool { + !(mct && component_index < 3) +} + pub(super) fn store_component( store: &J2kDirectStoreStep, bands: &[DirectCpuBand], diff --git a/crates/j2k-native/src/direct_cpu/referenced.rs b/crates/j2k-native/src/direct_cpu/referenced.rs index 16b75a2d7..63942ecf8 100644 --- a/crates/j2k-native/src/direct_cpu/referenced.rs +++ b/crates/j2k-native/src/direct_cpu/referenced.rs @@ -111,6 +111,7 @@ fn execute_referenced_htj2k_plan_with_payloads<'scratch>( classic_workspace, ht_workspace, &mut output_initialized[0], + true, )?; } else if let Some(geometry) = tile.color_geometry() { execute_color_components_referenced( diff --git a/crates/j2k-native/src/direct_cpu/referenced/component.rs b/crates/j2k-native/src/direct_cpu/referenced/component.rs index cd9cc03d2..376998b25 100644 --- a/crates/j2k-native/src/direct_cpu/referenced/component.rs +++ b/crates/j2k-native/src/direct_cpu/referenced/component.rs @@ -11,8 +11,8 @@ use crate::{ use super::super::referenced_classic::{execute_classic_sub_band_referenced, ClassicPayloadCursor}; use super::super::{ apply_inverse_mct_region, checked_sub_band_job_output_range, execute_idwt_step, - prepare_sub_band_output, store_component, DirectComponentBandScratch, DirectComponentPlane, - SubBandJobOutputRange, + prepare_sub_band_output, rounds_at_store, store_component, DirectComponentBandScratch, + DirectComponentPlane, SubBandJobOutputRange, }; use super::payload::ReferencedPayloadCursor; @@ -55,6 +55,7 @@ pub(super) fn execute_color_components_referenced( classic_workspace, ht_workspace, &mut output_initialized[component_index], + rounds_at_store(mct, component_index), )?; } if mct { @@ -65,7 +66,7 @@ pub(super) fn execute_color_components_referenced( transform, rgb_bit_depths, signed, - false, + true, destination, [plane0, plane1, plane2], )?; @@ -87,6 +88,7 @@ pub(super) fn execute_component_plan_referenced( classic_workspace: &mut crate::J2kCodeBlockDecodeWorkspace, ht_workspace: &mut HtCodeBlockDecodeWorkspace, output_initialized: &mut bool, + round_irreversible_output: bool, ) -> Result<()> { bands.reset(); let mut stored = false; @@ -112,7 +114,13 @@ pub(super) fn execute_component_plan_referenced( } J2kDirectGrayscaleStep::Idwt(step) => execute_idwt_step(step, bands)?, J2kDirectGrayscaleStep::Store(store) => { - store_component(store, bands.active(), output, output_initialized, false)?; + store_component( + store, + bands.active(), + output, + output_initialized, + round_irreversible_output, + )?; stored = true; } } diff --git a/crates/j2k-native/src/direct_cpu/referenced_classic.rs b/crates/j2k-native/src/direct_cpu/referenced_classic.rs index 323ddb7cf..e623e704b 100644 --- a/crates/j2k-native/src/direct_cpu/referenced_classic.rs +++ b/crates/j2k-native/src/direct_cpu/referenced_classic.rs @@ -14,8 +14,8 @@ use super::allocation::prepare_referenced_classic_scratch; use super::referenced::{decoded_color_components, decoded_plane, payload_slice}; use super::{ apply_inverse_mct_region, checked_sub_band_job_output_range, execute_idwt_step, - prepare_sub_band_output, store_component, DirectComponentBandScratch, DirectComponentPlane, - J2kDirectCpuScratch, J2kDirectDecodedComponents, SubBandJobOutputRange, + prepare_sub_band_output, rounds_at_store, store_component, DirectComponentBandScratch, + DirectComponentPlane, J2kDirectCpuScratch, J2kDirectDecodedComponents, SubBandJobOutputRange, }; /// Execute retained per-tile Gray/RGB/RGBA classic JPEG 2000 geometry @@ -106,6 +106,7 @@ fn execute_referenced_classic_plan_with_payloads<'scratch>( compressed_payload, classic_workspace, &mut output_initialized[0], + true, )?; } else if let Some(geometry) = tile.color_geometry() { execute_color_components_referenced( @@ -188,6 +189,7 @@ fn execute_color_components_referenced( compressed_payload, classic_workspace, &mut output_initialized[component_index], + rounds_at_store(mct, component_index), )?; } if mct { @@ -198,7 +200,7 @@ fn execute_color_components_referenced( transform, rgb_bit_depths, signed, - false, + true, destination, [plane0, plane1, plane2], )?; @@ -206,6 +208,10 @@ fn execute_color_components_referenced( Ok(()) } +#[expect( + clippy::too_many_arguments, + reason = "classic execution keeps its validated payload cursor, retained workspace, and store rounding explicit" +)] fn execute_component_plan_referenced( plan: &J2kDirectGrayscalePlan, bands: &mut DirectComponentBandScratch, @@ -214,6 +220,7 @@ fn execute_component_plan_referenced( compressed_payload: &mut alloc::vec::Vec, classic_workspace: &mut J2kCodeBlockDecodeWorkspace, output_initialized: &mut bool, + round_irreversible_output: bool, ) -> Result<()> { bands.reset(); let mut stored = false; @@ -235,7 +242,13 @@ fn execute_component_plan_referenced( } J2kDirectGrayscaleStep::Idwt(step) => execute_idwt_step(step, bands)?, J2kDirectGrayscaleStep::Store(store) => { - store_component(store, bands.active(), output, output_initialized, false)?; + store_component( + store, + bands.active(), + output, + output_initialized, + round_irreversible_output, + )?; stored = true; } } diff --git a/crates/j2k-native/src/direct_cpu/referenced_staged/finish.rs b/crates/j2k-native/src/direct_cpu/referenced_staged/finish.rs index 80d0ca940..4c3e91d94 100644 --- a/crates/j2k-native/src/direct_cpu/referenced_staged/finish.rs +++ b/crates/j2k-native/src/direct_cpu/referenced_staged/finish.rs @@ -9,8 +9,8 @@ use crate::{ use super::super::referenced::decoded_components as decoded_htj2k_components; use super::super::referenced_classic::decoded_components as decoded_classic_components; use super::super::{ - apply_inverse_mct_region, execute_idwt_step, store_component, J2kDirectCpuScratch, - J2kDirectDecodedComponents, StagedDirectRoute, + apply_inverse_mct_region, execute_idwt_step, rounds_at_store, store_component, + J2kDirectCpuScratch, J2kDirectDecodedComponents, StagedDirectRoute, }; use super::plan_access::{ classic_tile_color_transform, classic_tile_components, ht_tile_color_transform, @@ -110,6 +110,7 @@ pub(super) fn finish_tile_components( { bail!(DecodingError::CodeBlockDecodeFailure); } + let mct = color_transform.is_some_and(|(_, mct, _)| mct); for (component_index, component) in components.iter().enumerate() { let bands = &mut scratch.component_band_sets[component_index]; let output = &mut scratch.component_planes[component_index]; @@ -126,7 +127,7 @@ pub(super) fn finish_tile_components( bands.active(), output, &mut output_initialized, - false, + rounds_at_store(mct, component_index), )?; stored = true; } @@ -145,7 +146,7 @@ pub(super) fn finish_tile_components( transform, bit_depths, signed, - false, + true, destination, [plane0, plane1, plane2], )?; diff --git a/crates/j2k-native/src/image.rs b/crates/j2k-native/src/image.rs index 675fed101..5bcfb0597 100644 --- a/crates/j2k-native/src/image.rs +++ b/crates/j2k-native/src/image.rs @@ -473,23 +473,7 @@ impl<'a> Image<'a> { &self, decoder_context: &'ctx mut DecoderContext<'a>, ) -> Result> { - self.validate_component_plane_precision()?; - let decoded_image = self.decode_image( - decoder_context, - None, - None, - false, - self.retained_metadata_bytes()?, - )?; - let DecodedImage { - decoded_components, - boxes: _, - } = decoded_image; - self.try_borrow_component_planes( - decoded_components.as_slice(), - decoded_components.capacity(), - (self.width(), self.height()), - ) + self.decode_borrowed_component_planes(decoder_context, None, None, false) } /// Decode the image into owned native-bit-depth component planes. @@ -574,23 +558,7 @@ impl<'a> Image<'a> { decoder_context: &'ctx mut DecoderContext<'a>, ht_decoder: &mut dyn HtCodeBlockDecoder, ) -> Result> { - self.validate_component_plane_precision()?; - let decoded_image = self.decode_image( - decoder_context, - None, - Some(ht_decoder), - false, - self.retained_metadata_bytes()?, - )?; - let DecodedImage { - decoded_components, - boxes: _, - } = decoded_image; - self.try_borrow_component_planes( - decoded_components.as_slice(), - decoded_components.capacity(), - (self.width(), self.height()), - ) + self.decode_borrowed_component_planes(decoder_context, None, Some(ht_decoder), false) } /// Decode borrowed component planes for a requested region using a @@ -604,25 +572,7 @@ impl<'a> Image<'a> { roi: (u32, u32, u32, u32), decoder_context: &'ctx mut DecoderContext<'a>, ) -> Result> { - validate_roi((self.width(), self.height()), roi)?; - self.validate_component_plane_precision()?; - let (_x, _y, width, height) = roi; - let decoded_image = self.decode_image( - decoder_context, - Some(roi), - None, - false, - self.retained_metadata_bytes()?, - )?; - let DecodedImage { - decoded_components, - boxes: _, - } = decoded_image; - self.try_borrow_component_planes( - decoded_components.as_slice(), - decoded_components.capacity(), - (width, height), - ) + self.decode_borrowed_component_planes(decoder_context, Some(roi), None, false) } /// Decode a source-coordinate region into owned native-bit-depth component @@ -666,25 +616,7 @@ impl<'a> Image<'a> { roi: (u32, u32, u32, u32), ht_decoder: &mut dyn HtCodeBlockDecoder, ) -> Result> { - validate_roi((self.width(), self.height()), roi)?; - self.validate_component_plane_precision()?; - let (_x, _y, width, height) = roi; - let decoded_image = self.decode_image( - decoder_context, - Some(roi), - Some(ht_decoder), - false, - self.retained_metadata_bytes()?, - )?; - let DecodedImage { - decoded_components, - boxes: _, - } = decoded_image; - self.try_borrow_component_planes( - decoded_components.as_slice(), - decoded_components.capacity(), - (width, height), - ) + self.decode_borrowed_component_planes(decoder_context, Some(roi), Some(ht_decoder), false) } /// Decode a region of the image and return it as an 8-bit interleaved bitmap. @@ -859,11 +791,35 @@ impl PreparedRegionDecoder<'_, '_, '_> { pub fn decode_region_components( &mut self, roi: (u32, u32, u32, u32), + ) -> Result> { + self.decode_region_component_planes(roi, false) + } + + /// Rounded-for-integer-output counterpart of + /// [`Self::decode_region_components`]; see + /// [`Image::decode_components_for_integer_output_with_context`]. + /// + /// # Errors + /// + /// Returns an error when the region, component precision, or codestream is invalid. + #[doc(hidden)] + pub fn decode_region_components_for_integer_output( + &mut self, + roi: (u32, u32, u32, u32), + ) -> Result> { + self.decode_region_component_planes(roi, true) + } + + fn decode_region_component_planes( + &mut self, + roi: (u32, u32, u32, u32), + round_irreversible_output: bool, ) -> Result> { validate_roi((self.image.width(), self.image.height()), roi)?; self.image.validate_component_plane_precision()?; self.decoder_context.set_output_region(Some(roi)); - self.decoder_context.set_round_irreversible_output(false); + self.decoder_context + .set_round_irreversible_output(round_irreversible_output); let decode_result = j2c::decode_preparsed( &self.image.header, self.retained_image_bytes, @@ -871,6 +827,7 @@ impl PreparedRegionDecoder<'_, '_, '_> { self.decoder_context, ); self.decoder_context.set_output_region(None); + self.decoder_context.set_round_irreversible_output(false); decode_result?; let (_x, _y, width, height) = roi; let decoded_image = self diff --git a/crates/j2k-native/src/image/output_api.rs b/crates/j2k-native/src/image/output_api.rs index 3ae6c7ddf..e402575b7 100644 --- a/crates/j2k-native/src/image/output_api.rs +++ b/crates/j2k-native/src/image/output_api.rs @@ -7,9 +7,10 @@ use alloc::vec::Vec; use crate::color::{ComponentPlane, DecodedComponents, DecodedNativeComponents, RawBitmap}; use crate::error::{bail, DecodingError, Result, ValidationError}; use crate::j2c::{self, ComponentData, DecoderContext, Reversible53CoefficientImage}; +use crate::jp2::DecodedImage; use crate::{ checked_decode_byte_len3, checked_decode_byte_len4, checked_decode_sample_count, - native_bytes_per_sample, try_reserve_decode_elements, validate_roi, + native_bytes_per_sample, try_reserve_decode_elements, validate_roi, HtCodeBlockDecoder, }; use super::native::{try_clone_color_space, NativeOutputBudget}; @@ -405,3 +406,107 @@ impl<'a> Image<'a> { Ok(()) } } + +/// Borrowed component planes rounded for integer output. +/// +/// The plain component-plane methods return unrounded irreversible (9/7) +/// samples with the unsigned level shift already added. Rounding those planes +/// afterwards differs from [`Image::decode`] on exact ties, because adding the +/// shift costs one bit of f32 precision. These methods round each centered +/// sample ties-to-even before the shift, as the interleaved decode paths do, so +/// converting their planes to integer samples reproduces that output. +/// Reversible (5/3) samples are unchanged. +impl<'a> Image<'a> { + /// Rounded-for-integer-output counterpart of + /// [`Self::decode_components_with_context`]. + /// + /// # Errors + /// + /// Returns an error when component precision is unsupported or decoding fails. + #[doc(hidden)] + pub fn decode_components_for_integer_output_with_context<'ctx>( + &self, + decoder_context: &'ctx mut DecoderContext<'a>, + ) -> Result> { + self.decode_borrowed_component_planes(decoder_context, None, None, true) + } + + /// Rounded-for-integer-output counterpart of + /// [`Self::decode_region_components_with_context`]. + /// + /// # Errors + /// + /// Returns an error when the region is invalid, precision is unsupported, or decoding fails. + #[doc(hidden)] + pub fn decode_region_components_for_integer_output_with_context<'ctx>( + &self, + roi: (u32, u32, u32, u32), + decoder_context: &'ctx mut DecoderContext<'a>, + ) -> Result> { + self.decode_borrowed_component_planes(decoder_context, Some(roi), None, true) + } + + /// Rounded-for-integer-output counterpart of + /// [`Self::decode_components_with_ht_decoder`]. + /// + /// # Errors + /// + /// Returns an error when component precision is unsupported or decoding fails. + #[doc(hidden)] + pub fn decode_components_for_integer_output_with_ht_decoder<'ctx>( + &self, + decoder_context: &'ctx mut DecoderContext<'a>, + ht_decoder: &mut dyn HtCodeBlockDecoder, + ) -> Result> { + self.decode_borrowed_component_planes(decoder_context, None, Some(ht_decoder), true) + } + + /// Rounded-for-integer-output counterpart of + /// [`Self::decode_region_components_with_ht_decoder`]. + /// + /// # Errors + /// + /// Returns an error when the region is invalid, precision is unsupported, or decoding fails. + #[doc(hidden)] + pub fn decode_region_components_for_integer_output_with_ht_decoder<'ctx>( + &self, + decoder_context: &'ctx mut DecoderContext<'a>, + roi: (u32, u32, u32, u32), + ht_decoder: &mut dyn HtCodeBlockDecoder, + ) -> Result> { + self.decode_borrowed_component_planes(decoder_context, Some(roi), Some(ht_decoder), true) + } + + pub(super) fn decode_borrowed_component_planes<'ctx>( + &self, + decoder_context: &'ctx mut DecoderContext<'a>, + roi: Option<(u32, u32, u32, u32)>, + ht_decoder: Option<&mut dyn HtCodeBlockDecoder>, + round_irreversible_output: bool, + ) -> Result> { + let dimensions = match roi { + Some(roi) => { + validate_roi((self.width(), self.height()), roi)?; + (roi.2, roi.3) + } + None => (self.width(), self.height()), + }; + self.validate_component_plane_precision()?; + let decoded_image = self.decode_image( + decoder_context, + roi, + ht_decoder, + round_irreversible_output, + self.retained_metadata_bytes()?, + )?; + let DecodedImage { + decoded_components, + boxes: _, + } = decoded_image; + self.try_borrow_component_planes( + decoded_components.as_slice(), + decoded_components.capacity(), + dimensions, + ) + } +} diff --git a/crates/j2k-native/src/j2c/mct.rs b/crates/j2k-native/src/j2c/mct.rs index cb2937921..b98806b15 100644 --- a/crates/j2k-native/src/j2c/mct.rs +++ b/crates/j2k-native/src/j2c/mct.rs @@ -4,7 +4,7 @@ use super::codestream::{ComponentInfo, Header, WaveletTransform}; use super::decode::TileDecodeContext; use crate::error::{bail, err, ColorError, Result}; -use crate::math::{dispatch, f32x8, floor_f32, round_ties_even_then_add, Level, Simd}; +use crate::math::{dispatch, f32x8, floor_f32, mul_add, round_ties_even_then_add, Level, Simd}; use crate::{HtCodeBlockDecoder, J2kInverseMctJob, J2kWaveletTransform}; use j2k_codec_math::mct; @@ -242,9 +242,14 @@ fn apply_inner_impl( let src0 = *y0; let src1 = *y1; let src2 = *y2; - *y0 = src0 + mct::ICT_INV_R_CR * src2 + addends[0]; - *y1 = src0 + mct::ICT_INV_G_CB * src1 + mct::ICT_INV_G_CR * src2 + addends[1]; - *y2 = src0 + mct::ICT_INV_B_CB * src1 + addends[2]; + // Same fused expressions as the SIMD body above and Metal. + *y0 = mul_add(src2, mct::ICT_INV_R_CR, src0) + addends[0]; + *y1 = mul_add( + src2, + mct::ICT_INV_G_CR, + mul_add(src1, mct::ICT_INV_G_CB, src0), + ) + addends[1]; + *y2 = mul_add(src1, mct::ICT_INV_B_CB, src0) + addends[2]; } } // Reversible MCT, specified in G.2. @@ -344,6 +349,48 @@ mod tests { } } + #[test] + fn irreversible_scalar_tail_matches_simd_body_bits() { + // These inputs round differently fused and unfused, so a tail computed + // without FMA disagrees with the SIMD body and with Metal. + let y0 = f32::from_bits(0xc23e_1ece); + let y1 = f32::from_bits(0x4126_4c1f); + let y2 = f32::from_bits(0xc129_2742); + let expected = [ + crate::math::mul_add(y2, mct::ICT_INV_R_CR, y0), + crate::math::mul_add( + y2, + mct::ICT_INV_G_CR, + crate::math::mul_add(y1, mct::ICT_INV_G_CB, y0), + ), + crate::math::mul_add(y1, mct::ICT_INV_B_CB, y0), + ]; + assert_ne!( + expected[0].to_bits(), + (y0 + mct::ICT_INV_R_CR * y2).to_bits(), + "fixture must separate fused from unfused rounding" + ); + + let mut plane0 = [y0; 9]; + let mut plane1 = [y1; 9]; + let mut plane2 = [y2; 9]; + apply_inner( + WaveletTransform::Irreversible97, + &mut plane0, + &mut plane1, + &mut plane2, + [0.0; 3], + ); + + for index in 0..plane0.len() { + assert_eq!( + [plane0[index], plane1[index], plane2[index]].map(f32::to_bits), + expected.map(f32::to_bits), + "sample {index}" + ); + } + } + #[test] fn irreversible_output_rounds_before_unsigned_level_shift() { let mut samples = [f32::from_bits(0xc117_fffc), -9.5, -8.5]; diff --git a/crates/j2k-native/src/math.rs b/crates/j2k-native/src/math.rs index 8e42979f9..405d98744 100644 --- a/crates/j2k-native/src/math.rs +++ b/crates/j2k-native/src/math.rs @@ -78,7 +78,8 @@ mod inner { #[inline(always)] pub(crate) fn mul_add(self, mul: Self, addend: Self) -> Self { Self { - inner: self.inner.mul_add(mul.inner, addend.inner), + // Preserve the scalar path's single-rounding contract without hardware FMA. + inner: self.inner.mul_add_precise(mul.inner, addend.inner), } } @@ -895,6 +896,22 @@ mod simd_operator_tests { ); } } + + #[test] + fn vector_mul_add_preserves_fused_rounding() { + let a = [f32::from_bits(0x4526_ba09); SIMD_WIDTH]; + let b = [f32::from_bits(0xbf41_420c); SIMD_WIDTH]; + let c = [f32::from_bits(0x4470_e5c9); SIMD_WIDTH]; + let mut output = [0.0; SIMD_WIDTH]; + + dispatch!(Level::new(), simd => { + f32x8::from_slice(simd, &a) + .mul_add(f32x8::from_slice(simd, &b), f32x8::from_slice(simd, &c)) + .store(&mut output); + }); + + assert!(output.iter().all(|value| value.to_bits() == 0xc483_47a5)); + } } #[cfg(test)] diff --git a/crates/j2k-t803/fuzz/Cargo.lock b/crates/j2k-t803/fuzz/Cargo.lock index 81895c116..5df9dd59e 100644 --- a/crates/j2k-t803/fuzz/Cargo.lock +++ b/crates/j2k-t803/fuzz/Cargo.lock @@ -163,9 +163,9 @@ dependencies = [ [[package]] name = "fearless_simd" -version = "0.7.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" +checksum = "f3772b63c40606beea8fe1f7b06a60de27ceb9bfc2d9dad3427368ca41dc7e62" [[package]] name = "find-msvc-tools" diff --git a/crates/j2k-transcode/fuzz/Cargo.lock b/crates/j2k-transcode/fuzz/Cargo.lock index d217b8b9e..9816fe9b7 100644 --- a/crates/j2k-transcode/fuzz/Cargo.lock +++ b/crates/j2k-transcode/fuzz/Cargo.lock @@ -65,9 +65,9 @@ checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "fearless_simd" -version = "0.7.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" +checksum = "f3772b63c40606beea8fe1f7b06a60de27ceb9bfc2d9dad3427368ca41dc7e62" [[package]] name = "find-msvc-tools" diff --git a/crates/j2k-transcode/src/jpeg_to_htj2k/error/native_encode.rs b/crates/j2k-transcode/src/jpeg_to_htj2k/error/native_encode.rs index 3760a0700..2595cab13 100644 --- a/crates/j2k-transcode/src/jpeg_to_htj2k/error/native_encode.rs +++ b/crates/j2k-transcode/src/jpeg_to_htj2k/error/native_encode.rs @@ -132,12 +132,4 @@ mod tests { .is_some()); } } - - #[test] - fn opaque_native_encode_error_retains_structural_equality() { - assert_eq!( - Htj2kEncodeError::new(EncodeError::InvalidInput { what: "fixture" }), - Htj2kEncodeError::new(EncodeError::InvalidInput { what: "fixture" }) - ); - } } diff --git a/crates/j2k-transcode/tests/dct53_2d.rs b/crates/j2k-transcode/tests/dct53_2d.rs index f29f61b1b..88eae76ac 100644 --- a/crates/j2k-transcode/tests/dct53_2d.rs +++ b/crates/j2k-transcode/tests/dct53_2d.rs @@ -61,21 +61,6 @@ fn dct8x8_grid_to_2d_53_crosses_block_boundaries() { ); } -#[test] -fn dct8x8_grid_public_wrapper_is_repeatable_for_same_geometry() { - let blocks = synthetic_grid_blocks(2, 2); - let direct = - dct8x8_blocks_to_dwt53_float_linear(&blocks, 2, 2, 13, 11).expect("valid DCT grid"); - let stateless = - dct8x8_blocks_to_dwt53_float_linear(&blocks, 2, 2, 13, 11).expect("valid DCT grid"); - - let repeated = - dct8x8_blocks_to_dwt53_float_linear(&blocks, 2, 2, 13, 11).expect("valid DCT grid"); - - assert!(max_abs_diff_53(&direct, &stateless) <= 1.0e-9); - assert!(max_abs_diff_53(&repeated, &stateless) <= 1.0e-9); -} - fn dc_only_block() -> [[f64; 8]; 8] { let mut block = [[0.0; 8]; 8]; block[0][0] = 384.0; diff --git a/crates/j2k/fuzz/Cargo.lock b/crates/j2k/fuzz/Cargo.lock index 5be59b3de..42d2ca4ad 100644 --- a/crates/j2k/fuzz/Cargo.lock +++ b/crates/j2k/fuzz/Cargo.lock @@ -103,9 +103,9 @@ checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "fearless_simd" -version = "0.7.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" +checksum = "f3772b63c40606beea8fe1f7b06a60de27ceb9bfc2d9dad3427368ca41dc7e62" [[package]] name = "find-msvc-tools" diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs index 5ea7e347a..e05f84776 100644 --- a/crates/j2k/src/decode.rs +++ b/crates/j2k/src/decode.rs @@ -122,7 +122,10 @@ pub(crate) fn decode_image_region_into_with_native_context<'a>( match fmt { PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => { let components = image - .decode_region_components_with_context((roi.x, roi.y, roi.w, roi.h), native_context) + .decode_region_components_for_integer_output_with_context( + (roi.x, roi.y, roi.w, roi.h), + native_context, + ) .map_err(J2kError::from_native_decode_error)?; write_components_u8_output(&components, out, stride, fmt) } @@ -154,7 +157,7 @@ pub(crate) fn decode_prepared_image_region_into( roi: Rect, ) -> Result<(), J2kError> { let components = decoder - .decode_region_components((roi.x, roi.y, roi.w, roi.h)) + .decode_region_components_for_integer_output((roi.x, roi.y, roi.w, roi.h)) .map_err(J2kError::from_native_decode_error)?; match fmt { PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => { diff --git a/crates/j2k/src/owned_batch/cpu_materialize.rs b/crates/j2k/src/owned_batch/cpu_materialize.rs index 4ff72a9bd..c1d7e31f0 100644 --- a/crates/j2k/src/owned_batch/cpu_materialize.rs +++ b/crates/j2k/src/owned_batch/cpu_materialize.rs @@ -70,13 +70,15 @@ fn image_components<'ctx, 'input>( let decoded = backend::image(image.bytes(), options.settings, target_resolution)?; let output_rect = plan.output_rect(); let decoded_dims = (decoded.width(), decoded.height()); + // Every batch sample type is an integer output, so irreversible samples + // are rounded before the level shift exactly as the single-image decode. if output_rect == Rect::full(decoded_dims) { decoded - .decode_components_with_context(context) + .decode_components_for_integer_output_with_context(context) .map_err(J2kError::from_native_decode_error) } else { decoded - .decode_region_components_with_context( + .decode_region_components_for_integer_output_with_context( (output_rect.x, output_rect.y, output_rect.w, output_rect.h), context, ) diff --git a/crates/j2k/tests/irreversible_rounding.rs b/crates/j2k/tests/irreversible_rounding.rs new file mode 100644 index 000000000..927f74f5c --- /dev/null +++ b/crates/j2k/tests/irreversible_rounding.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Every CPU integer output of irreversible (9/7) data follows one rounding +//! contract: the centered sample is rounded ties-to-even before the unsigned +//! level shift is added, as the full decode and `OpenJPEG` do. Region, row, and +//! batch decodes must therefore equal the matching crop of the full decode, +//! including samples whose shifted value is an exact tie. + +use std::sync::Arc; + +use j2k::{ + BatchDecodeOptions, BatchLayout, CpuBatchDecoder, CpuBatchSamples, DecodeRequest, EncodedImage, + J2kDecoder, J2kError, J2kScratchPool, +}; +use j2k_core::{ImageDecodeRows, PixelFormat, Rect, RowSink}; +use j2k_native::{encode, encode_htj2k, EncodeOptions}; +use j2k_test_support::{crop_interleaved_bytes, PixelRect}; + +const SIZE: u32 = 512; + +#[derive(Clone, Copy, Debug)] +struct LossyCase { + components: u16, + ht: bool, + use_mct: bool, +} + +const CASES: [LossyCase; 3] = [ + LossyCase { + components: 3, + ht: true, + use_mct: true, + }, + LossyCase { + components: 3, + ht: false, + use_mct: false, + }, + LossyCase { + components: 1, + ht: true, + use_mct: false, + }, +]; + +#[derive(Default)] +struct CollectRows(Vec); + +impl RowSink for CollectRows { + type Error = J2kError; + + fn write_row(&mut self, _y: u32, row: &[u8]) -> Result<(), Self::Error> { + self.0.extend_from_slice(row); + Ok(()) + } +} + +fn lossy_fixture(case: LossyCase) -> Vec { + let components = u32::from(case.components); + let mut state = 0x1234_5678_u32; + let pixels: Vec = (0..SIZE * SIZE * components) + .map(|index| { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + let (pixel, component) = (index / components, index % components); + let (x, y) = (pixel % SIZE, pixel / SIZE); + let base = (x * 3 + y * 5 + component * 40) / 9 % 200 + 20; + u8::try_from(base + state % 17).expect("sample fits u8") + }) + .collect(); + let options = EncodeOptions { + reversible: false, + use_mct: case.use_mct, + irreversible_quantization_scale: 4.0, + ..EncodeOptions::default() + }; + if case.ht { + encode_htj2k(&pixels, SIZE, SIZE, case.components, 8, false, &options) + } else { + encode(&pixels, SIZE, SIZE, case.components, 8, false, &options) + } + .expect("encode lossy 9/7 fixture") +} + +/// Samples whose unrounded output lands exactly on `k + 0.5` after the level +/// shift: the inputs on which rounding after the shift breaks the contract. +#[expect( + clippy::float_cmp, + reason = "a tie is an exact property of the f32 value" +)] +fn shifted_tie_count(bytes: &[u8]) -> usize { + let image = j2k_native::Image::new(bytes, &j2k_native::DecodeSettings::default()) + .expect("inspect lossy fixture"); + let mut context = j2k_native::DecoderContext::default(); + let components = image + .decode_components_with_context(&mut context) + .expect("float component decode"); + components + .planes() + .iter() + .flat_map(j2k_native::ComponentPlane::samples) + .filter(|sample| { + let doubled = **sample * 2.0; + doubled == doubled.floor() && doubled.rem_euclid(2.0) == 1.0 + }) + .count() +} + +fn format_for(case: LossyCase) -> (PixelFormat, usize) { + if case.components == 1 { + (PixelFormat::Gray8, 1) + } else { + (PixelFormat::Rgb8, 3) + } +} + +fn batch_u8(bytes: &[u8], request: DecodeRequest) -> Vec { + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let result = CpuBatchDecoder::new(options) + .decode(Vec::from([EncodedImage::new(Arc::from(bytes), request)])) + .expect("CPU batch decode"); + assert!(result.errors().is_empty(), "{:?}", result.errors()); + let CpuBatchSamples::U8(samples) = result.groups()[0].samples() else { + panic!("8-bit lossy fixture must use U8 batch storage") + }; + samples.clone() +} + +fn mismatch_summary(label: &str, expected: &[u8], actual: &[u8]) -> Option { + if expected.len() != actual.len() { + return Some(format!( + "{label}: lengths {} / {}", + expected.len(), + actual.len() + )); + } + let mismatches = expected.iter().zip(actual).filter(|(a, b)| a != b).count(); + (mismatches > 0).then(|| format!("{label}: {mismatches} bytes differ from the full decode")) +} + +#[test] +fn irreversible_region_row_and_batch_outputs_match_the_full_decode() { + let roi = Rect { + x: 1, + y: 1, + w: SIZE - 1, + h: SIZE - 1, + }; + let mut failures = Vec::new(); + for case in CASES { + let bytes = lossy_fixture(case); + assert!( + shifted_tie_count(&bytes) > 0, + "{case:?}: fixture has no shifted ties, so it cannot detect the rounding order" + ); + let (fmt, channels) = format_for(case); + let width = SIZE as usize; + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut full: Vec = std::iter::repeat_n(0, width * width * channels).collect(); + decoder + .decode_into(&mut full, width * channels, fmt) + .expect("full decode"); + let expected_region = crop_interleaved_bytes( + &full, + width, + channels, + PixelRect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + ); + + let mut region: Vec = std::iter::repeat_n(0, expected_region.len()).collect(); + decoder + .decode_region_into( + &mut J2kScratchPool::new(), + &mut region, + roi.w as usize * channels, + fmt, + roi, + ) + .expect("region decode"); + + let mut rows = CollectRows::default(); + as ImageDecodeRows<'_, u8>>::decode_rows(&mut decoder, &mut rows) + .expect("row decode"); + + let checks = [ + ("region", &expected_region, region), + ("rows", &full, rows.0), + ("batch full", &full, batch_u8(&bytes, DecodeRequest::Full)), + ( + "batch region", + &expected_region, + batch_u8(&bytes, DecodeRequest::Region { roi }), + ), + ]; + for (label, expected, actual) in checks { + failures.extend(mismatch_summary( + &format!("{case:?} {label}"), + expected, + &actual, + )); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/docs/architecture.md b/docs/architecture.md index 92be21a68..7f713d4bd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -115,9 +115,9 @@ backend kind is not executable authority, and tests requesting a specialization must obtain the same runtime token as production. The `scalar-only` feature always selects `Scalar`. -AArch64 entry kernels use the safe `fearless_simd 0.7` kernel boundary. x86-64 +AArch64 entry kernels use the safe `fearless_simd 1.0` kernel boundary. x86-64 uses a project-private equivalent that enables exactly AVX2. This distinction -is intentional: the `fearless_simd::Avx2` token in 0.7 represents the broader +is intentional: the `fearless_simd::Avx2` token represents the broader x86-64-v3 feature set, including FMA, BMI, and other features. Requiring that token would silently remove acceleration from CPUs that satisfy the decoder's existing AVX2-plus-operating-system-state contract but not all of v3. diff --git a/docs/benchmark-evidence.md b/docs/benchmark-evidence.md index a634f1689..e4ecd0777 100644 --- a/docs/benchmark-evidence.md +++ b/docs/benchmark-evidence.md @@ -184,6 +184,94 @@ decode routes were slower, and the measured lossless and lossy encode medians were only about 4.2% and 7.8% faster than CPU. No `Auto` threshold was changed from that diagnostic. +### Local Metal lossless routing run - 2026-09-24 + +This run replaced the Metal decode cells; the combined run below has since +re-measured them. Its corpus is local, not a +publication corpus: 512x512 tiles from GDC whole-slide DICOM files, decoded with +`opj_decompress`, stitched into 256x256, 640x480, 1024x1024, and 2048x2048 RGB +images and 640x480 and 2048x2048 gray images, then re-encoded as Part 1 +lossless (OpenJPEG), HTJ2K lossless codestreams (OpenJPH `-reversible true`), +and HTJ2K lossless JPH files (Grok). There are 22 cases (14 decode, 8 encode); +manifest SHA-256 +`217d700a9f882c4cd63b76aed250de92124d74a0bf476d33855dbbe857d6befe`. +The run used `release-bench`, candidate `c1cc3ff1` with a dirty tree, and an +Apple M4 Pro (16 GPU cores) on macOS 27.0. The verifier accepted all 76 cells +and promoted 33. The verified artifact SHA-256 is +`69987ce1cee902e4ef664964250059021ef15b59e0828802098fbe84d192df77`. + +Lossy inputs were excluded from this run. `tests/auto_routing_parity.rs` found +that Metal 9/7 decode differed from the CPU by one code value in roughly 10 +samples per million on every third-party 9/7 file, while every 5/3 file matched. +Because promotion requires identical bytes, the earlier 9/7 Metal cells +(`metal_part1` lossy repeated, `metal_part15` lossy repeated and half-scale) +were withdrawn. The cause was found and fixed the same day; see "Local Metal +routing run with lossy inputs" below. + +The resulting Metal cells cover lossless inputs only. Repeated batches of 16 run +on Metal for HTJ2K RGB8 from 256x256 (JPH from 640x480), HTJ2K Gray8 from +640x480, Part 1 RGB8 from 640x480, and Part 1 Gray8 from 2048x2048. +Single full-image decodes run on Metal for HTJ2K from 640x480, for RGB8 and +Gray8 and for codestreams and JPH, when the source components match the output +format. Part 1 single-image decodes stay on the CPU because Metal measured 105% +to 1400% slower. ROI and half-scale decodes promoted only HTJ2K Gray8 at 2048x2048; +Metal Auto does not route those operations for gray sources, so they stay on the CPU. +The lossy RGB8 encode threshold is now 2048x2048 pixels, where HTJ2K and Part 1 +lossy encode measured 54% and 38% faster; at 640x480 they were within 4% of the CPU. + +### Local Metal routing run with lossy inputs - 2026-09-24 + +The 9/7 mismatch was in output rounding, not in the wavelet transform. The CPU +rounds each centered sample to the nearest integer, ties to even, and only then +adds the unsigned level shift, as OpenJPEG does (`opj_lrintf(value) + +dc_level_shift`). The Metal store, pack, and inverse-colour-transform kernels +added the shift first and rounded half up. Adding 128 in f32 drops one bit of +precision, so a value just below `k + 0.5` became a tie and rounded up: every +mismatch was Metal one code value high, at a sample whose shifted CPU value was +exactly `k + 0.5`. The fix makes Metal integer output round before the shift and +compute the inverse ICT with the CPU's fused expressions. Three CPU paths +disagreed with the CPU full decode in the same way (region decode, row +streaming, and `CpuBatchDecoder`, which built integer output from unrounded +component planes), as did the CPU inverse ICT's non-SIMD tail. They now follow +the same contract, so a CPU region decode equals the crop of the full decode. + +The first run of this corpus also exposed an unrelated Metal bug: the repeated +Part 1 batch path decoded into a recycled scratch buffer without zero-filling +code blocks that have no coding passes, so a lossy batch decoded after a +lossless one of the same geometry kept stale coefficients (39% of bytes wrong +for `gray8-2048x2048-part1-lossy`). That path now zero-fills like the +single-image, grouped, and distinct-batch paths. + +The corpus is the lossless run's 22 cases plus 12 lossy decode files made from +the same stitched sources: Part 1 lossy (OpenJPEG) and HTJ2K lossy codestreams +(OpenJPH) for every RGB and gray size. There are 34 cases (26 decode, 8 encode); +manifest SHA-256 +`dd6363374dccf8d6b34656f90b9557b5a01f8582b94f5014e3165dc61f1248ec`. +`tests/auto_routing_parity.rs` found no mismatch in any full, ROI, or half-scale +decode, and the bench's own parity check passed for every CPU, Metal, and Auto +route. The run used `release-bench`, candidate `c1cc3ff1` with a dirty tree, +and the same Apple M4 Pro on macOS 27.0. The verifier accepted all 124 cells and +promoted 49. The verified artifact SHA-256 is +`66f9d83f932efb7df6cab2de0048849f09343da00915e22794b0056b29f2aa5f`; its cells +are recorded under the `metal_local_combined` source. + +Every lossless cell re-measured to the same thresholds as the lossless run. The +new lossy cells are: + +- Repeated batches of 16: HTJ2K RGB8 from 256x256 (46% faster at 256x256), + Part 1 RGB8 from 1024x1024 (56%; 640x480 measured 10% slower), and HTJ2K and + Part 1 Gray8 from 2048x2048 (62% and 18%; HTJ2K Gray8 at 640x480 was only 8% + faster, below the 10% bar). +- Single full-image decodes: HTJ2K RGB8 and Gray8 from 640x480 (44% and 37% + faster at 640x480, 77% and 78% at 2048x2048). + +Part 1 lossy single-image decodes stay on the CPU (66% to 2400% slower on +Metal). No half-scale cell qualified: HTJ2K lossy RGB8 half-scale measured 3% +to 176% slower, so the earlier `metal_part15` half-scale cell stays withdrawn. +ROI and half-scale promoted only for HTJ2K Gray8, which Metal Auto does not +route. The lossy RGB8 encode threshold stays at 2048x2048 pixels (Part 1 and +HTJ2K 36% and 57% faster there). + ### External CUDA routing development run - 2026-08-05 The uninterrupted CUDA matrix used the same 12 external cases from diff --git a/docs/env-vars.md b/docs/env-vars.md index fc644fdbb..494c8c167 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -93,6 +93,7 @@ override it. | --- | --- | --- | --- | | `J2K_ML_BATCH_INPUT_MODE` | Selects content-distinct generated inputs (`distinct`) or one repeated `Arc` owner (`repeated`) for the `j2k-ml` batch benchmarks. One process uses one mode. | `distinct` | Benchmark | | `J2K_MPSGRAPH_BENCH_ITERATIONS` | Number of repeated samples per size/batch/path cell in the direct MPSGraph benchmark. | `5` | Benchmark | +| `J2K_JPEG_HARNESS_BASELINE_SHADERS` | Directory holding another copy of the JPEG Metal shader files (same names as `crates/j2k-jpeg-metal/src/shaders_*.metal`). The ignored `decode_kernel_stage_profile` test compiles it as a baseline and A/Bs it against the in-tree kernels in one process; the copy must match the current host ABI. | No baseline column | Benchmark | | `J2K_JPEG_TEXTURE_SMALL_BATCHES` | When set to any value, selects the small-batch and tile-size control matrix for the ignored JPEG Metal `texture_scheduling_experiment` test. | Regular 16-tile timing matrix | Benchmark | | `J2K_ML_BATCH_PROCESS_MODE` | Selects uninstrumented Criterion measurement (`criterion`) or the separate low-batch telemetry/profile process (`profile`) for CUDA and Metal batch benchmarks. | `criterion` | Benchmark | | `J2K_REQUIRE_OPENJPEG` | Makes OpenJPEG parity tests and benchmark comparator runs fail instead of skip when OpenJPEG tools are unavailable. | Skip unavailable comparator paths | Benchmark | diff --git a/docs/performance-experiments/P30-P31-metal-decode-notes.md b/docs/performance-experiments/P30-P31-metal-decode-notes.md new file mode 100644 index 000000000..d48364691 --- /dev/null +++ b/docs/performance-experiments/P30-P31-metal-decode-notes.md @@ -0,0 +1,149 @@ +# P30–P31: Metal HT cleanup and 9/7 lifting decode + +Measured on 2026-09-22 in place on `main` with a dirty working tree (unrelated +concurrent JPEG Metal, native SIMD, and HT-cleanup-only edits were preserved). +Apple M4 Pro, 16 GPU cores, 48 GiB RAM, Metal 4, Rust 1.96.0. No public API, +Auto routing policy, or allocation cap changed. The tables below are the +original Criterion estimates. JSON records from a fresh same-tree A/B on +2026-09-23, each change measured on its own against the current tree, are +`P30-metal-ht-cooperative-cleanup.json`, `P31-metal-idwt97-fused-lifting.json` +and, for the pool cap, `P35-metal-buffer-pool-cap.json`. + +## Decisions + +- **P30 promoted:** cleanup-only HT code blocks decode with two kernels instead + of one thread per block. +- **P31 promoted:** the four horizontal 9/7 lifting steps, and the vertical scale + plus four vertical lifting steps, each run as one fused dispatch instead of + one full-plane pass per step. +- **Pool cap promoted (2026-09-23):** each J2K Metal buffer pool now retains an + eighth of the device's `recommendedMaxWorkingSetSize`, clamped to + [256 MiB, 1 GiB] and to `maxBufferLength` (`buffer_pool/state.rs`, + `retained_bytes_for_device`). That is about 680 MB on an 8 GB Mac and 1 GiB + from 16 GB up. A session owns a private and a shared pool, so it can retain up + to twice the cap between decodes, until the session is dropped. + +## P30: two-kernel HT cleanup decode + +The previous kernel ran one thread per code block with 6 KiB of per-thread +scratch. A 512×512 RGB image has about 192 64×64 blocks, so a dispatch +occupied only six SIMD groups. Each lane serially decoded MEL, VLC, and all +4,096 MagSgn samples of its block. Stubbing that kernel removed 82% of GPU time +from the 16×512² HT batch. + +`ht_cleanup_simd.metal` splits the work: + +1. `j2k_decode_ht_cleanup_vlc_*` runs one thread per block and decodes the + serial MEL/VLC streams. It writes `(u_q << 16) | inf` for quad (q, r) into + the block's own output word at column 2q, row 2r, so no scratch allocation is + needed. For blocks at most 64 samples wide, the previous row's significance + context stays in registers. The reader advances once per quad pair: a pair + consumes at most 30 bits (two 7-bit codewords plus a 16-bit UVLC prefix and + suffix, computed from the tables), and every fetch leaves more than 32 valid + bits. MEL runs are decoded on demand from a 64-bit buffer. The host packs + 1–32 blocks per SIMD group, aiming for about 96 SIMD groups: packing trades + lane divergence against resident-group count. A 64×64 block's serial walk + costs about 0.18 ms on one lane. +2. `j2k_decode_ht_cleanup_magsgn_*` runs one SIMD group per block. Each lane + owns one quad, derives its MagSgn bit count from the stored VLC word and the + previous row's exponents, and locates its bits with `simd_prefix_exclusive_sum` + over a destuffed 8,192-bit threadgroup window that the group refills + cooperatively. + +Validation order, status codes, bit consumption, and every arithmetic +expression mirror `decode_ht_cleanup_common`. SigProp/MagRef jobs and +devices whose SIMD width is not 32 keep the previous kernels. + +## P31: fused 9/7 lifting + +`dispatch_irreversible97_stages_after_horizontal_scale` previously issued nine +full-plane passes per level. It now issues two. In +`j2k_idwt_irreversible97_horizontal_lift_fused`, each threadgroup owns four +whole rows; in `j2k_idwt_irreversible97_vertical_fused`, each owns a 32-column +strip. Each walks its tiles in order with a 4-sample halo. The halo before a +tile comes from a threadgroup carry of the previous tile's pre-lift samples, +because that tile's device copy was already overwritten; the halo after it has +not been written yet. An earlier prototype with independent overlapping tiles +raced on those halos and failed only at batch 16. The lifting expressions and +edge mirroring are those of the original per-step kernels, which are kept as +the test oracle. + +## Correctness + +- `cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit` covers eight + fixtures: gradient and noise; 8- and 16-bit; 131×67 odd dimensions; 32×32, + 64×64, 128×32 and 256×16 blocks; lossless and 9/7. It compares the + production dispatcher at 1, 12 and 96 replicas (1-, 8- and 32-lane packing) + with the CPU oracle and the legacy kernel, bit for bit. Two injected faults + (ignoring MagSgn unstuffing; dropping one exponent neighbour) each failed it. +- `irreversible97_fused_lifting_matches_full_grid_reference_bits` covers 15 + geometries, including multi-tile shapes with partial final tiles and odd + origins. Each runs at batch 1, 3 and 16 with both high-pass constants, + against the full-grid per-step reference. It passed six consecutive runs. +- Every `decode_stages` output hash is unchanged, and the geometry benchmark's + per-byte comparison with native CPU decode passes at all four geometries. +- Update 2026-09-24: a later fix to 9/7 integer output rounding (see + `docs/benchmark-evidence.md`, "Local Metal routing run with lossy inputs") + changes Metal 9/7 output on samples that land exactly on a shifted tie, so + 9/7 output hashes recorded here may not reproduce on the current tree. P30 + and P31 remain output-neutral; the fix is in the final store and inverse + colour transform, not in HT cleanup or lifting. + +## Results + +Same-tree A/B, `gpu-quick`. The baseline forced the legacy HT kernel and had +the per-step IDWT; the treatment has both P30 and P31. Criterion mean of 10 +samples. + +| Workload | Before | After | Change | +|---|---:|---:|---:| +| HT 5/3 RGB 512², batch 16 (broadcast), resident | 6.82 ms | 1.97 ms | −71% | +| HT 9/7 RGB 512², batch 16, resident | 6.62 ms | 1.67 ms | −75% | +| HT 9/7 distinct 128², batch 16, resident | 4.86 ms | 1.33 ms | −73% | +| HT 9/7 distinct 640×480, batch 16, resident | 11.35 ms | 6.08 ms | −46% | +| HT 9/7 distinct 1024², batch 16, resident | 31.74 ms | 28.16 ms | −11% | +| HT 9/7 512², batch 1, resident | 6.05 ms | 1.32 ms | −78% | +| Mixed distinct groups, prepared batch, resident | 3.03 ms | 0.83 ms | −73% | +| Classic 5/3 RGB 512², batch 16 (unchanged route) | 68.79 ms | 68.90 ms | noise | + +For 640×480, P30 alone gave 7.28 ms and P31 took it to 6.08 ms. `release-bench` +absolute results after both changes (20 samples) were 1.958 ms, 1.665 ms, +1.313 ms, 6.077 ms, 27.86 ms, 1.302 ms, and 0.822 ms, in table order. + +## Measured but not changed + +- **Large-batch buffer reuse (now promoted, see Decisions).** At 16×1024² RGB + the decode needs roughly 450 MB of scratch; with a 256 MiB pool and exact-size + reuse, buffers were reallocated every decode, and commit-to-GPU-start delay + was about 10 ms. Same-tree Criterion A/B, `gpu-quick`, 20 samples, 256 MiB vs + the device-scaled cap (1 GiB on this 48 GB machine): + + | Workload (batch 16 unless noted) | 256 MiB | Scaled | Change | + |---|---:|---:|---:| + | HT 9/7 distinct 1024², resident | 28.48 ms | 19.98 ms | −29.8% | + | HT 9/7 distinct 1024², readback | 31.73 ms | 23.06 ms | −27.3% | + | HT 9/7 distinct 640×480, resident | 6.08 ms | 6.12 ms | +0.7% (fits either cap) | + | HT 9/7 512², batch 1, resident | 1.306 ms | 1.307 ms | no change | + + Output hashes are unchanged (before the 2026-09-24 9/7 rounding fix noted + under Correctness). +- **Unbounded batched IDWT.** Removing P20's 20 MiB bound with the fused kernels + measured 28.2 → 26.6 ms at 1024² and no change at 640×480. +- **Classic Tier-1** is unchanged at 68.8 ms for 16×512² RGB (about 180 Mcoeff/s), + one serial MQ decoder per block. Its context formation re-reads eight + neighbour states per visit from per-thread memory. A flag-word rewrite is + possible, but serial MQ decoding bounds the GPU at roughly multi-core CPU + throughput, so the CPU Tier-1 hybrid route remains the stronger default. +- **JPEG one-shot calls.** A fresh `MetalSession` per call costs about 2.2 ms of + driver-side submission time (16×16: 2.66 ms Metal versus 2.3 µs CPU). A + retained session measured 0.30–0.74 ms per single decode. +- **JPEG 4:4:4 resident textures** take 2.50 ms for 16×256², versus 0.55 ms + for 4:2:0. That is 4.5× slower for 2× the samples; 4:4:4 still uses the + direct texture shader rather than component planes. + +## Reproduction + +```sh +J2K_REQUIRE_METAL_RUNTIME=1 cargo test --profile gpu-quick -p j2k-metal --lib -- cooperative irreversible97 --test-threads=1 +J2K_REQUIRE_METAL_BENCH=1 cargo bench --profile release-bench -p j2k-metal --bench decode_stages -- --sample-size 20 --warm-up-time 2 --measurement-time 5 +``` diff --git a/docs/performance-experiments/P30-metal-ht-cooperative-cleanup.json b/docs/performance-experiments/P30-metal-ht-cooperative-cleanup.json new file mode 100644 index 000000000..9507444ca --- /dev/null +++ b/docs/performance-experiments/P30-metal-ht-cooperative-cleanup.json @@ -0,0 +1,542 @@ +{ + "schema_version": 1, + "experiment_id": "P30-METAL-HT-COOPERATIVE-CLEANUP", + "status": "promoted", + "environment": { + "commit": "c1cc3ff15975d70f51d324a6e185209d630b15de", + "branch": "main", + "dirty": true, + "cpu": "Apple M4 Pro, 12 cores", + "gpu": "Apple M4 Pro, 16 cores, Metal 4", + "ram_bytes": 51539607552, + "os": "macOS 27.0 build 26A428, Darwin 27.0.0", + "driver_runtime": "Metal 4", + "rust_version": "rustc 1.96.0 (ac68faa20 2026-05-25)", + "llvm_version": "LLVM 22.1.2", + "gpu_toolchain": "Metal framework runtime source compilation (newLibraryWithSource); offline metal CLI not installed", + "build_profile": "gpu-quick", + "feature_flags": [], + "environment_variables": { + "J2K_REQUIRE_METAL_BENCH": "1" + }, + "input_corpus_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691", + "sample_count": 20, + "warm_up_seconds": 2.0, + "measurement_seconds": 5.0 + }, + "workloads": [ + { + "id": "ht53-rgb8-512x512-b16-broadcast-resident", + "transform": "reversible_5_3", + "entropy": "ht", + "code_block": "64x64", + "image": "512x512", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-512x512-b16-broadcast-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "512x512", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-128x128-b16-distinct-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "128x128", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-640x480-b16-distinct-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "640x480", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "above_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-1024x1024-b16-distinct-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "1024x1024", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_1024", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-512x512-b1-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "512x512", + "batch": 1, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht53-gray8-mixed4-prepared-batch-resident", + "transform": "reversible_5_3", + "entropy": "ht", + "code_block": "64x64", + "image": "representative_matrix", + "batch": 4, + "components": 1, + "output": "native", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "classic53-rgb8-512x512-b16-broadcast-resident", + "transform": "reversible_5_3", + "entropy": "classic", + "code_block": "64x64", + "image": "512x512", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_512", + "jpeg_sampling": null, + "jpeg_restart": null + } + ], + "measurements": [ + { + "workload_id": "ht53-rgb8-512x512-b16-broadcast-resident", + "variant": "baseline", + "wall_time_ns": 6862658.6, + "wall_time_ci_lower_ns": 6844852.9, + "wall_time_ci_upper_ns": 6887592.2, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Legacy cleanup-only HT kernel: one thread per code block (CooperativeHtCleanup disabled, as on devices whose SIMD width is not 32).", + "output_sha256": "6e6253e6d7cb92b3d39a87485816bcb1939198dda07122ba245fa14bb1292485", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "ht53-rgb8-512x512-b16-broadcast-resident", + "variant": "treatment", + "wall_time_ns": 1979515.6, + "wall_time_ci_lower_ns": 1978465.8, + "wall_time_ci_upper_ns": 1980888.5, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two-kernel cleanup: thread-per-block MEL/VLC, then one SIMD group per block for MagSgn.", + "output_sha256": "6e6253e6d7cb92b3d39a87485816bcb1939198dda07122ba245fa14bb1292485", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "ht97-rgb8-512x512-b16-broadcast-resident", + "variant": "baseline", + "wall_time_ns": 6796302.0, + "wall_time_ci_lower_ns": 6749748.1, + "wall_time_ci_upper_ns": 6856667.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Legacy cleanup-only HT kernel: one thread per code block (CooperativeHtCleanup disabled, as on devices whose SIMD width is not 32).", + "output_sha256": "ffe40f6a9fd711d0c8bd6f5a9298f041b6a6f098aafa53a8d80be88a9a74edb8", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "ht97-rgb8-512x512-b16-broadcast-resident", + "variant": "treatment", + "wall_time_ns": 1677829.1, + "wall_time_ci_lower_ns": 1674426.3, + "wall_time_ci_upper_ns": 1679813.4, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two-kernel cleanup: thread-per-block MEL/VLC, then one SIMD group per block for MagSgn.", + "output_sha256": "ffe40f6a9fd711d0c8bd6f5a9298f041b6a6f098aafa53a8d80be88a9a74edb8", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "ht97-rgb8-128x128-b16-distinct-resident", + "variant": "baseline", + "wall_time_ns": 4929136.6, + "wall_time_ci_lower_ns": 4885324.7, + "wall_time_ci_upper_ns": 4954451.9, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Legacy cleanup-only HT kernel: one thread per code block (CooperativeHtCleanup disabled, as on devices whose SIMD width is not 32).", + "output_sha256": "c01760c8ab5ebc8ffc60a4c5673f88742fb9366705c1307f0017bafe84c91dda", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "9166ceef4a83c1a8dac3d2956218b17ce2742bbb66f0bdd6d47f69ce47a30dbf" + }, + { + "workload_id": "ht97-rgb8-128x128-b16-distinct-resident", + "variant": "treatment", + "wall_time_ns": 1289397.2, + "wall_time_ci_lower_ns": 1286181.1, + "wall_time_ci_upper_ns": 1291870.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two-kernel cleanup: thread-per-block MEL/VLC, then one SIMD group per block for MagSgn.", + "output_sha256": "c01760c8ab5ebc8ffc60a4c5673f88742fb9366705c1307f0017bafe84c91dda", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "9166ceef4a83c1a8dac3d2956218b17ce2742bbb66f0bdd6d47f69ce47a30dbf" + }, + { + "workload_id": "ht97-rgb8-640x480-b16-distinct-resident", + "variant": "baseline", + "wall_time_ns": 10357875.4, + "wall_time_ci_lower_ns": 10197179.1, + "wall_time_ci_upper_ns": 10475889.5, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Legacy cleanup-only HT kernel: one thread per code block (CooperativeHtCleanup disabled, as on devices whose SIMD width is not 32).", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26" + }, + { + "workload_id": "ht97-rgb8-640x480-b16-distinct-resident", + "variant": "treatment", + "wall_time_ns": 6101207.8, + "wall_time_ci_lower_ns": 6095990.9, + "wall_time_ci_upper_ns": 6105597.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two-kernel cleanup: thread-per-block MEL/VLC, then one SIMD group per block for MagSgn.", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26" + }, + { + "workload_id": "ht97-rgb8-1024x1024-b16-distinct-resident", + "variant": "baseline", + "wall_time_ns": 24829699.1, + "wall_time_ci_lower_ns": 24759806.5, + "wall_time_ci_upper_ns": 24886927.6, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Legacy cleanup-only HT kernel: one thread per code block (CooperativeHtCleanup disabled, as on devices whose SIMD width is not 32).", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691" + }, + { + "workload_id": "ht97-rgb8-1024x1024-b16-distinct-resident", + "variant": "treatment", + "wall_time_ns": 19971334.9, + "wall_time_ci_lower_ns": 19954812.9, + "wall_time_ci_upper_ns": 19990478.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two-kernel cleanup: thread-per-block MEL/VLC, then one SIMD group per block for MagSgn.", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691" + }, + { + "workload_id": "ht97-rgb8-512x512-b1-resident", + "variant": "baseline", + "wall_time_ns": 6119559.2, + "wall_time_ci_lower_ns": 6082068.2, + "wall_time_ci_upper_ns": 6148460.2, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Legacy cleanup-only HT kernel: one thread per code block (CooperativeHtCleanup disabled, as on devices whose SIMD width is not 32).", + "output_sha256": "08096a50534fc7edec3f339a24207e9b5b03166892fc73547d9ea06e7f806009", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "06e813ca5512e054c7d79fc93e96468c67ccc74381259ae672257e2f62f7ca34" + }, + { + "workload_id": "ht97-rgb8-512x512-b1-resident", + "variant": "treatment", + "wall_time_ns": 1284570.7, + "wall_time_ci_lower_ns": 1282230.0, + "wall_time_ci_upper_ns": 1286694.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two-kernel cleanup: thread-per-block MEL/VLC, then one SIMD group per block for MagSgn.", + "output_sha256": "08096a50534fc7edec3f339a24207e9b5b03166892fc73547d9ea06e7f806009", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "06e813ca5512e054c7d79fc93e96468c67ccc74381259ae672257e2f62f7ca34" + }, + { + "workload_id": "ht53-gray8-mixed4-prepared-batch-resident", + "variant": "baseline", + "wall_time_ns": 3009335.6, + "wall_time_ci_lower_ns": 2986201.1, + "wall_time_ci_upper_ns": 3034431.6, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Legacy cleanup-only HT kernel: one thread per code block (CooperativeHtCleanup disabled, as on devices whose SIMD width is not 32).", + "output_sha256": "086ef73794e76cf94c548a826c3d7a938fe9f4459681a2fd79de05b1e50175d8", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "ht53-gray8-mixed4-prepared-batch-resident", + "variant": "treatment", + "wall_time_ns": 799168.5, + "wall_time_ci_lower_ns": 798467.7, + "wall_time_ci_upper_ns": 799869.5, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two-kernel cleanup: thread-per-block MEL/VLC, then one SIMD group per block for MagSgn.", + "output_sha256": "086ef73794e76cf94c548a826c3d7a938fe9f4459681a2fd79de05b1e50175d8", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "classic53-rgb8-512x512-b16-broadcast-resident", + "variant": "baseline", + "wall_time_ns": 68773221.7, + "wall_time_ci_lower_ns": 68719560.7, + "wall_time_ci_upper_ns": 68834285.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Legacy cleanup-only HT kernel: one thread per code block (CooperativeHtCleanup disabled, as on devices whose SIMD width is not 32). Unchanged-route control (Classic Tier-1).", + "output_sha256": "6e6253e6d7cb92b3d39a87485816bcb1939198dda07122ba245fa14bb1292485", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "classic53-rgb8-512x512-b16-broadcast-resident", + "variant": "treatment", + "wall_time_ns": 68672754.5, + "wall_time_ci_lower_ns": 68664974.0, + "wall_time_ci_upper_ns": 68678607.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two-kernel cleanup: thread-per-block MEL/VLC, then one SIMD group per block for MagSgn. Unchanged-route control (Classic Tier-1).", + "output_sha256": "6e6253e6d7cb92b3d39a87485816bcb1939198dda07122ba245fa14bb1292485", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + } + ], + "decision": { + "priority_workload_id": "ht53-rgb8-512x512-b16-broadcast-resident", + "confidence_interval_supports_improvement": true, + "representative_regression_percent": 0.0, + "complexity_is_proportional": true, + "rationale": "Promoted. Cleanup-only HT code blocks decode with two kernels instead of one thread per block: a thread-per-block MEL/VLC pass, then one SIMD group per block for MagSgn (ht_cleanup_simd.metal). The baseline disables CooperativeHtCleanup so every dispatcher takes the legacy kernel; both arms include P31. Same-tree A/B on the current tree: three decode_stages executables (P30 baseline, P31 baseline, treatment) were built with the baseline sources present only during compilation, then run in rotation for two rounds. Recorded values are Criterion slope estimates of the first round. metal_decode_stages, metal_decode_stages_idwt97 and metal_decode_stages_classic fix 10 samples in code; the other rows use 20. Hardware counters were not collected. HT rows improve 71-79% at 512x512 (batch 16 5/3 and 9/7, batch 1 9/7), 128x128 batch 16 and the mixed 5/3 groups, 41% at 640x480 and 16-20% at 1024x1024, with disjoint intervals in both rounds. The Classic control moved -0.1%. No row regressed. The 1024x1024 gain is smaller because Tier-1 is a smaller share of that decode. See P30-P31-metal-decode-notes.md." + } +} diff --git a/docs/performance-experiments/P31-metal-idwt97-fused-lifting.json b/docs/performance-experiments/P31-metal-idwt97-fused-lifting.json new file mode 100644 index 000000000..d0116afbb --- /dev/null +++ b/docs/performance-experiments/P31-metal-idwt97-fused-lifting.json @@ -0,0 +1,542 @@ +{ + "schema_version": 1, + "experiment_id": "P31-METAL-IDWT97-FUSED-LIFTING", + "status": "promoted", + "environment": { + "commit": "c1cc3ff15975d70f51d324a6e185209d630b15de", + "branch": "main", + "dirty": true, + "cpu": "Apple M4 Pro, 12 cores", + "gpu": "Apple M4 Pro, 16 cores, Metal 4", + "ram_bytes": 51539607552, + "os": "macOS 27.0 build 26A428, Darwin 27.0.0", + "driver_runtime": "Metal 4", + "rust_version": "rustc 1.96.0 (ac68faa20 2026-05-25)", + "llvm_version": "LLVM 22.1.2", + "gpu_toolchain": "Metal framework runtime source compilation (newLibraryWithSource); offline metal CLI not installed", + "build_profile": "gpu-quick", + "feature_flags": [], + "environment_variables": { + "J2K_REQUIRE_METAL_BENCH": "1" + }, + "input_corpus_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691", + "sample_count": 20, + "warm_up_seconds": 2.0, + "measurement_seconds": 5.0 + }, + "workloads": [ + { + "id": "ht53-rgb8-512x512-b16-broadcast-resident", + "transform": "reversible_5_3", + "entropy": "ht", + "code_block": "64x64", + "image": "512x512", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-512x512-b16-broadcast-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "512x512", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-128x128-b16-distinct-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "128x128", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-640x480-b16-distinct-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "640x480", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "above_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-1024x1024-b16-distinct-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "1024x1024", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_1024", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-512x512-b1-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "512x512", + "batch": 1, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht53-gray8-mixed4-prepared-batch-resident", + "transform": "reversible_5_3", + "entropy": "ht", + "code_block": "64x64", + "image": "representative_matrix", + "batch": 4, + "components": 1, + "output": "native", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "classic53-rgb8-512x512-b16-broadcast-resident", + "transform": "reversible_5_3", + "entropy": "classic", + "code_block": "64x64", + "image": "512x512", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_512", + "jpeg_sampling": null, + "jpeg_restart": null + } + ], + "measurements": [ + { + "workload_id": "ht53-rgb8-512x512-b16-broadcast-resident", + "variant": "baseline", + "wall_time_ns": 2458718.2, + "wall_time_ci_lower_ns": 2357310.4, + "wall_time_ci_upper_ns": 2576926.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD per-step 9/7 lifting: four horizontal step passes, a vertical scale pass and four vertical step passes per level, each over the full plane. Control: 5/3 route, no 9/7 lifting.", + "output_sha256": "6e6253e6d7cb92b3d39a87485816bcb1939198dda07122ba245fa14bb1292485", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "ht53-rgb8-512x512-b16-broadcast-resident", + "variant": "treatment", + "wall_time_ns": 1979515.6, + "wall_time_ci_lower_ns": 1978465.8, + "wall_time_ci_upper_ns": 1980888.5, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two fused tile dispatches per level: horizontal lifting in 4-row groups, vertical scale and lifting in 32-column strips with a carried 4-sample halo. Control: 5/3 route, no 9/7 lifting.", + "output_sha256": "6e6253e6d7cb92b3d39a87485816bcb1939198dda07122ba245fa14bb1292485", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "ht97-rgb8-512x512-b16-broadcast-resident", + "variant": "baseline", + "wall_time_ns": 1740622.7, + "wall_time_ci_lower_ns": 1694888.8, + "wall_time_ci_upper_ns": 1771167.7, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD per-step 9/7 lifting: four horizontal step passes, a vertical scale pass and four vertical step passes per level, each over the full plane.", + "output_sha256": "ffe40f6a9fd711d0c8bd6f5a9298f041b6a6f098aafa53a8d80be88a9a74edb8", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "ht97-rgb8-512x512-b16-broadcast-resident", + "variant": "treatment", + "wall_time_ns": 1677829.1, + "wall_time_ci_lower_ns": 1674426.3, + "wall_time_ci_upper_ns": 1679813.4, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two fused tile dispatches per level: horizontal lifting in 4-row groups, vertical scale and lifting in 32-column strips with a carried 4-sample halo.", + "output_sha256": "ffe40f6a9fd711d0c8bd6f5a9298f041b6a6f098aafa53a8d80be88a9a74edb8", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "ht97-rgb8-128x128-b16-distinct-resident", + "variant": "baseline", + "wall_time_ns": 1372423.2, + "wall_time_ci_lower_ns": 1358597.6, + "wall_time_ci_upper_ns": 1386325.9, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD per-step 9/7 lifting: four horizontal step passes, a vertical scale pass and four vertical step passes per level, each over the full plane.", + "output_sha256": "c01760c8ab5ebc8ffc60a4c5673f88742fb9366705c1307f0017bafe84c91dda", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "9166ceef4a83c1a8dac3d2956218b17ce2742bbb66f0bdd6d47f69ce47a30dbf" + }, + { + "workload_id": "ht97-rgb8-128x128-b16-distinct-resident", + "variant": "treatment", + "wall_time_ns": 1289397.2, + "wall_time_ci_lower_ns": 1286181.1, + "wall_time_ci_upper_ns": 1291870.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two fused tile dispatches per level: horizontal lifting in 4-row groups, vertical scale and lifting in 32-column strips with a carried 4-sample halo.", + "output_sha256": "c01760c8ab5ebc8ffc60a4c5673f88742fb9366705c1307f0017bafe84c91dda", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "9166ceef4a83c1a8dac3d2956218b17ce2742bbb66f0bdd6d47f69ce47a30dbf" + }, + { + "workload_id": "ht97-rgb8-640x480-b16-distinct-resident", + "variant": "baseline", + "wall_time_ns": 7435933.3, + "wall_time_ci_lower_ns": 7403006.4, + "wall_time_ci_upper_ns": 7481697.6, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD per-step 9/7 lifting: four horizontal step passes, a vertical scale pass and four vertical step passes per level, each over the full plane.", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26" + }, + { + "workload_id": "ht97-rgb8-640x480-b16-distinct-resident", + "variant": "treatment", + "wall_time_ns": 6101207.8, + "wall_time_ci_lower_ns": 6095990.9, + "wall_time_ci_upper_ns": 6105597.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two fused tile dispatches per level: horizontal lifting in 4-row groups, vertical scale and lifting in 32-column strips with a carried 4-sample halo.", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26" + }, + { + "workload_id": "ht97-rgb8-1024x1024-b16-distinct-resident", + "variant": "baseline", + "wall_time_ns": 20584640.2, + "wall_time_ci_lower_ns": 20545641.4, + "wall_time_ci_upper_ns": 20652865.5, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD per-step 9/7 lifting: four horizontal step passes, a vertical scale pass and four vertical step passes per level, each over the full plane.", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691" + }, + { + "workload_id": "ht97-rgb8-1024x1024-b16-distinct-resident", + "variant": "treatment", + "wall_time_ns": 19971334.9, + "wall_time_ci_lower_ns": 19954812.9, + "wall_time_ci_upper_ns": 19990478.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two fused tile dispatches per level: horizontal lifting in 4-row groups, vertical scale and lifting in 32-column strips with a carried 4-sample halo.", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691" + }, + { + "workload_id": "ht97-rgb8-512x512-b1-resident", + "variant": "baseline", + "wall_time_ns": 1270672.8, + "wall_time_ci_lower_ns": 1267895.2, + "wall_time_ci_upper_ns": 1272687.8, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD per-step 9/7 lifting: four horizontal step passes, a vertical scale pass and four vertical step passes per level, each over the full plane.", + "output_sha256": "08096a50534fc7edec3f339a24207e9b5b03166892fc73547d9ea06e7f806009", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "06e813ca5512e054c7d79fc93e96468c67ccc74381259ae672257e2f62f7ca34" + }, + { + "workload_id": "ht97-rgb8-512x512-b1-resident", + "variant": "treatment", + "wall_time_ns": 1284570.7, + "wall_time_ci_lower_ns": 1282230.0, + "wall_time_ci_upper_ns": 1286694.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two fused tile dispatches per level: horizontal lifting in 4-row groups, vertical scale and lifting in 32-column strips with a carried 4-sample halo.", + "output_sha256": "08096a50534fc7edec3f339a24207e9b5b03166892fc73547d9ea06e7f806009", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment.", + "input_sha256": "06e813ca5512e054c7d79fc93e96468c67ccc74381259ae672257e2f62f7ca34" + }, + { + "workload_id": "ht53-gray8-mixed4-prepared-batch-resident", + "variant": "baseline", + "wall_time_ns": 805529.4, + "wall_time_ci_lower_ns": 804550.5, + "wall_time_ci_upper_ns": 806371.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD per-step 9/7 lifting: four horizontal step passes, a vertical scale pass and four vertical step passes per level, each over the full plane. Control: 5/3 route, no 9/7 lifting.", + "output_sha256": "086ef73794e76cf94c548a826c3d7a938fe9f4459681a2fd79de05b1e50175d8", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "ht53-gray8-mixed4-prepared-batch-resident", + "variant": "treatment", + "wall_time_ns": 799168.5, + "wall_time_ci_lower_ns": 798467.7, + "wall_time_ci_upper_ns": 799869.5, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two fused tile dispatches per level: horizontal lifting in 4-row groups, vertical scale and lifting in 32-column strips with a carried 4-sample halo. Control: 5/3 route, no 9/7 lifting.", + "output_sha256": "086ef73794e76cf94c548a826c3d7a938fe9f4459681a2fd79de05b1e50175d8", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "classic53-rgb8-512x512-b16-broadcast-resident", + "variant": "baseline", + "wall_time_ns": 68734562.5, + "wall_time_ci_lower_ns": 68709328.3, + "wall_time_ci_upper_ns": 68776141.9, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD per-step 9/7 lifting: four horizontal step passes, a vertical scale pass and four vertical step passes per level, each over the full plane. Unchanged-route control (Classic 5/3).", + "output_sha256": "6e6253e6d7cb92b3d39a87485816bcb1939198dda07122ba245fa14bb1292485", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + }, + { + "workload_id": "classic53-rgb8-512x512-b16-broadcast-resident", + "variant": "treatment", + "wall_time_ns": 68672754.5, + "wall_time_ci_lower_ns": 68664974.0, + "wall_time_ci_upper_ns": 68678607.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Two fused tile dispatches per level: horizontal lifting in 4-row groups, vertical scale and lifting in 32-column strips with a carried 4-sample halo. Unchanged-route control (Classic 5/3).", + "output_sha256": "6e6253e6d7cb92b3d39a87485816bcb1939198dda07122ba245fa14bb1292485", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical across the P30-baseline, P31-baseline and treatment executables in both rounds. The geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; the mixed-group probe asserts the lossless source pixels; cooperative_cleanup_matches_cpu_and_legacy_kernel_bit_for_bit and irreversible97_fused_lifting_matches_full_grid_reference_bits pass on the treatment." + } + ], + "decision": { + "priority_workload_id": "ht97-rgb8-640x480-b16-distinct-resident", + "confidence_interval_supports_improvement": true, + "representative_regression_percent": 1.09, + "complexity_is_proportional": true, + "rationale": "Promoted. The per-step 9/7 inverse lifting issued nine full-plane passes per level; it now issues two fused tile dispatches. The baseline restores HEAD's three per-step kernels, their pipelines and HEAD's dispatch body in the current tree; both arms include P30. Same-tree A/B on the current tree: three decode_stages executables (P30 baseline, P31 baseline, treatment) were built with the baseline sources present only during compilation, then run in rotation for two rounds. Recorded values are Criterion slope estimates of the first round. metal_decode_stages, metal_decode_stages_idwt97 and metal_decode_stages_classic fix 10 samples in code; the other rows use 20. Hardware counters were not collected. 9/7 rows: 640x480 batch 16 -17.9% (repeat -20.9%), 128x128 batch 16 -6.0% (-6.4%), 1024x1024 batch 16 -3.0% (-2.9%), 512x512 batch 16 -3.6% (-0.5%), disjoint intervals except the 512x512 batch-16 repeat. 512x512 batch 1 moved +1.1% (+0.4%) with overlapping intervals. The 5/3 512x512 control read -19.5% in the first round because its baseline, the first benchmark in that process, ran 24% slower than in the repeat, where it read -1.9%; no 5/3 effect is claimed. The Classic control moved -0.1%. See P30-P31-metal-decode-notes.md." + } +} diff --git a/docs/performance-experiments/P32-metal-jpeg-decode-kernel-notes.md b/docs/performance-experiments/P32-metal-jpeg-decode-kernel-notes.md new file mode 100644 index 000000000..213c4473b --- /dev/null +++ b/docs/performance-experiments/P32-metal-jpeg-decode-kernel-notes.md @@ -0,0 +1,174 @@ +# P32: Metal JPEG decode kernel (Huffman and IDCT) + +Measured on 2026-09-23 in place on `main` (`c1cc3ff1`) with a dirty working +tree; the unrelated in-flight JPEG Metal, CPU JPEG, and J2K Metal edits were +preserved. Apple M4 Pro, 16 GPU cores, macOS 27.0, Rust 1.96.0. No public API, +Auto routing policy, or allocation cap changed. The tables below are harness +medians and Criterion estimates. The JSON record, from a fresh same-tree A/B +with P33 and P34 in both arms, is `P32-metal-jpeg-decode-kernel.json`. + +## Decisions + +- **Promoted: fused Huffman lookahead tables.** `PreparedHuffman` replaces the + separate `fast_symbol`/`fast_len` byte tables with one `fast` entry, + `(code length << 8) | symbol`, and adds `fast_ac`, + `(value << 8) | (run << 4) | total length`, for AC codes whose code and extra + bits fit the 9-bit lookahead. A hit decodes run, length, and the extended + value with one constant-memory load and no `receive_extend`. The canonical + search for longer codes starts at length 10, because every shorter code has + a lookahead entry. +- **Promoted: register-resident IDCT.** The column pass loads coefficients as + `short4` rows and transforms four columns at once as `int4`; the row pass + works from registers. `decode_idct_deposit_block` stores each row straight to + the plane, removing the `work[64]` and `pixels[64]` round trips. Every + `idct_islow` caller (texture, region, scaled, split) uses the same passes. + +## Why the results are exact + +- The fused AC path runs after `ensure_bits_padded(9)`, as the symbol path + did. The original would consume the same bits: `receive_extend` counts padded + bits as buffered, so it never refills or fails when `total <= 9` bits are + buffered. Error codes are unchanged; `status.position` is unchanged because + no refill order changed. +- The column pass has no per-column DC shortcut. With all AC terms zero, the + full expression is `(p0·2¹³ + 2¹⁰) >> 11 = p0 << 2` exactly, and i16 inputs + cannot overflow it. The row pass keeps its AC-zero shortcut: pass-1 DC values + can reach about 2²⁰, where the full expression's `p0 << 13` could overflow and + diverge from the CPU. + +## Harness + +`compute/tests/decode_kernel_harness.rs`: + +- Correctness, run with the normal suite: batch buffers (full, half/quarter/ + eighth scale, region, region-scaled), resident textures (full, half scale), + and one-shot decodes must match the CPU decoder byte for byte. Fixtures cover + 4:2:0, 4:2:2 and 4:4:4; bench, textured and noise content; quality 50 to 100; + odd sizes (133×77, 131×67, 130×66); restart intervals; and batch 1 and 16 of + distinct images. +- Stage profile (`#[ignore]`): probe builds replace `decode_idct_deposit_block` + with bodies that stop after Huffman decoding, coefficient materialization, + or the IDCT, and stub the pack kernel. GPU time comes from command-buffer + timestamps, sampled round-robin across variants after a 0.5 s warm-up so clock + ramping affects every variant alike. `J2K_JPEG_HARNESS_BASELINE_SHADERS` + compiles a second shader copy for a same-process A/B. +- Fault injection: consuming one bit too few for 12-bit codes, and a rounding + change in one IDCT output column, each failed the correctness tests. + +## Results + +GPU ms, median of 21, batch 16 of distinct images. The baseline is the original +algorithm on the new table layout, so it already includes merging the two +lookahead loads into one; the Criterion A/B below is against HEAD. + +| Buffer batch (fused decode + pack) | Baseline | After | Change | +|---|---:|---:|---:| +| 4:2:0 bench q90 512² | 0.811 | 0.692 | −15% | +| 4:2:0 textured q90 512² | 0.688 | 0.561 | −18% | +| 4:2:0 textured q75 512² | 0.520 | 0.409 | −21% | +| 4:2:0 textured q90 256² | 0.257 | 0.220 | −14% | +| 4:2:0 textured q90 1024² | 3.165 | 2.543 | −20% | +| 4:2:2 textured q90 512² | 0.900 | 0.681 | −24% | +| 4:2:0 noise q95 512² | 0.896 | 0.822 | −8% | + +| Resident texture batch | Baseline | After | Change | +|---|---:|---:|---:| +| 4:2:0 256² | 0.303 | 0.257 | −15% | +| 4:2:0 512² | 0.740 | 0.612 | −17% | +| 4:2:2 512² | 0.941 | 0.717 | −24% | +| 4:4:4 256² | 1.894 | 1.677 | −11% | +| 4:4:4 512² | 3.856 | 3.219 | −17% | + +Stage split for 4:2:0 textured q90 512², before → after (ms): Huffman +0.338 → 0.271, coefficients 0.052 → 0.067, IDCT and store 0.171 → 0.094, +pack 0.120 → 0.118. Huffman is now 61% of the decode kernel and 48% of the +full GPU time. + +Criterion, `compare` bench, `gpu-quick`, 20 samples; baseline is HEAD's +`abi.rs` and shaders swapped into the same tree: + +| Benchmark | HEAD | After | Change (95% CI) | +|---|---:|---:|---:| +| `wsi_tile_batch_rgb` fast420 256², batch 64 | 6.21 ms | 4.55 ms | −26% [−33, −20] | +| `wsi_tile_batch_rgb` fast420 restart2 256² | 5.33 ms | 4.47 ms | −22% [−29, −14] | +| `wsi_tile_batch_rgb` fast422 256² | 6.50 ms | 5.09 ms | −15% [−22, −8] | +| `wsi_tile_batch_rgb` fast444 256² | 6.67 ms | 6.38 ms | −4% [−7, −1] | +| resident textures batch 16, fast420 256² | 547 µs | 502 µs | −8% [−9, −7] | +| resident textures batch 16, fast420 restart2 | 840 µs | 789 µs | −6% [−7, −5] | +| resident textures batch 16, fast422 256² | 464 µs | 435 µs | −7% [−7, −6] | +| resident textures batch 16, fast444 256² | 2.51 ms | 2.34 ms | −7% [−7, −6] | + +## Measured and rejected + +Each was bit-exact and A/B'd against the promoted state; none helped, so none +was kept. Every change that added live state to the entropy loop was slower, +which points at register pressure rather than ALU or memory as its limit. + +| Candidate | Result | +|---|---| +| `simd_all`-uniform IDCT shortcuts (no divergence) | +1% to +4% | +| DC fast path through the fused table | ±1% | +| 32-bit `hi:lo` bit reader (no 64-bit shifts) | +2% to +5% | +| Huffman tables in `device` instead of `constant` space | +2% to +7% | +| Block-sparse IDCT (skip zero right half, prune zero bottom rows) | +1% to +3% | +| End-of-block and ZRL folded into the fused table | ±1% | +| Entropy word prefetched one refill ahead | +10% to +15% | + +### Follow-up: four output pixels per pack thread (rejected) + +The 4:2:0 and 4:2:2 RGB batch pack kernels were temporarily changed from two +to four horizontal output pixels per thread, with the host dispatch width +adjusted accordingly. The correctness harness remained byte-exact. Same-tree +Criterion A/B (`gpu-quick`, 20 samples, 64 × 256² tiles, retained session) +measured 4:2:0 at 1.552 ms before and 1.549 ms after (change interval −1.05% +to −0.07%, within Criterion's noise threshold). 4:2:2 measured 1.872 ms before +and 1.862 ms after (change interval −0.72% to +1.77%, no detected change). +The extra loop and reduced thread count did not produce a material end-to-end +gain, so the original two-pixel kernels and dispatch width were restored. +Vector output stores were not part of this experiment. + +A 10- or 11-bit lookahead was not built: measured fused-hit rates rise only +from 84–89% at 9 bits to 89–94% at 10 bits and under 1 point more at 11, and +folding EOB (up to 8% of symbols) into the table already showed that fewer +slow-path symbols did not move the time. Tables above 9 bits also exceed the +4 KB `setBytes` limit. + +## Defects found (fixed 2026-09-23, after this experiment) + +- **CPU 4:2:0 bottom row.** `component_row_triplet` + (`crates/j2k-jpeg/src/entropy/sequential/emit/upsample.rs`) took the lower + chroma neighbour of an odd final output row from MCU padding instead of + replicating the last real chroma row. `djpeg` (libjpeg-turbo) on a 130×66 + 4:2:0 image matched Metal byte for byte; the CPU differed in the last row + only. It affected whole-image outputs whose height is even but not a multiple + of the scaled MCU height, for example 1080-row images, and ROIs touching the + bottom edge. + **Fix:** the triplet takes the stripe's real chroma rows, + `ceil(stripe_rows / v_ratio)`, and replicates the last one, as libjpeg-turbo's + `set_bottom_pointers` does. The 12-bit writers had the same clamp to the + padded plane height and now clamp to `ceil(height / 2)`. The harness no + longer excludes any row and gained ROIs that touch the bottom edge; + `ybr420_bottom_rows_match_turbo_when_height_is_not_mcu_aligned` compares + heights 34, 64, 65, 66, 68, 72, 76 (restart none/1/7, full, RGBA and bottom + region) against libjpeg-turbo, with contrasting content in the MCU padding. +- **Resident texture batches of about 30 or more decoders failed.** Every + `Decoder` reported 17,926,448 retained bytes regardless of image size, because + `retained_allocation_bytes_excluding_cpu_checkpoint_cache` added the fixed + `MAX_DECODER_CONTEXT_ALLOCATION_BYTES`. 64 decoders requested 1.07 GiB against + the 512 MiB cap (`AllocationTooLarge`), so the batch-64 and batch-256 + `wsi_tile_batch_rgba_textures` benchmarks failed. + **Fix:** retained bytes now count only what the decoder owns (100,656 bytes + for a baseline 4:2:0 tile, mostly prepared Huffman tables). Decoders share one `DecoderContext` (the + thread-local default or a caller's), so its reserve is charged once per + budget: decode workspace planning (unchanged), CPU checkpoint growth, and + device-plan construction. Both benchmarks now complete; with 64 × 256² tiles, + 4:2:0 takes 1.22 ms, 4:2:2 1.34 ms, and 4:4:4 8.77 ms. + +## Reproduction + +```sh +J2K_REQUIRE_METAL_RUNTIME=1 cargo test --profile gpu-quick -p j2k-jpeg-metal --lib -- decode_kernel_harness +J2K_REQUIRE_METAL_RUNTIME=1 J2K_JPEG_HARNESS_BASELINE_SHADERS=/path/to/baseline-shaders \ + cargo test --profile gpu-quick -p j2k-jpeg-metal --lib -- decode_kernel_stage_profile \ + --include-ignored --nocapture --test-threads=1 +``` diff --git a/docs/performance-experiments/P32-metal-jpeg-decode-kernel.json b/docs/performance-experiments/P32-metal-jpeg-decode-kernel.json new file mode 100644 index 000000000..afbdfa766 --- /dev/null +++ b/docs/performance-experiments/P32-metal-jpeg-decode-kernel.json @@ -0,0 +1,807 @@ +{ + "schema_version": 1, + "experiment_id": "P32-METAL-JPEG-DECODE-KERNEL", + "status": "promoted", + "environment": { + "commit": "c1cc3ff15975d70f51d324a6e185209d630b15de", + "branch": "main", + "dirty": true, + "cpu": "Apple M4 Pro, 12 cores", + "gpu": "Apple M4 Pro, 16 cores, Metal 4", + "ram_bytes": 51539607552, + "os": "macOS 27.0 build 26A428, Darwin 27.0.0", + "driver_runtime": "Metal 4", + "rust_version": "rustc 1.96.0 (ac68faa20 2026-05-25)", + "llvm_version": "LLVM 22.1.2", + "gpu_toolchain": "Metal framework runtime source compilation (newLibraryWithSource); offline metal CLI not installed", + "build_profile": "gpu-quick", + "feature_flags": [], + "environment_variables": { + "J2K_REQUIRE_METAL_BENCH": "1", + "J2K_REQUIRE_METAL_RUNTIME": "1" + }, + "input_corpus_sha256": "f9ccb9adc1701689dc9747a3926f153b76842896746fb4c2dc6dd1b162cf30d0", + "sample_count": 50, + "warm_up_seconds": 3.0, + "measurement_seconds": 10.0 + }, + "workloads": [ + { + "id": "surfaces-rgb8-256x256-b64-420-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-420r2-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "present" + }, + { + "id": "surfaces-rgb8-256x256-b64-422-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:2", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-444-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-420-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-420r2-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "present" + }, + { + "id": "surfaces-rgb8-256x256-b64-422-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:2", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-444-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + }, + { + "id": "textures-rgba8-256x256-b16-420", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 16, + "components": 3, + "output": "rgba8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "none" + }, + { + "id": "textures-rgba8-256x256-b16-420r2", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 16, + "components": 3, + "output": "rgba8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "present" + }, + { + "id": "textures-rgba8-256x256-b16-422", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 16, + "components": 3, + "output": "rgba8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:2", + "jpeg_restart": "none" + }, + { + "id": "textures-rgba8-256x256-b16-444", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 16, + "components": 3, + "output": "rgba8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + } + ], + "measurements": [ + { + "workload_id": "surfaces-rgb8-256x256-b64-420-oneshot", + "variant": "baseline", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 5353279.7, + "wall_time_ci_lower_ns": 5108382.4, + "wall_time_ci_upper_ns": 5713163.2, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420-oneshot", + "variant": "treatment", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 4575736.0, + "wall_time_ci_lower_ns": 4550441.4, + "wall_time_ci_upper_ns": 4603263.8, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420r2-oneshot", + "variant": "baseline", + "input_sha256": "4a075eb7fc6a1aaabe6e0dbc49b751c0c165f33249e1ec2299fbd9159fbbf946", + "wall_time_ns": 5863483.3, + "wall_time_ci_lower_ns": 5314401.6, + "wall_time_ci_upper_ns": 6370931.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420r2-oneshot", + "variant": "treatment", + "input_sha256": "4a075eb7fc6a1aaabe6e0dbc49b751c0c165f33249e1ec2299fbd9159fbbf946", + "wall_time_ns": 4493629.8, + "wall_time_ci_lower_ns": 4459507.7, + "wall_time_ci_upper_ns": 4528215.8, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-oneshot", + "variant": "baseline", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 5511813.0, + "wall_time_ci_lower_ns": 5483877.9, + "wall_time_ci_upper_ns": 5545506.6, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-oneshot", + "variant": "treatment", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 5237533.6, + "wall_time_ci_lower_ns": 5204540.8, + "wall_time_ci_upper_ns": 5272175.8, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-oneshot", + "variant": "baseline", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 7076539.8, + "wall_time_ci_lower_ns": 6947177.0, + "wall_time_ci_upper_ns": 7229519.7, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-oneshot", + "variant": "treatment", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 6551582.8, + "wall_time_ci_lower_ns": 6502559.1, + "wall_time_ci_upper_ns": 6608411.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420-retained", + "variant": "baseline", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 1713152.9, + "wall_time_ci_lower_ns": 1676303.7, + "wall_time_ci_upper_ns": 1792392.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420-retained", + "variant": "treatment", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 1562005.2, + "wall_time_ci_lower_ns": 1560446.6, + "wall_time_ci_upper_ns": 1563672.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420r2-retained", + "variant": "baseline", + "input_sha256": "4a075eb7fc6a1aaabe6e0dbc49b751c0c165f33249e1ec2299fbd9159fbbf946", + "wall_time_ns": 1601865.0, + "wall_time_ci_lower_ns": 1599371.2, + "wall_time_ci_upper_ns": 1603971.6, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420r2-retained", + "variant": "treatment", + "input_sha256": "4a075eb7fc6a1aaabe6e0dbc49b751c0c165f33249e1ec2299fbd9159fbbf946", + "wall_time_ns": 1529999.0, + "wall_time_ci_lower_ns": 1527311.9, + "wall_time_ci_upper_ns": 1532131.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-retained", + "variant": "baseline", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 2070765.3, + "wall_time_ci_lower_ns": 2066700.5, + "wall_time_ci_upper_ns": 2077398.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-retained", + "variant": "treatment", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 1870932.1, + "wall_time_ci_lower_ns": 1867875.3, + "wall_time_ci_upper_ns": 1873431.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-retained", + "variant": "baseline", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 2925829.0, + "wall_time_ci_lower_ns": 2919864.4, + "wall_time_ci_upper_ns": 2933948.5, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-retained", + "variant": "treatment", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 2494321.6, + "wall_time_ci_lower_ns": 2488060.4, + "wall_time_ci_upper_ns": 2499770.4, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "textures-rgba8-256x256-b16-420", + "variant": "baseline", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 528022.7, + "wall_time_ci_lower_ns": 524842.3, + "wall_time_ci_upper_ns": 531681.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "c72a0fc32abff293a768babfb2ca8a779b3cf7f286da7e29d1ebb7b3ed35a2d3", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "textures-rgba8-256x256-b16-420", + "variant": "treatment", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 498758.0, + "wall_time_ci_lower_ns": 495148.4, + "wall_time_ci_upper_ns": 501668.5, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "c72a0fc32abff293a768babfb2ca8a779b3cf7f286da7e29d1ebb7b3ed35a2d3", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "textures-rgba8-256x256-b16-420r2", + "variant": "baseline", + "input_sha256": "4a075eb7fc6a1aaabe6e0dbc49b751c0c165f33249e1ec2299fbd9159fbbf946", + "wall_time_ns": 827327.3, + "wall_time_ci_lower_ns": 824673.5, + "wall_time_ci_upper_ns": 829394.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "c72a0fc32abff293a768babfb2ca8a779b3cf7f286da7e29d1ebb7b3ed35a2d3", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "textures-rgba8-256x256-b16-420r2", + "variant": "treatment", + "input_sha256": "4a075eb7fc6a1aaabe6e0dbc49b751c0c165f33249e1ec2299fbd9159fbbf946", + "wall_time_ns": 800921.8, + "wall_time_ci_lower_ns": 797978.3, + "wall_time_ci_upper_ns": 803250.9, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "c72a0fc32abff293a768babfb2ca8a779b3cf7f286da7e29d1ebb7b3ed35a2d3", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "textures-rgba8-256x256-b16-422", + "variant": "baseline", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 446828.5, + "wall_time_ci_lower_ns": 445976.1, + "wall_time_ci_upper_ns": 447587.2, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "2cc911ba6e6cde5c4077d0f5c6c455dcb6d8b61689e4bdf0e6627714ccf2a01a", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "textures-rgba8-256x256-b16-422", + "variant": "treatment", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 401996.4, + "wall_time_ci_lower_ns": 399029.9, + "wall_time_ci_upper_ns": 406145.6, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "2cc911ba6e6cde5c4077d0f5c6c455dcb6d8b61689e4bdf0e6627714ccf2a01a", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "textures-rgba8-256x256-b16-444", + "variant": "baseline", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 557293.9, + "wall_time_ci_lower_ns": 552249.8, + "wall_time_ci_upper_ns": 562071.9, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "HEAD Huffman lookahead (separate fast_symbol and fast_len byte tables, receive_extend for every AC value) and HEAD IDCT staged through work[64] and pixels[64]. GPU time and hardware counters were not collected.", + "output_sha256": "b81d3ff994c0a5ff53f48e42327269e84ab66e8d0c0c7ebdccc6180731c23edd", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + }, + { + "workload_id": "textures-rgba8-256x256-b16-444", + "variant": "treatment", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 483327.3, + "wall_time_ci_lower_ns": 482947.3, + "wall_time_ci_upper_ns": 483791.7, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Fused 9-bit lookahead tables (fast, fast_ac) and the register-resident IDCT storing rows straight to the plane. GPU time and hardware counters were not collected.", + "output_sha256": "b81d3ff994c0a5ff53f48e42327269e84ab66e8d0c0c7ebdccc6180731c23edd", + "exact_parity": true, + "conformance": "Pass: p32_tile_and_texture_batch_output_hashes, compiled separately for each arm, decodes every workload input (64 tiles three times on one session; 16 resident textures read back) and compares every byte with the CPU decoder; input and output SHA-256 are identical in both arms. The baseline evidence binary omits src/tests.rs's lookahead-table layout assertions, which apply only to the new table layout. The decode_kernel_harness correctness suite passes on the treatment." + } + ], + "decision": { + "priority_workload_id": "surfaces-rgb8-256x256-b64-420-retained", + "confidence_interval_supports_improvement": true, + "representative_regression_percent": 0.0, + "complexity_is_proportional": true, + "rationale": "Promoted. Two decode-kernel changes: fused 9-bit Huffman lookahead tables (PreparedHuffman `fast` = (length << 8) | symbol, and `fast_ac` = (value << 8) | (run << 4) | total length for AC codes whose code and extra bits fit), and a register-resident IDCT that stores rows straight to the plane, removing the work[64] and pixels[64] round trips. The baseline is HEAD's abi.rs and five decode shaders, plus P33's jpeg_decode_fast444_batch on HEAD's helper signature, compiled into separate bench and test executables while swapped into the current tree; both arms include P33 and P34. First protocol (20 samples, 4 s): one-shot rows had intervals up to +/-13% under background GPU load and point estimates between -28% and +5.5%, so the matrix was re-measured with 50 samples, 3 s warm-up and 10 s measurement in ABBA order (baseline then treatment, then treatment then baseline). Recorded values are the first re-measured pair. All 12 rows improve with disjoint intervals in both re-measured rounds: 64-tile retained-session batches -4.5% to -14.7% (4:2:0 -8.8%, repeat -6.7%), one-shot batches -5.0% to -23.4% (these include per-call session setup), batch-16 resident textures -3.2% to -13.3%. No row regressed. GPU time was not collected: the in-process harness A/B binds the treatment's table layout and cannot run HEAD's shaders. Earlier harness GPU medians and rejected variants are in P32-metal-jpeg-decode-kernel-notes.md." + } +} diff --git a/docs/performance-experiments/P33-metal-jpeg-444-component-planes.json b/docs/performance-experiments/P33-metal-jpeg-444-component-planes.json new file mode 100644 index 000000000..899284788 --- /dev/null +++ b/docs/performance-experiments/P33-metal-jpeg-444-component-planes.json @@ -0,0 +1,302 @@ +{ + "schema_version": 3, + "experiment_id": "P33-METAL-JPEG-444-COMPONENT-PLANES", + "status": "promoted", + "environment": { + "commit": "c1cc3ff15975d70f51d324a6e185209d630b15de", + "branch": "main", + "dirty": true, + "cpu": "Apple M4 Pro, 12 cores", + "gpu": "Apple M4 Pro, 16 cores, Metal 4", + "ram_bytes": 51539607552, + "os": "macOS 27.0 build 26A428, Darwin 27.0.0", + "driver_runtime": "Metal 4", + "rust_version": "rustc 1.96.0 (ac68faa20 2026-05-25)", + "llvm_version": "LLVM 22.1.2", + "gpu_toolchain": "Metal framework runtime source compilation (newLibraryWithSource); offline metal CLI not installed", + "build_profile": "gpu-quick", + "feature_flags": [], + "environment_variables": { + "J2K_REQUIRE_METAL_BENCH": "1" + }, + "input_corpus_sha256": "5f8b31d19105b428ba17a29dc06496fc510ebac31448b5393a60242ec1878977", + "sample_count": 20, + "warm_up_seconds": 2.0, + "measurement_seconds": 4.0 + }, + "workloads": [ + { + "id": "tex-rgba8-256-b16-fast444", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 16, + "components": 3, + "output": "rgba8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + }, + { + "id": "tex-rgba8-256-b64-fast444", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgba8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + }, + { + "id": "tex-rgba8-8-b16-fast444", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "8x8", + "batch": 16, + "components": 3, + "output": "rgba8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + }, + { + "id": "tex-rgba8-8-b64-fast444", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "8x8", + "batch": 64, + "components": 3, + "output": "rgba8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + } + ], + "measurements": [ + { + "workload_id": "tex-rgba8-256-b16-fast444", + "variant": "baseline", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 2318900.0, + "wall_time_ci_lower_ns": 2312800.0, + "wall_time_ci_upper_ns": 2324900.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": 16, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "One decode dispatch per tile with one thread per entropy segment, pre-P32 decode_block/idct_block with three private 64-byte pixel arrays, texel writes from the decode thread; no staging planes.", + "component_workspace_bytes": 0, + "output_sha256": "b81d3ff994c0a5ff53f48e42327269e84ab66e8d0c0c7ebdccc6180731c23edd", + "exact_parity": true, + "conformance": "Output textures hashed for both routes on the exact benchmark inputs and equal to the CPU decoder (p33_fast444_texture_route_output_hashes); decode_kernel_harness texture parity (4:4:4 at 128x128, 133x77, 96x64, 131x67, 130x66; q50-q100; noise) and an Adobe-RGB 4:4:4 batch pass byte for byte." + }, + { + "workload_id": "tex-rgba8-256-b16-fast444", + "variant": "treatment", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 529210.0, + "wall_time_ci_lower_ns": 525540.0, + "wall_time_ci_upper_ns": 532220.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": 17, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "One decode dispatch for all tiles (jpeg_decode_fast444_batch, fused decode_idct_deposit_block into private Y/Cb/Cr planes), then one pack encoder with one jpeg_pack_444_rgba_texture dispatch per tile. Hardware counters were not collected.", + "component_workspace_bytes": 3145728, + "output_sha256": "b81d3ff994c0a5ff53f48e42327269e84ab66e8d0c0c7ebdccc6180731c23edd", + "exact_parity": true, + "conformance": "Output textures hashed for both routes on the exact benchmark inputs and equal to the CPU decoder (p33_fast444_texture_route_output_hashes); decode_kernel_harness texture parity (4:4:4 at 128x128, 133x77, 96x64, 131x67, 130x66; q50-q100; noise) and an Adobe-RGB 4:4:4 batch pass byte for byte." + }, + { + "workload_id": "tex-rgba8-256-b64-fast444", + "variant": "baseline", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 8768500.0, + "wall_time_ci_lower_ns": 8710600.0, + "wall_time_ci_upper_ns": 8819000.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": 64, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "One decode dispatch per tile with one thread per entropy segment, pre-P32 decode_block/idct_block with three private 64-byte pixel arrays, texel writes from the decode thread; no staging planes.", + "component_workspace_bytes": 0, + "output_sha256": "a26a6e3d8b8e8cdbafb37a318009d51cb304fe900f0bca1e2ab1bce2e24a215e", + "exact_parity": true, + "conformance": "Output textures hashed for both routes on the exact benchmark inputs and equal to the CPU decoder (p33_fast444_texture_route_output_hashes); decode_kernel_harness texture parity (4:4:4 at 128x128, 133x77, 96x64, 131x67, 130x66; q50-q100; noise) and an Adobe-RGB 4:4:4 batch pass byte for byte." + }, + { + "workload_id": "tex-rgba8-256-b64-fast444", + "variant": "treatment", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 1646800.0, + "wall_time_ci_lower_ns": 1641500.0, + "wall_time_ci_upper_ns": 1652100.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": 65, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "One decode dispatch for all tiles (jpeg_decode_fast444_batch, fused decode_idct_deposit_block into private Y/Cb/Cr planes), then one pack encoder with one jpeg_pack_444_rgba_texture dispatch per tile. Hardware counters were not collected.", + "component_workspace_bytes": 12582912, + "output_sha256": "a26a6e3d8b8e8cdbafb37a318009d51cb304fe900f0bca1e2ab1bce2e24a215e", + "exact_parity": true, + "conformance": "Output textures hashed for both routes on the exact benchmark inputs and equal to the CPU decoder (p33_fast444_texture_route_output_hashes); decode_kernel_harness texture parity (4:4:4 at 128x128, 133x77, 96x64, 131x67, 130x66; q50-q100; noise) and an Adobe-RGB 4:4:4 batch pass byte for byte." + }, + { + "workload_id": "tex-rgba8-8-b16-fast444", + "variant": "baseline", + "input_sha256": "502872ae94045e2f47d63333745020003a83ed5c8258c3369ef687da019654c0", + "wall_time_ns": 972660.0, + "wall_time_ci_lower_ns": 967820.0, + "wall_time_ci_upper_ns": 976670.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": 16, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "One decode dispatch per tile with one thread per entropy segment, pre-P32 decode_block/idct_block with three private 64-byte pixel arrays, texel writes from the decode thread; no staging planes.", + "component_workspace_bytes": 0, + "output_sha256": "2fcc78e80a4eadd6a42ace56eda28a07aa15f8aec3f4265c0ac7ebd38ef67d74", + "exact_parity": true, + "conformance": "Output textures hashed for both routes on the exact benchmark inputs and equal to the CPU decoder (p33_fast444_texture_route_output_hashes); decode_kernel_harness texture parity (4:4:4 at 128x128, 133x77, 96x64, 131x67, 130x66; q50-q100; noise) and an Adobe-RGB 4:4:4 batch pass byte for byte." + }, + { + "workload_id": "tex-rgba8-8-b16-fast444", + "variant": "treatment", + "input_sha256": "502872ae94045e2f47d63333745020003a83ed5c8258c3369ef687da019654c0", + "wall_time_ns": 191950.0, + "wall_time_ci_lower_ns": 191110.0, + "wall_time_ci_upper_ns": 192880.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": 17, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "One decode dispatch for all tiles (jpeg_decode_fast444_batch, fused decode_idct_deposit_block into private Y/Cb/Cr planes), then one pack encoder with one jpeg_pack_444_rgba_texture dispatch per tile. Hardware counters were not collected.", + "component_workspace_bytes": 3072, + "output_sha256": "2fcc78e80a4eadd6a42ace56eda28a07aa15f8aec3f4265c0ac7ebd38ef67d74", + "exact_parity": true, + "conformance": "Output textures hashed for both routes on the exact benchmark inputs and equal to the CPU decoder (p33_fast444_texture_route_output_hashes); decode_kernel_harness texture parity (4:4:4 at 128x128, 133x77, 96x64, 131x67, 130x66; q50-q100; noise) and an Adobe-RGB 4:4:4 batch pass byte for byte." + }, + { + "workload_id": "tex-rgba8-8-b64-fast444", + "variant": "baseline", + "input_sha256": "502872ae94045e2f47d63333745020003a83ed5c8258c3369ef687da019654c0", + "wall_time_ns": 3172200.0, + "wall_time_ci_lower_ns": 3152800.0, + "wall_time_ci_upper_ns": 3188300.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": 64, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "One decode dispatch per tile with one thread per entropy segment, pre-P32 decode_block/idct_block with three private 64-byte pixel arrays, texel writes from the decode thread; no staging planes.", + "component_workspace_bytes": 0, + "output_sha256": "690d4d9cdc16b16cea94f3a5a54ef46cd0d63d220699d56055e11de1926f8497", + "exact_parity": true, + "conformance": "Output textures hashed for both routes on the exact benchmark inputs and equal to the CPU decoder (p33_fast444_texture_route_output_hashes); decode_kernel_harness texture parity (4:4:4 at 128x128, 133x77, 96x64, 131x67, 130x66; q50-q100; noise) and an Adobe-RGB 4:4:4 batch pass byte for byte." + }, + { + "workload_id": "tex-rgba8-8-b64-fast444", + "variant": "treatment", + "input_sha256": "502872ae94045e2f47d63333745020003a83ed5c8258c3369ef687da019654c0", + "wall_time_ns": 345970.0, + "wall_time_ci_lower_ns": 341300.0, + "wall_time_ci_upper_ns": 349790.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": 65, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "One decode dispatch for all tiles (jpeg_decode_fast444_batch, fused decode_idct_deposit_block into private Y/Cb/Cr planes), then one pack encoder with one jpeg_pack_444_rgba_texture dispatch per tile. Hardware counters were not collected.", + "component_workspace_bytes": 12288, + "output_sha256": "690d4d9cdc16b16cea94f3a5a54ef46cd0d63d220699d56055e11de1926f8497", + "exact_parity": true, + "conformance": "Output textures hashed for both routes on the exact benchmark inputs and equal to the CPU decoder (p33_fast444_texture_route_output_hashes); decode_kernel_harness texture parity (4:4:4 at 128x128, 133x77, 96x64, 131x67, 130x66; q50-q100; noise) and an Adobe-RGB 4:4:4 batch pass byte for byte." + } + ], + "decision": { + "priority_workload_id": "tex-rgba8-256-b64-fast444", + "confidence_interval_supports_improvement": true, + "representative_regression_percent": 0.0, + "complexity_is_proportional": true, + "rationale": "Promoted. Full-frame 4:4:4 resident texture batches now decode every tile in one jpeg_decode_fast444_batch dispatch on the shared JpegFast420BatchParams ABI (three fused decode_idct_deposit_block calls per MCU) into private component planes, then pack with the existing jpeg_pack_444_rgba_texture kernel, the route 4:2:0 and 4:2:2 already used. Baseline is the same tree with only the routing line restored to the direct jpeg_decode_fast444_rgba_texture_batch kernel. Criterion wall time falls 77-89% across batch 16/64 at 256x256 and 8x8, every interval disjoint. Harness GPU medians for 16 distinct textured q90 images: 1.661 -> 0.244 ms at 256x256 and 3.223 -> 0.962 ms at 512x512. The planes cost 3 x width x height x tiles bytes of pooled private scratch (12.6 MB at 64 x 256x256). The 4:4:4 RGB buffer route (full frame as a region-scaled batch) was measured at 0.255 ms (256x256) and 1.042 ms (512x512) GPU for 16 tiles, within 5-7% of the plane route, and was left unchanged." + } +} diff --git a/docs/performance-experiments/P34-metal-jpeg-surface-output-pool.json b/docs/performance-experiments/P34-metal-jpeg-surface-output-pool.json new file mode 100644 index 000000000..4a2772ceb --- /dev/null +++ b/docs/performance-experiments/P34-metal-jpeg-surface-output-pool.json @@ -0,0 +1,806 @@ +{ + "schema_version": 1, + "experiment_id": "P34-METAL-JPEG-SURFACE-OUTPUT-POOL", + "status": "promoted", + "environment": { + "commit": "c1cc3ff15975d70f51d324a6e185209d630b15de", + "branch": "main", + "dirty": true, + "cpu": "Apple M4 Pro, 12 cores", + "gpu": "Apple M4 Pro, 16 cores, Metal 4", + "ram_bytes": 51539607552, + "os": "macOS 27.0 build 26A428, Darwin 27.0.0", + "driver_runtime": "Metal 4", + "rust_version": "rustc 1.96.0 (ac68faa20 2026-05-25)", + "llvm_version": "LLVM 22.1.2", + "gpu_toolchain": "Metal framework runtime source compilation (newLibraryWithSource); offline metal CLI not installed", + "build_profile": "gpu-quick", + "feature_flags": [], + "environment_variables": { + "J2K_REQUIRE_METAL_BENCH": "1" + }, + "input_corpus_sha256": "cfc7455f8c0178061e5a2be9e8b30f52cc19b8bc4fbffce4e17c9d75dd0e9748", + "sample_count": 20, + "warm_up_seconds": 2.0, + "measurement_seconds": 4.0 + }, + "workloads": [ + { + "id": "surfaces-rgb8-256x256-b64-420-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-422-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:2", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-444-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-16x16-b64-420-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "16x16", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-16x8-b64-422-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "16x8", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:2", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-8x8-b64-444-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "8x8", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-420-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-422-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:2", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-444-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-16x16-b64-420-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "16x16", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-16x8-b64-422-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "16x8", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:2", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-8x8-b64-444-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "8x8", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + } + ], + "measurements": [ + { + "workload_id": "surfaces-rgb8-256x256-b64-420-retained", + "variant": "baseline", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 1834200.0, + "wall_time_ci_lower_ns": 1831300.0, + "wall_time_ci_upper_ns": 1837000.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420-retained", + "variant": "treatment", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 1550100.0, + "wall_time_ci_lower_ns": 1547600.0, + "wall_time_ci_upper_ns": 1552900.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "The 12.6 MB output buffer (64 x 256x256 RGB) is reused from the session pool once the previous batch's surfaces drop. In the 16 x 512x512 harness loop, driver scheduling (kernelStart to kernelEnd) fell from 0.283 to 0.030 ms per call and GPU time was unchanged (0.588 vs 0.564 ms). Hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-retained", + "variant": "baseline", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 2128400.0, + "wall_time_ci_lower_ns": 2124800.0, + "wall_time_ci_upper_ns": 2132000.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-retained", + "variant": "treatment", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 1837800.0, + "wall_time_ci_lower_ns": 1833900.0, + "wall_time_ci_upper_ns": 1841100.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "The 12.6 MB output buffer (64 x 256x256 RGB) is reused from the session pool once the previous batch's surfaces drop. In the 16 x 512x512 harness loop, driver scheduling (kernelStart to kernelEnd) fell from 0.283 to 0.030 ms per call and GPU time was unchanged (0.588 vs 0.564 ms). Hardware counters were not collected.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-retained", + "variant": "baseline", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 2738800.0, + "wall_time_ci_lower_ns": 2733700.0, + "wall_time_ci_upper_ns": 2743100.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-retained", + "variant": "treatment", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 2435800.0, + "wall_time_ci_lower_ns": 2432500.0, + "wall_time_ci_upper_ns": 2439500.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "The 12.6 MB output buffer (64 x 256x256 RGB) is reused from the session pool once the previous batch's surfaces drop. In the 16 x 512x512 harness loop, driver scheduling (kernelStart to kernelEnd) fell from 0.283 to 0.030 ms per call and GPU time was unchanged (0.588 vs 0.564 ms). Hardware counters were not collected.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-16x16-b64-420-retained", + "variant": "baseline", + "input_sha256": "f2ba3b7999773d904e7004122c4593d5ad9d1005ed0cc66b09693a1cfb693ddf", + "wall_time_ns": 226240.0, + "wall_time_ci_lower_ns": 225080.0, + "wall_time_ci_upper_ns": 227330.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "bd01c15297ee92cedb54e59f0e1d1aad82b5bcf811569041059e7cc3dcf43b8c", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-16x16-b64-420-retained", + "variant": "treatment", + "input_sha256": "f2ba3b7999773d904e7004122c4593d5ad9d1005ed0cc66b09693a1cfb693ddf", + "wall_time_ns": 225040.0, + "wall_time_ci_lower_ns": 223580.0, + "wall_time_ci_upper_ns": 226210.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control: outputs under 1 MiB are allocated per call as before.", + "output_sha256": "bd01c15297ee92cedb54e59f0e1d1aad82b5bcf811569041059e7cc3dcf43b8c", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-16x8-b64-422-retained", + "variant": "baseline", + "input_sha256": "eb9bad4e5df64c9324b913ed2ad2061667c37903d7ae3a7b671df65c4d42fb8b", + "wall_time_ns": 195830.0, + "wall_time_ci_lower_ns": 194390.0, + "wall_time_ci_upper_ns": 196920.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "53c8d1d1b9fd98db549a278cfd9bac689a8151a60779d35f3cc46d9f1428294e", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-16x8-b64-422-retained", + "variant": "treatment", + "input_sha256": "eb9bad4e5df64c9324b913ed2ad2061667c37903d7ae3a7b671df65c4d42fb8b", + "wall_time_ns": 197690.0, + "wall_time_ci_lower_ns": 196970.0, + "wall_time_ci_upper_ns": 198450.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control: outputs under 1 MiB are allocated per call as before.", + "output_sha256": "53c8d1d1b9fd98db549a278cfd9bac689a8151a60779d35f3cc46d9f1428294e", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-8x8-b64-444-retained", + "variant": "baseline", + "input_sha256": "502872ae94045e2f47d63333745020003a83ed5c8258c3369ef687da019654c0", + "wall_time_ns": 243780.0, + "wall_time_ci_lower_ns": 242160.0, + "wall_time_ci_upper_ns": 245190.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "3832cf43293c951decfa045227d526f43ffc693deb7e04261079702523f3438c", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-8x8-b64-444-retained", + "variant": "treatment", + "input_sha256": "502872ae94045e2f47d63333745020003a83ed5c8258c3369ef687da019654c0", + "wall_time_ns": 236070.0, + "wall_time_ci_lower_ns": 230070.0, + "wall_time_ci_upper_ns": 242960.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control: outputs under 1 MiB are allocated per call as before.", + "output_sha256": "3832cf43293c951decfa045227d526f43ffc693deb7e04261079702523f3438c", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420-oneshot", + "variant": "baseline", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 4818300.0, + "wall_time_ci_lower_ns": 4786600.0, + "wall_time_ci_upper_ns": 4859000.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420-oneshot", + "variant": "treatment", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 4833700.0, + "wall_time_ci_lower_ns": 4800400.0, + "wall_time_ci_upper_ns": 4878600.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control: a fresh MetalSession per iteration starts with an empty pool, so every batch still allocates.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-oneshot", + "variant": "baseline", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 5415000.0, + "wall_time_ci_lower_ns": 5384100.0, + "wall_time_ci_upper_ns": 5442200.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-oneshot", + "variant": "treatment", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 5456200.0, + "wall_time_ci_lower_ns": 5406500.0, + "wall_time_ci_upper_ns": 5518900.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control: a fresh MetalSession per iteration starts with an empty pool, so every batch still allocates.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-oneshot", + "variant": "baseline", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 6793200.0, + "wall_time_ci_lower_ns": 6748600.0, + "wall_time_ci_upper_ns": 6861700.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-oneshot", + "variant": "treatment", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 6744100.0, + "wall_time_ci_lower_ns": 6713900.0, + "wall_time_ci_upper_ns": 6787100.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control: a fresh MetalSession per iteration starts with an empty pool, so every batch still allocates.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-16x16-b64-420-oneshot", + "variant": "baseline", + "input_sha256": "f2ba3b7999773d904e7004122c4593d5ad9d1005ed0cc66b09693a1cfb693ddf", + "wall_time_ns": 2545400.0, + "wall_time_ci_lower_ns": 2536800.0, + "wall_time_ci_upper_ns": 2555500.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "bd01c15297ee92cedb54e59f0e1d1aad82b5bcf811569041059e7cc3dcf43b8c", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-16x16-b64-420-oneshot", + "variant": "treatment", + "input_sha256": "f2ba3b7999773d904e7004122c4593d5ad9d1005ed0cc66b09693a1cfb693ddf", + "wall_time_ns": 2556600.0, + "wall_time_ci_lower_ns": 2543300.0, + "wall_time_ci_upper_ns": 2568900.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control: a fresh MetalSession per iteration starts with an empty pool, so every batch still allocates.", + "output_sha256": "bd01c15297ee92cedb54e59f0e1d1aad82b5bcf811569041059e7cc3dcf43b8c", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-16x8-b64-422-oneshot", + "variant": "baseline", + "input_sha256": "eb9bad4e5df64c9324b913ed2ad2061667c37903d7ae3a7b671df65c4d42fb8b", + "wall_time_ns": 2508600.0, + "wall_time_ci_lower_ns": 2481900.0, + "wall_time_ci_upper_ns": 2546300.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "53c8d1d1b9fd98db549a278cfd9bac689a8151a60779d35f3cc46d9f1428294e", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-16x8-b64-422-oneshot", + "variant": "treatment", + "input_sha256": "eb9bad4e5df64c9324b913ed2ad2061667c37903d7ae3a7b671df65c4d42fb8b", + "wall_time_ns": 2523100.0, + "wall_time_ci_lower_ns": 2501900.0, + "wall_time_ci_upper_ns": 2558600.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control: a fresh MetalSession per iteration starts with an empty pool, so every batch still allocates.", + "output_sha256": "53c8d1d1b9fd98db549a278cfd9bac689a8151a60779d35f3cc46d9f1428294e", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-8x8-b64-444-oneshot", + "variant": "baseline", + "input_sha256": "502872ae94045e2f47d63333745020003a83ed5c8258c3369ef687da019654c0", + "wall_time_ns": 2817700.0, + "wall_time_ci_lower_ns": 2807700.0, + "wall_time_ci_upper_ns": 2829400.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Every call allocated a fresh shared output buffer for all tiles.", + "output_sha256": "3832cf43293c951decfa045227d526f43ffc693deb7e04261079702523f3438c", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + }, + { + "workload_id": "surfaces-rgb8-8x8-b64-444-oneshot", + "variant": "treatment", + "input_sha256": "502872ae94045e2f47d63333745020003a83ed5c8258c3369ef687da019654c0", + "wall_time_ns": 2838100.0, + "wall_time_ci_lower_ns": 2825700.0, + "wall_time_ci_upper_ns": 2853700.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control: a fresh MetalSession per iteration starts with an empty pool, so every batch still allocates.", + "output_sha256": "3832cf43293c951decfa045227d526f43ffc693deb7e04261079702523f3438c", + "exact_parity": true, + "conformance": "Pass: three consecutive 64-tile retained-session batches hash equal to the CPU decoder (p34_retained_session_output_hashes); surface_output_reuse tests verify that live surfaces keep their bytes while three later batches decode and that one kept surface pins its batch's buffer; injecting reuse of buffers with live surfaces was caught by both tests." + } + ], + "decision": { + "priority_workload_id": "surfaces-rgb8-256x256-b64-420-retained", + "confidence_interval_supports_improvement": true, + "representative_regression_percent": 0.95, + "complexity_is_proportional": true, + "rationale": "Promoted. Batch decodes that return Surfaces allocated a fresh shared output buffer per call; the driver spent about 0.26 ms of command-buffer scheduling mapping 12.6 MB of new pages, a quarter of a 16 x 512x512 call. Each MetalRuntime now keeps up to two pooled output buffers of 1-64 MiB and hands one out again only when the pool holds its sole reference (retainCount == 1): every live surface and every in-flight command buffer retains the buffer, so no output a caller can read is overwritten. Retained-session 64-tile 256x256 batches improve 10.7-15.4% with disjoint intervals; one-shot sessions and outputs under 1 MiB are unchanged controls. A first version without the 1 MiB floor slowed 64-tile 16x16 and 16x8 batches by 9-17%, so small outputs keep per-call allocation. An idle session retains at most 2 x 64 MiB of output buffers until it is dropped. The worst representative point estimate is +0.95% (4:2:2 16x8 retained control, within noise)." + } +} diff --git a/docs/performance-experiments/P35-metal-buffer-pool-cap.json b/docs/performance-experiments/P35-metal-buffer-pool-cap.json new file mode 100644 index 000000000..ebec61f31 --- /dev/null +++ b/docs/performance-experiments/P35-metal-buffer-pool-cap.json @@ -0,0 +1,358 @@ +{ + "schema_version": 1, + "experiment_id": "P35-METAL-BUFFER-POOL-CAP", + "status": "promoted", + "environment": { + "commit": "c1cc3ff15975d70f51d324a6e185209d630b15de", + "branch": "main", + "dirty": true, + "cpu": "Apple M4 Pro, 12 cores", + "gpu": "Apple M4 Pro, 16 cores, Metal 4", + "ram_bytes": 51539607552, + "os": "macOS 27.0 build 26A428, Darwin 27.0.0", + "driver_runtime": "Metal 4", + "rust_version": "rustc 1.96.0 (ac68faa20 2026-05-25)", + "llvm_version": "LLVM 22.1.2", + "gpu_toolchain": "Metal framework runtime source compilation (newLibraryWithSource); offline metal CLI not installed", + "build_profile": "gpu-quick", + "feature_flags": [], + "environment_variables": { + "J2K_REQUIRE_METAL_BENCH": "1" + }, + "input_corpus_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691", + "sample_count": 20, + "warm_up_seconds": 2.0, + "measurement_seconds": 5.0 + }, + "workloads": [ + { + "id": "idwt97-1024x1024-b16-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "1024x1024", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_1024", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "idwt97-1024x1024-b16-readback", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "1024x1024", + "batch": 16, + "components": 3, + "output": "rgb8_host", + "operation": "full", + "axis_class": "at_1024", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "idwt97-640x480-b16-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "640x480", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "above_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "idwt97-640x480-b16-readback", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "640x480", + "batch": 16, + "components": 3, + "output": "rgb8_host", + "operation": "full", + "axis_class": "above_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "idwt97-512x512-b1-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "512x512", + "batch": 1, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_512", + "jpeg_sampling": null, + "jpeg_restart": null + } + ], + "measurements": [ + { + "workload_id": "idwt97-1024x1024-b16-resident", + "variant": "baseline", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691", + "wall_time_ns": 29136378.8, + "wall_time_ci_lower_ns": 28833688.2, + "wall_time_ci_upper_ns": 29414913.6, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-pool retained-byte cap fixed at 256 MiB (MAX_RETAINED_BYTES_PER_POOL pinned to the clamp's lower bound). Pool hit counters and hardware counters were not collected.", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: in each arm the decode_stages geometry probe compares every output byte with native CPU decode before timing, and a cold-input decode on the same session, drawing on the warmed pools, is compared again after the timed rows; output SHA-256 is identical in both arms and both rounds. The retained_bytes_for_device unit tests (production_limits) pass." + }, + { + "workload_id": "idwt97-1024x1024-b16-resident", + "variant": "treatment", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691", + "wall_time_ns": 20333858.2, + "wall_time_ci_lower_ns": 20161529.2, + "wall_time_ci_upper_ns": 20531964.6, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-pool cap is an eighth of recommendedMaxWorkingSetSize clamped to [256 MiB, 1 GiB]: 1 GiB on this 48 GB device. Pool hit counters and hardware counters were not collected.", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: in each arm the decode_stages geometry probe compares every output byte with native CPU decode before timing, and a cold-input decode on the same session, drawing on the warmed pools, is compared again after the timed rows; output SHA-256 is identical in both arms and both rounds. The retained_bytes_for_device unit tests (production_limits) pass." + }, + { + "workload_id": "idwt97-1024x1024-b16-readback", + "variant": "baseline", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691", + "wall_time_ns": 32461444.9, + "wall_time_ci_lower_ns": 32002385.0, + "wall_time_ci_upper_ns": 32796378.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-pool retained-byte cap fixed at 256 MiB (MAX_RETAINED_BYTES_PER_POOL pinned to the clamp's lower bound). Pool hit counters and hardware counters were not collected.", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: in each arm the decode_stages geometry probe compares every output byte with native CPU decode before timing, and a cold-input decode on the same session, drawing on the warmed pools, is compared again after the timed rows; output SHA-256 is identical in both arms and both rounds. The retained_bytes_for_device unit tests (production_limits) pass." + }, + { + "workload_id": "idwt97-1024x1024-b16-readback", + "variant": "treatment", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691", + "wall_time_ns": 23665450.8, + "wall_time_ci_lower_ns": 23473032.7, + "wall_time_ci_upper_ns": 23840006.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-pool cap is an eighth of recommendedMaxWorkingSetSize clamped to [256 MiB, 1 GiB]: 1 GiB on this 48 GB device. Pool hit counters and hardware counters were not collected.", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: in each arm the decode_stages geometry probe compares every output byte with native CPU decode before timing, and a cold-input decode on the same session, drawing on the warmed pools, is compared again after the timed rows; output SHA-256 is identical in both arms and both rounds. The retained_bytes_for_device unit tests (production_limits) pass." + }, + { + "workload_id": "idwt97-640x480-b16-resident", + "variant": "baseline", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26", + "wall_time_ns": 6069223.3, + "wall_time_ci_lower_ns": 6048300.8, + "wall_time_ci_upper_ns": 6088372.7, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-pool retained-byte cap fixed at 256 MiB (MAX_RETAINED_BYTES_PER_POOL pinned to the clamp's lower bound). Control workload. Pool hit counters and hardware counters were not collected.", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: in each arm the decode_stages geometry probe compares every output byte with native CPU decode before timing, and a cold-input decode on the same session, drawing on the warmed pools, is compared again after the timed rows; output SHA-256 is identical in both arms and both rounds. The retained_bytes_for_device unit tests (production_limits) pass." + }, + { + "workload_id": "idwt97-640x480-b16-resident", + "variant": "treatment", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26", + "wall_time_ns": 6102928.4, + "wall_time_ci_lower_ns": 6077908.3, + "wall_time_ci_upper_ns": 6131567.9, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-pool cap is an eighth of recommendedMaxWorkingSetSize clamped to [256 MiB, 1 GiB]: 1 GiB on this 48 GB device. Control workload. Pool hit counters and hardware counters were not collected.", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: in each arm the decode_stages geometry probe compares every output byte with native CPU decode before timing, and a cold-input decode on the same session, drawing on the warmed pools, is compared again after the timed rows; output SHA-256 is identical in both arms and both rounds. The retained_bytes_for_device unit tests (production_limits) pass." + }, + { + "workload_id": "idwt97-640x480-b16-readback", + "variant": "baseline", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26", + "wall_time_ns": 7005222.1, + "wall_time_ci_lower_ns": 6961324.8, + "wall_time_ci_upper_ns": 7058607.9, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-pool retained-byte cap fixed at 256 MiB (MAX_RETAINED_BYTES_PER_POOL pinned to the clamp's lower bound). Control workload. Pool hit counters and hardware counters were not collected.", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: in each arm the decode_stages geometry probe compares every output byte with native CPU decode before timing, and a cold-input decode on the same session, drawing on the warmed pools, is compared again after the timed rows; output SHA-256 is identical in both arms and both rounds. The retained_bytes_for_device unit tests (production_limits) pass." + }, + { + "workload_id": "idwt97-640x480-b16-readback", + "variant": "treatment", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26", + "wall_time_ns": 7115938.2, + "wall_time_ci_lower_ns": 7054210.9, + "wall_time_ci_upper_ns": 7166620.4, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-pool cap is an eighth of recommendedMaxWorkingSetSize clamped to [256 MiB, 1 GiB]: 1 GiB on this 48 GB device. Control workload. Pool hit counters and hardware counters were not collected.", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: in each arm the decode_stages geometry probe compares every output byte with native CPU decode before timing, and a cold-input decode on the same session, drawing on the warmed pools, is compared again after the timed rows; output SHA-256 is identical in both arms and both rounds. The retained_bytes_for_device unit tests (production_limits) pass." + }, + { + "workload_id": "idwt97-512x512-b1-resident", + "variant": "baseline", + "input_sha256": "06e813ca5512e054c7d79fc93e96468c67ccc74381259ae672257e2f62f7ca34", + "wall_time_ns": 1318545.7, + "wall_time_ci_lower_ns": 1309809.3, + "wall_time_ci_upper_ns": 1326050.2, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-pool retained-byte cap fixed at 256 MiB (MAX_RETAINED_BYTES_PER_POOL pinned to the clamp's lower bound). Control workload. Pool hit counters and hardware counters were not collected.", + "output_sha256": "08096a50534fc7edec3f339a24207e9b5b03166892fc73547d9ea06e7f806009", + "exact_parity": true, + "conformance": "Pass: in each arm the decode_stages geometry probe compares every output byte with native CPU decode before timing, and a cold-input decode on the same session, drawing on the warmed pools, is compared again after the timed rows; output SHA-256 is identical in both arms and both rounds. The retained_bytes_for_device unit tests (production_limits) pass." + }, + { + "workload_id": "idwt97-512x512-b1-resident", + "variant": "treatment", + "input_sha256": "06e813ca5512e054c7d79fc93e96468c67ccc74381259ae672257e2f62f7ca34", + "wall_time_ns": 1296111.4, + "wall_time_ci_lower_ns": 1289562.8, + "wall_time_ci_upper_ns": 1305087.6, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-pool cap is an eighth of recommendedMaxWorkingSetSize clamped to [256 MiB, 1 GiB]: 1 GiB on this 48 GB device. Control workload. Pool hit counters and hardware counters were not collected.", + "output_sha256": "08096a50534fc7edec3f339a24207e9b5b03166892fc73547d9ea06e7f806009", + "exact_parity": true, + "conformance": "Pass: in each arm the decode_stages geometry probe compares every output byte with native CPU decode before timing, and a cold-input decode on the same session, drawing on the warmed pools, is compared again after the timed rows; output SHA-256 is identical in both arms and both rounds. The retained_bytes_for_device unit tests (production_limits) pass." + } + ], + "decision": { + "priority_workload_id": "idwt97-1024x1024-b16-resident", + "confidence_interval_supports_improvement": true, + "representative_regression_percent": 1.58, + "complexity_is_proportional": true, + "rationale": "Promoted. Each J2K Metal buffer pool caches completed buffers for exact-size reuse; a session owns a private and a shared pool. The P30-P31 investigation found that with a fixed 256 MiB per-pool cap the scratch of a 16 x 1024x1024 RGB HT 9/7 decode did not fit, so buffers were reallocated on every decode; this record re-measures the change without relying on those numbers. The cap is now an eighth of recommendedMaxWorkingSetSize clamped to [256 MiB, 1 GiB] and never above maxBufferLength (retained_bytes_for_device): 1 GiB on this 48 GB M4 Pro, about 680 MB on an 8 GB Mac. Same-tree A/B: the baseline executable was compiled with MAX_RETAINED_BYTES_PER_POOL pinned to 256 MiB, the treatment from the unmodified tree; baseline then treatment, then a second round in the same order. 1024x1024 batch-16 resident -30.2% (repeat -32.4%), readback -27.1% (repeat -28.4%), disjoint intervals in both rounds. In the first round the controls (640x480 batch 16, 512x512 batch 1) moved by less than 2%; in the repeat they ranged from -6.6% (640x480 resident, whose baseline ran about 7% slower than in the first round) to +0.2%. The control signs are inconsistent between rounds, so no control effect is claimed. The worst recorded point estimate is +1.58% (idwt97-640x480-b16-readback), -1.28% in the repeat. Cost: an idle session on a large device can retain up to 2 GiB (two pools x 1 GiB) until it is dropped. Complexity: one function and two constants. Recorded values are Criterion slope estimates of the first round." + } +} diff --git a/docs/performance-experiments/P36-metal-idwt97-batched-large.json b/docs/performance-experiments/P36-metal-idwt97-batched-large.json new file mode 100644 index 000000000..ce792bbe1 --- /dev/null +++ b/docs/performance-experiments/P36-metal-idwt97-batched-large.json @@ -0,0 +1,484 @@ +{ + "schema_version": 1, + "experiment_id": "P36-METAL-IDWT97-BATCHED-LARGE", + "status": "promoted", + "environment": { + "commit": "c1cc3ff15975d70f51d324a6e185209d630b15de", + "branch": "main", + "dirty": true, + "cpu": "Apple M4 Pro, 12 cores", + "gpu": "Apple M4 Pro, 16 cores, Metal 4", + "ram_bytes": 51539607552, + "os": "macOS 27.0 build 26A428, Darwin 27.0.0", + "driver_runtime": "Metal 4", + "rust_version": "rustc 1.96.0 (ac68faa20 2026-05-25)", + "llvm_version": "LLVM 22.1.2", + "gpu_toolchain": "Metal framework runtime source compilation (newLibraryWithSource); offline metal CLI not installed", + "build_profile": "gpu-quick", + "feature_flags": [], + "environment_variables": { + "J2K_REQUIRE_METAL_BENCH": "1" + }, + "input_corpus_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691", + "sample_count": 20, + "warm_up_seconds": 2.0, + "measurement_seconds": 5.0 + }, + "workloads": [ + { + "id": "ht97-rgb8-1024x1024-b16-distinct-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "1024x1024", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_1024", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-1024x1024-b16-distinct-readback", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "1024x1024", + "batch": 16, + "components": 3, + "output": "rgb8_host", + "operation": "full", + "axis_class": "at_1024", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-640x480-b16-distinct-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "640x480", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "above_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-640x480-b16-distinct-readback", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "640x480", + "batch": 16, + "components": 3, + "output": "rgb8_host", + "operation": "full", + "axis_class": "above_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-128x128-b16-distinct-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "128x128", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-512x512-b1-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "512x512", + "batch": 1, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_512", + "jpeg_sampling": null, + "jpeg_restart": null + }, + { + "id": "ht97-rgb8-512x512-b16-broadcast-resident", + "transform": "irreversible_9_7", + "entropy": "ht", + "code_block": "64x64", + "image": "512x512", + "batch": 16, + "components": 3, + "output": "rgb8_resident", + "operation": "full", + "axis_class": "at_512", + "jpeg_sampling": null, + "jpeg_restart": null + } + ], + "measurements": [ + { + "workload_id": "ht97-rgb8-1024x1024-b16-distinct-resident", + "variant": "baseline", + "wall_time_ns": 20516063.8, + "wall_time_ci_lower_ns": 20435366.3, + "wall_time_ci_upper_ns": 20595348.5, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-image 9/7 IDWT: each image, component and level dispatched separately (working set above 20 MiB). Hardware counters were not collected.", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691" + }, + { + "workload_id": "ht97-rgb8-1024x1024-b16-distinct-resident", + "variant": "treatment", + "wall_time_ns": 18132902.2, + "wall_time_ci_lower_ns": 18094929.0, + "wall_time_ci_upper_ns": 18180717.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Batched 9/7 IDWT: one dispatch per component and level for all 16 images. Hardware counters were not collected.", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691" + }, + { + "workload_id": "ht97-rgb8-1024x1024-b16-distinct-readback", + "variant": "baseline", + "wall_time_ns": 23884830.0, + "wall_time_ci_lower_ns": 23753557.2, + "wall_time_ci_upper_ns": 23993486.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Per-image 9/7 IDWT: each image, component and level dispatched separately (working set above 20 MiB). Hardware counters were not collected.", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691" + }, + { + "workload_id": "ht97-rgb8-1024x1024-b16-distinct-readback", + "variant": "treatment", + "wall_time_ns": 21561922.6, + "wall_time_ci_lower_ns": 21476654.5, + "wall_time_ci_upper_ns": 21683991.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Batched 9/7 IDWT: one dispatch per component and level for all 16 images. Hardware counters were not collected.", + "output_sha256": "41413b9adec9a1b6c3b1c09c615e9f4434af5dbfb7727adf75213bc9d658a2b9", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "073234a592b09347b002a0f9c422cda74f0bf0037c6fbf690d62fd60c828e691" + }, + { + "workload_id": "ht97-rgb8-640x480-b16-distinct-resident", + "variant": "baseline", + "wall_time_ns": 6345625.1, + "wall_time_ci_lower_ns": 6292497.3, + "wall_time_ci_upper_ns": 6417856.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control (batched below 20 MiB, or a single image, in both arms). Hardware counters were not collected.", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26" + }, + { + "workload_id": "ht97-rgb8-640x480-b16-distinct-resident", + "variant": "treatment", + "wall_time_ns": 6177297.5, + "wall_time_ci_lower_ns": 6166186.6, + "wall_time_ci_upper_ns": 6190673.8, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control (batched below 20 MiB, or a single image, in both arms). Hardware counters were not collected.", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26" + }, + { + "workload_id": "ht97-rgb8-640x480-b16-distinct-readback", + "variant": "baseline", + "wall_time_ns": 7332789.2, + "wall_time_ci_lower_ns": 7236028.9, + "wall_time_ci_upper_ns": 7428079.7, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control (batched below 20 MiB, or a single image, in both arms). Hardware counters were not collected.", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26" + }, + { + "workload_id": "ht97-rgb8-640x480-b16-distinct-readback", + "variant": "treatment", + "wall_time_ns": 7172790.8, + "wall_time_ci_lower_ns": 7141379.7, + "wall_time_ci_upper_ns": 7214250.9, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control (batched below 20 MiB, or a single image, in both arms). Hardware counters were not collected.", + "output_sha256": "e420fc7f4fcd882ccbb6a784e4e59b9bebe835fd82320c74c1c708fa643ccbd5", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "370ef19e2bd9a50c4969656ac730efd5518a69ce85aec1f6dd5ca34e6d3b4b26" + }, + { + "workload_id": "ht97-rgb8-128x128-b16-distinct-resident", + "variant": "baseline", + "wall_time_ns": 1308058.4, + "wall_time_ci_lower_ns": 1301218.0, + "wall_time_ci_upper_ns": 1318390.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control (batched below 20 MiB, or a single image, in both arms). Hardware counters were not collected.", + "output_sha256": "c01760c8ab5ebc8ffc60a4c5673f88742fb9366705c1307f0017bafe84c91dda", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "9166ceef4a83c1a8dac3d2956218b17ce2742bbb66f0bdd6d47f69ce47a30dbf" + }, + { + "workload_id": "ht97-rgb8-128x128-b16-distinct-resident", + "variant": "treatment", + "wall_time_ns": 1298848.5, + "wall_time_ci_lower_ns": 1292149.9, + "wall_time_ci_upper_ns": 1307811.9, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control (batched below 20 MiB, or a single image, in both arms). Hardware counters were not collected.", + "output_sha256": "c01760c8ab5ebc8ffc60a4c5673f88742fb9366705c1307f0017bafe84c91dda", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "9166ceef4a83c1a8dac3d2956218b17ce2742bbb66f0bdd6d47f69ce47a30dbf" + }, + { + "workload_id": "ht97-rgb8-512x512-b1-resident", + "variant": "baseline", + "wall_time_ns": 1286783.2, + "wall_time_ci_lower_ns": 1276115.5, + "wall_time_ci_upper_ns": 1297896.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control (batched below 20 MiB, or a single image, in both arms). Hardware counters were not collected.", + "output_sha256": "08096a50534fc7edec3f339a24207e9b5b03166892fc73547d9ea06e7f806009", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "06e813ca5512e054c7d79fc93e96468c67ccc74381259ae672257e2f62f7ca34" + }, + { + "workload_id": "ht97-rgb8-512x512-b1-resident", + "variant": "treatment", + "wall_time_ns": 1310428.4, + "wall_time_ci_lower_ns": 1302898.4, + "wall_time_ci_upper_ns": 1315576.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control (batched below 20 MiB, or a single image, in both arms). Hardware counters were not collected.", + "output_sha256": "08096a50534fc7edec3f339a24207e9b5b03166892fc73547d9ea06e7f806009", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes.", + "input_sha256": "06e813ca5512e054c7d79fc93e96468c67ccc74381259ae672257e2f62f7ca34" + }, + { + "workload_id": "ht97-rgb8-512x512-b16-broadcast-resident", + "variant": "baseline", + "wall_time_ns": 1820466.7, + "wall_time_ci_lower_ns": 1751244.4, + "wall_time_ci_upper_ns": 1889424.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control (batched below 20 MiB, or a single image, in both arms). Hardware counters were not collected.", + "output_sha256": "ffe40f6a9fd711d0c8bd6f5a9298f041b6a6f098aafa53a8d80be88a9a74edb8", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes." + }, + { + "workload_id": "ht97-rgb8-512x512-b16-broadcast-resident", + "variant": "treatment", + "wall_time_ns": 1851057.1, + "wall_time_ci_lower_ns": 1726629.1, + "wall_time_ci_upper_ns": 1988899.4, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "Unchanged-route control (batched below 20 MiB, or a single image, in both arms). Hardware counters were not collected.", + "output_sha256": "ffe40f6a9fd711d0c8bd6f5a9298f041b6a6f098aafa53a8d80be88a9a74edb8", + "exact_parity": true, + "conformance": "Pass: output SHA-256 is identical in both executables and both rounds; the geometry probes compare every output byte with native CPU decode before timing and again after the timed rows; irreversible97_fused_lifting_matches_full_grid_reference_bits (batch 1, 3 and 16) passes." + } + ], + "decision": { + "priority_workload_id": "ht97-rgb8-1024x1024-b16-distinct-resident", + "confidence_interval_supports_improvement": true, + "representative_regression_percent": 1.84, + "complexity_is_proportional": true, + "rationale": "Promoted. decode_stage_profile (test-only truncation of the resident color route, round-robin GPU timestamps) attributed the 16 x 1024x1024 HT 9/7 decode's 19.0 ms of GPU time as zero-fill+VLC 5.9 ms, MagSgn 2.3 ms, IDWT 9.7 ms and ICT+store 1.2 ms. Above a 20 MiB working set the batch reconstructed each image, component and level separately (P20's bound, set when the per-step kernels regressed there). With P31's fused lifting the batched route is faster: same-process GPU probe IDWT 9.8 -> 7.8 ms, full 18.7 -> 16.6 ms. The bound is removed; single images keep the per-image route. Same-tree A/B: baseline executable compiled with the 20 MiB bound restored, ABBA order, 20 samples. 1024x1024 batch 16 resident -11.6% (repeat -8.9%), readback -9.7% (repeat -7.8%), disjoint intervals in both rounds. Unchanged-route controls moved between -3.6% and +2.1% with signs varying by round; the worst first-round point estimate is +1.84% (ht97-rgb8-512x512-b1-resident), -3.64% in the repeat. Scratch size is unchanged: the output plane is allocated for the whole batch in both routes. Complexity: one condition." + } +} diff --git a/docs/performance-experiments/P37-metal-jpeg-default-session.json b/docs/performance-experiments/P37-metal-jpeg-default-session.json new file mode 100644 index 000000000..15f8b3d3b --- /dev/null +++ b/docs/performance-experiments/P37-metal-jpeg-default-session.json @@ -0,0 +1,550 @@ +{ + "schema_version": 1, + "experiment_id": "P37-METAL-JPEG-DEFAULT-SESSION", + "status": "measured", + "environment": { + "commit": "c1cc3ff15975d70f51d324a6e185209d630b15de", + "branch": "main", + "dirty": true, + "cpu": "Apple M4 Pro, 12 cores", + "gpu": "Apple M4 Pro, 16 cores, Metal 4", + "ram_bytes": 51539607552, + "os": "macOS 27.0 build 26A428, Darwin 27.0.0", + "driver_runtime": "Metal 4", + "rust_version": "rustc 1.96.0 (ac68faa20 2026-05-25)", + "llvm_version": "LLVM 22.1.2", + "gpu_toolchain": "Metal framework runtime source compilation (newLibraryWithSource); offline metal CLI not installed", + "build_profile": "gpu-quick", + "feature_flags": [], + "environment_variables": { + "J2K_REQUIRE_METAL_BENCH": "1" + }, + "input_corpus_sha256": "cfc7455f8c0178061e5a2be9e8b30f52cc19b8bc4fbffce4e17c9d75dd0e9748", + "sample_count": 20, + "warm_up_seconds": 2.0, + "measurement_seconds": 4.0 + }, + "workloads": [ + { + "id": "surfaces-rgb8-256x256-b64-420-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-420-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-422-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:2", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-422-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:2", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-444-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-256x256-b64-444-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "256x256", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:4:4", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-16x16-b64-420-16x16-oneshot", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "16x16", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "none" + }, + { + "id": "surfaces-rgb8-16x16-b64-420-16x16-retained", + "transform": "jpeg_idct", + "entropy": "huffman", + "code_block": "8x8", + "image": "16x16", + "batch": 64, + "components": 3, + "output": "rgb8", + "operation": "full", + "axis_class": "below_512", + "jpeg_sampling": "4:2:0", + "jpeg_restart": "none" + } + ], + "measurements": [ + { + "workload_id": "surfaces-rgb8-256x256-b64-420-oneshot", + "variant": "baseline", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 4676507.2, + "wall_time_ci_lower_ns": 4582482.5, + "wall_time_ci_upper_ns": 4806473.4, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() built a new runtime (command queue, empty pools) per session. Hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420-oneshot", + "variant": "treatment", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 2011986.2, + "wall_time_ci_lower_ns": 1868147.7, + "wall_time_ci_upper_ns": 2251576.6, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() uses the process-wide default runtime shared by every session created without a backend. Hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420-retained", + "variant": "baseline", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 1782052.7, + "wall_time_ci_lower_ns": 1575683.1, + "wall_time_ci_upper_ns": 2008158.5, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() built a new runtime (command queue, empty pools) per session. Hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-420-retained", + "variant": "treatment", + "input_sha256": "188d3f1f8093f950e16e291da59f812a5e5fececa144f72eac81f294668a2b37", + "wall_time_ns": 1568973.8, + "wall_time_ci_lower_ns": 1563962.9, + "wall_time_ci_upper_ns": 1573021.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() uses the process-wide default runtime shared by every session created without a backend. Hardware counters were not collected.", + "output_sha256": "cc59c797ce4dc8ccd3489a4671a9995a71768075190d63158fd31723fc1b8a36", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-oneshot", + "variant": "baseline", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 5138696.2, + "wall_time_ci_lower_ns": 5054484.6, + "wall_time_ci_upper_ns": 5222280.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() built a new runtime (command queue, empty pools) per session. Hardware counters were not collected.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-oneshot", + "variant": "treatment", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 2250657.3, + "wall_time_ci_lower_ns": 2244814.7, + "wall_time_ci_upper_ns": 2256406.1, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() uses the process-wide default runtime shared by every session created without a backend. Hardware counters were not collected.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-retained", + "variant": "baseline", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 1853346.9, + "wall_time_ci_lower_ns": 1847676.7, + "wall_time_ci_upper_ns": 1860211.7, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() built a new runtime (command queue, empty pools) per session. Hardware counters were not collected.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-422-retained", + "variant": "treatment", + "input_sha256": "ba502b42a95544e3e205be17d6650ee09f7c373d6d35b2c5a30fe9ae8c63f43a", + "wall_time_ns": 1887651.2, + "wall_time_ci_lower_ns": 1884388.9, + "wall_time_ci_upper_ns": 1891052.2, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() uses the process-wide default runtime shared by every session created without a backend. Hardware counters were not collected.", + "output_sha256": "f66eefe3f045a3e64859c190e75ae0ce081962e5ff865c2dd95b494cd8c779d9", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-oneshot", + "variant": "baseline", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 6912965.8, + "wall_time_ci_lower_ns": 6581496.9, + "wall_time_ci_upper_ns": 7293076.5, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() built a new runtime (command queue, empty pools) per session. Hardware counters were not collected.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-oneshot", + "variant": "treatment", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 3679165.0, + "wall_time_ci_lower_ns": 3544714.8, + "wall_time_ci_upper_ns": 3968949.9, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() uses the process-wide default runtime shared by every session created without a backend. Hardware counters were not collected.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-retained", + "variant": "baseline", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 2790362.9, + "wall_time_ci_lower_ns": 2566438.3, + "wall_time_ci_upper_ns": 3073185.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() built a new runtime (command queue, empty pools) per session. Hardware counters were not collected.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-256x256-b64-444-retained", + "variant": "treatment", + "input_sha256": "24ff4713bffd67d685f04307b6d287a1c570cd2a17e9a9d5046a0d09897ebd12", + "wall_time_ns": 2516524.8, + "wall_time_ci_lower_ns": 2512437.3, + "wall_time_ci_upper_ns": 2521313.3, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() uses the process-wide default runtime shared by every session created without a backend. Hardware counters were not collected.", + "output_sha256": "fc06878cc866b5c644d3d82a3e702b8f2738ac4fb8db439596e40b9806b0fc89", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-16x16-b64-420-16x16-oneshot", + "variant": "baseline", + "input_sha256": "f2ba3b7999773d904e7004122c4593d5ad9d1005ed0cc66b09693a1cfb693ddf", + "wall_time_ns": 3334962.6, + "wall_time_ci_lower_ns": 2416406.3, + "wall_time_ci_upper_ns": 4411391.4, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() built a new runtime (command queue, empty pools) per session. Hardware counters were not collected.", + "output_sha256": "bd01c15297ee92cedb54e59f0e1d1aad82b5bcf811569041059e7cc3dcf43b8c", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-16x16-b64-420-16x16-oneshot", + "variant": "treatment", + "input_sha256": "f2ba3b7999773d904e7004122c4593d5ad9d1005ed0cc66b09693a1cfb693ddf", + "wall_time_ns": 244486.2, + "wall_time_ci_lower_ns": 228095.8, + "wall_time_ci_upper_ns": 271786.8, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() uses the process-wide default runtime shared by every session created without a backend. Hardware counters were not collected.", + "output_sha256": "bd01c15297ee92cedb54e59f0e1d1aad82b5bcf811569041059e7cc3dcf43b8c", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-16x16-b64-420-16x16-retained", + "variant": "baseline", + "input_sha256": "f2ba3b7999773d904e7004122c4593d5ad9d1005ed0cc66b09693a1cfb693ddf", + "wall_time_ns": 224342.4, + "wall_time_ci_lower_ns": 220382.6, + "wall_time_ci_upper_ns": 231602.0, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() built a new runtime (command queue, empty pools) per session. Hardware counters were not collected.", + "output_sha256": "bd01c15297ee92cedb54e59f0e1d1aad82b5bcf811569041059e7cc3dcf43b8c", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + }, + { + "workload_id": "surfaces-rgb8-16x16-b64-420-16x16-retained", + "variant": "treatment", + "input_sha256": "f2ba3b7999773d904e7004122c4593d5ad9d1005ed0cc66b09693a1cfb693ddf", + "wall_time_ns": 244520.4, + "wall_time_ci_lower_ns": 228923.5, + "wall_time_ci_upper_ns": 267544.8, + "gpu_time_ns": null, + "stage_times_ns": {}, + "dispatch_count": null, + "host_to_device_bytes": null, + "device_to_host_bytes": null, + "device_read_bytes": null, + "device_write_bytes": null, + "registers_per_thread": null, + "private_bytes_per_thread": null, + "shared_bytes_per_group": null, + "occupancy_percent": null, + "spill_loads": null, + "spill_stores": null, + "cache_observation": "MetalSession::default() uses the process-wide default runtime shared by every session created without a backend. Hardware counters were not collected.", + "output_sha256": "bd01c15297ee92cedb54e59f0e1d1aad82b5bcf811569041059e7cc3dcf43b8c", + "exact_parity": true, + "conformance": "Pass: the session backend does not change any kernel or output path; output hashes are those of p34_retained_session_output_hashes and p32_tile_and_texture_batch_output_hashes, both of which decode through MetalSession::default() on the shared runtime and match the CPU decoder; metal_sessions_without_a_backend_share_one_default_runtime and release_forgets_pooled_outputs_and_keeps_leased_scratch pass." + } + ], + "decision": { + "priority_workload_id": "surfaces-rgb8-256x256-b64-420-oneshot", + "confidence_interval_supports_improvement": true, + "representative_regression_percent": 8.99, + "complexity_is_proportional": true, + "rationale": "Measured; kept at the user's direction (cache one default session for one-shot APIs), but not promoted under the 2% rule. MetalSession::default() used to build a new MetalRuntime per session; one-shot tile batches paid about 3 ms of queue creation, driver setup and cold pools per call. Sessions created without a backend now share one process-wide default runtime; pooled buffers are released by release_default_session_buffers() and automatically on libdispatch memory-pressure warnings. Same-tree A/B, compile-only baseline, ABBA, 20 samples: one-shot 64-tile batches 4:2:0 256x256 -57.0% (repeat -55.6%), 4:2:2 -56.2% (repeat -49.1%), 4:4:4 -46.8% (repeat -47.3%), 16x16 -92.7% (repeat -90.6%), disjoint intervals. Retained-session controls were re-measured with 50 samples x 10 s (ABBA): 256x256 rows moved -3.8% to +3.1% with signs varying by round; the 16x16 retained row was slower in both re-measured rounds (+5.3%, +2.2%, disjoint), about 5-11 us on 220 us, cause not identified. Recorded values are the first 20-sample round; its worst point estimate is +8.99% (surfaces-rgb8-16x16-b64-420-16x16-retained). Callers that need an isolated runtime can pass an explicit MetalBackendSession, which is unchanged." + } +} diff --git a/docs/routing-promotion-evidence.json b/docs/routing-promotion-evidence.json index 439fd2f19..f3196b923 100644 --- a/docs/routing-promotion-evidence.json +++ b/docs/routing-promotion-evidence.json @@ -31,6 +31,13 @@ "artifact_sha256": "c98f11c0b2a2a96853953ceee7ea672e0e5044bdb8abbd397c8c36eb82fe53b8", "operation_coverage": ["full_decode", "roi_decode", "scaled_decode", "batch_decode", "lossless_encode", "lossy_encode"] } +, + { + "name": "metal_local_combined", + "backend": "metal", + "artifact_sha256": "66f9d83f932efb7df6cab2de0048849f09343da00915e22794b0056b29f2aa5f", + "operation_coverage": ["full_decode", "roi_decode", "scaled_decode", "batch_decode", "lossless_encode", "lossy_encode"] + } ], "cuda_decode": [ {"source":"cuda_part1","source_components":3,"format":"Rgb8","transfer_syntax":"Jpeg2000Lossless","payload_kind":"Jpeg2000Codestream","operation":"Full","minimum_width":256,"minimum_height":149,"minimum_pixels":0,"minimum_count":1}, @@ -57,17 +64,25 @@ {"source":"cuda_part15","source_components":3,"format":"Rgb8","transfer_syntax":"HtJpeg2000Lossless","payload_kind":"JphFile","operation":"Repeated","minimum_width":768,"minimum_height":512,"minimum_pixels":0,"minimum_count":16} ], "metal_decode": [ - {"source":"metal_part1","source_components":1,"format":"Gray8","transfer_syntax":"Jpeg2000Lossy","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":0,"minimum_height":0,"minimum_pixels":2960793,"minimum_count":16}, - {"source":"metal_part1","source_components":3,"format":"Rgb8","transfer_syntax":"Jpeg2000Lossy","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":0,"minimum_height":0,"minimum_pixels":307200,"minimum_count":16}, - {"source":"metal_part1","source_components":3,"format":"Rgb8","transfer_syntax":"Jpeg2000Lossless","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":0,"minimum_height":0,"minimum_pixels":5038848,"minimum_count":16}, - {"source":"metal_part15","source_components":3,"format":"Rgb8","transfer_syntax":"HtJpeg2000Lossy","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":640,"minimum_height":480,"minimum_pixels":0,"minimum_count":16}, - {"source":"metal_part15","source_components":3,"format":"Rgb8","transfer_syntax":"HtJpeg2000Lossless","payload_kind":"JphFile","operation":"Repeated","minimum_width":768,"minimum_height":512,"minimum_pixels":0,"minimum_count":16}, - {"source":"metal_part15","source_components":3,"format":"Rgb8","transfer_syntax":"HtJpeg2000Lossy","payload_kind":"Jpeg2000Codestream","operation":"ScaledHalf","minimum_width":320,"minimum_height":240,"minimum_pixels":0,"minimum_count":1} + {"source":"metal_local_combined","source_components":3,"format":"Rgb8","transfer_syntax":"HtJpeg2000Lossless","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":256,"minimum_height":256,"minimum_pixels":65536,"minimum_count":16}, + {"source":"metal_local_combined","source_components":3,"format":"Rgb8","transfer_syntax":"HtJpeg2000Lossless","payload_kind":"JphFile","operation":"Repeated","minimum_width":640,"minimum_height":480,"minimum_pixels":307200,"minimum_count":16}, + {"source":"metal_local_combined","source_components":1,"format":"Gray8","transfer_syntax":"HtJpeg2000Lossless","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":640,"minimum_height":480,"minimum_pixels":307200,"minimum_count":16}, + {"source":"metal_local_combined","source_components":3,"format":"Rgb8","transfer_syntax":"Jpeg2000Lossless","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":640,"minimum_height":480,"minimum_pixels":307200,"minimum_count":16}, + {"source":"metal_local_combined","source_components":1,"format":"Gray8","transfer_syntax":"Jpeg2000Lossless","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":2048,"minimum_height":2048,"minimum_pixels":4194304,"minimum_count":16}, + {"source":"metal_local_combined","source_components":3,"format":"Rgb8","transfer_syntax":"HtJpeg2000Lossless","payload_kind":"Jpeg2000Codestream","operation":"Full","minimum_width":640,"minimum_height":480,"minimum_pixels":307200,"minimum_count":1}, + {"source":"metal_local_combined","source_components":3,"format":"Rgb8","transfer_syntax":"HtJpeg2000Lossless","payload_kind":"JphFile","operation":"Full","minimum_width":640,"minimum_height":480,"minimum_pixels":307200,"minimum_count":1}, + {"source":"metal_local_combined","source_components":1,"format":"Gray8","transfer_syntax":"HtJpeg2000Lossless","payload_kind":"Jpeg2000Codestream","operation":"Full","minimum_width":640,"minimum_height":480,"minimum_pixels":307200,"minimum_count":1}, + {"source":"metal_local_combined","source_components":3,"format":"Rgb8","transfer_syntax":"HtJpeg2000Lossy","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":256,"minimum_height":256,"minimum_pixels":65536,"minimum_count":16}, + {"source":"metal_local_combined","source_components":3,"format":"Rgb8","transfer_syntax":"Jpeg2000Lossy","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":1024,"minimum_height":1024,"minimum_pixels":1048576,"minimum_count":16}, + {"source":"metal_local_combined","source_components":1,"format":"Gray8","transfer_syntax":"HtJpeg2000Lossy","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":2048,"minimum_height":2048,"minimum_pixels":4194304,"minimum_count":16}, + {"source":"metal_local_combined","source_components":1,"format":"Gray8","transfer_syntax":"Jpeg2000Lossy","payload_kind":"Jpeg2000Codestream","operation":"Repeated","minimum_width":2048,"minimum_height":2048,"minimum_pixels":4194304,"minimum_count":16}, + {"source":"metal_local_combined","source_components":3,"format":"Rgb8","transfer_syntax":"HtJpeg2000Lossy","payload_kind":"Jpeg2000Codestream","operation":"Full","minimum_width":640,"minimum_height":480,"minimum_pixels":307200,"minimum_count":1}, + {"source":"metal_local_combined","source_components":1,"format":"Gray8","transfer_syntax":"HtJpeg2000Lossy","payload_kind":"Jpeg2000Codestream","operation":"Full","minimum_width":640,"minimum_height":480,"minimum_pixels":307200,"minimum_count":1} ], "metal_host_output": [ {"source":"metal_host_output","source_components":3,"width":1024,"height":1024}, {"source":"metal_host_output","source_components":1,"width":2048,"height":2048}, {"source":"metal_host_output","source_components":3,"width":2048,"height":2048} ], - "metal_lossy_rgb8": {"source":"metal_part1","minimum_pixels":5038848} + "metal_lossy_rgb8": {"source":"metal_local_combined","minimum_pixels":4194304} } diff --git a/docs/stable-api-1.0.implementation-public-api.txt b/docs/stable-api-1.0.implementation-public-api.txt index c20de08a0..6783771b3 100644 --- a/docs/stable-api-1.0.implementation-public-api.txt +++ b/docs/stable-api-1.0.implementation-public-api.txt @@ -2355,9 +2355,13 @@ pub fn j2k_native::Image<'a>::build_direct_grayscale_plan_region_with_context(&s pub fn j2k_native::Image<'a>::build_direct_grayscale_plan_with_context(&self, &mut j2k_native::DecoderContext<'a>) -> j2k_native::Result pub fn j2k_native::Image<'a>::build_referenced_classic_plan_region_with_context(&self, &mut j2k_native::DecoderContext<'a>, (u32, u32, u32, u32)) -> j2k_native::Result pub fn j2k_native::Image<'a>::build_referenced_htj2k_plan_region_with_context(&self, &mut j2k_native::DecoderContext<'a>, (u32, u32, u32, u32)) -> j2k_native::Result +pub fn j2k_native::Image<'a>::decode_components_for_integer_output_with_context<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>) -> j2k_native::Result> +pub fn j2k_native::Image<'a>::decode_components_for_integer_output_with_ht_decoder<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>, &mut dyn j2k_native::HtCodeBlockDecoder) -> j2k_native::Result> pub fn j2k_native::Image<'a>::decode_components_with_ht_decoder<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>, &mut dyn j2k_native::HtCodeBlockDecoder) -> j2k_native::Result> pub fn j2k_native::Image<'a>::decode_native_components_with_retained_capacity(&self, usize) -> j2k_native::Result pub fn j2k_native::Image<'a>::decode_native_with_retained_capacity(&self, usize) -> j2k_native::Result +pub fn j2k_native::Image<'a>::decode_region_components_for_integer_output_with_context<'ctx>(&self, (u32, u32, u32, u32), &'ctx mut j2k_native::DecoderContext<'a>) -> j2k_native::Result> +pub fn j2k_native::Image<'a>::decode_region_components_for_integer_output_with_ht_decoder<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>, (u32, u32, u32, u32), &mut dyn j2k_native::HtCodeBlockDecoder) -> j2k_native::Result> pub fn j2k_native::Image<'a>::decode_region_components_with_ht_decoder<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>, (u32, u32, u32, u32), &mut dyn j2k_native::HtCodeBlockDecoder) -> j2k_native::Result> pub fn j2k_native::Image<'a>::decode_reversible_53_coefficients(&self) -> j2k_native::Result pub fn j2k_native::Image<'a>::decode_reversible_53_coefficients_with_context(&self, &mut j2k_native::DecoderContext<'a>) -> j2k_native::Result @@ -2387,6 +2391,7 @@ pub fn j2k_native::J2kRequiredBandRegion::width(self) -> u32 pub fn j2k_native::NativeComponentPlane::allocated_bytes(&self) -> usize pub fn j2k_native::NativeComponentPlane::into_parts(self) -> j2k_native::NativeComponentPlaneParts pub fn j2k_native::PreparedRegionDecoder<'_, '_, '_>::decode_region_components(&mut self, (u32, u32, u32, u32)) -> j2k_native::Result> +pub fn j2k_native::PreparedRegionDecoder<'_, '_, '_>::decode_region_components_for_integer_output(&mut self, (u32, u32, u32, u32)) -> j2k_native::Result> pub fn j2k_native::ResidentHtj2kEncodeError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub fn j2k_native::ResidentHtj2kEncodeError::source(&self) -> core::option::Option<&(dyn core::error::Error + 'static)> pub fn j2k_native::Reversible53CoefficientImage::encode_htj2k(&self, &j2k_native::EncodeOptions) -> j2k_native::EncodeResult> diff --git a/xtask/src/promotion_codegen.rs b/xtask/src/promotion_codegen.rs index cd8a7c1b4..591321c83 100644 --- a/xtask/src/promotion_codegen.rs +++ b/xtask/src/promotion_codegen.rs @@ -275,29 +275,47 @@ fn render_metal(manifest: &Manifest) -> Result { Ok(out) } +/// rustfmt's `max_width`; generated lines that would exceed it are wrapped +/// the way rustfmt wraps them, so the output is already formatted. +const MAX_LINE_WIDTH: usize = 100; + fn render_sources(out: &mut String, manifest: &Manifest, backend: &str) { - let sources = manifest + let names = manifest .sources .iter() .filter(|source| source.backend == backend) + .map(|source| (rust_source_name(&source.name), &source.artifact_sha256)) .collect::>(); - for source in &sources { - writeln!( - out, - "const {}: &str = \"{}\";", - rust_source_name(&source.name), - source.artifact_sha256 - ) + for (name, sha256) in &names { + let line = format!("const {name}: &str = \"{sha256}\";"); + if line.len() <= MAX_LINE_WIDTH { + writeln!(out, "{line}") + } else { + writeln!(out, "const {name}: &str =\n \"{sha256}\";") + } .expect("writing to a String cannot fail"); } - out.push_str("\npub(crate) const SOURCE_EVIDENCE: &[&str] = &["); - for (index, source) in sources.iter().enumerate() { - if index != 0 { - out.push_str(", "); + let list = names + .iter() + .map(|(name, _)| name.as_str()) + .collect::>(); + let line = format!( + "pub(crate) const SOURCE_EVIDENCE: &[&str] = &[{}];", + list.join(", ") + ); + if line.len() <= MAX_LINE_WIDTH { + writeln!(out, "\n{line}") + } else { + let mut items = String::new(); + for name in &list { + writeln!(items, " {name},").expect("writing to a String cannot fail"); } - out.push_str(&rust_source_name(&source.name)); + writeln!( + out, + "\npub(crate) const SOURCE_EVIDENCE: &[&str] = &[\n{items}];" + ) } - out.push_str("];\n"); + .expect("writing to a String cannot fail"); } fn render_cuda_cell(out: &mut String, cell: &DecodeCell) -> Result<(), String> { From 1505419bd9e1f36d2ccab20e41f7a40425ff090c Mon Sep 17 00:00:00 2001 From: GF Date: Thu, 24 Sep 2026 16:49:06 -0400 Subject: [PATCH 2/6] ci: inventory new Metal profiling tests --- xtask/src/metal.rs | 2 ++ xtask/src/metal/tests.rs | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/xtask/src/metal.rs b/xtask/src/metal.rs index afe2b0e1d..59981daa2 100644 --- a/xtask/src/metal.rs +++ b/xtask/src/metal.rs @@ -62,6 +62,8 @@ const METAL_OPTIONAL_IGNORED_TESTS: &[&str] = &[ "engine::decode_dispatch::idwt::irreversible::performance::metal_irreversible97_interleave_horizontal_scale_gpu_timing", "engine::direct_buffers::performance::metal_checked_buffer_copy_into_perf", "engine::decode_dispatch::idwt::irreversible::performance::metal_irreversible97_stage_gpu_timing", + "engine::tests::decode_stage_profile::batched_idwt_route_profile", + "engine::tests::decode_stage_profile::decode_stage_profile", ]; struct MetalTestSuite { diff --git a/xtask/src/metal/tests.rs b/xtask/src/metal/tests.rs index 79d2dfc14..ca0bcd9a5 100644 --- a/xtask/src/metal/tests.rs +++ b/xtask/src/metal/tests.rs @@ -180,7 +180,7 @@ fn ignored_inventory_is_unique_and_has_expected_size() { .copied() .collect::>(); assert_eq!(required.len(), 21); - assert_eq!(optional.len(), 7); + assert_eq!(optional.len(), 9); assert!(optional.contains("idwt::tests::metal_host_slice_idwt_decode_perf")); assert!(optional .contains("engine::direct_buffers::performance::metal_checked_buffer_copy_into_perf")); @@ -192,6 +192,8 @@ fn ignored_inventory_is_unique_and_has_expected_size() { assert!(optional.contains( "engine::runtime::resource_profile_tests::benchmark_cold_and_repeated_session_kernel_initialization" )); + assert!(optional.contains("engine::tests::decode_stage_profile::batched_idwt_route_profile")); + assert!(optional.contains("engine::tests::decode_stage_profile::decode_stage_profile")); assert_eq!(required.len(), J2K_METAL_REQUIRED_IGNORED_TESTS.len()); assert_eq!(optional.len(), METAL_OPTIONAL_IGNORED_TESTS.len()); assert!(required.is_disjoint(&optional)); From d4f51050b8d0caea9fe39932696161998d771943 Mon Sep 17 00:00:00 2001 From: GF Date: Thu, 24 Sep 2026 17:05:04 -0400 Subject: [PATCH 3/6] ci: account for new debug assertions --- xtask/src/panic_surface/source_inventory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtask/src/panic_surface/source_inventory.rs b/xtask/src/panic_surface/source_inventory.rs index fa89f82af..2f1cd9fe2 100644 --- a/xtask/src/panic_surface/source_inventory.rs +++ b/xtask/src/panic_surface/source_inventory.rs @@ -15,7 +15,7 @@ const PANIC_MACRO_BASELINE: PanicMacroInventory = PanicMacroInventory { assert: 8, assert_eq: 3, assert_ne: 0, - debug_assert: 91, + debug_assert: 95, debug_assert_eq: 66, debug_assert_ne: 0, }; From 5312ff40ddc7c0d69718e8e886c4cf35ca962fa2 Mon Sep 17 00:00:00 2001 From: GF Date: Thu, 24 Sep 2026 19:24:28 -0400 Subject: [PATCH 4/6] docs: inventory TurboJPEG test FFI --- docs/unsafe-audit.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/unsafe-audit.md b/docs/unsafe-audit.md index 452d99b10..df31da360 100644 --- a/docs/unsafe-audit.md +++ b/docs/unsafe-audit.md @@ -201,6 +201,7 @@ moved, or removed. | `crates/j2k-jpeg/src/simd/neon_memory.rs` | Private fixed-size AArch64 NEON load/store leaves. | Callers are already inside a `fearless_simd::Neon` kernel; array references prove complete initialized extents, AArch64 vector memory operations permit the represented alignment, and shared/exclusive references preserve aliasing rules. | Backend parity across empty, short, tail, crop, missing-row, and unaligned cases; 2,000-case-per-class IDCT properties; NEON hot-path tests. | | `crates/j2k-jpeg/src/simd/x86.rs` | Private exact-AVX2 capability token and safe target-feature bridge. | The token is constructible only after cached AVX2 plus OS register-state detection; safe references preserve bounds, alignment, aliasing, and initialization contracts across the bridge. It intentionally does not imply the broader x86-64-v3 feature set. | Capability-selection/scalar-fallback tests, x86 cross-build, AVX2 backend parity, and native x86 release-lane execution. | | `crates/j2k-jpeg/src/simd/x86_memory.rs` | Private fixed-size x86 AVX2 load/store leaves and validated three-row load cursor. | Array leaves prove complete initialized extents. The cursor carries an exact-AVX2 token and the three source lifetimes, derives its chunk count from the shortest row, and advances only while a full eight-byte chunk remains; every operation is unaligned-capable and shared/exclusive references preserve aliasing rules. | Cursor exhaustion and unaligned-load tests, AVX2 row and IDCT differential tests including unaligned subslices, x86 cross-build, and native Windows AVX2 execution. | +| `crates/j2k-jpeg/tests/libjpeg_turbo_compare.rs` | Test-only TurboJPEG 3 FFI used by JPEG differential and 12-bit decoder tests. | Handles are destroyed once; input/output pointers use the declared sample geometry and live owners; TurboJPEG-owned JPEG buffers are copied and freed through the paired API; integer dimensions are checked before FFI conversion. | Optional libjpeg-turbo differential suite, including generated 8-bit encode fixtures and 12-bit decode comparisons. | | `crates/j2k-metal-support/src/allocation.rs` | Typed objc2-metal buffer and texture allocation plus the pointer-based copying buffer initializer. | Checked products and device/repository caps precede dispatch; typed APIs return retained owners and nil is rejected; the only unsafe call receives exactly the initialized source-slice extent and Metal copies it synchronously, so no borrowed no-copy storage escapes. | Pure cap/overflow/zero-sized-ABI tests, real Metal allocation/texture tests, autorelease ownership tests, and strict all-target Clippy. | | `crates/j2k-metal-support/src/buffer_access.rs` | Raw CPU access to shared Metal-buffer contents. | Typed ranges require padding-free `GpuAbi`, checked byte arithmetic, alignment, bounds, and CPU-visible storage; callers must prove command completion and exclusive access. | Checked read/write/fill regressions, nil/range/alignment tests, adapter alias-order policies, and real Metal tests. | | `crates/j2k-metal-support/src/resident.rs` | Opaque immutable Metal image ownership, unsafe raw adoption/access, and completion-retaining image submissions. | Layout construction and adoption validate dimensions, pitch, overflow, allocation bounds, and device identity; safe clones and subviews expose no mutable handle; producers transfer exclusive outputs and every raw-bound input into a submission that retains them until successful completion or a blocking drop. | Layout/bounds/device regressions, input-retention and drop-before-wait tests, resident JPEG/J2K encode parity, unsafe API snapshots, and strict Metal validation. | From 91ea6c714220a2bc87adedc588776efc63b49ebc Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 26 Sep 2026 20:21:40 -0400 Subject: [PATCH 5/6] ci: fetch approved GPU workflow history before policy checks --- .github/workflows/ci.yml | 3 +++ .github/workflows/full-validation.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48c8e0e46..ed094c830 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: fetch-depth: 0 + - name: Fetch approved GPU workflow definitions + # The approved immutable revision predates the rewritten public history. + run: git fetch --no-tags origin 7307750c0e4ea4af071258a63962b0bfccf7973d - uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c with: toolchain: "1.96" diff --git a/.github/workflows/full-validation.yml b/.github/workflows/full-validation.yml index f17f25803..1cf9b81f9 100644 --- a/.github/workflows/full-validation.yml +++ b/.github/workflows/full-validation.yml @@ -188,6 +188,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: fetch-depth: 0 + - name: Fetch approved GPU workflow definitions + # The approved immutable revision predates the rewritten public history. + run: git fetch --no-tags origin 7307750c0e4ea4af071258a63962b0bfccf7973d - uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c with: toolchain: stable From 3b72328d8be9a8f1148c24056c2ff4777666ae13 Mon Sep 17 00:00:00 2001 From: GF Date: Sat, 26 Sep 2026 20:33:41 -0400 Subject: [PATCH 6/6] ci: pin approved GPU runners to reachable public history --- .github/workflows/ci.yml | 3 --- .github/workflows/full-validation.yml | 3 --- .github/workflows/gpu-benchmarks.yml | 2 +- .github/workflows/gpu-validation.yml | 2 +- 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed094c830..48c8e0e46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,9 +75,6 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: fetch-depth: 0 - - name: Fetch approved GPU workflow definitions - # The approved immutable revision predates the rewritten public history. - run: git fetch --no-tags origin 7307750c0e4ea4af071258a63962b0bfccf7973d - uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c with: toolchain: "1.96" diff --git a/.github/workflows/full-validation.yml b/.github/workflows/full-validation.yml index 1cf9b81f9..f17f25803 100644 --- a/.github/workflows/full-validation.yml +++ b/.github/workflows/full-validation.yml @@ -188,9 +188,6 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: fetch-depth: 0 - - name: Fetch approved GPU workflow definitions - # The approved immutable revision predates the rewritten public history. - run: git fetch --no-tags origin 7307750c0e4ea4af071258a63962b0bfccf7973d - uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c with: toolchain: stable diff --git a/.github/workflows/gpu-benchmarks.yml b/.github/workflows/gpu-benchmarks.yml index ea69139ff..c03b0fbbf 100644 --- a/.github/workflows/gpu-benchmarks.yml +++ b/.github/workflows/gpu-benchmarks.yml @@ -36,7 +36,7 @@ concurrency: jobs: run: name: Benchmarks - uses: frames-sg/j2k/.github/workflows/gpu-benchmarks-runner.yml@7307750c0e4ea4af071258a63962b0bfccf7973d + uses: frames-sg/j2k/.github/workflows/gpu-benchmarks-runner.yml@e12ff458a55d99c2689626578e7560d7adc75185 with: lane: ${{ inputs.lane }} suite: ${{ inputs.suite }} diff --git a/.github/workflows/gpu-validation.yml b/.github/workflows/gpu-validation.yml index 4e4f5873f..71c6ca1e6 100644 --- a/.github/workflows/gpu-validation.yml +++ b/.github/workflows/gpu-validation.yml @@ -31,7 +31,7 @@ concurrency: jobs: run: name: GPU - uses: frames-sg/j2k/.github/workflows/gpu-validation-runner.yml@7307750c0e4ea4af071258a63962b0bfccf7973d + uses: frames-sg/j2k/.github/workflows/gpu-validation-runner.yml@e12ff458a55d99c2689626578e7560d7adc75185 with: target: ${{ inputs.target }} mode: ${{ inputs.mode }}