From 1b9eb318d182a14b1dfc1902aa16594ba1e318f4 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Mon, 24 Aug 2026 22:57:18 +0100 Subject: [PATCH 1/4] Make precise f32-to-i32 conversion branchless --- fearless_simd/src/generated/avx2.rs | 38 +++++++++------------------ fearless_simd/src/generated/sse4_2.rs | 19 +++++--------- fearless_simd_gen/src/mk_x86.rs | 32 ++++++++++------------ 3 files changed, 32 insertions(+), 57 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 2798f316..718ae700 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -613,19 +613,12 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: f32x4) -> i32x4 { let a = a.into(); - let mut converted = _mm_cvttps_epi32(a); - let in_range = _mm_cmplt_ps(a, _mm_set1_ps(2147483648.0)); - let all_in_range = _mm_movemask_ps(in_range) == 0b1111; - if !all_in_range { - converted = _mm_blendv_epi8( - _mm_set1_epi32(i32::MAX), - converted, - _mm_castps_si128(in_range), - ); - let is_not_nan = _mm_castps_si128(_mm_cmpord_ps(a, a)); - converted = _mm_and_si128(converted, is_not_nan); - } - converted.simd_into(token) + let converted = _mm_cvttps_epi32(a); + let positive_overflow = + _mm_castps_si128(_mm_cmple_ps(_mm_set1_ps(2147483648.0), a)); + let converted = _mm_xor_si128(converted, positive_overflow); + let is_not_nan = _mm_castps_si128(_mm_cmpord_ps(a, a)); + _mm_and_si128(converted, is_not_nan).simd_into(token) } ); kernel(self, a) @@ -5468,19 +5461,12 @@ impl Simd for Avx2 { #[inline(always)] fn kernel(token: Avx2, a: f32x8) -> i32x8 { let a = a.into(); - let mut converted = _mm256_cvttps_epi32(a); - let in_range = _mm256_cmp_ps::<17i32>(a, _mm256_set1_ps(2147483648.0)); - let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111; - if !all_in_range { - converted = _mm256_blendv_epi8( - _mm256_set1_epi32(i32::MAX), - converted, - _mm256_castps_si256(in_range), - ); - let is_not_nan = _mm256_castps_si256(_mm256_cmp_ps::<7i32>(a, a)); - converted = _mm256_and_si256(converted, is_not_nan); - } - converted.simd_into(token) + let converted = _mm256_cvttps_epi32(a); + let positive_overflow = + _mm256_castps_si256(_mm256_cmp_ps::<18i32>(_mm256_set1_ps(2147483648.0), a)); + let converted = _mm256_xor_si256(converted, positive_overflow); + let is_not_nan = _mm256_castps_si256(_mm256_cmp_ps::<7i32>(a, a)); + _mm256_and_si256(converted, is_not_nan).simd_into(token) } ); kernel(self, a) diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index 9ecf9619..c5ec3a6d 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -763,19 +763,12 @@ impl Simd for Sse4_2 { #[inline(always)] fn kernel(token: Sse4_2, a: f32x4) -> i32x4 { let a = a.into(); - let mut converted = _mm_cvttps_epi32(a); - let in_range = _mm_cmplt_ps(a, _mm_set1_ps(2147483648.0)); - let all_in_range = _mm_movemask_ps(in_range) == 0b1111; - if !all_in_range { - converted = _mm_blendv_epi8( - _mm_set1_epi32(i32::MAX), - converted, - _mm_castps_si128(in_range), - ); - let is_not_nan = _mm_castps_si128(_mm_cmpord_ps(a, a)); - converted = _mm_and_si128(converted, is_not_nan); - } - converted.simd_into(token) + let converted = _mm_cvttps_epi32(a); + let positive_overflow = + _mm_castps_si128(_mm_cmple_ps(_mm_set1_ps(2147483648.0), a)); + let converted = _mm_xor_si128(converted, positive_overflow); + let is_not_nan = _mm_castps_si128(_mm_cmpord_ps(a, a)); + _mm_and_si128(converted, is_not_nan).simd_into(token) } ); kernel(self, a) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 51d59764..61c64ca3 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -3866,6 +3866,7 @@ impl X86 { let max = simple_intrinsic("max", vec_ty); let set0 = intrinsic_ident("setzero", coarse_type(vec_ty), vec_ty.n_bits()); let cmplt = float_compare_method("simd_lt", vec_ty); + let cmple = float_compare_method("simd_le", vec_ty); let cmpord = float_compare_method("ord", vec_ty); let set1_float = set1_intrinsic(vec_ty); let set1_int = set1_intrinsic(&target_ty); @@ -3885,6 +3886,7 @@ impl X86 { ); let blend = intrinsic_ident("blendv", "epi8", vec_ty.n_bits()); let and = intrinsic_ident("and", coarse_type(&target_ty), vec_ty.n_bits()); + let xor = intrinsic_ident("xor", coarse_type(&target_ty), vec_ty.n_bits()); let andnot = simple_intrinsic("andnot", vec_ty); let add_int = simple_sign_unaware_intrinsic("add", &target_ty); let sub_float = simple_intrinsic("sub", vec_ty); @@ -3916,24 +3918,18 @@ impl X86 { (Int, Precise) => { quote! { let a = a.into(); - - let mut converted = #convert(a); - - // In the common case where everything is in range, we don't need to do anything else. - let in_range = #cmplt(a, #set1_float(2147483648.0)); - let all_in_range = #movemask(in_range) == #all_ones; - - if !all_in_range { - // If we are above i32::MAX (2147483647), clamp to it. - converted = #blend(#set1_int(i32::MAX), converted, #cast_to_int(in_range)); - // Set NaN to 0. Using `and` seems slightly faster than `blend`. - let is_not_nan = #cast_to_int(#cmpord(a, a)); - converted = #and(converted, is_not_nan); - // We don't need to handle negative overflow because Intel's "invalid result" sentinel - // value is -2147483648, which is what we want anyway. - } - - converted.simd_into(#token) + let converted = #convert(a); + // The truncating instruction returns i32::MIN for every invalid lane. + // Flipping its bits for positive overflow turns that sentinel into + // i32::MAX. Negative overflow already has the desired value. + let positive_overflow = #cast_to_int(#cmple( + #set1_float(2147483648.0), + a, + )); + let converted = #xor(converted, positive_overflow); + // The ordered mask is false only for NaN, which Rust converts to zero. + let is_not_nan = #cast_to_int(#cmpord(a, a)); + #and(converted, is_not_nan).simd_into(#token) } } (Unsigned, Precise) => { From ad881b2e6662a4031877e32bf4a6d29c9d3bdfcd Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Mon, 24 Aug 2026 23:41:01 +0100 Subject: [PATCH 2/4] Optimize 64-bit int<->float conversions on AVX2, add more tests for these ops --- fearless_simd/src/generated/avx2.rs | 293 +++++++++++++++--- fearless_simd/src/generated/sse2.rs | 14 + fearless_simd/src/generated/sse4_2.rs | 46 ++- fearless_simd_gen/src/mk_x86.rs | 198 +++++++++++- .../tests/harness/ops/cvt_f64.rs | 44 +++ .../tests/harness/ops/cvt_i64.rs | 36 +++ .../tests/harness/ops/cvt_i64_precise.rs | 32 ++ .../tests/harness/ops/cvt_u64.rs | 34 ++ .../tests/harness/ops/cvt_u64_precise.rs | 32 ++ 9 files changed, 676 insertions(+), 53 deletions(-) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 718ae700..a24797d3 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -4066,19 +4066,103 @@ impl Simd for Avx2 { } #[inline(always)] fn cvt_u64_f64x2(self, a: f64x2) -> u64x2 { - [a[0usize] as u64, a[1usize] as u64].simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x2) -> u64x2 { + let a = a.into(); + let bits = _mm_castpd_si128(a); + let exponent = _mm_srli_epi64::<52>(bits); + let significand = _mm_or_si128( + _mm_and_si128(bits, _mm_set1_epi64x(0x000f_ffff_ffff_ffff)), + _mm_set1_epi64x(0x0010_0000_0000_0000), + ); + let shift_bias = _mm_set1_epi64x(1075); + let right_count = _mm_sub_epi64(shift_bias, exponent); + let left_count = _mm_sub_epi64(exponent, shift_bias); + let converted = _mm_or_si128( + _mm_srlv_epi64(significand, right_count), + _mm_sllv_epi64(significand, left_count), + ); + converted.simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn cvt_u64_precise_f64x2(self, a: f64x2) -> u64x2 { - [a[0usize] as u64, a[1usize] as u64].simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x2) -> u64x2 { + let a = a.into(); + let bits = _mm_castpd_si128(a); + let exponent = _mm_srli_epi64::<52>(bits); + let significand = _mm_or_si128( + _mm_and_si128(bits, _mm_set1_epi64x(0x000f_ffff_ffff_ffff)), + _mm_set1_epi64x(0x0010_0000_0000_0000), + ); + let shift_bias = _mm_set1_epi64x(1075); + let right_count = _mm_sub_epi64(shift_bias, exponent); + let left_count = _mm_sub_epi64(exponent, shift_bias); + let converted = _mm_or_si128( + _mm_srlv_epi64(significand, right_count), + _mm_sllv_epi64(significand, left_count), + ); + let overflow = + _mm_castpd_si128(_mm_cmpge_pd(a, _mm_set1_pd(18_446_744_073_709_551_616.0))); + _mm_or_si128(converted, overflow).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] + #[cfg(target_arch = "x86_64")] + fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x2) -> i64x2 { + let a = a.into(); + let low = _mm_cvttsd_si64(a); + let high = _mm_cvttsd_si64(_mm_unpackhi_pd(a, a)); + _mm_set_epi64x(high, low).simd_into(token) + } + ); + kernel(self, a) + } + #[cfg(target_arch = "x86")] fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { [a[0usize] as i64, a[1usize] as i64].simd_into(self) } #[inline(always)] fn cvt_i64_precise_f64x2(self, a: f64x2) -> i64x2 { - [a[0usize] as i64, a[1usize] as i64].simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x2) -> i64x2 { + let a = a.into(); + let bits = _mm_castpd_si128(a); + let absolute = _mm_and_si128(bits, _mm_set1_epi64x(i64::MAX)); + let exponent = _mm_srli_epi64::<52>(absolute); + let significand = _mm_or_si128( + _mm_and_si128(bits, _mm_set1_epi64x(0x000f_ffff_ffff_ffff)), + _mm_set1_epi64x(0x0010_0000_0000_0000), + ); + let shift_bias = _mm_set1_epi64x(1075); + let right_count = _mm_sub_epi64(shift_bias, exponent); + let left_count = _mm_sub_epi64(exponent, shift_bias); + let magnitude = _mm_or_si128( + _mm_srlv_epi64(significand, right_count), + _mm_sllv_epi64(significand, left_count), + ); + let sign = _mm_cmpgt_epi64(_mm_setzero_si128(), bits); + let converted = _mm_sub_epi64(_mm_xor_si128(magnitude, sign), sign); + let overflow = _mm_castpd_si128(_mm_cmpge_pd( + _mm_castsi128_pd(absolute), + _mm_set1_pd(9_223_372_036_854_775_808.0), + )); + let bound = _mm_xor_si128(_mm_set1_epi64x(i64::MAX), sign); + _mm_blendv_epi8(converted, bound, overflow).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn splat_i64x2(self, val: i64) -> i64x2 { @@ -4452,7 +4536,21 @@ impl Simd for Avx2 { } #[inline(always)] fn cvt_f64_i64x2(self, a: i64x2) -> f64x2 { - [a[0usize] as f64, a[1usize] as f64].simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x2) -> f64x2 { + let a = a.into(); + let low = _mm_blend_epi16::<0xcc>(a, _mm_set1_epi64x(0x4330_0000_0000_0000)); + let high = _mm_srli_epi64::<32>(a); + let high = _mm_xor_si128(high, _mm_set1_epi64x(4985484789646622720i64)); + let high = _mm_sub_pd( + _mm_castsi128_pd(high), + _mm_set1_pd(f64::from_bits(4985484789647671296u64)), + ); + _mm_add_pd(_mm_castsi128_pd(low), high).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { @@ -4812,7 +4910,21 @@ impl Simd for Avx2 { } #[inline(always)] fn cvt_f64_u64x2(self, a: u64x2) -> f64x2 { - [a[0usize] as f64, a[1usize] as f64].simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x2) -> f64x2 { + let a = a.into(); + let low = _mm_blend_epi16::<0xcc>(a, _mm_set1_epi64x(0x4330_0000_0000_0000)); + let high = _mm_srli_epi64::<32>(a); + let high = _mm_xor_si128(high, _mm_set1_epi64x(4985484787499139072i64)); + let high = _mm_sub_pd( + _mm_castsi128_pd(high), + _mm_set1_pd(f64::from_bits(4985484787500187648u64)), + ); + _mm_add_pd(_mm_castsi128_pd(low), high).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { @@ -9090,43 +9202,114 @@ impl Simd for Avx2 { } #[inline(always)] fn cvt_u64_f64x4(self, a: f64x4) -> u64x4 { - [ - a[0usize] as u64, - a[1usize] as u64, - a[2usize] as u64, - a[3usize] as u64, - ] - .simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> u64x4 { + let a = a.into(); + let bits = _mm256_castpd_si256(a); + let exponent = _mm256_srli_epi64::<52>(bits); + let significand = _mm256_or_si256( + _mm256_and_si256(bits, _mm256_set1_epi64x(0x000f_ffff_ffff_ffff)), + _mm256_set1_epi64x(0x0010_0000_0000_0000), + ); + let shift_bias = _mm256_set1_epi64x(1075); + let right_count = _mm256_sub_epi64(shift_bias, exponent); + let left_count = _mm256_sub_epi64(exponent, shift_bias); + let converted = _mm256_or_si256( + _mm256_srlv_epi64(significand, right_count), + _mm256_sllv_epi64(significand, left_count), + ); + converted.simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn cvt_u64_precise_f64x4(self, a: f64x4) -> u64x4 { - [ - a[0usize] as u64, - a[1usize] as u64, - a[2usize] as u64, - a[3usize] as u64, - ] - .simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> u64x4 { + let a = a.into(); + let bits = _mm256_castpd_si256(a); + let exponent = _mm256_srli_epi64::<52>(bits); + let significand = _mm256_or_si256( + _mm256_and_si256(bits, _mm256_set1_epi64x(0x000f_ffff_ffff_ffff)), + _mm256_set1_epi64x(0x0010_0000_0000_0000), + ); + let shift_bias = _mm256_set1_epi64x(1075); + let right_count = _mm256_sub_epi64(shift_bias, exponent); + let left_count = _mm256_sub_epi64(exponent, shift_bias); + let converted = _mm256_or_si256( + _mm256_srlv_epi64(significand, right_count), + _mm256_sllv_epi64(significand, left_count), + ); + let overflow = _mm256_castpd_si256(_mm256_cmp_pd::<29i32>( + a, + _mm256_set1_pd(18_446_744_073_709_551_616.0), + )); + _mm256_or_si256(converted, overflow).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn cvt_i64_f64x4(self, a: f64x4) -> i64x4 { - [ - a[0usize] as i64, - a[1usize] as i64, - a[2usize] as i64, - a[3usize] as i64, - ] - .simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> i64x4 { + let a = a.into(); + let bits = _mm256_castpd_si256(a); + let absolute = _mm256_and_si256(bits, _mm256_set1_epi64x(i64::MAX)); + let exponent = _mm256_srli_epi64::<52>(absolute); + let significand = _mm256_or_si256( + _mm256_and_si256(bits, _mm256_set1_epi64x(0x000f_ffff_ffff_ffff)), + _mm256_set1_epi64x(0x0010_0000_0000_0000), + ); + let shift_bias = _mm256_set1_epi64x(1075); + let right_count = _mm256_sub_epi64(shift_bias, exponent); + let left_count = _mm256_sub_epi64(exponent, shift_bias); + let magnitude = _mm256_or_si256( + _mm256_srlv_epi64(significand, right_count), + _mm256_sllv_epi64(significand, left_count), + ); + let sign = _mm256_cmpgt_epi64(_mm256_setzero_si256(), bits); + let converted = _mm256_sub_epi64(_mm256_xor_si256(magnitude, sign), sign); + converted.simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn cvt_i64_precise_f64x4(self, a: f64x4) -> i64x4 { - [ - a[0usize] as i64, - a[1usize] as i64, - a[2usize] as i64, - a[3usize] as i64, - ] - .simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4) -> i64x4 { + let a = a.into(); + let bits = _mm256_castpd_si256(a); + let absolute = _mm256_and_si256(bits, _mm256_set1_epi64x(i64::MAX)); + let exponent = _mm256_srli_epi64::<52>(absolute); + let significand = _mm256_or_si256( + _mm256_and_si256(bits, _mm256_set1_epi64x(0x000f_ffff_ffff_ffff)), + _mm256_set1_epi64x(0x0010_0000_0000_0000), + ); + let shift_bias = _mm256_set1_epi64x(1075); + let right_count = _mm256_sub_epi64(shift_bias, exponent); + let left_count = _mm256_sub_epi64(exponent, shift_bias); + let magnitude = _mm256_or_si256( + _mm256_srlv_epi64(significand, right_count), + _mm256_sllv_epi64(significand, left_count), + ); + let sign = _mm256_cmpgt_epi64(_mm256_setzero_si256(), bits); + let converted = _mm256_sub_epi64(_mm256_xor_si256(magnitude, sign), sign); + let overflow = _mm256_castpd_si256(_mm256_cmp_pd::<29i32>( + _mm256_castsi256_pd(absolute), + _mm256_set1_pd(9_223_372_036_854_775_808.0), + )); + let bound = _mm256_xor_si256(_mm256_set1_epi64x(i64::MAX), sign); + _mm256_blendv_epi8(converted, bound, overflow).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn splat_i64x4(self, val: i64) -> i64x4 { @@ -9506,13 +9689,21 @@ impl Simd for Avx2 { } #[inline(always)] fn cvt_f64_i64x4(self, a: i64x4) -> f64x4 { - [ - a[0usize] as f64, - a[1usize] as f64, - a[2usize] as f64, - a[3usize] as f64, - ] - .simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i64x4) -> f64x4 { + let a = a.into(); + let low = _mm256_blend_epi32::<0xaa>(a, _mm256_set1_epi64x(0x4330_0000_0000_0000)); + let high = _mm256_srli_epi64::<32>(a); + let high = _mm256_xor_si256(high, _mm256_set1_epi64x(4985484789646622720i64)); + let high = _mm256_sub_pd( + _mm256_castsi256_pd(high), + _mm256_set1_pd(f64::from_bits(4985484789647671296u64)), + ); + _mm256_add_pd(_mm256_castsi256_pd(low), high).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn splat_u64x4(self, val: u64) -> u64x4 { @@ -9876,13 +10067,21 @@ impl Simd for Avx2 { } #[inline(always)] fn cvt_f64_u64x4(self, a: u64x4) -> f64x4 { - [ - a[0usize] as f64, - a[1usize] as f64, - a[2usize] as f64, - a[3usize] as f64, - ] - .simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u64x4) -> f64x4 { + let a = a.into(); + let low = _mm256_blend_epi32::<0xaa>(a, _mm256_set1_epi64x(0x4330_0000_0000_0000)); + let high = _mm256_srli_epi64::<32>(a); + let high = _mm256_xor_si256(high, _mm256_set1_epi64x(4985484787499139072i64)); + let high = _mm256_sub_pd( + _mm256_castsi256_pd(high), + _mm256_set1_pd(f64::from_bits(4985484787500187648u64)), + ); + _mm256_add_pd(_mm256_castsi256_pd(low), high).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn splat_mask64x4(self, val: bool) -> mask64x4 { diff --git a/fearless_simd/src/generated/sse2.rs b/fearless_simd/src/generated/sse2.rs index 4cc7fd51..a9eb5bdd 100644 --- a/fearless_simd/src/generated/sse2.rs +++ b/fearless_simd/src/generated/sse2.rs @@ -4392,6 +4392,20 @@ impl Simd for Sse2 { [a[0usize] as u64, a[1usize] as u64].simd_into(self) } #[inline(always)] + #[cfg(target_arch = "x86_64")] + fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse2, a: f64x2) -> i64x2 { + let a = a.into(); + let low = _mm_cvttsd_si64(a); + let high = _mm_cvttsd_si64(_mm_unpackhi_pd(a, a)); + _mm_set_epi64x(high, low).simd_into(token) + } + ); + kernel(self, a) + } + #[cfg(target_arch = "x86")] fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { [a[0usize] as i64, a[1usize] as i64].simd_into(self) } diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index c5ec3a6d..cf57ed52 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -4299,6 +4299,20 @@ impl Simd for Sse4_2 { [a[0usize] as u64, a[1usize] as u64].simd_into(self) } #[inline(always)] + #[cfg(target_arch = "x86_64")] + fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f64x2) -> i64x2 { + let a = a.into(); + let low = _mm_cvttsd_si64(a); + let high = _mm_cvttsd_si64(_mm_unpackhi_pd(a, a)); + _mm_set_epi64x(high, low).simd_into(token) + } + ); + kernel(self, a) + } + #[cfg(target_arch = "x86")] fn cvt_i64_f64x2(self, a: f64x2) -> i64x2 { [a[0usize] as i64, a[1usize] as i64].simd_into(self) } @@ -4676,7 +4690,21 @@ impl Simd for Sse4_2 { } #[inline(always)] fn cvt_f64_i64x2(self, a: i64x2) -> f64x2 { - [a[0usize] as f64, a[1usize] as f64].simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i64x2) -> f64x2 { + let a = a.into(); + let low = _mm_blend_epi16::<0xcc>(a, _mm_set1_epi64x(0x4330_0000_0000_0000)); + let high = _mm_srli_epi64::<32>(a); + let high = _mm_xor_si128(high, _mm_set1_epi64x(4985484789646622720i64)); + let high = _mm_sub_pd( + _mm_castsi128_pd(high), + _mm_set1_pd(f64::from_bits(4985484789647671296u64)), + ); + _mm_add_pd(_mm_castsi128_pd(low), high).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn splat_u64x2(self, val: u64) -> u64x2 { @@ -5039,7 +5067,21 @@ impl Simd for Sse4_2 { } #[inline(always)] fn cvt_f64_u64x2(self, a: u64x2) -> f64x2 { - [a[0usize] as f64, a[1usize] as f64].simd_into(self) + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u64x2) -> f64x2 { + let a = a.into(); + let low = _mm_blend_epi16::<0xcc>(a, _mm_set1_epi64x(0x4330_0000_0000_0000)); + let high = _mm_srli_epi64::<32>(a); + let high = _mm_xor_si128(high, _mm_set1_epi64x(4985484787499139072i64)); + let high = _mm_sub_pd( + _mm_castsi128_pd(high), + _mm_set1_pd(f64::from_bits(4985484787500187648u64)), + ); + _mm_add_pd(_mm_castsi128_pd(low), high).simd_into(token) + } + ); + kernel(self, a) } #[inline(always)] fn splat_mask64x2(self, val: bool) -> mask64x2 { diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 61c64ca3..be8bfa90 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -3679,13 +3679,50 @@ impl X86 { precision, ); + // For a two-lane signed approximate conversion, scalar casts outperform SIMD formulations. + // CVTTSD2SI cast is not required to saturate, so it is cheaper than `as` casts too. + // + // CVTTSD2SI is available starting with SSE2, so use it for every pre-AVX-512 x86 level. + // The SIMD implementation below is faster for u64, for precise semantics, + // and for vectors larger than 2 elements. + // + // CVTTSD2SI with a 64-bit destination only exists in 64-bit mode. + // 32-bit x86 is not an optimization target in 2026, so route it through scalar fallback. + if matches!( + conversion, + ( + Self::Sse2 | Self::Sse4_2 | Self::Avx2, + Float, + Int, + 64, + 128, + Approx, + ) + ) { + let unpack_high = simple_intrinsic("unpackhi", vec_ty); + let specialized = self.kernel_method(op, vec_ty, |token| { + quote! { + let a = a.into(); + let low = _mm_cvttsd_si64(a); + let high = _mm_cvttsd_si64(#unpack_high(a, a)); + _mm_set_epi64x(high, low).simd_into(#token) + } + }); + let fallback = fallback_method(op, vec_ty); + return quote! { + #[cfg(target_arch = "x86_64")] + #specialized + #[cfg(target_arch = "x86")] + #fallback + }; + } + // Deal with whole-method scalar fallbacks before constructing an intrinsic-backed kernel. - // Pre-AVX-512 x86 has no packed 64-bit conversion instructions. Some SSE2 32-bit cases are - // also absent or slower than converting the lanes individually. + // Some conversions have no native SIMD versions, or are slower than scalar fallbacks. if matches!( conversion, - (Self::Sse2 | Self::Sse4_2, _, _, 64, 128, _) - | (Self::Avx2, _, _, 64, 128 | 256, _) + (Self::Sse2, _, _, 64, 128, _) + | (Self::Sse4_2, Float, Int | Unsigned, 64, 128, _) | (Self::Sse2, _, _, 32, 128, Precise) | (Self::Sse2, Unsigned, _, 32, 128, _) | (Self::Sse2, _, Unsigned, 32, 128, _) @@ -3844,6 +3881,159 @@ impl X86 { #convert(a.into()).simd_into(#token) } } + (Self::Sse4_2, source @ (Int | Unsigned), Float, 64, 128, _) + | (Self::Avx2, source @ (Int | Unsigned), Float, 64, 128 | 256, _) => { + // Befire AVX-512 there's no packed 64-bit integer -> f64 instruction. + // Split each integer into 32-bit halves, embed those halves in the fraction fields + // of exact powers of two, then remove the biases. This is LLVM's u64 expansion, + // extended to signed values by biasing the high half around i32::MIN. + // + // The subtraction is exact; the final addition performs the one rounding required + // by the integer -> f64 conversion. + // https://github.com/llvm/llvm-project/blob/llvmorg-22.1.8/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp#L8670-L8726 + let target_ty = vec_ty.cast(target_scalar); + let bits = vec_ty.n_bits(); + let set1_int = set1_intrinsic(vec_ty); + let set1_float = set1_intrinsic(&target_ty); + let srli = intrinsic_ident("srli", "epi64", bits); + let xor = intrinsic_ident("xor", coarse_type(vec_ty), bits); + let cast_to_float = cast_ident(Int, Float, 64, 64, bits); + let sub = simple_intrinsic("sub", &target_ty); + let add = simple_intrinsic("add", &target_ty); + let (blend, blend_mask) = match bits { + 128 => (intrinsic_ident("blend", "epi16", bits), quote! { 0xcc }), + 256 => (intrinsic_ident("blend", "epi32", bits), quote! { 0xaa }), + _ => unreachable!(), + }; + let (high_bias_bits, combined_bias_bits) = match source { + Unsigned => (0x4530_0000_0000_0000_i64, 0x4530_0000_0010_0000_u64), + Int => (0x4530_0000_8000_0000_i64, 0x4530_0000_8010_0000_u64), + _ => unreachable!(), + }; + + quote! { + let a = a.into(); + let low = #blend::<#blend_mask>(a, #set1_int(0x4330_0000_0000_0000)); + let high = #srli::<32>(a); + let high = #xor(high, #set1_int(#high_bias_bits)); + let high = #sub( + #cast_to_float(high), + #set1_float(f64::from_bits(#combined_bias_bits)), + ); + #add(#cast_to_float(low), high).simd_into(#token) + } + } + ( + Self::Avx2, + Float, + target @ (Int | Unsigned), + 64, + 128 | 256, + precision, + ) => { + // Recover the integer magnitude directly from the binary64 exponent and + // significand. AVX2 variable shifts produce zero for counts above 63, so doing + // both shift directions and ORing them avoids a per-lane branch or blend: + // + // magnitude = (significand >> (1075 - exponent)) + // | (significand << (exponent - 1075)) + // + // Subnormals, values below one, infinities, and NaNs naturally become zero. + // This adapts LLVM's/compiler-rt's scalar significand-and-exponent expansion to + // AVX2's packed variable shifts. + // https://github.com/llvm/llvm-project/blob/llvmorg-22.1.8/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp#L8497-L8565 + let target_ty = vec_ty.cast(target); + let bits = vec_ty.n_bits(); + let set1_int = set1_intrinsic(&target_ty); + let set1_float = set1_intrinsic(vec_ty); + let setzero = intrinsic_ident("setzero", coarse_type(&target_ty), bits); + let and = intrinsic_ident("and", coarse_type(&target_ty), bits); + let or = intrinsic_ident("or", coarse_type(&target_ty), bits); + let xor = intrinsic_ident("xor", coarse_type(&target_ty), bits); + let sub = intrinsic_ident("sub", "epi64", bits); + let srli = intrinsic_ident("srli", "epi64", bits); + let srlv = intrinsic_ident("srlv", "epi64", bits); + let sllv = intrinsic_ident("sllv", "epi64", bits); + let cmpgt = intrinsic_ident("cmpgt", "epi64", bits); + let blend = intrinsic_ident("blendv", "epi8", bits); + let cast_to_int = cast_ident(Float, target, 64, 64, bits); + let cast_to_float = cast_ident(target, Float, 64, 64, bits); + let cmpge = float_compare_method("simd_ge", vec_ty); + + match (target, precision) { + (Unsigned, precision) => { + let finish = match precision { + Approx => quote! { converted }, + Precise => quote! { + // Keeping the sign bit in `exponent` makes all negative lanes zero. + // NaN is also already zero; only positive overflow needs correction. + let overflow = #cast_to_int(#cmpge( + a, + #set1_float(18_446_744_073_709_551_616.0), + )); + #or(converted, overflow) + }, + }; + + quote! { + let a = a.into(); + let bits = #cast_to_int(a); + // Deliberately retain the sign bit. For a negative lane it makes both + // variable-shift counts exceed 63, producing zero. + let exponent = #srli::<52>(bits); + let significand = #or( + #and(bits, #set1_int(0x000f_ffff_ffff_ffff)), + #set1_int(0x0010_0000_0000_0000), + ); + let shift_bias = #set1_int(1075); + let right_count = #sub(shift_bias, exponent); + let left_count = #sub(exponent, shift_bias); + let converted = #or( + #srlv(significand, right_count), + #sllv(significand, left_count), + ); + #finish.simd_into(#token) + } + } + (Int, precision) => { + let finish = match precision { + Approx => quote! { converted }, + Precise => quote! { + // At |a| >= 2^63, choose MIN or MAX from the original sign. The + // ordered comparison is false for NaN, whose magnitude is already 0. + let overflow = #cast_to_int(#cmpge( + #cast_to_float(absolute), + #set1_float(9_223_372_036_854_775_808.0), + )); + let bound = #xor(#set1_int(i64::MAX), sign); + #blend(converted, bound, overflow) + }, + }; + + quote! { + let a = a.into(); + let bits = #cast_to_int(a); + let absolute = #and(bits, #set1_int(i64::MAX)); + let exponent = #srli::<52>(absolute); + let significand = #or( + #and(bits, #set1_int(0x000f_ffff_ffff_ffff)), + #set1_int(0x0010_0000_0000_0000), + ); + let shift_bias = #set1_int(1075); + let right_count = #sub(shift_bias, exponent); + let left_count = #sub(exponent, shift_bias); + let magnitude = #or( + #srlv(significand, right_count), + #sllv(significand, left_count), + ); + let sign = #cmpgt(#setzero(), bits); + let converted = #sub(#xor(magnitude, sign), sign); + #finish.simd_into(#token) + } + } + _ => unreachable!(), + } + } ( Self::Sse2 | Self::Sse4_2, Float, diff --git a/fearless_simd_tests/tests/harness/ops/cvt_f64.rs b/fearless_simd_tests/tests/harness/ops/cvt_f64.rs index acc2bf28..fb97fe45 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_f64.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_f64.rs @@ -72,3 +72,47 @@ fn cvt_f64_u64x8(simd: S) { let result = f64x8::float_from(a); assert_eq!(*result, values.map(|x| x as f64)); } + +#[simd_test] +#[ignore = "randomly checks 10 million signed and unsigned 64-bit integers"] +// Run with: cargo test --release cvt_f64_i64_u64_random -- --ignored +fn cvt_f64_i64_u64_random(simd: S) { + simd.vectorize( + #[inline(always)] + || { + let mut rng = fastrand::Rng::with_seed(0x243f_6a88_85a3_08d3); + + for iteration in 0..2_500_000 { + let signed = [rng.i64(..), rng.i64(..), rng.i64(..), rng.i64(..)]; + let expected_signed = signed.map(|value| (value as f64).to_bits()); + let signed_x4 = i64x4::from_slice(simd, &signed).to_float::>(); + assert_eq!( + (*signed_x4).map(f64::to_bits), + expected_signed, + "signed x4 iteration {iteration}", + ); + let signed_x2 = i64x2::from_slice(simd, &signed[..2]).to_float::>(); + assert_eq!( + (*signed_x2).map(f64::to_bits), + [expected_signed[0], expected_signed[1]], + "signed x2 iteration {iteration}", + ); + + let unsigned = [rng.u64(..), rng.u64(..), rng.u64(..), rng.u64(..)]; + let expected_unsigned = unsigned.map(|value| (value as f64).to_bits()); + let unsigned_x4 = u64x4::from_slice(simd, &unsigned).to_float::>(); + assert_eq!( + (*unsigned_x4).map(f64::to_bits), + expected_unsigned, + "unsigned x4 iteration {iteration}", + ); + let unsigned_x2 = u64x2::from_slice(simd, &unsigned[..2]).to_float::>(); + assert_eq!( + (*unsigned_x2).map(f64::to_bits), + [expected_unsigned[0], expected_unsigned[1]], + "unsigned x2 iteration {iteration}", + ); + } + }, + ); +} diff --git a/fearless_simd_tests/tests/harness/ops/cvt_i64.rs b/fearless_simd_tests/tests/harness/ops/cvt_i64.rs index a975b5c4..a2e2317a 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_i64.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_i64.rs @@ -84,3 +84,39 @@ fn cvt_i64_f64x8(simd: S) { let result = i64x8::truncate_from(a); assert_eq!(*result, values.map(|x| x as i64)); } + +#[simd_test] +#[ignore = "randomly checks 10 million in-range f64 bit patterns"] +// Run with: cargo test --release cvt_i64_f64_random -- --ignored +fn cvt_i64_f64_random(simd: S) { + simd.vectorize( + #[inline(always)] + || { + let mut rng = fastrand::Rng::with_seed(0x1319_8a2e_0370_7344); + + for iteration in 0..2_500_000 { + let values: [f64; 4] = core::array::from_fn(|_| { + loop { + let value = f64::from_bits(rng.u64(..)); + if (-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0) + .contains(&value) + { + break value; + } + } + }); + let expected = values.map(|value| value as i64); + + let result_x4 = f64x4::from_slice(simd, &values).to_int::>(); + assert_eq!(*result_x4, expected, "x4 iteration {iteration}"); + + let result_x2 = f64x2::from_slice(simd, &values[..2]).to_int::>(); + assert_eq!( + *result_x2, + [expected[0], expected[1]], + "x2 iteration {iteration}", + ); + } + }, + ); +} diff --git a/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs b/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs index 857a41de..67a6456b 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs @@ -96,3 +96,35 @@ fn cvt_i64_precise_f64x8(simd: S) { let result = i64x8::truncate_from_precise(a); assert_eq!(*result, values.map(|x| x as i64)); } + +#[simd_test] +#[ignore = "randomly checks 10 million arbitrary f64 bit patterns"] +// Run with: cargo test --release cvt_i64_precise_f64_random -- --ignored +fn cvt_i64_precise_f64_random(simd: S) { + simd.vectorize( + #[inline(always)] + || { + let mut rng = fastrand::Rng::with_seed(0xa409_3822_299f_31d0); + + for iteration in 0..2_500_000 { + let values = [ + f64::from_bits(rng.u64(..)), + f64::from_bits(rng.u64(..)), + f64::from_bits(rng.u64(..)), + f64::from_bits(rng.u64(..)), + ]; + let expected = values.map(|value| value as i64); + + let result_x4 = f64x4::from_slice(simd, &values).to_int_precise::>(); + assert_eq!(*result_x4, expected, "x4 iteration {iteration}"); + + let result_x2 = f64x2::from_slice(simd, &values[..2]).to_int_precise::>(); + assert_eq!( + *result_x2, + [expected[0], expected[1]], + "x2 iteration {iteration}", + ); + } + }, + ); +} diff --git a/fearless_simd_tests/tests/harness/ops/cvt_u64.rs b/fearless_simd_tests/tests/harness/ops/cvt_u64.rs index ca3081b5..f09aa2c1 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_u64.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_u64.rs @@ -93,3 +93,37 @@ fn cvt_u64_f64x8(simd: S) { let result = u64x8::truncate_from(a); assert_eq!(*result, values.map(|x| x as u64)); } + +#[simd_test] +#[ignore = "randomly checks 10 million in-range f64 bit patterns"] +// Run with: cargo test --release cvt_u64_f64_random -- --ignored +fn cvt_u64_f64_random(simd: S) { + simd.vectorize( + #[inline(always)] + || { + let mut rng = fastrand::Rng::with_seed(0x082e_fa98_ec4e_6c89); + + for iteration in 0..2_500_000 { + let values: [f64; 4] = core::array::from_fn(|_| { + loop { + let value = f64::from_bits(rng.u64(..)); + if (0.0..18_446_744_073_709_551_616.0).contains(&value) { + break value; + } + } + }); + let expected = values.map(|value| value as u64); + + let result_x4 = f64x4::from_slice(simd, &values).to_int::>(); + assert_eq!(*result_x4, expected, "x4 iteration {iteration}"); + + let result_x2 = f64x2::from_slice(simd, &values[..2]).to_int::>(); + assert_eq!( + *result_x2, + [expected[0], expected[1]], + "x2 iteration {iteration}", + ); + } + }, + ); +} diff --git a/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs b/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs index b56e7a33..051ad725 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs @@ -96,3 +96,35 @@ fn cvt_u64_precise_f64x8(simd: S) { let result = u64x8::truncate_from_precise(a); assert_eq!(*result, values.map(|x| x as u64)); } + +#[simd_test] +#[ignore = "randomly checks 10 million arbitrary f64 bit patterns"] +// Run with: cargo test --release cvt_u64_precise_f64_random -- --ignored +fn cvt_u64_precise_f64_random(simd: S) { + simd.vectorize( + #[inline(always)] + || { + let mut rng = fastrand::Rng::with_seed(0x4528_21e6_38d0_1377); + + for iteration in 0..2_500_000 { + let values = [ + f64::from_bits(rng.u64(..)), + f64::from_bits(rng.u64(..)), + f64::from_bits(rng.u64(..)), + f64::from_bits(rng.u64(..)), + ]; + let expected = values.map(|value| value as u64); + + let result_x4 = f64x4::from_slice(simd, &values).to_int_precise::>(); + assert_eq!(*result_x4, expected, "x4 iteration {iteration}"); + + let result_x2 = f64x2::from_slice(simd, &values[..2]).to_int_precise::>(); + assert_eq!( + *result_x2, + [expected[0], expected[1]], + "x2 iteration {iteration}", + ); + } + }, + ); +} From 116284fec454b3c7bf2c796bef96b2c397590e5f Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Tue, 25 Aug 2026 00:22:21 +0100 Subject: [PATCH 3/4] Add adversarial tests for i64/u64 <-> f64 conversions for interesting bit patterns --- .../tests/harness/ops/cvt_f64.rs | 394 +++++++++++++++++- .../tests/harness/ops/cvt_i64.rs | 130 +++++- .../tests/harness/ops/cvt_i64_precise.rs | 115 ++++- .../tests/harness/ops/cvt_u64.rs | 102 ++++- .../tests/harness/ops/cvt_u64_precise.rs | 115 ++++- 5 files changed, 849 insertions(+), 7 deletions(-) diff --git a/fearless_simd_tests/tests/harness/ops/cvt_f64.rs b/fearless_simd_tests/tests/harness/ops/cvt_f64.rs index fb97fe45..800dab5d 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_f64.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_f64.rs @@ -74,9 +74,323 @@ fn cvt_f64_u64x8(simd: S) { } #[simd_test] -#[ignore = "randomly checks 10 million signed and unsigned 64-bit integers"] +fn cvt_f64_i64_u64_adversarial(simd: S) { + // Keep the oracle in integer arithmetic. In release builds, LLVM can vectorize a scalar + // u64-to-f64 map into the same split/bias algorithm that this test is meant to check. + fn u64_to_f64_bits(value: u64) -> u64 { + if value == 0 { + return 0; + } + + let fraction_mask = (1_u64 << 52) - 1; + let mut exponent = 63 - value.leading_zeros(); + if exponent <= 52 { + let fraction = (value << (52 - exponent)) & fraction_mask; + return ((u64::from(exponent) + 1023) << 52) | fraction; + } + + let shift = exponent - 52; + let mut significand = value >> shift; + let remainder = value & ((1_u64 << shift) - 1); + let halfway = 1_u64 << (shift - 1); + if remainder > halfway || (remainder == halfway && significand & 1 != 0) { + significand += 1; + if significand == 1_u64 << 53 { + significand >>= 1; + exponent += 1; + } + } + + ((u64::from(exponent) + 1023) << 52) | (significand & fraction_mask) + } + + fn i64_to_f64_bits(value: i64) -> u64 { + ((value < 0) as u64) << 63 | u64_to_f64_bits(value.unsigned_abs()) + } + + simd.vectorize( + #[inline(always)] + || { + // These values exercise the 32-bit split, signed high-half bias, cancellation, + // carry between halves, and rounding carries at the ends of the integer ranges. + let unsigned_boundaries = [ + 0, + 1, + 0x0000_0000_7fff_ffff, + 0x0000_0000_8000_0000, + 0x0000_0000_ffff_fffe, + 0x0000_0000_ffff_ffff, + 0x0000_0001_0000_0000, + 0x0000_0001_0000_0001, + 0x0000_0001_ffff_ffff, + 0x7fff_ffff_0000_0001, + 0x8000_0000_ffff_ffff, + 0xffff_ffff_0000_0001, + (1_u64 << 53) - 1, + 1_u64 << 53, + (1_u64 << 53) + 1, + (1_u64 << 53) + 3, + 1_u64 << 63, + (1_u64 << 63) + 1023, + (1_u64 << 63) + 1024, + (1_u64 << 63) + 1025, + u64::MAX - 2047, + u64::MAX - 1024, + u64::MAX - 1023, + u64::MAX - 1022, + u64::MAX - 1, + u64::MAX, + 0x5555_aaaa_ffff_0001, + 0xaaaa_5555_0000_ffff, + ]; + for (group_index, values) in unsigned_boundaries.chunks_exact(4).enumerate() { + let values = [values[0], values[1], values[2], values[3]]; + let expected = values.map(u64_to_f64_bits); + + let result_x4 = u64x4::from_slice(simd, &values).to_float::>(); + assert_eq!( + (*result_x4).map(f64::to_bits), + expected, + "unsigned split/end-point group {group_index}", + ); + + let result_x2 = u64x2::from_slice(simd, &values[..2]).to_float::>(); + assert_eq!( + (*result_x2).map(f64::to_bits), + [expected[0], expected[1]], + "unsigned x2 split/end-point group {group_index}", + ); + } + + let signed_boundaries = [ + i64::MIN, + i64::MIN + 511, + i64::MIN + 512, + i64::MIN + 513, + i64::MIN + 1023, + i64::MIN + 1024, + i64::MAX - 512, + i64::MAX - 511, + i64::MAX - 510, + i64::MAX - 1, + i64::MAX, + -((1_i64 << 53) + 3), + -((1_i64 << 32) + 1), + -(1_i64 << 32), + -((1_i64 << 32) - 1), + -1, + 0, + 1, + (1_i64 << 32) - 1, + 1_i64 << 32, + (1_i64 << 32) + 1, + -((1_i64 << 53) + 1), + (1_i64 << 53) + 1, + (1_i64 << 53) + 3, + 0x0000_0001_ffff_ffff, + 0x7fff_ffff_0000_0001, + 0x5555_aaaa_ffff_0001, + -0x5555_aaaa_ffff_0001, + ]; + for (group_index, values) in signed_boundaries.chunks_exact(4).enumerate() { + let values = [values[0], values[1], values[2], values[3]]; + let expected = values.map(i64_to_f64_bits); + + let result_x4 = i64x4::from_slice(simd, &values).to_float::>(); + assert_eq!( + (*result_x4).map(f64::to_bits), + expected, + "signed split/end-point group {group_index}", + ); + + let result_x2 = i64x2::from_slice(simd, &values[..2]).to_float::>(); + assert_eq!( + (*result_x2).map(f64::to_bits), + [expected[0], expected[1]], + "signed x2 split/end-point group {group_index}", + ); + } + + // Guarantee that the two-lane kernels see the normalization-carry midpoints too; + // their positions in the larger boundary tables are primarily chosen for x4. + let unsigned_x2_carry_values = [u64::MAX - 1023, u64::MAX - 1022]; + let unsigned_x2_carry = + u64x2::from_slice(simd, &unsigned_x2_carry_values).to_float::>(); + assert_eq!( + (*unsigned_x2_carry).map(f64::to_bits), + unsigned_x2_carry_values.map(u64_to_f64_bits), + "unsigned x2 normalization carry", + ); + + let signed_x2_carry_values = [i64::MIN + 512, i64::MAX - 511]; + let signed_x2_carry = + i64x2::from_slice(simd, &signed_x2_carry_values).to_float::>(); + assert_eq!( + (*signed_x2_carry).map(f64::to_bits), + signed_x2_carry_values.map(i64_to_f64_bits), + "signed x2 normalization carries", + ); + + // Sweep every discarded-bit pattern in two adjacent output intervals. The first + // retained significand is even and the second is odd, so this exhaustively covers + // both directions of ties-to-even for every integer-to-f64 spacing from 2 to 2048. + const EVEN_SIGNIFICAND: u64 = 0x0015_5555_aaaa_aaaa; + for shift in 1..=11 { + let spacing = 1_u64 << shift; + let base = EVEN_SIGNIFICAND << shift; + let mut offset = 0; + while offset <= 2 * spacing { + let values: [u64; 4] = + core::array::from_fn(|lane| base + (offset + lane as u64).min(2 * spacing)); + let expected = values.map(u64_to_f64_bits); + + let result_x4 = u64x4::from_slice(simd, &values).to_float::>(); + assert_eq!( + (*result_x4).map(f64::to_bits), + expected, + "unsigned residue sweep shift {shift}, offset {offset}", + ); + let result_x2 = u64x2::from_slice(simd, &values[..2]).to_float::>(); + assert_eq!( + (*result_x2).map(f64::to_bits), + [expected[0], expected[1]], + "unsigned x2 residue sweep shift {shift}, offset {offset}", + ); + let result_x2_high = + u64x2::from_slice(simd, &values[2..]).to_float::>(); + assert_eq!( + (*result_x2_high).map(f64::to_bits), + [expected[2], expected[3]], + "unsigned x2 high residue sweep shift {shift}, offset {offset}", + ); + + if shift <= 10 { + let positive = values.map(|value| value as i64); + let expected_positive = positive.map(i64_to_f64_bits); + let result_positive = + i64x4::from_slice(simd, &positive).to_float::>(); + assert_eq!( + (*result_positive).map(f64::to_bits), + expected_positive, + "positive signed residue sweep shift {shift}, offset {offset}", + ); + let result_positive_x2 = + i64x2::from_slice(simd, &positive[..2]).to_float::>(); + assert_eq!( + (*result_positive_x2).map(f64::to_bits), + [expected_positive[0], expected_positive[1]], + "positive signed x2 residue sweep shift {shift}, offset {offset}", + ); + let result_positive_x2_high = + i64x2::from_slice(simd, &positive[2..]).to_float::>(); + assert_eq!( + (*result_positive_x2_high).map(f64::to_bits), + [expected_positive[2], expected_positive[3]], + "positive signed x2 high residue sweep shift {shift}, offset {offset}", + ); + + let negative = positive.map(|value| -value); + let expected_negative = negative.map(i64_to_f64_bits); + let result_negative = + i64x4::from_slice(simd, &negative).to_float::>(); + assert_eq!( + (*result_negative).map(f64::to_bits), + expected_negative, + "negative signed residue sweep shift {shift}, offset {offset}", + ); + let result_negative_x2 = + i64x2::from_slice(simd, &negative[..2]).to_float::>(); + assert_eq!( + (*result_negative_x2).map(f64::to_bits), + [expected_negative[0], expected_negative[1]], + "negative signed x2 residue sweep shift {shift}, offset {offset}", + ); + let result_negative_x2_high = + i64x2::from_slice(simd, &negative[2..]).to_float::>(); + assert_eq!( + (*result_negative_x2_high).map(f64::to_bits), + [expected_negative[2], expected_negative[3]], + "negative signed x2 high residue sweep shift {shift}, offset {offset}", + ); + } + + offset += 4; + } + } + + let unsigned_x8_values = [ + 0, + 0xffff_ffff, + 0x1_0000_0000, + (1_u64 << 53) + 3, + 1_u64 << 63, + u64::MAX - 1023, + u64::MAX - 1, + u64::MAX, + ]; + let unsigned_x8 = u64x8::from_slice(simd, &unsigned_x8_values).to_float::>(); + assert_eq!( + (*unsigned_x8).map(f64::to_bits), + unsigned_x8_values.map(u64_to_f64_bits), + "unsigned x8 lane order", + ); + + let signed_x8_values = [ + i64::MIN, + i64::MIN + 512, + -((1_i64 << 53) + 3), + -(1_i64 << 32), + (1_i64 << 32) - 1, + (1_i64 << 53) + 3, + i64::MAX - 511, + i64::MAX, + ]; + let signed_x8 = i64x8::from_slice(simd, &signed_x8_values).to_float::>(); + assert_eq!( + (*signed_x8).map(f64::to_bits), + signed_x8_values.map(i64_to_f64_bits), + "signed x8 lane order", + ); + }, + ); +} + +#[simd_test] +#[ignore = "checks uniform integers plus every rounding binade and tie parity"] // Run with: cargo test --release cvt_f64_i64_u64_random -- --ignored fn cvt_f64_i64_u64_random(simd: S) { + // Keep the oracle independent of LLVM's vectorized u64-to-f64 lowering. + fn u64_to_f64_bits(value: u64) -> u64 { + if value == 0 { + return 0; + } + + let fraction_mask = (1_u64 << 52) - 1; + let mut exponent = 63 - value.leading_zeros(); + if exponent <= 52 { + let fraction = (value << (52 - exponent)) & fraction_mask; + return ((u64::from(exponent) + 1023) << 52) | fraction; + } + + let shift = exponent - 52; + let mut significand = value >> shift; + let remainder = value & ((1_u64 << shift) - 1); + let halfway = 1_u64 << (shift - 1); + if remainder > halfway || (remainder == halfway && significand & 1 != 0) { + significand += 1; + if significand == 1_u64 << 53 { + significand >>= 1; + exponent += 1; + } + } + + ((u64::from(exponent) + 1023) << 52) | (significand & fraction_mask) + } + + fn i64_to_f64_bits(value: i64) -> u64 { + ((value < 0) as u64) << 63 | u64_to_f64_bits(value.unsigned_abs()) + } + simd.vectorize( #[inline(always)] || { @@ -84,7 +398,7 @@ fn cvt_f64_i64_u64_random(simd: S) { for iteration in 0..2_500_000 { let signed = [rng.i64(..), rng.i64(..), rng.i64(..), rng.i64(..)]; - let expected_signed = signed.map(|value| (value as f64).to_bits()); + let expected_signed = signed.map(i64_to_f64_bits); let signed_x4 = i64x4::from_slice(simd, &signed).to_float::>(); assert_eq!( (*signed_x4).map(f64::to_bits), @@ -99,7 +413,7 @@ fn cvt_f64_i64_u64_random(simd: S) { ); let unsigned = [rng.u64(..), rng.u64(..), rng.u64(..), rng.u64(..)]; - let expected_unsigned = unsigned.map(|value| (value as f64).to_bits()); + let expected_unsigned = unsigned.map(u64_to_f64_bits); let unsigned_x4 = u64x4::from_slice(simd, &unsigned).to_float::>(); assert_eq!( (*unsigned_x4).map(f64::to_bits), @@ -112,6 +426,80 @@ fn cvt_f64_i64_u64_random(simd: S) { [expected_unsigned[0], expected_unsigned[1]], "unsigned x2 iteration {iteration}", ); + + // Uniform integers mostly land in the largest one or two binades. Construct a + // random retained significand in every inexact binade and force residues around + // both an even and an odd halfway case. + let unsigned_shift = 1 + iteration as u32 % 11; + let unsigned_spacing = 1_u64 << unsigned_shift; + let unsigned_halfway = unsigned_spacing / 2; + let unsigned_significand = + ((1_u64 << 52) | (rng.u64(..) & ((1_u64 << 52) - 1))) & !1; + let unsigned_even = unsigned_significand << unsigned_shift; + let unsigned_odd = (unsigned_significand + 1) << unsigned_shift; + let unsigned_targeted = [ + unsigned_even + unsigned_halfway - 1, + unsigned_even + unsigned_halfway, + unsigned_even + unsigned_halfway + 1, + unsigned_odd + unsigned_halfway, + ]; + let expected_unsigned_targeted = unsigned_targeted.map(u64_to_f64_bits); + let result_unsigned_targeted = + u64x4::from_slice(simd, &unsigned_targeted).to_float::>(); + assert_eq!( + (*result_unsigned_targeted).map(f64::to_bits), + expected_unsigned_targeted, + "targeted unsigned iteration {iteration}, shift {unsigned_shift}", + ); + let result_unsigned_targeted_x2_low = + u64x2::from_slice(simd, &unsigned_targeted[..2]).to_float::>(); + assert_eq!( + (*result_unsigned_targeted_x2_low).map(f64::to_bits), + [expected_unsigned_targeted[0], expected_unsigned_targeted[1]], + "targeted unsigned x2 low iteration {iteration}, shift {unsigned_shift}", + ); + let result_unsigned_targeted_x2_high = + u64x2::from_slice(simd, &unsigned_targeted[2..]).to_float::>(); + assert_eq!( + (*result_unsigned_targeted_x2_high).map(f64::to_bits), + [expected_unsigned_targeted[2], expected_unsigned_targeted[3]], + "targeted unsigned x2 high iteration {iteration}, shift {unsigned_shift}", + ); + + let signed_shift = 1 + iteration as u32 % 10; + let signed_spacing = 1_u64 << signed_shift; + let signed_halfway = signed_spacing / 2; + let signed_significand = ((1_u64 << 52) | (rng.u64(..) & ((1_u64 << 52) - 1))) & !1; + let signed_even = (signed_significand << signed_shift) + signed_halfway; + let signed_odd = ((signed_significand + 1) << signed_shift) + signed_halfway; + let signed_targeted = [ + signed_even as i64, + -(signed_even as i64), + signed_odd as i64, + -(signed_odd as i64), + ]; + let expected_signed_targeted = signed_targeted.map(i64_to_f64_bits); + let result_signed_targeted = + i64x4::from_slice(simd, &signed_targeted).to_float::>(); + assert_eq!( + (*result_signed_targeted).map(f64::to_bits), + expected_signed_targeted, + "targeted signed iteration {iteration}, shift {signed_shift}", + ); + let result_signed_targeted_x2_low = + i64x2::from_slice(simd, &signed_targeted[..2]).to_float::>(); + assert_eq!( + (*result_signed_targeted_x2_low).map(f64::to_bits), + [expected_signed_targeted[0], expected_signed_targeted[1]], + "targeted signed x2 even iteration {iteration}, shift {signed_shift}", + ); + let result_signed_targeted_x2_high = + i64x2::from_slice(simd, &signed_targeted[2..]).to_float::>(); + assert_eq!( + (*result_signed_targeted_x2_high).map(f64::to_bits), + [expected_signed_targeted[2], expected_signed_targeted[3]], + "targeted signed x2 odd iteration {iteration}, shift {signed_shift}", + ); } }, ); diff --git a/fearless_simd_tests/tests/harness/ops/cvt_i64.rs b/fearless_simd_tests/tests/harness/ops/cvt_i64.rs index a2e2317a..d619e228 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_i64.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_i64.rs @@ -86,7 +86,112 @@ fn cvt_i64_f64x8(simd: S) { } #[simd_test] -#[ignore = "randomly checks 10 million in-range f64 bit patterns"] +fn cvt_i64_f64_exponent_and_significand_boundaries(simd: S) { + simd.vectorize( + #[inline(always)] + || { + const FRACTIONS: [u64; 4] = [0, 1, 0x0005_5555_aaaa_aaaa, 0x000f_ffff_ffff_ffff]; + + // Every exponent field whose positive and negative values are both in range. This + // covers both variable-shift directions, including counts 64/63, the count-zero + // pivot at exponent field 1075, and every left shift through the signed limit. + for exponent in 0_u64..=1085 { + let positive = FRACTIONS.map(|fraction| f64::from_bits(exponent << 52 | fraction)); + let expected_positive = positive.map(|value| value as i64); + let result_positive = f64x4::from_slice(simd, &positive).to_int::>(); + assert_eq!( + *result_positive, expected_positive, + "positive exponent field {exponent}", + ); + let result_positive_x2 = + f64x2::from_slice(simd, &positive[..2]).to_int::>(); + assert_eq!( + *result_positive_x2, + [expected_positive[0], expected_positive[1]], + "positive x2 exponent field {exponent}", + ); + let result_positive_x2_high = + f64x2::from_slice(simd, &positive[2..]).to_int::>(); + assert_eq!( + *result_positive_x2_high, + [expected_positive[2], expected_positive[3]], + "positive x2 high fractions, exponent field {exponent}", + ); + + let negative = FRACTIONS + .map(|fraction| f64::from_bits(1_u64 << 63 | exponent << 52 | fraction)); + let expected_negative = negative.map(|value| value as i64); + let result_negative = f64x4::from_slice(simd, &negative).to_int::>(); + assert_eq!( + *result_negative, expected_negative, + "negative exponent field {exponent}", + ); + let result_negative_x2 = + f64x2::from_slice(simd, &negative[..2]).to_int::>(); + assert_eq!( + *result_negative_x2, + [expected_negative[0], expected_negative[1]], + "negative x2 exponent field {exponent}", + ); + let result_negative_x2_high = + f64x2::from_slice(simd, &negative[2..]).to_int::>(); + assert_eq!( + *result_negative_x2_high, + [expected_negative[2], expected_negative[3]], + "negative x2 high fractions, exponent field {exponent}", + ); + } + + // Put the lowest fraction bit that survives truncation immediately before and after + // its transition. This catches off-by-one shift counts that broad random bits hide. + for exponent in 1024_u64..=1074 { + let shift = 1075 - exponent; + let retained_bit = 1_u64 << shift; + let values = [ + f64::from_bits((exponent << 52) | (retained_bit - 1)), + f64::from_bits(exponent << 52 | retained_bit), + f64::from_bits(1_u64 << 63 | exponent << 52 | (retained_bit - 1)), + f64::from_bits(1_u64 << 63 | exponent << 52 | retained_bit), + ]; + let expected = values.map(|value| value as i64); + let result = f64x4::from_slice(simd, &values).to_int::>(); + assert_eq!( + *result, expected, + "retained-bit transition at exponent field {exponent}, shift {shift}", + ); + let result_positive_x2 = f64x2::from_slice(simd, &values[..2]).to_int::>(); + assert_eq!( + *result_positive_x2, + [expected[0], expected[1]], + "positive x2 retained-bit transition at exponent field {exponent}", + ); + let result_negative_x2 = f64x2::from_slice(simd, &values[2..]).to_int::>(); + assert_eq!( + *result_negative_x2, + [expected[2], expected[3]], + "negative x2 retained-bit transition at exponent field {exponent}", + ); + } + + let x8_bits = [ + 0x0000_0000_0000_0001, + 0x8000_0000_0000_0001, + 0x3fef_ffff_ffff_ffff, + 0xbfef_ffff_ffff_ffff, + 0x3ff0_0000_0000_0000, + 0xbff0_0000_0000_0000, + 0x43df_ffff_ffff_ffff, + 0xc3e0_0000_0000_0000, + ]; + let x8_values = x8_bits.map(f64::from_bits); + let x8_result = f64x8::from_slice(simd, &x8_values).to_int::>(); + assert_eq!(*x8_result, x8_values.map(|value| value as i64)); + }, + ); +} + +#[simd_test] +#[ignore = "checks raw-bit and exponent-stratified in-range f64 values"] // Run with: cargo test --release cvt_i64_f64_random -- --ignored fn cvt_i64_f64_random(simd: S) { simd.vectorize( @@ -116,6 +221,29 @@ fn cvt_i64_f64_random(simd: S) { [expected[0], expected[1]], "x2 iteration {iteration}", ); + + // Accepted uniform bit patterns overwhelmingly have magnitude below one. Cycle + // through every exponent that can affect a nonzero in-range result, while still + // randomizing the full significand and sign. + let stratified: [f64; 4] = core::array::from_fn(|lane| { + let exponent = 1011 + (iteration * 4 + lane) as u64 % 75; + let sign = if rng.bool() { 1_u64 << 63 } else { 0 }; + let fraction = rng.u64(..) & 0x000f_ffff_ffff_ffff; + f64::from_bits(sign | exponent << 52 | fraction) + }); + let expected_stratified = stratified.map(|value| value as i64); + let result_stratified = f64x4::from_slice(simd, &stratified).to_int::>(); + assert_eq!( + *result_stratified, expected_stratified, + "stratified x4 iteration {iteration}", + ); + let result_stratified_x2 = + f64x2::from_slice(simd, &stratified[..2]).to_int::>(); + assert_eq!( + *result_stratified_x2, + [expected_stratified[0], expected_stratified[1]], + "stratified x2 iteration {iteration}", + ); } }, ); diff --git a/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs b/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs index 67a6456b..c6241ad4 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_i64_precise.rs @@ -98,12 +98,103 @@ fn cvt_i64_precise_f64x8(simd: S) { } #[simd_test] -#[ignore = "randomly checks 10 million arbitrary f64 bit patterns"] +fn cvt_i64_precise_f64_all_exponents_and_special_values(simd: S) { + simd.vectorize( + #[inline(always)] + || { + const FRACTIONS: [u64; 4] = [0, 1, 0x0005_5555_aaaa_aaaa, 0x000f_ffff_ffff_ffff]; + + // Cover every exponent with zero, minimal, alternating, and maximal fraction fields, + // for both signs. Besides every reconstruction shift, exponent 2047 supplies both + // infinities and signaling/quiet NaNs with different payloads. + for exponent in 0_u64..=2047 { + let positive = FRACTIONS.map(|fraction| f64::from_bits(exponent << 52 | fraction)); + let expected_positive = positive.map(|value| value as i64); + let result_positive = + f64x4::from_slice(simd, &positive).to_int_precise::>(); + assert_eq!( + *result_positive, expected_positive, + "positive exponent field {exponent}", + ); + let result_positive_x2 = + f64x2::from_slice(simd, &positive[..2]).to_int_precise::>(); + assert_eq!( + *result_positive_x2, + [expected_positive[0], expected_positive[1]], + "positive x2 exponent field {exponent}", + ); + let result_positive_x2_high = + f64x2::from_slice(simd, &positive[2..]).to_int_precise::>(); + assert_eq!( + *result_positive_x2_high, + [expected_positive[2], expected_positive[3]], + "positive x2 high fractions, exponent field {exponent}", + ); + + let negative = FRACTIONS + .map(|fraction| f64::from_bits(1_u64 << 63 | exponent << 52 | fraction)); + let expected_negative = negative.map(|value| value as i64); + let result_negative = + f64x4::from_slice(simd, &negative).to_int_precise::>(); + assert_eq!( + *result_negative, expected_negative, + "negative exponent field {exponent}", + ); + let result_negative_x2 = + f64x2::from_slice(simd, &negative[..2]).to_int_precise::>(); + assert_eq!( + *result_negative_x2, + [expected_negative[0], expected_negative[1]], + "negative x2 exponent field {exponent}", + ); + let result_negative_x2_high = + f64x2::from_slice(simd, &negative[2..]).to_int_precise::>(); + assert_eq!( + *result_negative_x2_high, + [expected_negative[2], expected_negative[3]], + "negative x2 high fractions, exponent field {exponent}", + ); + } + + let boundary_bits = [ + 0xc3e0_0000_0000_0001, + 0xc3e0_0000_0000_0000, + 0xc3df_ffff_ffff_ffff, + 0xbff0_0000_0000_0000, + 0x43df_ffff_ffff_ffff, + 0x43e0_0000_0000_0000, + 0x43e0_0000_0000_0001, + 0x7fef_ffff_ffff_ffff, + 0xfff0_0000_0000_0000, + 0x7ff0_0000_0000_0000, + 0x7ff0_0000_0000_0001, + 0xfff0_0000_0000_0001, + 0x7ff8_0000_0000_0000, + 0xfff8_0000_0000_0000, + 0x7fff_ffff_ffff_ffff, + 0xffff_ffff_ffff_ffff, + ]; + for (group_index, bits) in boundary_bits.chunks_exact(8).enumerate() { + let values: [f64; 8] = core::array::from_fn(|lane| f64::from_bits(bits[lane])); + let expected = values.map(|value| value as i64); + let result = f64x8::from_slice(simd, &values).to_int_precise::>(); + assert_eq!(*result, expected, "boundary/special x8 group {group_index}"); + } + }, + ); +} + +#[simd_test] +#[ignore = "checks arbitrary bits plus randomized critical exponent fields"] // Run with: cargo test --release cvt_i64_precise_f64_random -- --ignored fn cvt_i64_precise_f64_random(simd: S) { simd.vectorize( #[inline(always)] || { + const CRITICAL_EXPONENTS: [u64; 16] = [ + 0, 1, 1011, 1012, 1022, 1023, 1024, 1074, 1075, 1076, 1085, 1086, 1087, 1088, 2046, + 2047, + ]; let mut rng = fastrand::Rng::with_seed(0xa409_3822_299f_31d0); for iteration in 0..2_500_000 { @@ -124,6 +215,28 @@ fn cvt_i64_precise_f64_random(simd: S) { [expected[0], expected[1]], "x2 iteration {iteration}", ); + + let targeted: [f64; 4] = core::array::from_fn(|lane| { + let exponent = + CRITICAL_EXPONENTS[(iteration * 5 + lane) % CRITICAL_EXPONENTS.len()]; + let sign = if rng.bool() { 1_u64 << 63 } else { 0 }; + let fraction = rng.u64(..) & 0x000f_ffff_ffff_ffff; + f64::from_bits(sign | exponent << 52 | fraction) + }); + let expected_targeted = targeted.map(|value| value as i64); + let result_targeted = + f64x4::from_slice(simd, &targeted).to_int_precise::>(); + assert_eq!( + *result_targeted, expected_targeted, + "targeted x4 iteration {iteration}", + ); + let result_targeted_x2 = + f64x2::from_slice(simd, &targeted[..2]).to_int_precise::>(); + assert_eq!( + *result_targeted_x2, + [expected_targeted[0], expected_targeted[1]], + "targeted x2 iteration {iteration}", + ); } }, ); diff --git a/fearless_simd_tests/tests/harness/ops/cvt_u64.rs b/fearless_simd_tests/tests/harness/ops/cvt_u64.rs index f09aa2c1..f0272e03 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_u64.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_u64.rs @@ -95,7 +95,85 @@ fn cvt_u64_f64x8(simd: S) { } #[simd_test] -#[ignore = "randomly checks 10 million in-range f64 bit patterns"] +fn cvt_u64_f64_exponent_and_significand_boundaries(simd: S) { + simd.vectorize( + #[inline(always)] + || { + const FRACTIONS: [u64; 4] = [0, 1, 0x0005_5555_aaaa_aaaa, 0x000f_ffff_ffff_ffff]; + + // Sweep every nonnegative exponent field that is still below 2^64. In particular, + // this covers variable shift counts 64/63, the count-zero pivot at 2^52, and every + // left shift up through the unsigned-only [2^63, 2^64) interval. + for exponent in 0_u64..=1086 { + let values = FRACTIONS.map(|fraction| f64::from_bits(exponent << 52 | fraction)); + let expected = values.map(|value| value as u64); + let result_x4 = f64x4::from_slice(simd, &values).to_int::>(); + assert_eq!(*result_x4, expected, "exponent field {exponent}"); + + let result_x2 = f64x2::from_slice(simd, &values[..2]).to_int::>(); + assert_eq!( + *result_x2, + [expected[0], expected[1]], + "x2 exponent field {exponent}", + ); + let result_x2_high = f64x2::from_slice(simd, &values[2..]).to_int::>(); + assert_eq!( + *result_x2_high, + [expected[2], expected[3]], + "x2 high fractions, exponent field {exponent}", + ); + } + + // Put the lowest fraction bit that survives truncation immediately before and after + // its transition. This catches off-by-one shift counts that broad random bits hide. + for exponent in 1024_u64..=1074 { + let shift = 1075 - exponent; + let retained_bit = 1_u64 << shift; + let values = [ + f64::from_bits((exponent << 52) | (retained_bit - 1)), + f64::from_bits(exponent << 52 | retained_bit), + f64::from_bits(exponent << 52 | retained_bit | 1), + f64::from_bits(exponent << 52 | 0x000f_ffff_ffff_ffff), + ]; + let expected = values.map(|value| value as u64); + let result = f64x4::from_slice(simd, &values).to_int::>(); + assert_eq!( + *result, expected, + "retained-bit transition at exponent field {exponent}, shift {shift}", + ); + let result_x2_low = f64x2::from_slice(simd, &values[..2]).to_int::>(); + assert_eq!( + *result_x2_low, + [expected[0], expected[1]], + "x2 low retained-bit transition at exponent field {exponent}", + ); + let result_x2_high = f64x2::from_slice(simd, &values[2..]).to_int::>(); + assert_eq!( + *result_x2_high, + [expected[2], expected[3]], + "x2 high retained-bit transition at exponent field {exponent}", + ); + } + + let x8_bits = [ + 0x0000_0000_0000_0001, + 0x000f_ffff_ffff_ffff, + 0x3fef_ffff_ffff_ffff, + 0x3ff0_0000_0000_0000, + 0x432f_ffff_ffff_ffff, + 0x4330_0000_0000_0000, + 0x43df_ffff_ffff_ffff, + 0x43ef_ffff_ffff_ffff, + ]; + let x8_values = x8_bits.map(f64::from_bits); + let x8_result = f64x8::from_slice(simd, &x8_values).to_int::>(); + assert_eq!(*x8_result, x8_values.map(|value| value as u64)); + }, + ); +} + +#[simd_test] +#[ignore = "checks raw-bit and exponent-stratified in-range f64 values"] // Run with: cargo test --release cvt_u64_f64_random -- --ignored fn cvt_u64_f64_random(simd: S) { simd.vectorize( @@ -123,6 +201,28 @@ fn cvt_u64_f64_random(simd: S) { [expected[0], expected[1]], "x2 iteration {iteration}", ); + + // Accepted uniform bit patterns overwhelmingly have magnitude below one. Cycle + // through every exponent that can affect a nonzero in-range result, while still + // randomizing the full significand. + let stratified: [f64; 4] = core::array::from_fn(|lane| { + let exponent = 1011 + (iteration * 5 + lane) as u64 % 76; + let fraction = rng.u64(..) & 0x000f_ffff_ffff_ffff; + f64::from_bits(exponent << 52 | fraction) + }); + let expected_stratified = stratified.map(|value| value as u64); + let result_stratified = f64x4::from_slice(simd, &stratified).to_int::>(); + assert_eq!( + *result_stratified, expected_stratified, + "stratified x4 iteration {iteration}", + ); + let result_stratified_x2 = + f64x2::from_slice(simd, &stratified[..2]).to_int::>(); + assert_eq!( + *result_stratified_x2, + [expected_stratified[0], expected_stratified[1]], + "stratified x2 iteration {iteration}", + ); } }, ); diff --git a/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs b/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs index 051ad725..fbb4fce8 100644 --- a/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs +++ b/fearless_simd_tests/tests/harness/ops/cvt_u64_precise.rs @@ -98,12 +98,103 @@ fn cvt_u64_precise_f64x8(simd: S) { } #[simd_test] -#[ignore = "randomly checks 10 million arbitrary f64 bit patterns"] +fn cvt_u64_precise_f64_all_exponents_and_special_values(simd: S) { + simd.vectorize( + #[inline(always)] + || { + const FRACTIONS: [u64; 4] = [0, 1, 0x0005_5555_aaaa_aaaa, 0x000f_ffff_ffff_ffff]; + + // Cover every exponent with zero, minimal, alternating, and maximal fraction fields, + // for both signs. Besides every reconstruction shift, exponent 2047 supplies both + // infinities and signaling/quiet NaNs with different payloads. + for exponent in 0_u64..=2047 { + let positive = FRACTIONS.map(|fraction| f64::from_bits(exponent << 52 | fraction)); + let expected_positive = positive.map(|value| value as u64); + let result_positive = + f64x4::from_slice(simd, &positive).to_int_precise::>(); + assert_eq!( + *result_positive, expected_positive, + "positive exponent field {exponent}", + ); + let result_positive_x2 = + f64x2::from_slice(simd, &positive[..2]).to_int_precise::>(); + assert_eq!( + *result_positive_x2, + [expected_positive[0], expected_positive[1]], + "positive x2 exponent field {exponent}", + ); + let result_positive_x2_high = + f64x2::from_slice(simd, &positive[2..]).to_int_precise::>(); + assert_eq!( + *result_positive_x2_high, + [expected_positive[2], expected_positive[3]], + "positive x2 high fractions, exponent field {exponent}", + ); + + let negative = FRACTIONS + .map(|fraction| f64::from_bits(1_u64 << 63 | exponent << 52 | fraction)); + let expected_negative = negative.map(|value| value as u64); + let result_negative = + f64x4::from_slice(simd, &negative).to_int_precise::>(); + assert_eq!( + *result_negative, expected_negative, + "negative exponent field {exponent}", + ); + let result_negative_x2 = + f64x2::from_slice(simd, &negative[..2]).to_int_precise::>(); + assert_eq!( + *result_negative_x2, + [expected_negative[0], expected_negative[1]], + "negative x2 exponent field {exponent}", + ); + let result_negative_x2_high = + f64x2::from_slice(simd, &negative[2..]).to_int_precise::>(); + assert_eq!( + *result_negative_x2_high, + [expected_negative[2], expected_negative[3]], + "negative x2 high fractions, exponent field {exponent}", + ); + } + + let boundary_bits = [ + 0x8000_0000_0000_0001, + 0xbff0_0000_0000_0000, + 0x0000_0000_0000_0000, + 0x3ff0_0000_0000_0001, + 0x43ef_ffff_ffff_ffff, + 0x43f0_0000_0000_0000, + 0x43f0_0000_0000_0001, + 0x7fef_ffff_ffff_ffff, + 0xfff0_0000_0000_0000, + 0x7ff0_0000_0000_0000, + 0x7ff0_0000_0000_0001, + 0xfff0_0000_0000_0001, + 0x7ff8_0000_0000_0000, + 0xfff8_0000_0000_0000, + 0x7fff_ffff_ffff_ffff, + 0xffff_ffff_ffff_ffff, + ]; + for (group_index, bits) in boundary_bits.chunks_exact(8).enumerate() { + let values: [f64; 8] = core::array::from_fn(|lane| f64::from_bits(bits[lane])); + let expected = values.map(|value| value as u64); + let result = f64x8::from_slice(simd, &values).to_int_precise::>(); + assert_eq!(*result, expected, "boundary/special x8 group {group_index}"); + } + }, + ); +} + +#[simd_test] +#[ignore = "checks arbitrary bits plus randomized critical exponent fields"] // Run with: cargo test --release cvt_u64_precise_f64_random -- --ignored fn cvt_u64_precise_f64_random(simd: S) { simd.vectorize( #[inline(always)] || { + const CRITICAL_EXPONENTS: [u64; 16] = [ + 0, 1, 1011, 1012, 1022, 1023, 1024, 1074, 1075, 1076, 1085, 1086, 1087, 1088, 2046, + 2047, + ]; let mut rng = fastrand::Rng::with_seed(0x4528_21e6_38d0_1377); for iteration in 0..2_500_000 { @@ -124,6 +215,28 @@ fn cvt_u64_precise_f64_random(simd: S) { [expected[0], expected[1]], "x2 iteration {iteration}", ); + + let targeted: [f64; 4] = core::array::from_fn(|lane| { + let exponent = + CRITICAL_EXPONENTS[(iteration * 5 + lane) % CRITICAL_EXPONENTS.len()]; + let sign = if rng.bool() { 1_u64 << 63 } else { 0 }; + let fraction = rng.u64(..) & 0x000f_ffff_ffff_ffff; + f64::from_bits(sign | exponent << 52 | fraction) + }); + let expected_targeted = targeted.map(|value| value as u64); + let result_targeted = + f64x4::from_slice(simd, &targeted).to_int_precise::>(); + assert_eq!( + *result_targeted, expected_targeted, + "targeted x4 iteration {iteration}", + ); + let result_targeted_x2 = + f64x2::from_slice(simd, &targeted[..2]).to_int_precise::>(); + assert_eq!( + *result_targeted_x2, + [expected_targeted[0], expected_targeted[1]], + "targeted x2 iteration {iteration}", + ); } }, ); From 91509103cab39ce3f460f35f6c26ce3f7b8abfd0 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Tue, 25 Aug 2026 00:35:05 +0100 Subject: [PATCH 4/4] Fix typo --- fearless_simd_gen/src/mk_x86.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index be8bfa90..1369aa75 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -3883,7 +3883,7 @@ impl X86 { } (Self::Sse4_2, source @ (Int | Unsigned), Float, 64, 128, _) | (Self::Avx2, source @ (Int | Unsigned), Float, 64, 128 | 256, _) => { - // Befire AVX-512 there's no packed 64-bit integer -> f64 instruction. + // Before AVX-512 there's no packed 64-bit integer -> f64 instruction. // Split each integer into 32-bit halves, embed those halves in the fraction fields // of exact powers of two, then remove the biases. This is LLVM's u64 expansion, // extended to signed values by biasing the high half around i32::MIN.