From 77054180927f62f113c5484c02b1f84a5040a6fd Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 5 Sep 2026 20:25:25 +0200 Subject: [PATCH 1/3] add regression tests for vector shift codegen issues --- tests/assembly-llvm/x86-vendor-intrinsics.rs | 63 ++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/assembly-llvm/x86-vendor-intrinsics.rs b/tests/assembly-llvm/x86-vendor-intrinsics.rs index b600b74c66ecd..ba9502d045d20 100644 --- a/tests/assembly-llvm/x86-vendor-intrinsics.rs +++ b/tests/assembly-llvm/x86-vendor-intrinsics.rs @@ -19,3 +19,66 @@ extern "C" fn test_packus_epi16(a: __m128i, b: __m128i) -> __m128i { // CHECK-NEXT: ret _mm_packus_epi16(a, b) } + +// See for context. +// CHECK-LABEL: shlv_u16x16: +#[unsafe(no_mangle)] +#[target_feature(enable = "avx2")] +extern "C" fn shlv_u16x16(a: __m256i, count: __m256i) -> __m256i { + // CHECK: .cfi_startproc + // CHECK-NOT: vpcmp + // CHECK-NOT: vmov + // CHECK: vpsllvw + // CHECK-NOT: vpcmp + // CHECK-NOT: vmov + // CHECK: ret + let low_words = _mm256_set1_epi32(0x0000_ffff); + let low_count = _mm256_and_si256(count, low_words); + let high_count = _mm256_srli_epi32::<16>(count); + + let high_values = _mm256_andnot_si256(low_words, a); + let low_shifted = _mm256_sllv_epi32(a, low_count); + let high_shifted = _mm256_sllv_epi32(high_values, high_count); + let low_shifted = _mm256_and_si256(low_shifted, low_words); + + _mm256_or_si256(low_shifted, high_shifted) +} + +// See for context. +// CHECK-LABEL: shrv_u16x16: +#[unsafe(no_mangle)] +#[target_feature(enable = "avx2")] +extern "C" fn shrv_u16x16(a: __m256i, count: __m256i) -> __m256i { + // CHECK: .cfi_startproc + // CHECK-NOT: vpcmp + // CHECK-NOT: vmov + // CHECK: vpsrlvw + // CHECK-NOT: vpcmp + // CHECK-NOT: vmov + // CHECK: ret + let low_words = _mm256_set1_epi32(0x0000_ffff); + let low_count = _mm256_and_si256(count, low_words); + let high_count = _mm256_srli_epi32::<16>(count); + + let low_values = _mm256_and_si256(a, low_words); + let low_shifted = _mm256_srlv_epi32(low_values, low_count); + let high_shifted = _mm256_srlv_epi32(a, high_count); + let high_shifted = _mm256_andnot_si256(low_words, high_shifted); + + _mm256_or_si256(low_shifted, high_shifted) +} + +// See for context. +// CHECK-LABEL: test_sllv_srlv: +#[unsafe(no_mangle)] +#[target_feature(enable = "avx512bw")] +extern "C" fn test_sllv_srlv(win: __m512i, x: __m512i, v: __m512i) -> __m512i { + // CHECK: .cfi_startproc + // CHECK-NEXT: vpsllvw + // CHECK-NEXT: vporq + // CHECK-NEXT: vpandd + // CHECK-NEXT: vpsrlvw + // CHECK-NEXT: ret + let w = _mm512_or_si512(win, _mm512_sllv_epi16(x, v)); + _mm512_srlv_epi16(w, _mm512_and_si512(w, _mm512_set1_epi16(7))) +} From 1e34c4299f6c1d351c0d1531bd4439813e7caba6 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 5 Sep 2026 20:42:26 +0200 Subject: [PATCH 2/3] add regression test for _mm_mulhi_epu16 issue --- tests/assembly-llvm/x86-vendor-intrinsics.rs | 40 ++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/assembly-llvm/x86-vendor-intrinsics.rs b/tests/assembly-llvm/x86-vendor-intrinsics.rs index ba9502d045d20..41e333fef6cc3 100644 --- a/tests/assembly-llvm/x86-vendor-intrinsics.rs +++ b/tests/assembly-llvm/x86-vendor-intrinsics.rs @@ -82,3 +82,43 @@ extern "C" fn test_sllv_srlv(win: __m512i, x: __m512i, v: __m512i) -> __m512i { let w = _mm512_or_si512(win, _mm512_sllv_epi16(x, v)); _mm512_srlv_epi16(w, _mm512_and_si512(w, _mm512_set1_epi16(7))) } + +// See for context. +// CHECK-LABEL: test_mulhi_loop: +#[no_mangle] +#[target_feature(enable = "sse2")] +extern "C" fn test_mulhi_loop(buf: &mut [__m128i], factor: i16) { + // CHECK: .cfi_startproc + let factor = _mm_set1_epi16(factor); + // CHECK-NOT: pmuludq + // CHECK: pmulhuw + // CHECK-NOT: pmuludq + for i in 0..buf.len() { + buf[i] = _mm_mulhi_epu16(buf[i], factor); + } + // CHECK: ret +} + +// CHECK-LABEL: mul_and_shift: +#[no_mangle] +#[target_feature(enable = "sse2")] +extern "C" fn mul_and_shift(a: __m128i, b: __m128i) -> __m128i { + // CHECK: .cfi_startproc + // CHECK-NEXT: pmulhuw + // CHECK-NEXT: psrlw + // CHECK-NEXT: ret + unsafe { _mm_srli_epi16(_mm_mulhi_epu16(a, b), 1) } +} + +// See for context. +// CHECK-LABEL: test_mulhi: +#[no_mangle] +#[target_feature(enable = "avx2")] +extern "C" fn test_mulhi(a: __m256i) -> __m256i { + // CHECK: .cfi_startproc + // CHECK-NEXT: vpand + // CHECK-NEXT: vpmulhw + // CHECK-NEXT: ret + let a = _mm256_and_si256(a, _mm256_set1_epi16(0x7FFF)); + _mm256_mulhi_epi16(a, _mm256_set1_epi16(1000)) +} From 50511f78001989f554a210eb1198b048532d10f5 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Tue, 29 Sep 2026 08:16:10 +0200 Subject: [PATCH 3/3] add regression test for _mm256_avg_epu8 issue --- tests/assembly-llvm/x86-vendor-intrinsics.rs | 50 ++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/assembly-llvm/x86-vendor-intrinsics.rs b/tests/assembly-llvm/x86-vendor-intrinsics.rs index 41e333fef6cc3..4e6af3a3a97c2 100644 --- a/tests/assembly-llvm/x86-vendor-intrinsics.rs +++ b/tests/assembly-llvm/x86-vendor-intrinsics.rs @@ -122,3 +122,53 @@ extern "C" fn test_mulhi(a: __m256i) -> __m256i { let a = _mm256_and_si256(a, _mm256_set1_epi16(0x7FFF)); _mm256_mulhi_epi16(a, _mm256_set1_epi16(1000)) } + +// See for context. +// CHECK-LABEL: test_avg_epu8: +#[no_mangle] +#[target_feature(enable = "avx2")] +pub unsafe fn test_avg_epu8( + x: __m256i, + ch: __m256i, + ct: __m256i, + dh: __m256i, + dt: __m256i, +) -> Result<__m256i, ()> { + // CHECK: .cfi_startproc + let shr3 = _mm256_srli_epi32::<3>(x); + + // Things get moved around as `h2` is only needed if we don't return `Err`. + // There is an early return path in bewteen the two `_mm256_avg_epu8` calls. + // But we should still see `vpavgb` for both of them. + // CHECK: vpavgb + // CHECK: ret + let h1 = _mm256_avg_epu8(shr3, _mm256_shuffle_epi8(ch, x)); + // CHECK: vpavgb + // CHECK: ret + let h2 = _mm256_avg_epu8(shr3, _mm256_shuffle_epi8(dh, x)); + + let o1 = _mm256_shuffle_epi8(ct, h1); + let o2 = _mm256_shuffle_epi8(dt, h2); + + let c1 = _mm256_adds_epi8(x, o1); + let c2 = _mm256_add_epi8(x, o2); + + if _mm256_movemask_epi8(c1) != 0 { + return Err(()); + } + + Ok(c2) +} + +// See for context. +// CHECK-LABEL: test_movemask_avg_epu8: +#[no_mangle] +#[target_feature(enable = "avx2")] +pub unsafe extern "C" fn test_movemask_avg_epu8(x: __m256i, y: __m256i) -> i32 { + // CHECK: .cfi_startproc + // CHECK-NEXT: vpavgb + // CHECK-NEXT: vpmovmskb + // CHECK-NEXT: vzeroupper + // CHECK-NEXT: ret + _mm256_movemask_epi8(_mm256_avg_epu8(x, y)) +}