diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index c0a1208a8c7d0..d21129e75f514 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -2336,7 +2336,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } // Unary integer intrinsics - if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz) { + if matches!( + name, + sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop | sym::simd_cttz + ) { let vec_ty = bx.cx.type_vector( match *in_elem.kind() { ty::Int(i) => bx.cx.type_int_from_ty(i), @@ -2354,31 +2357,38 @@ fn generic_simd_intrinsic<'ll, 'tcx>( sym::simd_bswap => "bswap", sym::simd_bitreverse => "bitreverse", sym::simd_ctlz => "ctlz", + sym::simd_ctpop => "ctpop", sym::simd_cttz => "cttz", _ => unreachable!(), }; let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits(); let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,); - return if name == sym::simd_bswap && int_size == 8 { + return match name { // byte swap is no-op for i8/u8 - Ok(args[0].immediate()) - } else if matches!(name, sym::simd_ctlz | sym::simd_cttz) { - let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty); - let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - Ok(bx.call( - fn_ty, - None, - None, - f, - &[args[0].immediate(), bx.const_int(bx.type_i1(), 0)], - None, - None, - )) - } else { - let fn_ty = bx.type_func(&[vec_ty], vec_ty); - let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None)) + sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()), + sym::simd_ctlz | sym::simd_cttz => { + // this fun bonus i1 arg means "poison if the arg vector contains zero" + let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + Ok(bx.call( + fn_ty, + None, + None, + f, + // simd_ctlz and simd_cttz are exposed to safe code, so let's not poison anything + &[args[0].immediate(), bx.const_int(bx.type_i1(), 0)], + None, + None, + )) + } + sym::simd_bswap | simd::simd_bitreverse | simd::simd_ctpop => { + // simple unary argument cases + let fn_ty = bx.type_func(&[vec_ty], vec_ty); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None)) + } + _ => unreachable!(), }; } diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index d829e720d9e18..50fe20346cf45 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -607,6 +607,7 @@ pub fn check_intrinsic_type( | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz + | sym::simd_ctpop | sym::simd_fsqrt | sym::simd_fsin | sym::simd_fcos diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 68b1b32baf2dc..203d24185c631 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1681,6 +1681,7 @@ symbols! { simd_cast_ptr, simd_ceil, simd_ctlz, + simd_ctpop, simd_cttz, simd_div, simd_eq, diff --git a/library/core/src/intrinsics/simd.rs b/library/core/src/intrinsics/simd.rs index ceea67901294a..d1be534eaf083 100644 --- a/library/core/src/intrinsics/simd.rs +++ b/library/core/src/intrinsics/simd.rs @@ -569,6 +569,13 @@ extern "rust-intrinsic" { #[rustc_nounwind] pub fn simd_ctlz(x: T) -> T; + /// Count the number of ones in each element. + /// + /// `T` must be a vector of integers. + #[rustc_nounwind] + #[cfg(not(bootstrap))] + pub fn simd_ctpop(x: T) -> T; + /// Count the trailing zeros of each element. /// /// `T` must be a vector of integers. diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-2.rs b/tests/ui/simd/intrinsic/generic-arithmetic-2.rs index 4ad98d5671135..fc3087cbf7542 100644 --- a/tests/ui/simd/intrinsic/generic-arithmetic-2.rs +++ b/tests/ui/simd/intrinsic/generic-arithmetic-2.rs @@ -30,6 +30,7 @@ extern "rust-intrinsic" { fn simd_bswap(x: T) -> T; fn simd_bitreverse(x: T) -> T; fn simd_ctlz(x: T) -> T; + fn simd_ctpop(x: T) -> T; fn simd_cttz(x: T) -> T; } @@ -77,7 +78,6 @@ fn main() { simd_cttz(x); simd_cttz(y); - simd_add(0, 0); //~^ ERROR expected SIMD input type, found non-SIMD `i32` simd_sub(0, 0); @@ -108,24 +108,25 @@ fn main() { simd_cttz(0); //~^ ERROR expected SIMD input type, found non-SIMD `i32` - simd_shl(z, z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_shr(z, z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_and(z, z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_or(z, z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_xor(z, z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_bswap(z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_bitreverse(z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_ctlz(z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` + simd_ctpop(z); + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_cttz(z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` } } diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr b/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr index db26f3417c95e..6f5f86d7d374c 100644 --- a/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr +++ b/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr @@ -83,59 +83,65 @@ LL | simd_cttz(0); | ^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shl` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:112:9 + --> $DIR/generic-arithmetic-2.rs:111:9 | LL | simd_shl(z, z); | ^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shr` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:114:9 + --> $DIR/generic-arithmetic-2.rs:113:9 | LL | simd_shr(z, z); | ^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_and` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:116:9 + --> $DIR/generic-arithmetic-2.rs:115:9 | LL | simd_and(z, z); | ^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_or` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:118:9 + --> $DIR/generic-arithmetic-2.rs:117:9 | LL | simd_or(z, z); | ^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_xor` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:120:9 + --> $DIR/generic-arithmetic-2.rs:119:9 | LL | simd_xor(z, z); | ^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_bswap` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:122:9 + --> $DIR/generic-arithmetic-2.rs:121:9 | LL | simd_bswap(z); | ^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_bitreverse` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:124:9 + --> $DIR/generic-arithmetic-2.rs:123:9 | LL | simd_bitreverse(z); | ^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_ctlz` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:126:9 + --> $DIR/generic-arithmetic-2.rs:125:9 | LL | simd_ctlz(z); | ^^^^^^^^^^^^ +error[E0511]: invalid monomorphization of `simd_ctpop` intrinsic: unsupported operation on `f32x4` with element `f32` + --> $DIR/generic-arithmetic-2.rs:127:9 + | +LL | simd_ctpop(z); + | ^^^^^^^^^^^^^ + error[E0511]: invalid monomorphization of `simd_cttz` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:128:9 + --> $DIR/generic-arithmetic-2.rs:129:9 | LL | simd_cttz(z); | ^^^^^^^^^^^^ -error: aborting due to 23 previous errors +error: aborting due to 24 previous errors For more information about this error, try `rustc --explain E0511`. diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs b/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs index 33143b1f7b53b..60dfa62741488 100644 --- a/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs +++ b/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs @@ -47,6 +47,7 @@ extern "rust-intrinsic" { fn simd_bswap(x: T) -> T; fn simd_bitreverse(x: T) -> T; fn simd_ctlz(x: T) -> T; + fn simd_ctpop(x: T) -> T; fn simd_cttz(x: T) -> T; } @@ -57,6 +58,8 @@ fn main() { let x2 = i32x4(2, 3, 4, 5); let y2 = U32::<4>([2, 3, 4, 5]); let z2 = f32x4(2.0, 3.0, 4.0, 5.0); + let x3 = i32x4(0, i32::MAX, i32::MIN, -1_i32); + let y3 = U32::<4>([0, i32::MAX as _, i32::MIN as _, -1_i32 as _]); unsafe { all_eq!(simd_add(x1, x2), i32x4(3, 5, 7, 9)); @@ -147,6 +150,13 @@ fn main() { all_eq!(simd_ctlz(x1), i32x4(31, 30, 30, 29)); all_eq_!(simd_ctlz(y1), U32::<4>([31, 30, 30, 29])); + all_eq!(simd_ctpop(x1), i32x4(1, 1, 2, 1)); + all_eq_!(simd_ctpop(y1), U32::<4>([1, 1, 2, 1])); + all_eq!(simd_ctpop(x2), i32x4(1, 2, 1, 2)); + all_eq_!(simd_ctpop(y2), U32::<4>([1, 2, 1, 2])); + all_eq!(simd_ctpop(x3), i32x4(0, 31, 1, 32)); + all_eq_!(simd_ctpop(y3), U32::<4>([0, 31, 1, 32])); + all_eq!(simd_cttz(x1), i32x4(0, 1, 0, 2)); all_eq_!(simd_cttz(y1), U32::<4>([0, 1, 0, 2])); }