diff --git a/library/core/src/intrinsics/simd/mod.rs b/library/core/src/intrinsics/simd/mod.rs index 9311dcc9bd00e..f083d46e6d387 100644 --- a/library/core/src/intrinsics/simd/mod.rs +++ b/library/core/src/intrinsics/simd/mod.rs @@ -15,6 +15,7 @@ use crate::marker::ConstParamTy; /// `idx` must be in-bounds of the vector. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_insert(x: T, idx: u32, val: U) -> T; /// Extracts an element from a vector. @@ -26,6 +27,7 @@ pub const unsafe fn simd_insert(x: T, idx: u32, val: U) -> T; /// `idx` must be const and in-bounds of the vector. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_extract(x: T, idx: u32) -> U; /// Inserts an element into a vector, returning the updated vector. @@ -39,6 +41,7 @@ pub const unsafe fn simd_extract(x: T, idx: u32) -> U; /// `idx` must be in-bounds of the vector. #[rustc_nounwind] #[rustc_intrinsic] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_insert_dyn(x: T, idx: u32, val: U) -> T; /// Extracts an element from a vector. @@ -52,6 +55,7 @@ pub const unsafe fn simd_insert_dyn(x: T, idx: u32, val: U) -> T; /// `idx` must be in-bounds of the vector. #[rustc_nounwind] #[rustc_intrinsic] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_extract_dyn(x: T, idx: u32) -> U; /// Creates a vector where every lane has the provided value. @@ -59,6 +63,7 @@ pub const unsafe fn simd_extract_dyn(x: T, idx: u32) -> U; /// `T` must be a vector with element type `U`. #[rustc_nounwind] #[rustc_intrinsic] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_splat(value: U) -> T; /// Adds two simd vectors elementwise. @@ -67,6 +72,7 @@ pub const unsafe fn simd_splat(value: U) -> T; /// For integers, wrapping arithmetic is used. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_add(x: T, y: T) -> T; /// Subtracts `rhs` from `lhs` elementwise. @@ -75,6 +81,7 @@ pub const unsafe fn simd_add(x: T, y: T) -> T; /// For integers, wrapping arithmetic is used. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_sub(lhs: T, rhs: T) -> T; /// Multiplies two simd vectors elementwise. @@ -83,6 +90,7 @@ pub const unsafe fn simd_sub(lhs: T, rhs: T) -> T; /// For integers, wrapping arithmetic is used. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_mul(x: T, y: T) -> T; /// Divides `lhs` by `rhs` elementwise. @@ -94,6 +102,7 @@ pub const unsafe fn simd_mul(x: T, y: T) -> T; /// Additionally for signed integers, `::MIN / -1` is undefined behavior. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_div(lhs: T, rhs: T) -> T; /// Returns remainder of two vectors elementwise. @@ -105,6 +114,7 @@ pub const unsafe fn simd_div(lhs: T, rhs: T) -> T; /// Additionally for signed integers, `::MIN / -1` is undefined behavior. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_rem(lhs: T, rhs: T) -> T; /// Shifts vector left elementwise, with UB on overflow. @@ -118,6 +128,7 @@ pub const unsafe fn simd_rem(lhs: T, rhs: T) -> T; /// Each element of `rhs` must be less than `::BITS`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_shl(lhs: T, rhs: T) -> T; /// Shifts vector right elementwise, with UB on overflow. @@ -131,6 +142,7 @@ pub const unsafe fn simd_shl(lhs: T, rhs: T) -> T; /// Each element of `rhs` must be less than `::BITS`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_shr(lhs: T, rhs: T) -> T; /// Funnel Shifts vector left elementwise, with UB on overflow. @@ -148,6 +160,7 @@ pub const unsafe fn simd_shr(lhs: T, rhs: T) -> T; /// Each element of `shift` must be less than `::BITS`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_funnel_shl(a: T, b: T, shift: T) -> T; /// Funnel Shifts vector right elementwise, with UB on overflow. @@ -165,6 +178,7 @@ pub const unsafe fn simd_funnel_shl(a: T, b: T, shift: T) -> T; /// Each element of `shift` must be less than `::BITS`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_funnel_shr(a: T, b: T, shift: T) -> T; /// Compute the carry-less product. @@ -184,6 +198,7 @@ pub unsafe fn simd_carryless_mul(a: T, b: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_and(x: T, y: T) -> T; /// "Ors" vectors elementwise. @@ -191,6 +206,7 @@ pub const unsafe fn simd_and(x: T, y: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_or(x: T, y: T) -> T; /// "Exclusive ors" vectors elementwise. @@ -198,6 +214,7 @@ pub const unsafe fn simd_or(x: T, y: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_xor(x: T, y: T) -> T; /// Numerically casts a vector, elementwise. @@ -219,6 +236,7 @@ pub const unsafe fn simd_xor(x: T, y: T) -> T; /// * Be representable in the return type, after truncating off its fractional part #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_cast(x: T) -> U; /// Numerically casts a vector, elementwise. @@ -233,6 +251,7 @@ pub const unsafe fn simd_cast(x: T) -> U; /// Otherwise, truncates or extends the value, maintaining the sign for signed integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_as(x: T) -> U; /// Negates a vector elementwise. @@ -241,6 +260,7 @@ pub const unsafe fn simd_as(x: T) -> U; /// For integers, wrapping arithmetic is used. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_neg(x: T) -> T; /// Returns absolute value of a vector, elementwise. @@ -248,6 +268,7 @@ pub const unsafe fn simd_neg(x: T) -> T; /// `T` must be a vector of floating-point primitive types. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_fabs(x: T) -> T; /// Returns the minimum of two vectors, elementwise. @@ -261,6 +282,7 @@ pub const unsafe fn simd_fabs(x: T) -> T; /// and `-0.0`), either input may be returned non-deterministically. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_minimum_number_nsz(x: T, y: T) -> T; /// Returns the maximum of two vectors, elementwise. @@ -274,6 +296,7 @@ pub const unsafe fn simd_minimum_number_nsz(x: T, y: T) -> T; /// and `-0.0`), either input may be returned non-deterministically. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_maximum_number_nsz(x: T, y: T) -> T; /// Tests elementwise equality of two vectors. @@ -285,6 +308,7 @@ pub const unsafe fn simd_maximum_number_nsz(x: T, y: T) -> T; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_eq(x: T, y: T) -> U; /// Tests elementwise inequality equality of two vectors. @@ -296,6 +320,7 @@ pub const unsafe fn simd_eq(x: T, y: T) -> U; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_ne(x: T, y: T) -> U; /// Tests if `x` is less than `y`, elementwise. @@ -307,6 +332,7 @@ pub const unsafe fn simd_ne(x: T, y: T) -> U; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_lt(x: T, y: T) -> U; /// Tests if `x` is less than or equal to `y`, elementwise. @@ -318,6 +344,7 @@ pub const unsafe fn simd_lt(x: T, y: T) -> U; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_le(x: T, y: T) -> U; /// Tests if `x` is greater than `y`, elementwise. @@ -329,6 +356,7 @@ pub const unsafe fn simd_le(x: T, y: T) -> U; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_gt(x: T, y: T) -> U; /// Tests if `x` is greater than or equal to `y`, elementwise. @@ -340,6 +368,7 @@ pub const unsafe fn simd_gt(x: T, y: T) -> U; /// Returns `0` for false and `!0` for true. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_ge(x: T, y: T) -> U; /// Shuffles two vectors by const indices. @@ -356,6 +385,7 @@ pub const unsafe fn simd_ge(x: T, y: T) -> U; /// of `xy`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_shuffle(x: T, y: T, idx: U) -> V; /// Reads a vector of pointers. @@ -377,6 +407,7 @@ pub const unsafe fn simd_shuffle(x: T, y: T, idx: U) -> V; /// `mask` must only contain `0` or `!0` values. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_gather(val: T, ptr: U, mask: V) -> T; /// Writes to a vector of pointers. @@ -401,6 +432,7 @@ pub const unsafe fn simd_gather(val: T, ptr: U, mask: V) -> T; /// `mask` must only contain `0` or `!0` values. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_scatter(val: T, ptr: U, mask: V); /// A type for alignment options for SIMD masked load/store intrinsics. @@ -436,6 +468,7 @@ pub enum SimdAlign { /// `mask` must only contain `0` or `!0` values. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_masked_load(mask: V, ptr: U, val: T) -> T; @@ -458,6 +491,7 @@ pub const unsafe fn simd_masked_load(mask: V, p /// `mask` must only contain `0` or `!0` values. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_masked_store(mask: V, ptr: U, val: T); /// Adds two simd vectors elementwise, with saturation. @@ -465,6 +499,7 @@ pub const unsafe fn simd_masked_store(mask: V, /// `T` must be a vector of integer primitive types. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_saturating_add(x: T, y: T) -> T; /// Subtracts two simd vectors elementwise, with saturation. @@ -474,6 +509,7 @@ pub const unsafe fn simd_saturating_add(x: T, y: T) -> T; /// Subtract `rhs` from `lhs`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_saturating_sub(lhs: T, rhs: T) -> T; /// Adds elements within a vector from left to right. @@ -485,6 +521,7 @@ pub const unsafe fn simd_saturating_sub(lhs: T, rhs: T) -> T; /// Starting with the value `y`, add the elements of `x` and accumulate. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_add_ordered(x: T, y: U) -> U; /// Adds elements within a vector in arbitrary order. May also be re-associated with @@ -506,6 +543,7 @@ pub unsafe fn simd_reduce_add_unordered(x: T) -> U; /// Starting with the value `y`, multiply the elements of `x` and accumulate. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_mul_ordered(x: T, y: U) -> U; /// Multiplies elements within a vector in arbitrary order. May also be re-associated with @@ -526,6 +564,7 @@ pub unsafe fn simd_reduce_mul_unordered(x: T) -> U; /// `x` must contain only `0` or `!0`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_all(x: T) -> bool; /// Checks if any mask value is true. @@ -536,6 +575,7 @@ pub const unsafe fn simd_reduce_all(x: T) -> bool; /// `x` must contain only `0` or `!0`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_any(x: T) -> bool; /// Returns the maximum element of a vector. @@ -545,6 +585,7 @@ pub const unsafe fn simd_reduce_any(x: T) -> bool; /// `U` must be the element type of `T`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_max(x: T) -> U; /// Returns the minimum element of a vector. @@ -554,6 +595,7 @@ pub const unsafe fn simd_reduce_max(x: T) -> U; /// `U` must be the element type of `T`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_min(x: T) -> U; /// Logical "and"s all elements together. @@ -563,6 +605,7 @@ pub const unsafe fn simd_reduce_min(x: T) -> U; /// `U` must be the element type of `T`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_and(x: T) -> U; /// Logical "ors" all elements together. @@ -572,6 +615,7 @@ pub const unsafe fn simd_reduce_and(x: T) -> U; /// `U` must be the element type of `T`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_or(x: T) -> U; /// Logical "exclusive ors" all elements together. @@ -581,6 +625,7 @@ pub const unsafe fn simd_reduce_or(x: T) -> U; /// `U` must be the element type of `T`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_reduce_xor(x: T) -> U; /// Truncates an integer vector to a bitmask. @@ -618,6 +663,7 @@ pub const unsafe fn simd_reduce_xor(x: T) -> U; /// `x` must contain only `0` and `!0`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_bitmask(x: T) -> U; /// Selects elements from a mask. @@ -634,6 +680,7 @@ pub const unsafe fn simd_bitmask(x: T) -> U; /// `mask` must only contain `0` and `!0`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_select(mask: M, if_true: T, if_false: T) -> T; /// Selects elements from a bitmask. @@ -650,6 +697,7 @@ pub const unsafe fn simd_select(mask: M, if_true: T, if_false: T) -> T; /// The bitmask bit order matches `simd_bitmask`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_select_bitmask(m: M, yes: T, no: T) -> T; /// Calculates the offset from a pointer vector elementwise, potentially @@ -662,6 +710,7 @@ pub const unsafe fn simd_select_bitmask(m: M, yes: T, no: T) -> T; /// Operates as if by `::wrapping_offset`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_arith_offset(ptr: T, offset: U) -> T; /// Casts a vector of pointers. @@ -669,6 +718,7 @@ pub const unsafe fn simd_arith_offset(ptr: T, offset: U) -> T; /// `T` and `U` must be vectors of pointers with the same number of elements. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_cast_ptr(ptr: T) -> U; /// Exposes a vector of pointers as a vector of addresses. @@ -687,6 +737,7 @@ pub unsafe fn simd_expose_provenance(ptr: T) -> U; /// `U` must be a vector of pointers, with the same length as `T`. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_with_exposed_provenance(addr: T) -> U; /// Swaps bytes of each element. @@ -694,6 +745,7 @@ pub const unsafe fn simd_with_exposed_provenance(addr: T) -> U; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_bswap(x: T) -> T; /// Reverses bits of each element. @@ -701,6 +753,7 @@ pub const unsafe fn simd_bswap(x: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_bitreverse(x: T) -> T; /// Counts the leading zeros of each element. @@ -708,6 +761,7 @@ pub const unsafe fn simd_bitreverse(x: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_ctlz(x: T) -> T; /// Counts the number of ones in each element. @@ -715,6 +769,7 @@ pub const unsafe fn simd_ctlz(x: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_ctpop(x: T) -> T; /// Counts the trailing zeros of each element. @@ -722,6 +777,7 @@ pub const unsafe fn simd_ctpop(x: T) -> T; /// `T` must be a vector of integers. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_cttz(x: T) -> T; /// Rounds up each element to the next highest integer-valued float. @@ -729,6 +785,7 @@ pub const unsafe fn simd_cttz(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_ceil(x: T) -> T; /// Rounds down each element to the next lowest integer-valued float. @@ -736,6 +793,7 @@ pub const unsafe fn simd_ceil(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_floor(x: T) -> T; /// Rounds each element to the closest integer-valued float. @@ -744,6 +802,7 @@ pub const unsafe fn simd_floor(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_round(x: T) -> T; /// Rounds each element to the closest integer-valued float. @@ -752,6 +811,7 @@ pub const unsafe fn simd_round(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_round_ties_even(x: T) -> T; /// Returns the integer part of each element as an integer-valued float. @@ -760,6 +820,7 @@ pub const unsafe fn simd_round_ties_even(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_trunc(x: T) -> T; /// Takes the square root of each element. @@ -774,6 +835,7 @@ pub unsafe fn simd_fsqrt(x: T) -> T; /// `T` must be a vector of floats. #[rustc_intrinsic] #[rustc_nounwind] +#[rustc_intrinsic_const_stable_indirect] pub const unsafe fn simd_fma(x: T, y: T, z: T) -> T; /// Computes `(x*y) + z` for each element, non-deterministically executing either