diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index dd2477503fd6d..955467b8a285f 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -8,9 +8,9 @@ use either::Either; use rustc_abi::{FIRST_VARIANT, FieldIdx}; use rustc_data_structures::fx::FxHashSet; use rustc_index::IndexSlice; -use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::ty::{self, FnSig, Instance, Ty}; use rustc_middle::{bug, mir, span_bug}; -use rustc_span::Spanned; +use rustc_span::{Span, Spanned}; use rustc_target::callconv::FnAbi; use tracing::field::Empty; use tracing::{info, instrument, trace}; @@ -19,13 +19,13 @@ use super::{ EnteredTraceSpan, FnArg, FnVal, ImmTy, Immediate, InterpCx, InterpResult, Machine, MemPlaceMeta, PlaceTy, Projectable, RetagMode, interp_ok, throw_ub, throw_unsup_format, }; +use crate::interpret::OpTy; use crate::{enter_trace_span, util}; struct EvaluatedCalleeAndArgs<'tcx, M: Machine<'tcx>> { + func: OpTy<'tcx, M::Provenance>, callee: FnVal<'tcx, M::ExtraFnVal>, args: Vec>, - fn_sig: ty::FnSig<'tcx>, - fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, /// True if the function is marked as `#[track_caller]` ([`ty::InstanceKind::requires_caller_location`]) with_caller_location: bool, } @@ -467,6 +467,31 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { .map(|arg| self.eval_fn_call_argument(&arg.node, move_definitely_disjoint)) .collect::>>()?; + let (callee, with_caller_location) = match *func.layout.ty.kind() { + ty::FnPtr(..) => { + let fn_ptr = self.read_pointer(&func)?; + let fn_val = self.get_ptr_fn(fn_ptr)?; + (fn_val, false) + } + ty::FnDef(def_id, args) => { + let instance = self.resolve(def_id, args.no_bound_vars().unwrap())?; + (FnVal::Instance(instance), instance.def.requires_caller_location(*self.tcx)) + } + _ => { + span_bug!(terminator.source_info.span, "invalid callee of type {}", func.layout.ty) + } + }; + + interp_ok(EvaluatedCalleeAndArgs { func, callee, args, with_caller_location }) + } + + fn get_fn_abi( + &self, + func: &OpTy<'tcx, M::Provenance>, + callee: &FnVal<'tcx, M::ExtraFnVal>, + args: &[FnArg<'tcx, M::Provenance>], + span: Span, + ) -> InterpResult<'tcx, (FnSig<'tcx>, &'tcx FnAbi<'tcx, Ty<'tcx>>)> { let fn_sig_binder = { let _trace = enter_trace_span!(M, "fn_sig", ty = ?func.layout.ty.kind()); func.layout.ty.fn_sig(*self.tcx) @@ -476,26 +501,18 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let extra_args = self.tcx.mk_type_list_from_iter(extra_args.iter().map(|arg| arg.layout().ty)); - let (callee, fn_abi, with_caller_location) = match *func.layout.ty.kind() { - ty::FnPtr(..) => { - let fn_ptr = self.read_pointer(&func)?; - let fn_val = self.get_ptr_fn(fn_ptr)?; - (fn_val, self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, false) - } - ty::FnDef(def_id, args) => { - let instance = self.resolve(def_id, args.no_bound_vars().unwrap())?; - ( - FnVal::Instance(instance), - self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args)?, - instance.def.requires_caller_location(*self.tcx), - ) + let fn_abi = match *func.layout.ty.kind() { + ty::FnPtr(..) => self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, + ty::FnDef(..) => { + let FnVal::Instance(instance) = *callee else { unreachable!() }; + self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args)? } _ => { - span_bug!(terminator.source_info.span, "invalid callee of type {}", func.layout.ty) + span_bug!(span, "invalid callee of type {}", func.layout.ty) } }; - interp_ok(EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location }) + interp_ok((fn_sig, fn_abi)) } fn eval_terminator(&mut self, terminator: &mir::Terminator<'tcx>) -> InterpResult<'tcx> { @@ -554,18 +571,47 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // Evaluation order consistent with assignment: destination first. let dest_place = self.eval_place(destination)?; - let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = + let EvaluatedCalleeAndArgs { func, callee, args, with_caller_location } = self.eval_callee_and_args(terminator, func, args, &destination)?; - self.init_fn_call( - callee, - (fn_sig.abi(), fn_abi), - &args, - with_caller_location, - &dest_place, - target, - if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable }, - )?; + let _trace = enter_trace_span!( + M, + step::init_fn_call, + tracing_separate_thread = Empty, + ?callee + ) + .or_if_tracing_disabled(|| trace!("init_fn_call: {:#?}", callee)); + + match callee { + FnVal::Instance( + instance + @ ty::Instance { def: ty::InstanceKind::LlvmIntrinsic(_), args: _ }, + ) => { + // FIXME: Should `InPlace` arguments be reset to uninit? + M::call_llvm_intrinsic( + self, + instance, + &Self::copy_fn_args(&args), + &dest_place, + target, + )?; + } + _ => { + let (fn_sig, fn_abi) = + self.get_fn_abi(&func, &callee, &args, terminator.source_info.span)?; + + self.init_fn_call( + callee, + (fn_sig.abi(), fn_abi), + &args, + with_caller_location, + &dest_place, + target, + if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable }, + )?; + } + }; + // Sanity-check that `eval_fn_call` either pushed a new frame or // did a jump to another block. We disable the sanity check for functions that // can't return, since Miri sometimes does have to keep the location the same @@ -579,8 +625,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { TailCall { ref func, ref args, fn_span: _ } => { let old_frame_idx = self.frame_idx(); - let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = + let EvaluatedCalleeAndArgs { func, callee, args, with_caller_location } = self.eval_callee_and_args(terminator, func, args, &mir::Place::return_place())?; + let (fn_sig, fn_abi) = + self.get_fn_abi(&func, &callee, &args, terminator.source_info.span)?; self.init_fn_tail_call( callee, diff --git a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs index 4479ce2dba08b..4211a2443d8b8 100644 --- a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs +++ b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs @@ -222,6 +222,9 @@ fn check_call_site_abi<'tcx>( args.no_bound_vars().unwrap(), DUMMY_SP, ); + if let InstanceKind::LlvmIntrinsic(..) = instance.def { + return; + } tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty()))) } _ => { diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index 578a101b5f29a..32ac2d9c7f132 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -426,20 +426,6 @@ impl<'a, Ty> ArgAbi<'a, Ty> { PassMode::Indirect { attrs, meta_attrs, on_stack: false } } - /// Pass this argument directly instead. Should NOT be used! - /// Only exists because of past ABI mistakes that will take time to fix - /// (see ). - #[track_caller] - pub fn make_direct_deprecated(&mut self) { - match self.mode { - PassMode::Indirect { .. } => { - self.mode = PassMode::Direct(ArgAttributes::new()); - } - PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => {} // already direct - _ => panic!("Tried to make {:?} direct", self.mode), - } - } - /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead. /// This is valid for both sized and unsized arguments. #[track_caller] diff --git a/compiler/rustc_target/src/spec/abi_map.rs b/compiler/rustc_target/src/spec/abi_map.rs index dd69afec47975..3f28b90c9095e 100644 --- a/compiler/rustc_target/src/spec/abi_map.rs +++ b/compiler/rustc_target/src/spec/abi_map.rs @@ -97,6 +97,8 @@ impl AbiMap { // infallible lowerings (ExternAbi::C { .. }, _) => CanonAbi::C, (ExternAbi::Rust | ExternAbi::RustCall, _) => CanonAbi::Rust, + + // Dummy mapping to prevent reporting an error in the frontend (ExternAbi::Unadjusted, _) => CanonAbi::C, (ExternAbi::RustCold, _) if self.os == OsKind::Windows => CanonAbi::Rust, diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index ad8918f70a582..91cbe76c64ec4 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -433,7 +433,6 @@ fn fn_abi_sanity_check<'tcx>( fn fn_arg_sanity_check<'tcx>( cx: &LayoutCx<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, spec_abi: ExternAbi, arg: &ArgAbi<'tcx, Ty<'tcx>>, is_ret: bool, @@ -466,8 +465,7 @@ fn fn_abi_sanity_check<'tcx>( PassMode::Direct(attrs) => { // Here the Rust type is used to determine the actual ABI, so we have to be very // careful. Scalar/Vector is fine, since backends will generally use - // `layout.backend_repr` and ignore everything else. We should just reject - //`Aggregate` entirely here, but some targets need to be fixed first. + // `layout.backend_repr` and ignore everything else. match arg.layout.backend_repr { BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } @@ -475,19 +473,9 @@ fn fn_abi_sanity_check<'tcx>( BackendRepr::ScalarPair { .. } => { panic!("`PassMode::Direct` used for ScalarPair type {}", arg.layout.ty) } - BackendRepr::Memory { sized } => { - // For an unsized type we'd only pass the sized prefix, so there is no universe - // in which we ever want to allow this. - assert!(sized, "`PassMode::Direct` for unsized type in ABI: {:#?}", fn_abi); - - // This really shouldn't happen even for sized aggregates, since - // `immediate_llvm_type` will use `layout.fields` to turn this Rust type into an - // LLVM type. This means all sorts of Rust type details leak into the ABI. - // The unadjusted ABI however uses Direct for all args. It is ill-specified, - // but unfortunately we need it for calling certain LLVM intrinsics. - assert!( - matches!(spec_abi, ExternAbi::Unadjusted), - "`PassMode::Direct` for aggregates only allowed for \"unadjusted\"\n\ + BackendRepr::Memory { .. } => { + panic!( + "`PassMode::Direct` for aggregates not allowed\n\ Problematic type: {:#?}", arg.layout, ); @@ -539,9 +527,9 @@ fn fn_abi_sanity_check<'tcx>( } for arg in fn_abi.args.iter() { - fn_arg_sanity_check(cx, fn_abi, spec_abi, arg, false); + fn_arg_sanity_check(cx, spec_abi, arg, false); } - fn_arg_sanity_check(cx, fn_abi, spec_abi, &fn_abi.ret, true); + fn_arg_sanity_check(cx, spec_abi, &fn_abi.ret, true); } #[tracing::instrument( @@ -636,26 +624,13 @@ fn fn_abi_adjust_for_abi<'tcx>( fn_abi: &mut FnAbi<'tcx, Ty<'tcx>>, abi: ExternAbi, ) { - if abi == ExternAbi::Unadjusted { - // The "unadjusted" ABI passes aggregates in "direct" mode. That's fragile but needed for - // some LLVM intrinsics. - fn unadjust<'tcx>(arg: &mut ArgAbi<'tcx, Ty<'tcx>>) { - // This still uses `PassMode::Pair` for ScalarPair types. That's unlikely to be intended, - // but who knows what breaks if we change this now. - if matches!(arg.layout.backend_repr, BackendRepr::Memory { .. }) { - assert!( - arg.layout.backend_repr.is_sized(), - "'unadjusted' ABI does not support unsized arguments" - ); - } - arg.make_direct_deprecated(); - } + assert_ne!( + abi, + ExternAbi::Unadjusted, + "fn_abi_of_instance should not be called on LLVM intrinsics" + ); - unadjust(&mut fn_abi.ret); - for arg in fn_abi.args.iter_mut() { - unadjust(arg); - } - } else if abi.is_rustic_abi() { + if abi.is_rustic_abi() { fn_abi.adjust_for_rust_abi(cx); } else { fn_abi.adjust_for_foreign_abi(cx, abi); diff --git a/tests/codegen-llvm/const-vector.rs b/tests/codegen-llvm/const-vector.rs index 0459d0145e26f..40bfc58f8d584 100644 --- a/tests/codegen-llvm/const-vector.rs +++ b/tests/codegen-llvm/const-vector.rs @@ -1,4 +1,5 @@ //@ revisions: OPT0 OPT0_S390X +//@ min-llvm-version: 22 //@ [OPT0] ignore-s390x //@ [OPT0_S390X] only-s390x //@ [OPT0] compile-flags: -C no-prepopulate-passes -Copt-level=0 @@ -9,6 +10,7 @@ #![crate_type = "lib"] #![feature(abi_unadjusted)] #![feature(const_trait_impl)] +#![feature(link_llvm_intrinsics)] #![feature(repr_simd)] #![feature(rustc_attrs)] #![feature(simd_ffi)] @@ -19,26 +21,26 @@ #[path = "../auxiliary/minisimd.rs"] mod minisimd; -use minisimd::{PackedSimd as Simd, f32x2, i8x2}; +use minisimd::{PackedSimd, Simd, f32x2, i8x2}; // The following functions are required for the tests to ensure // that they are called with a const vector extern "unadjusted" { - fn test_i8x2(a: i8x2); - fn test_i8x2_two_args(a: i8x2, b: i8x2); - fn test_i8x2_mixed_args(a: i8x2, c: i32, b: i8x2); - fn test_i8x2_arr(a: i8x2); - fn test_f32x2(a: f32x2); - fn test_f32x2_arr(a: f32x2); - fn test_simd(a: Simd); - fn test_simd_unaligned(a: Simd); + #[link_name = "llvm.vector.reduce.add.v2i8"] + fn test_i8x2(a: i8x2) -> i8; + #[link_name = "llvm.vector.partial.reduce.add.v2i8.v2i8.v2i8"] + fn test_i8x2_two_args(a: i8x2, b: i8x2) -> i8x2; + #[link_name = "llvm.vector.insert.v2i8.v2i8"] + fn test_i8x2_mixed_args(a: i8x2, b: i8x2, c: u64) -> i8x2; + #[link_name = "llvm.vector.reduce.fadd.v2f32"] + fn test_f32x2(a: f32, b: f32x2) -> f32; + #[link_name = "llvm.vector.reduce.add.v4i32"] + fn test_simd4(a: PackedSimd) -> i32; + #[link_name = "llvm.vector.reduce.add.v3i32"] + fn test_simd3(a: Simd) -> i32; } -// Ensure the packed variant of the simd struct does not become a const vector -// if the size is not a power of 2 -// CHECK: %"minisimd::PackedSimd" = type { [3 x i32] } - #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))] #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))] #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))] @@ -46,36 +48,29 @@ extern "unadjusted" { #[cfg_attr(target_arch = "riscv64", target_feature(enable = "v"))] pub fn do_call() { unsafe { - // CHECK: call void @test_i8x2(<2 x i8> + // CHECK: call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> test_i8x2(const { i8x2::from_array([32, 64]) }); - // CHECK: call void @test_i8x2_two_args(<2 x i8> , <2 x i8> + // CHECK: call <2 x i8> @llvm.vector.partial.reduce.add.v2i8.v2i8(<2 x i8> , <2 x i8> ) test_i8x2_two_args( const { i8x2::from_array([32, 64]) }, const { i8x2::from_array([8, 16]) }, ); - // CHECK: call void @test_i8x2_mixed_args(<2 x i8> , i32 43, <2 x i8> + // CHECK: call <2 x i8> @llvm.vector.insert.v2i8.v2i8(<2 x i8> , <2 x i8> , i64 0 test_i8x2_mixed_args( const { i8x2::from_array([32, 64]) }, - 43, const { i8x2::from_array([8, 16]) }, + 0, ); - // CHECK: call void @test_i8x2_arr(<2 x i8> - test_i8x2_arr(const { i8x2::from_array([32, 64]) }); - - // CHECK: call void @test_f32x2(<2 x float> - test_f32x2(const { f32x2::from_array([0.32, 0.64]) }); - - // CHECK: void @test_f32x2_arr(<2 x float> - test_f32x2_arr(const { f32x2::from_array([0.32, 0.64]) }); + // CHECK: call float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> + test_f32x2(0.0, const { f32x2::from_array([0.32, 0.64]) }); - // CHECK: call void @test_simd(<4 x i32> - test_simd(const { Simd::([2, 4, 6, 8]) }); + // CHECK: call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> + test_simd4(const { PackedSimd::([2, 4, 6, 8]) }); - // CHECK: [[UNALIGNED_ARG:%.*]] = load %"minisimd::PackedSimd", ptr @anon{{.*}} - // CHECK-NEXT: call void @test_simd_unaligned(%"minisimd::PackedSimd" [[UNALIGNED_ARG]] - test_simd_unaligned(const { Simd::([2, 4, 6]) }); + // CHECK: call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> + test_simd3(const { Simd::([2, 4, 6]) }); } } diff --git a/tests/codegen-llvm/simd/unpadded-simd.rs b/tests/codegen-llvm/simd/unpadded-simd.rs index ef067a15702c3..64f8921b1fd75 100644 --- a/tests/codegen-llvm/simd/unpadded-simd.rs +++ b/tests/codegen-llvm/simd/unpadded-simd.rs @@ -1,19 +1,21 @@ // Make sure that no 0-sized padding is inserted in structs and that // structs are represented as expected by Neon intrinsics in LLVM. // See #87254. +//@ only-aarch64 +//@ compile-flags: -Cno-prepopulate-passes #![crate_type = "lib"] -#![feature(repr_simd, abi_unadjusted)] +#![feature(abi_unadjusted, link_llvm_intrinsics)] -#[derive(Copy, Clone)] -#[repr(simd)] -pub struct int16x4_t(pub [i16; 4]); +use std::arch::aarch64::int16x4x2_t; -#[derive(Copy, Clone)] -pub struct int16x4x2_t(pub int16x4_t, pub int16x4_t); +unsafe extern "unadjusted" { + #[link_name = "llvm.aarch64.neon.ld1x2.v4i16.p0"] + fn vld1_s16_x2(t: *const i16) -> int16x4x2_t; +} -// CHECK: %int16x4x2_t = type { <4 x i16>, <4 x i16> } #[no_mangle] -extern "unadjusted" fn takes_int16x4x2_t(t: int16x4x2_t) -> int16x4x2_t { - t +unsafe extern "C" fn returns_int16x4x2_t(a: *const i16) -> int16x4x2_t { + // CHECK: call { <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld1x2.v4i16.p0 + vld1_s16_x2(a) }