Skip to content

Don't compute FnAbi for LLVM intrinsics #160077

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bjorn3 wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Don't compute FnAbi for LLVM intrinsics #160077

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b095cab
Don't use extern "unadjusted" outside of LLVM intrinsics in tests
bjorn3 Jan 7, 2026
176bbd8
Avoid computing FnAbi for llvm intrinsics in miri
bjorn3 Jul 12, 2026
0bee31f
Don't compute FnAbi for LLVM intrinsics
bjorn3 Dec 28, 2025
98537ee
Require LLVM 22 for const-vector.rs test
bjorn3 Jul 28, 2026
0bab389
Fix unpadded-simd test
bjorn3 Jul 29, 2026
eacedff
Fix test
bjorn3 Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 78 additions & 30 deletions compiler/rustc_const_eval/src/interpret/step.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ use either::Either;
use rustc_abi::{FIRST_VARIANT, FieldIdx};
use rustc_data_structures::fx::FxHashSet;
use rustc_index::IndexSlice;
use rustc_middle::ty::{self, Instance, Ty};
use rustc_middle::ty::{self, FnSig, Instance, Ty};
use rustc_middle::{bug, mir, span_bug};
use rustc_span::Spanned;
use rustc_span::{Span, Spanned};
use rustc_target::callconv::FnAbi;
use tracing::field::Empty;
use tracing::{info, instrument, trace};
Expand All @@ -19,13 +19,13 @@ use super::{
EnteredTraceSpan, FnArg, FnVal, ImmTy, Immediate, InterpCx, InterpResult, Machine,
MemPlaceMeta, PlaceTy, Projectable, RetagMode, interp_ok, throw_ub, throw_unsup_format,
};
use crate::interpret::OpTy;
use crate::{enter_trace_span, util};

struct EvaluatedCalleeAndArgs<'tcx, M: Machine<'tcx>> {
func: OpTy<'tcx, M::Provenance>,
callee: FnVal<'tcx, M::ExtraFnVal>,
Comment on lines +26 to 27

@RalfJung RalfJung Jul 28, 2026 •
edited by rustbot
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This introduces redundancy (func and callee conceptually contain the same data). So I am not a fan of this.

What's the rationale for these non-trivial interpreter changes? I would understand making the fn_abi field an Option for calls that don't need an ABI, but this goes much beyond that it seems, by entirely removing fn_abi (and fn_sig).

View changes since the review

@bjorn3 bjorn3 Jul 29, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I could replace the func field with just the Ty. Making fn_abi optional would mean I did have to match on the instance inside eval_callee_and_args and do an unwrap outside of it. I can do it if you prefer it that way.

@RalfJung RalfJung Jul 29, 2026 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I could replace the func field with just the Ty

That would be better, albeit still a bit odd.

Making fn_abi optional would mean I did have to match on the instance inside eval_callee_and_args and do an unwrap outside of it. I can do it if you prefer it that way.

Well I don't even know yet why you need to change all those other things. I could reverse engineer it from your PR but it'd be easier if you just explained it to me.

args: Vec<FnArg<'tcx, M::Provenance>>,
fn_sig: ty::FnSig<'tcx>,
fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
/// True if the function is marked as `#[track_caller]` ([`ty::InstanceKind::requires_caller_location`])
with_caller_location: bool,
}
Expand Down Expand Up @@ -467,6 +467,31 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
.map(|arg| self.eval_fn_call_argument(&arg.node, move_definitely_disjoint))
.collect::<InterpResult<'tcx, Vec<_>>>()?;

let (callee, with_caller_location) = match *func.layout.ty.kind() {
ty::FnPtr(..) => {
let fn_ptr = self.read_pointer(&func)?;
let fn_val = self.get_ptr_fn(fn_ptr)?;
(fn_val, false)
}
ty::FnDef(def_id, args) => {
let instance = self.resolve(def_id, args.no_bound_vars().unwrap())?;
(FnVal::Instance(instance), instance.def.requires_caller_location(*self.tcx))
}
_ => {
span_bug!(terminator.source_info.span, "invalid callee of type {}", func.layout.ty)
}
};

interp_ok(EvaluatedCalleeAndArgs { func, callee, args, with_caller_location })
}

fn get_fn_abi(
&self,
func: &OpTy<'tcx, M::Provenance>,
callee: &FnVal<'tcx, M::ExtraFnVal>,
args: &[FnArg<'tcx, M::Provenance>],
span: Span,
) -> InterpResult<'tcx, (FnSig<'tcx>, &'tcx FnAbi<'tcx, Ty<'tcx>>)> {
let fn_sig_binder = {
let _trace = enter_trace_span!(M, "fn_sig", ty = ?func.layout.ty.kind());
func.layout.ty.fn_sig(*self.tcx)
Expand All @@ -476,26 +501,18 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let extra_args =
self.tcx.mk_type_list_from_iter(extra_args.iter().map(|arg| arg.layout().ty));

let (callee, fn_abi, with_caller_location) = match *func.layout.ty.kind() {
ty::FnPtr(..) => {
let fn_ptr = self.read_pointer(&func)?;
let fn_val = self.get_ptr_fn(fn_ptr)?;
(fn_val, self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, false)
}
ty::FnDef(def_id, args) => {
let instance = self.resolve(def_id, args.no_bound_vars().unwrap())?;
(
FnVal::Instance(instance),
self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args)?,
instance.def.requires_caller_location(*self.tcx),
)
let fn_abi = match *func.layout.ty.kind() {
ty::FnPtr(..) => self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?,
ty::FnDef(..) => {
let FnVal::Instance(instance) = *callee else { unreachable!() };
self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args)?
}
_ => {
span_bug!(terminator.source_info.span, "invalid callee of type {}", func.layout.ty)
span_bug!(span, "invalid callee of type {}", func.layout.ty)
}
};

interp_ok(EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location })
interp_ok((fn_sig, fn_abi))
}

fn eval_terminator(&mut self, terminator: &mir::Terminator<'tcx>) -> InterpResult<'tcx> {
Expand Down Expand Up @@ -554,18 +571,47 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {

// Evaluation order consistent with assignment: destination first.
let dest_place = self.eval_place(destination)?;
let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } =
let EvaluatedCalleeAndArgs { func, callee, args, with_caller_location } =
self.eval_callee_and_args(terminator, func, args, &destination)?;

self.init_fn_call(
callee,
(fn_sig.abi(), fn_abi),
&args,
with_caller_location,
&dest_place,
target,
if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable },
)?;
let _trace = enter_trace_span!(
M,
step::init_fn_call,
tracing_separate_thread = Empty,
?callee
)
.or_if_tracing_disabled(|| trace!("init_fn_call: {:#?}", callee));

match callee {
FnVal::Instance(
instance
@ ty::Instance { def: ty::InstanceKind::LlvmIntrinsic(_), args: _ },
) => {
// FIXME: Should `InPlace` arguments be reset to uninit?
M::call_llvm_intrinsic(
self,
instance,
&Self::copy_fn_args(&args),
&dest_place,
target,
)?;
}
_ => {
let (fn_sig, fn_abi) =
self.get_fn_abi(&func, &callee, &args, terminator.source_info.span)?;

self.init_fn_call(
callee,
(fn_sig.abi(), fn_abi),
&args,
with_caller_location,
&dest_place,
target,
if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable },
)?;
}
};

// Sanity-check that `eval_fn_call` either pushed a new frame or
// did a jump to another block. We disable the sanity check for functions that
// can't return, since Miri sometimes does have to keep the location the same
Expand All @@ -579,8 +625,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
TailCall { ref func, ref args, fn_span: _ } => {
let old_frame_idx = self.frame_idx();

let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } =
let EvaluatedCalleeAndArgs { func, callee, args, with_caller_location } =
self.eval_callee_and_args(terminator, func, args, &mir::Place::return_place())?;
let (fn_sig, fn_abi) =
self.get_fn_abi(&func, &callee, &args, terminator.source_info.span)?;

self.init_fn_tail_call(
callee,
Expand Down
3 changes: 3 additions & 0 deletions compiler/rustc_monomorphize/src/mono_checks/abi_check.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,9 @@ fn check_call_site_abi<'tcx>(
args.no_bound_vars().unwrap(),
DUMMY_SP,
);
if let InstanceKind::LlvmIntrinsic(..) = instance.def {
return;
}
tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty())))
}
_ => {
Expand Down
14 changes: 0 additions & 14 deletions compiler/rustc_target/src/callconv/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -426,20 +426,6 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
PassMode::Indirect { attrs, meta_attrs, on_stack: false }
}

/// Pass this argument directly instead. Should NOT be used!
/// Only exists because of past ABI mistakes that will take time to fix
/// (see <https://github.com/rust-lang/rust/issues/115666>).
#[track_caller]
pub fn make_direct_deprecated(&mut self) {
match self.mode {
PassMode::Indirect { .. } => {
self.mode = PassMode::Direct(ArgAttributes::new());
}
PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => {} // already direct
_ => panic!("Tried to make {:?} direct", self.mode),
}
}

/// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
/// This is valid for both sized and unsized arguments.
#[track_caller]
Expand Down
2 changes: 2 additions & 0 deletions compiler/rustc_target/src/spec/abi_map.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ impl AbiMap {
// infallible lowerings
(ExternAbi::C { .. }, _) => CanonAbi::C,
(ExternAbi::Rust | ExternAbi::RustCall, _) => CanonAbi::Rust,

// Dummy mapping to prevent reporting an error in the frontend
(ExternAbi::Unadjusted, _) => CanonAbi::C,

(ExternAbi::RustCold, _) if self.os == OsKind::Windows => CanonAbi::Rust,
Expand Down
49 changes: 12 additions & 37 deletions compiler/rustc_ty_utils/src/abi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,6 @@ fn fn_abi_sanity_check<'tcx>(

fn fn_arg_sanity_check<'tcx>(
cx: &LayoutCx<'tcx>,
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
spec_abi: ExternAbi,
arg: &ArgAbi<'tcx, Ty<'tcx>>,
is_ret: bool,
Expand Down Expand Up @@ -466,28 +465,17 @@ fn fn_abi_sanity_check<'tcx>(
PassMode::Direct(attrs) => {
// Here the Rust type is used to determine the actual ABI, so we have to be very
// careful. Scalar/Vector is fine, since backends will generally use
// `layout.backend_repr` and ignore everything else. We should just reject
//`Aggregate` entirely here, but some targets need to be fixed first.
// `layout.backend_repr` and ignore everything else.
match arg.layout.backend_repr {
BackendRepr::Scalar(_)
| BackendRepr::SimdVector { .. }
| BackendRepr::SimdScalableVector { .. } => {}
BackendRepr::ScalarPair { .. } => {
panic!("`PassMode::Direct` used for ScalarPair type {}", arg.layout.ty)
}
BackendRepr::Memory { sized } => {
// For an unsized type we'd only pass the sized prefix, so there is no universe
// in which we ever want to allow this.
assert!(sized, "`PassMode::Direct` for unsized type in ABI: {:#?}", fn_abi);

// This really shouldn't happen even for sized aggregates, since
// `immediate_llvm_type` will use `layout.fields` to turn this Rust type into an
// LLVM type. This means all sorts of Rust type details leak into the ABI.
// The unadjusted ABI however uses Direct for all args. It is ill-specified,
// but unfortunately we need it for calling certain LLVM intrinsics.
assert!(
matches!(spec_abi, ExternAbi::Unadjusted),
"`PassMode::Direct` for aggregates only allowed for \"unadjusted\"\n\
BackendRepr::Memory { .. } => {
panic!(
"`PassMode::Direct` for aggregates not allowed\n\
Problematic type: {:#?}",
arg.layout,
);
Expand Down Expand Up @@ -539,9 +527,9 @@ fn fn_abi_sanity_check<'tcx>(
}

for arg in fn_abi.args.iter() {
fn_arg_sanity_check(cx, fn_abi, spec_abi, arg, false);
fn_arg_sanity_check(cx, spec_abi, arg, false);
}
fn_arg_sanity_check(cx, fn_abi, spec_abi, &fn_abi.ret, true);
fn_arg_sanity_check(cx, spec_abi, &fn_abi.ret, true);
}

#[tracing::instrument(
Expand Down Expand Up @@ -636,26 +624,13 @@ fn fn_abi_adjust_for_abi<'tcx>(
fn_abi: &mut FnAbi<'tcx, Ty<'tcx>>,
abi: ExternAbi,
) {
if abi == ExternAbi::Unadjusted {
// The "unadjusted" ABI passes aggregates in "direct" mode. That's fragile but needed for
// some LLVM intrinsics.
fn unadjust<'tcx>(arg: &mut ArgAbi<'tcx, Ty<'tcx>>) {
// This still uses `PassMode::Pair` for ScalarPair types. That's unlikely to be intended,
// but who knows what breaks if we change this now.
if matches!(arg.layout.backend_repr, BackendRepr::Memory { .. }) {
assert!(
arg.layout.backend_repr.is_sized(),
"'unadjusted' ABI does not support unsized arguments"
);
}
arg.make_direct_deprecated();
}
assert_ne!(
abi,
ExternAbi::Unadjusted,
"fn_abi_of_instance should not be called on LLVM intrinsics"
);

unadjust(&mut fn_abi.ret);
for arg in fn_abi.args.iter_mut() {
unadjust(arg);
}
} else if abi.is_rustic_abi() {
if abi.is_rustic_abi() {
fn_abi.adjust_for_rust_abi(cx);
} else {
fn_abi.adjust_for_foreign_abi(cx, abi);
Expand Down
55 changes: 25 additions & 30 deletions tests/codegen-llvm/const-vector.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//@ revisions: OPT0 OPT0_S390X
//@ min-llvm-version: 22
//@ [OPT0] ignore-s390x
//@ [OPT0_S390X] only-s390x
//@ [OPT0] compile-flags: -C no-prepopulate-passes -Copt-level=0
Expand All @@ -9,6 +10,7 @@
#![crate_type = "lib"]
#![feature(abi_unadjusted)]
#![feature(const_trait_impl)]
#![feature(link_llvm_intrinsics)]
#![feature(repr_simd)]
#![feature(rustc_attrs)]
#![feature(simd_ffi)]
Expand All @@ -19,63 +21,56 @@

#[path = "../auxiliary/minisimd.rs"]
mod minisimd;
use minisimd::{PackedSimd as Simd, f32x2, i8x2};
use minisimd::{PackedSimd, Simd, f32x2, i8x2};

// The following functions are required for the tests to ensure
// that they are called with a const vector

extern "unadjusted" {
fn test_i8x2(a: i8x2);
fn test_i8x2_two_args(a: i8x2, b: i8x2);
fn test_i8x2_mixed_args(a: i8x2, c: i32, b: i8x2);
fn test_i8x2_arr(a: i8x2);
fn test_f32x2(a: f32x2);
fn test_f32x2_arr(a: f32x2);
fn test_simd(a: Simd<i32, 4>);
fn test_simd_unaligned(a: Simd<i32, 3>);

@bjorn3 bjorn3 Jul 29, 2026 •
edited by rustbot
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is no way to test this specific case anymore it seems. LLVM doesn't accept PackedSimd on intrinsics, extern "unadjusted" requires LLVM intrinsics and any other ABI doesn't pass non-power-of-2 vectors as { [3 x i32] }.

View changes since the review

#[link_name = "llvm.vector.reduce.add.v2i8"]
fn test_i8x2(a: i8x2) -> i8;
#[link_name = "llvm.vector.partial.reduce.add.v2i8.v2i8.v2i8"]
fn test_i8x2_two_args(a: i8x2, b: i8x2) -> i8x2;
#[link_name = "llvm.vector.insert.v2i8.v2i8"]
fn test_i8x2_mixed_args(a: i8x2, b: i8x2, c: u64) -> i8x2;
#[link_name = "llvm.vector.reduce.fadd.v2f32"]
fn test_f32x2(a: f32, b: f32x2) -> f32;
#[link_name = "llvm.vector.reduce.add.v4i32"]
fn test_simd4(a: PackedSimd<i32, 4>) -> i32;
#[link_name = "llvm.vector.reduce.add.v3i32"]
fn test_simd3(a: Simd<i32, 3>) -> i32;
}

// Ensure the packed variant of the simd struct does not become a const vector
// if the size is not a power of 2
// CHECK: %"minisimd::PackedSimd<i32, 3>" = type { [3 x i32] }

#[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
#[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
#[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))]
#[cfg_attr(target_arch = "mips", target_feature(enable = "msa"))]
#[cfg_attr(target_arch = "riscv64", target_feature(enable = "v"))]
pub fn do_call() {
unsafe {
// CHECK: call void @test_i8x2(<2 x i8> <i8 32, i8 64>
// CHECK: call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> <i8 32, i8 64>
test_i8x2(const { i8x2::from_array([32, 64]) });

// CHECK: call void @test_i8x2_two_args(<2 x i8> <i8 32, i8 64>, <2 x i8> <i8 8, i8 16>
// CHECK: call <2 x i8> @llvm.vector.partial.reduce.add.v2i8.v2i8(<2 x i8> <i8 32, i8 64>, <2 x i8> <i8 8, i8 16>)
test_i8x2_two_args(
const { i8x2::from_array([32, 64]) },
const { i8x2::from_array([8, 16]) },
);

// CHECK: call void @test_i8x2_mixed_args(<2 x i8> <i8 32, i8 64>, i32 43, <2 x i8> <i8 8, i8 16>
// CHECK: call <2 x i8> @llvm.vector.insert.v2i8.v2i8(<2 x i8> <i8 32, i8 64>, <2 x i8> <i8 8, i8 16>, i64 0
test_i8x2_mixed_args(
const { i8x2::from_array([32, 64]) },
43,
const { i8x2::from_array([8, 16]) },
0,
);

// CHECK: call void @test_i8x2_arr(<2 x i8> <i8 32, i8 64>
test_i8x2_arr(const { i8x2::from_array([32, 64]) });

// CHECK: call void @test_f32x2(<2 x float> <float {{0x3FD47AE140000000|3.200000e-01}}, float {{0x3FE47AE140000000|6.400000e-01}}>
test_f32x2(const { f32x2::from_array([0.32, 0.64]) });

// CHECK: void @test_f32x2_arr(<2 x float> <float {{0x3FD47AE140000000|3.200000e-01}}, float {{0x3FE47AE140000000|6.400000e-01}}>
test_f32x2_arr(const { f32x2::from_array([0.32, 0.64]) });
// CHECK: call float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> <float 0x3FD47AE140000000, float 0x3FE47AE140000000>
test_f32x2(0.0, const { f32x2::from_array([0.32, 0.64]) });

// CHECK: call void @test_simd(<4 x i32> <i32 2, i32 4, i32 6, i32 8>
test_simd(const { Simd::<i32, 4>([2, 4, 6, 8]) });
// CHECK: call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> <i32 2, i32 4, i32 6, i32 8>
test_simd4(const { PackedSimd::<i32, 4>([2, 4, 6, 8]) });

// CHECK: [[UNALIGNED_ARG:%.*]] = load %"minisimd::PackedSimd<i32, 3>", ptr @anon{{.*}}
// CHECK-NEXT: call void @test_simd_unaligned(%"minisimd::PackedSimd<i32, 3>" [[UNALIGNED_ARG]]
test_simd_unaligned(const { Simd::<i32, 3>([2, 4, 6]) });
// CHECK: call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> <i32 2, i32 4, i32 6>
test_simd3(const { Simd::<i32, 3>([2, 4, 6]) });
}
}
Loading
Loading