Mirrors/rust
1
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2026-05-30 13:06:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rollup merge of #157022 - RalfJung:inline-intrinsics-v2, r=oli-obk

Browse Source 
MIR inlining: allow backends to opt-in to inlining intrinsics

This is the same as https://github.com/rust-lang/rust/pull/156398. Github stopped processing updates on that other branch.

r? @oli-obk
This commit is contained in:
Guillaume Gomez
2026-05-27 20:45:10 +02:00
committed by GitHub
parent 1159cb541c 59428e76e8
commit dc05df7ebe
17 changed files with 108 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/rustc_codegen_cranelift/src/lib.rs
+4
View File
@@ -237,6 +237,10 @@ fn join_codegen(
) -> (CompiledModules, FxIndexMap<WorkProductId, WorkProduct>) {
ongoing_codegen.downcast::<driver::aot::OngoingCodegen>().unwrap().join(sess, outputs)
}
fn fallback_intrinsics(&self) -> Vec<Symbol> {
vec![sym::type_id_eq]
}
}
/// Determine if the Cranelift ir verifier should run.
compiler/rustc_codegen_gcc/src/lib.rs
+5 -1
View File
@@ -98,7 +98,7 @@
use rustc_middle::util::Providers;
use rustc_session::Session;
use rustc_session::config::{OptLevel, OutputFilenames};
use rustc_span::Symbol;
use rustc_span::{Symbol, sym};
use rustc_target::spec::{Arch, RelocModel};
use tempfile::TempDir;
@@ -311,6 +311,10 @@ fn join_codegen(
fn target_config(&self, sess: &Session) -> TargetConfig {
target_config(sess, &self.target_info)
}
fn fallback_intrinsics(&self) -> Vec<Symbol> {
vec![sym::type_id_eq]
}
}
fn new_context<'gcc, 'tcx>(tcx: TyCtxt<'tcx>) -> Context<'gcc> {
compiler/rustc_codegen_llvm/src/lib.rs
+8
View File
@@ -333,6 +333,14 @@ fn replaced_intrinsics(&self) -> Vec<Symbol> {
will_not_use_fallback
}
fn fallback_intrinsics(&self) -> Vec<Symbol> {
// `type_id_eq` is a safe choice since *all* backends use the fallback body for that.
// When adding more intrinsics, keep in mind that the distributed standard library
// is compiled with the LLVM backend but might later be included in a project built
// with cranelift or GCC.
vec![sym::type_id_eq]
}
fn target_cpu(&self, sess: &Session) -> String {
crate::llvm_util::target_cpu(sess).to_string()
}
compiler/rustc_codegen_ssa/src/traits/backend.rs
+6
View File
@@ -79,6 +79,12 @@ fn replaced_intrinsics(&self) -> Vec<Symbol> {
vec![]
}
/// Returns a list of all intrinsics that this backend definitely
/// does *not* replace, which means their fallback bodies can be MIR-inlined.
fn fallback_intrinsics(&self) -> Vec<Symbol> {
vec![]
}
/// Is ThinLTO supported by this backend?
fn thin_lto_supported(&self) -> bool {
true
compiler/rustc_interface/src/interface.rs
+1
View File
@@ -449,6 +449,7 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
};
codegen_backend.init(&sess);
sess.replaced_intrinsics = FxHashSet::from_iter(codegen_backend.replaced_intrinsics());
sess.fallback_intrinsics = FxHashSet::from_iter(codegen_backend.fallback_intrinsics());
sess.thin_lto_supported = codegen_backend.thin_lto_supported();
let cfg = parse_cfg(sess.dcx(), config.crate_cfg);
compiler/rustc_mir_transform/src/check_inline.rs
+6 -6
View File
@@ -59,12 +59,12 @@ pub(super) fn is_inline_valid_on_fn<'tcx>(
return Err("cold");
}
// Intrinsic fallback bodies are automatically made cross-crate inlineable,
// but at this stage we don't know whether codegen knows the intrinsic,
// so just conservatively don't inline it. This also ensures that we do not
// accidentally inline the body of an intrinsic that *must* be overridden.
if find_attr!(tcx, def_id, RustcIntrinsic) {
return Err("callee is an intrinsic");
// Intrinsics without fallback body cannot be inlined. The logic for which intrinsics *with*
// body can be inlined is in the inlining pass.
if let Some(intrinsic) = tcx.intrinsic(def_id)
&& intrinsic.must_be_overridden
{
return Err("callee is an intrinsic without fallback body");
}
Ok(())
compiler/rustc_mir_transform/src/inline.rs
+14 -2
View File
@@ -565,12 +565,24 @@ fn resolve_callsite<'tcx, I: Inliner<'tcx>>(
let args = tcx
.try_normalize_erasing_regions(inliner.typing_env(), Unnormalized::new_wip(args))
.ok()?;
let callee =
let mut callee =
Instance::try_resolve(tcx, inliner.typing_env(), def_id, args).ok().flatten()?;
if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def {
if let InstanceKind::Virtual(..) = callee.def {
return None;
}
if let InstanceKind::Intrinsic(..) = callee.def {
let intrinsic = tcx.intrinsic(def_id).unwrap();
if intrinsic.must_be_overridden {
return None; // intrinsic without fallback body
}
if !tcx.sess.fallback_intrinsics.contains(&intrinsic.name) {
return None; // intrinsic that the backend may want to overwrite
}
// The callee is the fallback body.
debug!("callsite is fallback body: {def_id:?}");
callee = ty::Instance { def: ty::InstanceKind::Item(def_id), args: callee.args };
}
if inliner.history().contains(&callee.def_id()) {
return None;
compiler/rustc_session/src/session.rs
+4
View File
@@ -164,6 +164,9 @@ pub struct Session {
/// The names of intrinsics that the current codegen backend replaces
/// with its own implementations.
pub replaced_intrinsics: FxHashSet<Symbol>,
/// The names of intrinsics that the current codegen backend does *not* replace
/// with its own implementations.
pub fallback_intrinsics: FxHashSet<Symbol>,
/// Does the codegen backend support ThinLTO?
pub thin_lto_supported: bool,
@@ -1124,6 +1127,7 @@ pub fn build_session(
target_filesearch,
host_filesearch,
replaced_intrinsics: FxHashSet::default(), // filled by `run_compiler`
fallback_intrinsics: FxHashSet::default(), // filled by `run_compiler`
thin_lto_supported: true, // filled by `run_compiler`
mir_opt_bisect_eval_count: AtomicUsize::new(0),
used_features: Lock::default(),
library/core/src/any.rs
+1 -24
View File
@@ -742,30 +742,7 @@ unsafe impl Sync for TypeId {}
impl const PartialEq for TypeId {
#[inline]
fn eq(&self, other: &Self) -> bool {
#[cfg(miri)]
return crate::intrinsics::type_id_eq(*self, *other);
#[cfg(not(miri))]
{
let this = self;
crate::intrinsics::const_eval_select!(
@capture { this: &TypeId, other: &TypeId } -> bool:
if const {
crate::intrinsics::type_id_eq(*this, *other)
} else {
// Ideally we would just invoke `type_id_eq` unconditionally here,
// but since we do not MIR inline intrinsics, because backends
// may want to override them (and miri does!), MIR opts do not
// clean up this call sufficiently for LLVM to turn repeated calls
// of `TypeId` comparisons against one specific `TypeId` into
// a lookup table.
// SAFETY: We know that at runtime none of the bits have provenance and all bits
// are initialized. So we can just convert the whole thing to a `u128` and compare that.
unsafe {
crate::mem::transmute::<_, u128>(*this) == crate::mem::transmute::<_, u128>(*other)
}
}
)
}
crate::intrinsics::type_id_eq(*self, *other)
}
}
library/core/src/intrinsics/mod.rs
+3 -1
View File
@@ -2934,7 +2934,9 @@ pub const fn type_of(_id: crate::any::TypeId) -> crate::mem::type_info::Type {
#[rustc_intrinsic]
#[rustc_do_not_const_check]
pub const fn type_id_eq(a: crate::any::TypeId, b: crate::any::TypeId) -> bool {
a.data == b.data
// SAFETY: we know `TypeId` is 16 bytes of initialized data.
// This is runtime-only code so we do not have to worry about provenance.
unsafe { crate::mem::transmute::<_, u128>(a) == crate::mem::transmute::<_, u128>(b) }
}
/// Gets the size of the type represented by this `TypeId`.
tests/mir-opt/inline/type_id_eq.call.Inline.diff
+36
View File
@@ -0,0 +1,36 @@
- // MIR for `call` before Inline
+ // MIR for `call` after Inline
fn call(_1: TypeId, _2: TypeId) -> bool {
debug a => _1;
debug b => _2;
let mut _0: bool;
let mut _3: std::any::TypeId;
let mut _4: std::any::TypeId;
+ scope 1 (inlined type_id_eq) {
+ let mut _5: u128;
+ let mut _6: u128;
+ }
bb0: {
StorageLive(_3);
_3 = copy _1;
StorageLive(_4);
_4 = copy _2;
- _0 = type_id_eq(move _3, move _4) -> [return: bb1, unwind unreachable];
- }
-
- bb1: {
+ StorageLive(_5);
+ _5 = copy _3 as u128 (Transmute);
+ StorageLive(_6);
+ _6 = copy _4 as u128 (Transmute);
+ _0 = Eq(move _5, move _6);
+ StorageDead(_6);
+ StorageDead(_5);
StorageDead(_4);
StorageDead(_3);
return;
}
}
tests/mir-opt/inline/type_id_eq.rs
+20
View File
@@ -0,0 +1,20 @@
#![feature(core_intrinsics)]
//@ test-mir-pass: Inline
//@ compile-flags: --crate-type=lib -C panic=abort
use std::any::{Any, TypeId};
use std::intrinsics::type_id_eq;
struct A<T: ?Sized + 'static> {
a: i32,
b: T,
}
// EMIT_MIR type_id_eq.call.Inline.diff
// CHECK-LABEL: fn call(
pub fn call(a: TypeId, b: TypeId) -> bool {
// CHECK: as u128 (Transmute)
// CHECK: as u128 (Transmute)
// CHECK: Eq
type_id_eq(a, b)
}
tests/ui/consts/const_transmute_type_id2.stderr
-7
View File
@@ -5,14 +5,7 @@ LL | assert!(a == b);
| ^^^^^^ evaluation of `_` failed inside this call
|
note: inside `<TypeId as PartialEq>::eq`
--> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL
--> $SRC_DIR/core/src/any.rs:LL:COL
::: $SRC_DIR/core/src/any.rs:LL:COL
|
= note: in this macro invocation
note: inside `<TypeId as PartialEq>::eq::compiletime`
--> $SRC_DIR/core/src/any.rs:LL:COL
= note: this error originates in the macro `$crate::intrinsics::const_eval_select` which comes from the expansion of the macro `crate::intrinsics::const_eval_select` (in Nightly builds, run with -Z macro-backtrace for more info)
error: aborting due to 1 previous error
tests/ui/consts/const_transmute_type_id3.stderr
-7
View File
@@ -5,14 +5,7 @@ LL | assert!(a == b);
| ^^^^^^ evaluation of `_` failed inside this call
|
note: inside `<TypeId as PartialEq>::eq`
--> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL
--> $SRC_DIR/core/src/any.rs:LL:COL
::: $SRC_DIR/core/src/any.rs:LL:COL
|
= note: in this macro invocation
note: inside `<TypeId as PartialEq>::eq::compiletime`
--> $SRC_DIR/core/src/any.rs:LL:COL
= note: this error originates in the macro `$crate::intrinsics::const_eval_select` which comes from the expansion of the macro `crate::intrinsics::const_eval_select` (in Nightly builds, run with -Z macro-backtrace for more info)
error: aborting due to 1 previous error
tests/ui/consts/const_transmute_type_id4.stderr
-7
View File
@@ -5,14 +5,7 @@ LL | assert!(a == b);
| ^^^^^^ evaluation of `_` failed inside this call
|
note: inside `<TypeId as PartialEq>::eq`
--> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL
--> $SRC_DIR/core/src/any.rs:LL:COL
::: $SRC_DIR/core/src/any.rs:LL:COL
|
= note: in this macro invocation
note: inside `<TypeId as PartialEq>::eq::compiletime`
--> $SRC_DIR/core/src/any.rs:LL:COL
= note: this error originates in the macro `$crate::intrinsics::const_eval_select` which comes from the expansion of the macro `crate::intrinsics::const_eval_select` (in Nightly builds, run with -Z macro-backtrace for more info)
error: aborting due to 1 previous error
tests/ui/consts/const_transmute_type_id5.stderr
-7
View File
@@ -5,14 +5,7 @@ LL | assert!(b == b);
| ^^^^^^ evaluation of `_` failed inside this call
|
note: inside `<TypeId as PartialEq>::eq`
--> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL
--> $SRC_DIR/core/src/any.rs:LL:COL
::: $SRC_DIR/core/src/any.rs:LL:COL
|
= note: in this macro invocation
note: inside `<TypeId as PartialEq>::eq::compiletime`
--> $SRC_DIR/core/src/any.rs:LL:COL
= note: this error originates in the macro `$crate::intrinsics::const_eval_select` which comes from the expansion of the macro `crate::intrinsics::const_eval_select` (in Nightly builds, run with -Z macro-backtrace for more info)
error: aborting due to 1 previous error
tests/ui/consts/const_transmute_type_id6.stderr
-7
View File
@@ -5,14 +5,7 @@ LL | id == id
| ^^^^^^^^ evaluation of `X` failed inside this call
|
note: inside `<TypeId as PartialEq>::eq`
--> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL
--> $SRC_DIR/core/src/any.rs:LL:COL
::: $SRC_DIR/core/src/any.rs:LL:COL
|
= note: in this macro invocation
note: inside `<TypeId as PartialEq>::eq::compiletime`
--> $SRC_DIR/core/src/any.rs:LL:COL
= note: this error originates in the macro `$crate::intrinsics::const_eval_select` which comes from the expansion of the macro `crate::intrinsics::const_eval_select` (in Nightly builds, run with -Z macro-backtrace for more info)
error: aborting due to 1 previous error