diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index acfe8188e360..48858e20381d 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -237,6 +237,10 @@ fn join_codegen( ) -> (CompiledModules, FxIndexMap) { ongoing_codegen.downcast::().unwrap().join(sess, outputs) } + + fn fallback_intrinsics(&self) -> Vec { + vec![sym::type_id_eq] + } } /// Determine if the Cranelift ir verifier should run. diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs index 6ca2ef88ef29..8e7611ed4939 100644 --- a/compiler/rustc_codegen_gcc/src/lib.rs +++ b/compiler/rustc_codegen_gcc/src/lib.rs @@ -98,7 +98,7 @@ use rustc_middle::util::Providers; use rustc_session::Session; use rustc_session::config::{OptLevel, OutputFilenames}; -use rustc_span::Symbol; +use rustc_span::{Symbol, sym}; use rustc_target::spec::{Arch, RelocModel}; use tempfile::TempDir; @@ -311,6 +311,10 @@ fn join_codegen( fn target_config(&self, sess: &Session) -> TargetConfig { target_config(sess, &self.target_info) } + + fn fallback_intrinsics(&self) -> Vec { + vec![sym::type_id_eq] + } } fn new_context<'gcc, 'tcx>(tcx: TyCtxt<'tcx>) -> Context<'gcc> { diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index a697f8fe70bb..31ef5b8fdc4f 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -333,6 +333,14 @@ fn replaced_intrinsics(&self) -> Vec { will_not_use_fallback } + fn fallback_intrinsics(&self) -> Vec { + // `type_id_eq` is a safe choice since *all* backends use the fallback body for that. + // When adding more intrinsics, keep in mind that the distributed standard library + // is compiled with the LLVM backend but might later be included in a project built + // with cranelift or GCC. + vec![sym::type_id_eq] + } + fn target_cpu(&self, sess: &Session) -> String { crate::llvm_util::target_cpu(sess).to_string() } diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs index cbb75836f979..26c80a0afc18 100644 --- a/compiler/rustc_codegen_ssa/src/traits/backend.rs +++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs @@ -79,6 +79,12 @@ fn replaced_intrinsics(&self) -> Vec { vec![] } + /// Returns a list of all intrinsics that this backend definitely + /// does *not* replace, which means their fallback bodies can be MIR-inlined. + fn fallback_intrinsics(&self) -> Vec { + vec![] + } + /// Is ThinLTO supported by this backend? fn thin_lto_supported(&self) -> bool { true diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index 875ed4ae5d30..95aeb1f7a234 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -449,6 +449,7 @@ pub fn run_compiler(config: Config, f: impl FnOnce(&Compiler) -> R + Se }; codegen_backend.init(&sess); sess.replaced_intrinsics = FxHashSet::from_iter(codegen_backend.replaced_intrinsics()); + sess.fallback_intrinsics = FxHashSet::from_iter(codegen_backend.fallback_intrinsics()); sess.thin_lto_supported = codegen_backend.thin_lto_supported(); let cfg = parse_cfg(sess.dcx(), config.crate_cfg); diff --git a/compiler/rustc_mir_transform/src/check_inline.rs b/compiler/rustc_mir_transform/src/check_inline.rs index 4158869f265b..50ef94f623dc 100644 --- a/compiler/rustc_mir_transform/src/check_inline.rs +++ b/compiler/rustc_mir_transform/src/check_inline.rs @@ -59,12 +59,12 @@ pub(super) fn is_inline_valid_on_fn<'tcx>( return Err("cold"); } - // Intrinsic fallback bodies are automatically made cross-crate inlineable, - // but at this stage we don't know whether codegen knows the intrinsic, - // so just conservatively don't inline it. This also ensures that we do not - // accidentally inline the body of an intrinsic that *must* be overridden. - if find_attr!(tcx, def_id, RustcIntrinsic) { - return Err("callee is an intrinsic"); + // Intrinsics without fallback body cannot be inlined. The logic for which intrinsics *with* + // body can be inlined is in the inlining pass. + if let Some(intrinsic) = tcx.intrinsic(def_id) + && intrinsic.must_be_overridden + { + return Err("callee is an intrinsic without fallback body"); } Ok(()) diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 31871c62fa7a..434f96996eea 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -565,12 +565,24 @@ fn resolve_callsite<'tcx, I: Inliner<'tcx>>( let args = tcx .try_normalize_erasing_regions(inliner.typing_env(), Unnormalized::new_wip(args)) .ok()?; - let callee = + let mut callee = Instance::try_resolve(tcx, inliner.typing_env(), def_id, args).ok().flatten()?; - if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def { + if let InstanceKind::Virtual(..) = callee.def { return None; } + if let InstanceKind::Intrinsic(..) = callee.def { + let intrinsic = tcx.intrinsic(def_id).unwrap(); + if intrinsic.must_be_overridden { + return None; // intrinsic without fallback body + } + if !tcx.sess.fallback_intrinsics.contains(&intrinsic.name) { + return None; // intrinsic that the backend may want to overwrite + } + // The callee is the fallback body. + debug!("callsite is fallback body: {def_id:?}"); + callee = ty::Instance { def: ty::InstanceKind::Item(def_id), args: callee.args }; + } if inliner.history().contains(&callee.def_id()) { return None; diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index 003164e8f905..c83bb62324e7 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -164,6 +164,9 @@ pub struct Session { /// The names of intrinsics that the current codegen backend replaces /// with its own implementations. pub replaced_intrinsics: FxHashSet, + /// The names of intrinsics that the current codegen backend does *not* replace + /// with its own implementations. + pub fallback_intrinsics: FxHashSet, /// Does the codegen backend support ThinLTO? pub thin_lto_supported: bool, @@ -1124,6 +1127,7 @@ pub fn build_session( target_filesearch, host_filesearch, replaced_intrinsics: FxHashSet::default(), // filled by `run_compiler` + fallback_intrinsics: FxHashSet::default(), // filled by `run_compiler` thin_lto_supported: true, // filled by `run_compiler` mir_opt_bisect_eval_count: AtomicUsize::new(0), used_features: Lock::default(), diff --git a/library/core/src/any.rs b/library/core/src/any.rs index 62300d8b70a9..54a6408a0632 100644 --- a/library/core/src/any.rs +++ b/library/core/src/any.rs @@ -742,30 +742,7 @@ unsafe impl Sync for TypeId {} impl const PartialEq for TypeId { #[inline] fn eq(&self, other: &Self) -> bool { - #[cfg(miri)] - return crate::intrinsics::type_id_eq(*self, *other); - #[cfg(not(miri))] - { - let this = self; - crate::intrinsics::const_eval_select!( - @capture { this: &TypeId, other: &TypeId } -> bool: - if const { - crate::intrinsics::type_id_eq(*this, *other) - } else { - // Ideally we would just invoke `type_id_eq` unconditionally here, - // but since we do not MIR inline intrinsics, because backends - // may want to override them (and miri does!), MIR opts do not - // clean up this call sufficiently for LLVM to turn repeated calls - // of `TypeId` comparisons against one specific `TypeId` into - // a lookup table. - // SAFETY: We know that at runtime none of the bits have provenance and all bits - // are initialized. So we can just convert the whole thing to a `u128` and compare that. - unsafe { - crate::mem::transmute::<_, u128>(*this) == crate::mem::transmute::<_, u128>(*other) - } - } - ) - } + crate::intrinsics::type_id_eq(*self, *other) } } diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 1aeb1a0eb397..9ef9c226f3cd 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -2934,7 +2934,9 @@ pub const fn type_of(_id: crate::any::TypeId) -> crate::mem::type_info::Type { #[rustc_intrinsic] #[rustc_do_not_const_check] pub const fn type_id_eq(a: crate::any::TypeId, b: crate::any::TypeId) -> bool { - a.data == b.data + // SAFETY: we know `TypeId` is 16 bytes of initialized data. + // This is runtime-only code so we do not have to worry about provenance. + unsafe { crate::mem::transmute::<_, u128>(a) == crate::mem::transmute::<_, u128>(b) } } /// Gets the size of the type represented by this `TypeId`. diff --git a/tests/mir-opt/inline/type_id_eq.call.Inline.diff b/tests/mir-opt/inline/type_id_eq.call.Inline.diff new file mode 100644 index 000000000000..85090e0a2738 --- /dev/null +++ b/tests/mir-opt/inline/type_id_eq.call.Inline.diff @@ -0,0 +1,36 @@ +- // MIR for `call` before Inline ++ // MIR for `call` after Inline + + fn call(_1: TypeId, _2: TypeId) -> bool { + debug a => _1; + debug b => _2; + let mut _0: bool; + let mut _3: std::any::TypeId; + let mut _4: std::any::TypeId; ++ scope 1 (inlined type_id_eq) { ++ let mut _5: u128; ++ let mut _6: u128; ++ } + + bb0: { + StorageLive(_3); + _3 = copy _1; + StorageLive(_4); + _4 = copy _2; +- _0 = type_id_eq(move _3, move _4) -> [return: bb1, unwind unreachable]; +- } +- +- bb1: { ++ StorageLive(_5); ++ _5 = copy _3 as u128 (Transmute); ++ StorageLive(_6); ++ _6 = copy _4 as u128 (Transmute); ++ _0 = Eq(move _5, move _6); ++ StorageDead(_6); ++ StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + return; + } + } + diff --git a/tests/mir-opt/inline/type_id_eq.rs b/tests/mir-opt/inline/type_id_eq.rs new file mode 100644 index 000000000000..72348447cdc4 --- /dev/null +++ b/tests/mir-opt/inline/type_id_eq.rs @@ -0,0 +1,20 @@ +#![feature(core_intrinsics)] +//@ test-mir-pass: Inline +//@ compile-flags: --crate-type=lib -C panic=abort + +use std::any::{Any, TypeId}; +use std::intrinsics::type_id_eq; + +struct A { + a: i32, + b: T, +} + +// EMIT_MIR type_id_eq.call.Inline.diff +// CHECK-LABEL: fn call( +pub fn call(a: TypeId, b: TypeId) -> bool { + // CHECK: as u128 (Transmute) + // CHECK: as u128 (Transmute) + // CHECK: Eq + type_id_eq(a, b) +} diff --git a/tests/ui/consts/const_transmute_type_id2.stderr b/tests/ui/consts/const_transmute_type_id2.stderr index b420deaa49ce..9c103c5c2ba6 100644 --- a/tests/ui/consts/const_transmute_type_id2.stderr +++ b/tests/ui/consts/const_transmute_type_id2.stderr @@ -5,14 +5,7 @@ LL | assert!(a == b); | ^^^^^^ evaluation of `_` failed inside this call | note: inside `::eq` - --> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL --> $SRC_DIR/core/src/any.rs:LL:COL - ::: $SRC_DIR/core/src/any.rs:LL:COL - | - = note: in this macro invocation -note: inside `::eq::compiletime` - --> $SRC_DIR/core/src/any.rs:LL:COL - = note: this error originates in the macro `$crate::intrinsics::const_eval_select` which comes from the expansion of the macro `crate::intrinsics::const_eval_select` (in Nightly builds, run with -Z macro-backtrace for more info) error: aborting due to 1 previous error diff --git a/tests/ui/consts/const_transmute_type_id3.stderr b/tests/ui/consts/const_transmute_type_id3.stderr index 9f796cda6145..679b048c1ab7 100644 --- a/tests/ui/consts/const_transmute_type_id3.stderr +++ b/tests/ui/consts/const_transmute_type_id3.stderr @@ -5,14 +5,7 @@ LL | assert!(a == b); | ^^^^^^ evaluation of `_` failed inside this call | note: inside `::eq` - --> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL --> $SRC_DIR/core/src/any.rs:LL:COL - ::: $SRC_DIR/core/src/any.rs:LL:COL - | - = note: in this macro invocation -note: inside `::eq::compiletime` - --> $SRC_DIR/core/src/any.rs:LL:COL - = note: this error originates in the macro `$crate::intrinsics::const_eval_select` which comes from the expansion of the macro `crate::intrinsics::const_eval_select` (in Nightly builds, run with -Z macro-backtrace for more info) error: aborting due to 1 previous error diff --git a/tests/ui/consts/const_transmute_type_id4.stderr b/tests/ui/consts/const_transmute_type_id4.stderr index c844b10d78cf..4060f086cbac 100644 --- a/tests/ui/consts/const_transmute_type_id4.stderr +++ b/tests/ui/consts/const_transmute_type_id4.stderr @@ -5,14 +5,7 @@ LL | assert!(a == b); | ^^^^^^ evaluation of `_` failed inside this call | note: inside `::eq` - --> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL --> $SRC_DIR/core/src/any.rs:LL:COL - ::: $SRC_DIR/core/src/any.rs:LL:COL - | - = note: in this macro invocation -note: inside `::eq::compiletime` - --> $SRC_DIR/core/src/any.rs:LL:COL - = note: this error originates in the macro `$crate::intrinsics::const_eval_select` which comes from the expansion of the macro `crate::intrinsics::const_eval_select` (in Nightly builds, run with -Z macro-backtrace for more info) error: aborting due to 1 previous error diff --git a/tests/ui/consts/const_transmute_type_id5.stderr b/tests/ui/consts/const_transmute_type_id5.stderr index 9a7384eb95b4..ac057d829562 100644 --- a/tests/ui/consts/const_transmute_type_id5.stderr +++ b/tests/ui/consts/const_transmute_type_id5.stderr @@ -5,14 +5,7 @@ LL | assert!(b == b); | ^^^^^^ evaluation of `_` failed inside this call | note: inside `::eq` - --> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL --> $SRC_DIR/core/src/any.rs:LL:COL - ::: $SRC_DIR/core/src/any.rs:LL:COL - | - = note: in this macro invocation -note: inside `::eq::compiletime` - --> $SRC_DIR/core/src/any.rs:LL:COL - = note: this error originates in the macro `$crate::intrinsics::const_eval_select` which comes from the expansion of the macro `crate::intrinsics::const_eval_select` (in Nightly builds, run with -Z macro-backtrace for more info) error: aborting due to 1 previous error diff --git a/tests/ui/consts/const_transmute_type_id6.stderr b/tests/ui/consts/const_transmute_type_id6.stderr index c0b35f3d1081..fd3ac663505f 100644 --- a/tests/ui/consts/const_transmute_type_id6.stderr +++ b/tests/ui/consts/const_transmute_type_id6.stderr @@ -5,14 +5,7 @@ LL | id == id | ^^^^^^^^ evaluation of `X` failed inside this call | note: inside `::eq` - --> $SRC_DIR/core/src/intrinsics/mod.rs:LL:COL --> $SRC_DIR/core/src/any.rs:LL:COL - ::: $SRC_DIR/core/src/any.rs:LL:COL - | - = note: in this macro invocation -note: inside `::eq::compiletime` - --> $SRC_DIR/core/src/any.rs:LL:COL - = note: this error originates in the macro `$crate::intrinsics::const_eval_select` which comes from the expansion of the macro `crate::intrinsics::const_eval_select` (in Nightly builds, run with -Z macro-backtrace for more info) error: aborting due to 1 previous error