Merge from rustc

compiler/rustc_ast_lowering/src/index.rs

@@ -55,6 +55,7 @@ pub(super) fn index_hir<'hir>(
         OwnerNode::TraitItem(item) => collector.visit_trait_item(item),
         OwnerNode::ImplItem(item) => collector.visit_impl_item(item),
         OwnerNode::ForeignItem(item) => collector.visit_foreign_item(item),
+        OwnerNode::AssocOpaqueTy(..) => unreachable!(),
     };
 
     for (local_id, node) in collector.nodes.iter_enumerated() {

compiler/rustc_ast_lowering/src/lib.rs

@@ -642,23 +642,8 @@ fn make_owner_info(&mut self, node: hir::OwnerNode<'hir>) -> &'hir hir::OwnerInf
         let bodies = SortedMap::from_presorted_elements(bodies);
 
         // Don't hash unless necessary, because it's expensive.
-        let (opt_hash_including_bodies, attrs_hash) = if self.tcx.needs_crate_hash() {
-            self.tcx.with_stable_hashing_context(|mut hcx| {
-                let mut stable_hasher = StableHasher::new();
-                node.hash_stable(&mut hcx, &mut stable_hasher);
-                // Bodies are stored out of line, so we need to pull them explicitly in the hash.
-                bodies.hash_stable(&mut hcx, &mut stable_hasher);
-                let h1 = stable_hasher.finish();
-
-                let mut stable_hasher = StableHasher::new();
-                attrs.hash_stable(&mut hcx, &mut stable_hasher);
-                let h2 = stable_hasher.finish();
-
-                (Some(h1), Some(h2))
-            })
-        } else {
-            (None, None)
-        };
+        let (opt_hash_including_bodies, attrs_hash) =
+            self.tcx.hash_owner_nodes(node, &bodies, &attrs);
         let num_nodes = self.item_local_id_counter.as_usize();
         let (nodes, parenting) = index::index_hir(self.tcx, node, &bodies, num_nodes);
         let nodes = hir::OwnerNodes { opt_hash_including_bodies, nodes, bodies };

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -71,7 +71,7 @@ pub(crate) fn eval_mir_constant<'tcx>(
     // This cannot fail because we checked all required_consts in advance.
     let val = cv
         .eval(fx.tcx, ty::ParamEnv::reveal_all(), Some(constant.span))
-        .expect("erroneous constant not captured by required_consts");
+        .expect("erroneous constant missed by mono item collection");
     (val, cv.ty())
 }
 

compiler/rustc_codegen_llvm/src/abi.rs

@@ -203,57 +203,63 @@ fn store(
         val: &'ll Value,
         dst: PlaceRef<'tcx, &'ll Value>,
     ) {
-        if self.is_ignore() {
-            return;
-        }
-        if self.is_sized_indirect() {
-            OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst)
-        } else if self.is_unsized_indirect() {
-            bug!("unsized `ArgAbi` must be handled through `store_fn_arg`");
-        } else if let PassMode::Cast { cast, pad_i32: _ } = &self.mode {
-            // FIXME(eddyb): Figure out when the simpler Store is safe, clang
-            // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}.
-            let can_store_through_cast_ptr = false;
-            if can_store_through_cast_ptr {
-                bx.store(val, dst.llval, self.layout.align.abi);
-            } else {
-                // The actual return type is a struct, but the ABI
-                // adaptation code has cast it into some scalar type. The
-                // code that follows is the only reliable way I have
-                // found to do a transform like i64 -> {i32,i32}.
-                // Basically we dump the data onto the stack then memcpy it.
-                //
-                // Other approaches I tried:
-                // - Casting rust ret pointer to the foreign type and using Store
-                //   is (a) unsafe if size of foreign type > size of rust type and
-                //   (b) runs afoul of strict aliasing rules, yielding invalid
-                //   assembly under -O (specifically, the store gets removed).
-                // - Truncating foreign type to correct integral type and then
-                //   bitcasting to the struct type yields invalid cast errors.
-
-                // We instead thus allocate some scratch space...
-                let scratch_size = cast.size(bx);
-                let scratch_align = cast.align(bx);
-                let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align);
-                bx.lifetime_start(llscratch, scratch_size);
-
-                // ... where we first store the value...
-                bx.store(val, llscratch, scratch_align);
-
-                // ... and then memcpy it to the intended destination.
-                bx.memcpy(
-                    dst.llval,
-                    self.layout.align.abi,
-                    llscratch,
-                    scratch_align,
-                    bx.const_usize(self.layout.size.bytes()),
-                    MemFlags::empty(),
-                );
-
-                bx.lifetime_end(llscratch, scratch_size);
+        match &self.mode {
+            PassMode::Ignore => {}
+            // Sized indirect arguments
+            PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
+                let align = attrs.pointee_align.unwrap_or(self.layout.align.abi);
+                OperandValue::Ref(val, None, align).store(bx, dst);
+            }
+            // Unsized indirect qrguments
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+                bug!("unsized `ArgAbi` must be handled through `store_fn_arg`");
+            }
+            PassMode::Cast { cast, pad_i32: _ } => {
+                // FIXME(eddyb): Figure out when the simpler Store is safe, clang
+                // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}.
+                let can_store_through_cast_ptr = false;
+                if can_store_through_cast_ptr {
+                    bx.store(val, dst.llval, self.layout.align.abi);
+                } else {
+                    // The actual return type is a struct, but the ABI
+                    // adaptation code has cast it into some scalar type. The
+                    // code that follows is the only reliable way I have
+                    // found to do a transform like i64 -> {i32,i32}.
+                    // Basically we dump the data onto the stack then memcpy it.
+                    //
+                    // Other approaches I tried:
+                    // - Casting rust ret pointer to the foreign type and using Store
+                    //   is (a) unsafe if size of foreign type > size of rust type and
+                    //   (b) runs afoul of strict aliasing rules, yielding invalid
+                    //   assembly under -O (specifically, the store gets removed).
+                    // - Truncating foreign type to correct integral type and then
+                    //   bitcasting to the struct type yields invalid cast errors.
+
+                    // We instead thus allocate some scratch space...
+                    let scratch_size = cast.size(bx);
+                    let scratch_align = cast.align(bx);
+                    let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align);
+                    bx.lifetime_start(llscratch, scratch_size);
+
+                    // ... where we first store the value...
+                    bx.store(val, llscratch, scratch_align);
+
+                    // ... and then memcpy it to the intended destination.
+                    bx.memcpy(
+                        dst.llval,
+                        self.layout.align.abi,
+                        llscratch,
+                        scratch_align,
+                        bx.const_usize(self.layout.size.bytes()),
+                        MemFlags::empty(),
+                    );
+
+                    bx.lifetime_end(llscratch, scratch_size);
+                }
+            }
+            _ => {
+                OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst);
             }
-        } else {
-            OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst);
         }
     }
 

compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs

@@ -164,6 +164,15 @@ pub(crate) fn from_mapping(
                 end_line,
                 end_col,
             ),
+            MappingKind::Branch { true_term, false_term } => Self::branch_region(
+                Counter::from_term(true_term),
+                Counter::from_term(false_term),
+                local_file_id,
+                start_line,
+                start_col,
+                end_line,
+                end_col,
+            ),
         }
     }
 
@@ -188,9 +197,6 @@ pub(crate) fn code_region(
         }
     }
 
-    // This function might be used in the future; the LLVM API is still evolving, as is coverage
-    // support.
-    #[allow(dead_code)]
     pub(crate) fn branch_region(
         counter: Counter,
         false_counter: Counter,

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -88,7 +88,7 @@ fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage) {
         match coverage.kind {
             // Marker statements have no effect during codegen,
             // so return early and don't create `func_coverage`.
-            CoverageKind::SpanMarker => return,
+            CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => return,
             // Match exhaustively to ensure that newly-added kinds are classified correctly.
             CoverageKind::CounterIncrement { .. } | CoverageKind::ExpressionUsed { .. } => {}
         }
@@ -108,7 +108,7 @@ fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage) {
 
         let Coverage { kind } = coverage;
         match *kind {
-            CoverageKind::SpanMarker => unreachable!(
+            CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!(
                 "unexpected marker statement {kind:?} should have caused an early return"
             ),
             CoverageKind::CounterIncrement { id } => {

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -1081,6 +1081,21 @@ fn is_illegal_instruction(_status: &ExitStatus) -> bool {
         }
     }
 
+    if sess.target.is_like_aix {
+        let stripcmd = "/usr/bin/strip";
+        match strip {
+            Strip::Debuginfo => {
+                // FIXME: AIX's strip utility only offers option to strip line number information.
+                strip_symbols_with_external_utility(sess, stripcmd, out_filename, Some("-l"))
+            }
+            Strip::Symbols => {
+                // Must be noted this option might remove symbol __aix_rust_metadata and thus removes .info section which contains metadata.
+                strip_symbols_with_external_utility(sess, stripcmd, out_filename, Some("-r"))
+            }
+            Strip::None => {}
+        }
+    }
+
     Ok(())
 }
 

compiler/rustc_codegen_ssa/src/back/linker.rs

@@ -1640,16 +1640,7 @@ fn control_flow_guard(&mut self) {}
 
     fn ehcont_guard(&mut self) {}
 
-    fn debuginfo(&mut self, strip: Strip, _: &[PathBuf]) {
-        match strip {
-            Strip::None => {}
-            // FIXME: -s strips the symbol table, line number information
-            // and relocation information.
-            Strip::Debuginfo | Strip::Symbols => {
-                self.cmd.arg("-s");
-            }
-        }
-    }
+    fn debuginfo(&mut self, _: Strip, _: &[PathBuf]) {}
 
     fn no_crt_objects(&mut self) {}
 

compiler/rustc_codegen_ssa/src/mir/constant.rs

@@ -21,11 +21,11 @@ pub fn eval_mir_constant_to_operand(
     }
 
     pub fn eval_mir_constant(&self, constant: &mir::ConstOperand<'tcx>) -> mir::ConstValue<'tcx> {
-        // `MirUsedCollector` visited all constants before codegen began, so if we got here there
-        // can be no more constants that fail to evaluate.
+        // `MirUsedCollector` visited all required_consts before codegen began, so if we got here
+        // there can be no more constants that fail to evaluate.
         self.monomorphize(constant.const_)
             .eval(self.cx.tcx(), ty::ParamEnv::reveal_all(), Some(constant.span))
-            .expect("erroneous constant not captured by required_consts")
+            .expect("erroneous constant missed by mono item collection")
     }
 
     /// This is a convenience helper for `simd_shuffle_indices`. It has the precondition

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -211,7 +211,8 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
     // It may seem like we should iterate over `required_consts` to ensure they all successfully
     // evaluate; however, the `MirUsedCollector` already did that during the collection phase of
-    // monomorphization so we don't have to do it again.
+    // monomorphization, and if there is an error during collection then codegen never starts -- so
+    // we don't have to do it again.
 
     fx.per_local_var_debug_info = fx.compute_per_local_var_debug_info(&mut start_bx);
 
@@ -376,29 +377,45 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 }
             }
 
-            if arg.is_sized_indirect() {
-                // Don't copy an indirect argument to an alloca, the caller
-                // already put it in a temporary alloca and gave it up.
-                // FIXME: lifetimes
-                let llarg = bx.get_param(llarg_idx);
-                llarg_idx += 1;
-                LocalRef::Place(PlaceRef::new_sized(llarg, arg.layout))
-            } else if arg.is_unsized_indirect() {
-                // As the storage for the indirect argument lives during
-                // the whole function call, we just copy the fat pointer.
-                let llarg = bx.get_param(llarg_idx);
-                llarg_idx += 1;
-                let llextra = bx.get_param(llarg_idx);
-                llarg_idx += 1;
-                let indirect_operand = OperandValue::Pair(llarg, llextra);
+            match arg.mode {
+                // Sized indirect arguments
+                PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
+                    // Don't copy an indirect argument to an alloca, the caller already put it
+                    // in a temporary alloca and gave it up.
+                    // FIXME: lifetimes
+                    if let Some(pointee_align) = attrs.pointee_align
+                        && pointee_align < arg.layout.align.abi
+                    {
+                        // ...unless the argument is underaligned, then we need to copy it to
+                        // a higher-aligned alloca.
+                        let tmp = PlaceRef::alloca(bx, arg.layout);
+                        bx.store_fn_arg(arg, &mut llarg_idx, tmp);
+                        LocalRef::Place(tmp)
+                    } else {
+                        let llarg = bx.get_param(llarg_idx);
+                        llarg_idx += 1;
+                        LocalRef::Place(PlaceRef::new_sized(llarg, arg.layout))
+                    }
+                }
+                // Unsized indirect qrguments
+                PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+                    // As the storage for the indirect argument lives during
+                    // the whole function call, we just copy the fat pointer.
+                    let llarg = bx.get_param(llarg_idx);
+                    llarg_idx += 1;
+                    let llextra = bx.get_param(llarg_idx);
+                    llarg_idx += 1;
+                    let indirect_operand = OperandValue::Pair(llarg, llextra);
 
-                let tmp = PlaceRef::alloca_unsized_indirect(bx, arg.layout);
-                indirect_operand.store(bx, tmp);
-                LocalRef::UnsizedPlace(tmp)
-            } else {
-                let tmp = PlaceRef::alloca(bx, arg.layout);
-                bx.store_fn_arg(arg, &mut llarg_idx, tmp);
-                LocalRef::Place(tmp)
+                    let tmp = PlaceRef::alloca_unsized_indirect(bx, arg.layout);
+                    indirect_operand.store(bx, tmp);
+                    LocalRef::UnsizedPlace(tmp)
+                }
+                _ => {
+                    let tmp = PlaceRef::alloca(bx, arg.layout);
+                    bx.store_fn_arg(arg, &mut llarg_idx, tmp);
+                    LocalRef::Place(tmp)
+                }
             }
         })
         .collect::<Vec<_>>();

compiler/rustc_const_eval/messages.ftl

@@ -374,12 +374,6 @@ const_eval_unallowed_op_in_const_context =
 const_eval_unavailable_target_features_for_fn =
     calling a function that requires unavailable target features: {$unavailable_feats}
 
-const_eval_undefined_behavior =
-    it is undefined behavior to use this value
-
-const_eval_undefined_behavior_note =
-    The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
-
 const_eval_uninhabited_enum_variant_read =
     read discriminant of an uninhabited enum variant
 const_eval_uninhabited_enum_variant_written =
@@ -434,6 +428,12 @@ const_eval_validation_expected_raw_ptr = expected a raw pointer
 const_eval_validation_expected_ref = expected a reference
 const_eval_validation_expected_str = expected a string
 
+const_eval_validation_failure =
+    it is undefined behavior to use this value
+
+const_eval_validation_failure_note =
+    The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
+
 const_eval_validation_front_matter_invalid_value = constructing invalid value
 const_eval_validation_front_matter_invalid_value_with_path = constructing invalid value at {$path}
 

compiler/rustc_const_eval/src/const_eval/error.rs

@@ -2,15 +2,16 @@
 
 use rustc_errors::{DiagArgName, DiagArgValue, DiagMessage, Diagnostic, IntoDiagArg};
 use rustc_hir::CRATE_HIR_ID;
+use rustc_middle::mir::interpret::Provenance;
 use rustc_middle::mir::AssertKind;
 use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::TyCtxt;
 use rustc_middle::ty::{layout::LayoutError, ConstInt};
 use rustc_span::{Span, Symbol, DUMMY_SP};
 
-use super::{CompileTimeInterpreter, InterpCx};
+use super::CompileTimeInterpreter;
 use crate::errors::{self, FrameNote, ReportErrorExt};
-use crate::interpret::{ErrorHandled, InterpError, InterpErrorInfo, MachineStopType};
+use crate::interpret::{ErrorHandled, Frame, InterpError, InterpErrorInfo, MachineStopType};
 
 /// The CTFE machine has some custom error kinds.
 #[derive(Clone, Debug)]
@@ -58,15 +59,12 @@ fn into(self) -> InterpErrorInfo<'tcx> {
 
 pub fn get_span_and_frames<'tcx, 'mir>(
     tcx: TyCtxtAt<'tcx>,
-    machine: &CompileTimeInterpreter<'mir, 'tcx>,
+    stack: &[Frame<'mir, 'tcx, impl Provenance, impl Sized>],
 ) -> (Span, Vec<errors::FrameNote>)
 where
     'tcx: 'mir,
 {
-    let mut stacktrace =
-        InterpCx::<CompileTimeInterpreter<'mir, 'tcx>>::generate_stacktrace_from_stack(
-            &machine.stack,
-        );
+    let mut stacktrace = Frame::generate_stacktrace_from_stack(stack);
     // Filter out `requires_caller_location` frames.
     stacktrace.retain(|frame| !frame.instance.def.requires_caller_location(*tcx));
     let span = stacktrace.first().map(|f| f.span).unwrap_or(tcx.span);
@@ -170,7 +168,7 @@ pub(super) fn lint<'tcx, 'mir, L>(
 ) where
     L: for<'a> rustc_errors::LintDiagnostic<'a, ()>,
 {
-    let (span, frames) = get_span_and_frames(tcx, machine);
+    let (span, frames) = get_span_and_frames(tcx, &machine.stack);
 
     tcx.emit_node_span_lint(
         lint,

compiler/rustc_const_eval/src/const_eval/eval_queries.rs

@@ -18,18 +18,18 @@
 use crate::errors::ConstEvalError;
 use crate::interpret::eval_nullary_intrinsic;
 use crate::interpret::{
-    create_static_alloc, intern_const_alloc_recursive, take_static_root_alloc, CtfeValidationMode,
-    GlobalId, Immediate, InternKind, InterpCx, InterpError, InterpResult, MPlaceTy, MemoryKind,
-    OpTy, RefTracking, StackPopCleanup,
+    create_static_alloc, intern_const_alloc_recursive, CtfeValidationMode, GlobalId, Immediate,
+    InternKind, InterpCx, InterpError, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking,
+    StackPopCleanup,
 };
 
 // Returns a pointer to where the result lives
-#[instrument(level = "trace", skip(ecx, body), ret)]
-fn eval_body_using_ecx<'mir, 'tcx>(
+#[instrument(level = "trace", skip(ecx, body))]
+fn eval_body_using_ecx<'mir, 'tcx, R: InterpretationResult<'tcx>>(
     ecx: &mut CompileTimeEvalContext<'mir, 'tcx>,
     cid: GlobalId<'tcx>,
     body: &'mir mir::Body<'tcx>,
-) -> InterpResult<'tcx, MPlaceTy<'tcx>> {
+) -> InterpResult<'tcx, R> {
     trace!(?ecx.param_env);
     let tcx = *ecx.tcx;
     assert!(
@@ -84,7 +84,10 @@ fn eval_body_using_ecx<'mir, 'tcx>(
     // Intern the result
     intern_const_alloc_recursive(ecx, intern_kind, &ret)?;
 
-    Ok(ret)
+    // Since evaluation had no errors, validate the resulting constant.
+    const_validate_mplace(&ecx, &ret, cid)?;
+
+    Ok(R::make_result(ret, ecx))
 }
 
 /// The `InterpCx` is only meant to be used to do field and index projections into constants for
@@ -282,18 +285,26 @@ pub fn eval_static_initializer_provider<'tcx>(
 
     let instance = ty::Instance::mono(tcx, def_id.to_def_id());
     let cid = rustc_middle::mir::interpret::GlobalId { instance, promoted: None };
-    let mut ecx = InterpCx::new(
-        tcx,
-        tcx.def_span(def_id),
-        ty::ParamEnv::reveal_all(),
-        // Statics (and promoteds inside statics) may access other statics, because unlike consts
-        // they do not have to behave "as if" they were evaluated at runtime.
-        CompileTimeInterpreter::new(CanAccessMutGlobal::Yes, CheckAlignment::Error),
-    );
-    let alloc_id = eval_in_interpreter(&mut ecx, cid, true)?.alloc_id;
-    let alloc = take_static_root_alloc(&mut ecx, alloc_id);
-    let alloc = tcx.mk_const_alloc(alloc);
-    Ok(alloc)
+    eval_in_interpreter(tcx, cid, ty::ParamEnv::reveal_all())
+}
+
+pub trait InterpretationResult<'tcx> {
+    /// This function takes the place where the result of the evaluation is stored
+    /// and prepares it for returning it in the appropriate format needed by the specific
+    /// evaluation query.
+    fn make_result<'mir>(
+        mplace: MPlaceTy<'tcx>,
+        ecx: &mut InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>,
+    ) -> Self;
+}
+
+impl<'tcx> InterpretationResult<'tcx> for ConstAlloc<'tcx> {
+    fn make_result<'mir>(
+        mplace: MPlaceTy<'tcx>,
+        _ecx: &mut InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>,
+    ) -> Self {
+        ConstAlloc { alloc_id: mplace.ptr().provenance.unwrap().alloc_id(), ty: mplace.layout.ty }
+    }
 }
 
 #[instrument(skip(tcx), level = "debug")]
@@ -319,92 +330,64 @@ pub fn eval_to_allocation_raw_provider<'tcx>(
         trace!("const eval: {:?} ({})", key, instance);
     }
 
-    let cid = key.value;
+    eval_in_interpreter(tcx, key.value, key.param_env)
+}
+
+fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
+    tcx: TyCtxt<'tcx>,
+    cid: GlobalId<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+) -> Result<R, ErrorHandled> {
     let def = cid.instance.def.def_id();
     let is_static = tcx.is_static(def);
 
     let mut ecx = InterpCx::new(
         tcx,
         tcx.def_span(def),
-        key.param_env,
+        param_env,
         // Statics (and promoteds inside statics) may access mutable global memory, because unlike consts
         // they do not have to behave "as if" they were evaluated at runtime.
         // For consts however we want to ensure they behave "as if" they were evaluated at runtime,
         // so we have to reject reading mutable global memory.
         CompileTimeInterpreter::new(CanAccessMutGlobal::from(is_static), CheckAlignment::Error),
     );
-    eval_in_interpreter(&mut ecx, cid, is_static)
-}
-
-pub fn eval_in_interpreter<'mir, 'tcx>(
-    ecx: &mut InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>,
-    cid: GlobalId<'tcx>,
-    is_static: bool,
-) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> {
-    // `is_static` just means "in static", it could still be a promoted!
-    debug_assert_eq!(is_static, ecx.tcx.static_mutability(cid.instance.def_id()).is_some());
-
     let res = ecx.load_mir(cid.instance.def, cid.promoted);
-    match res.and_then(|body| eval_body_using_ecx(ecx, cid, body)) {
-        Err(error) => {
-            let (error, backtrace) = error.into_parts();
-            backtrace.print_backtrace();
+    res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body)).map_err(|error| {
+        let (error, backtrace) = error.into_parts();
+        backtrace.print_backtrace();
 
-            let (kind, instance) = if is_static {
-                ("static", String::new())
+        let (kind, instance) = if ecx.tcx.is_static(cid.instance.def_id()) {
+            ("static", String::new())
+        } else {
+            // If the current item has generics, we'd like to enrich the message with the
+            // instance and its args: to show the actual compile-time values, in addition to
+            // the expression, leading to the const eval error.
+            let instance = &cid.instance;
+            if !instance.args.is_empty() {
+                let instance = with_no_trimmed_paths!(instance.to_string());
+                ("const_with_path", instance)
             } else {
-                // If the current item has generics, we'd like to enrich the message with the
-                // instance and its args: to show the actual compile-time values, in addition to
-                // the expression, leading to the const eval error.
-                let instance = &cid.instance;
-                if !instance.args.is_empty() {
-                    let instance = with_no_trimmed_paths!(instance.to_string());
-                    ("const_with_path", instance)
-                } else {
-                    ("const", String::new())
-                }
-            };
-
-            Err(super::report(
-                *ecx.tcx,
-                error,
-                None,
-                || super::get_span_and_frames(ecx.tcx, &ecx.machine),
-                |span, frames| ConstEvalError {
-                    span,
-                    error_kind: kind,
-                    instance,
-                    frame_notes: frames,
-                },
-            ))
-        }
-        Ok(mplace) => {
-            // Since evaluation had no errors, validate the resulting constant.
-
-            // Temporarily allow access to the static_root_ids for the purpose of validation.
-            let static_root_ids = ecx.machine.static_root_ids.take();
-            let res = const_validate_mplace(&ecx, &mplace, cid);
-            ecx.machine.static_root_ids = static_root_ids;
-
-            let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();
-
-            // Validation failed, report an error.
-            if let Err(error) = res {
-                Err(const_report_error(&ecx, error, alloc_id))
-            } else {
-                // Convert to raw constant
-                Ok(ConstAlloc { alloc_id, ty: mplace.layout.ty })
+                ("const", String::new())
             }
-        }
-    }
+        };
+
+        super::report(
+            *ecx.tcx,
+            error,
+            None,
+            || super::get_span_and_frames(ecx.tcx, ecx.stack()),
+            |span, frames| ConstEvalError { span, error_kind: kind, instance, frame_notes: frames },
+        )
+    })
 }
 
 #[inline(always)]
-pub fn const_validate_mplace<'mir, 'tcx>(
+fn const_validate_mplace<'mir, 'tcx>(
     ecx: &InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>,
     mplace: &MPlaceTy<'tcx>,
     cid: GlobalId<'tcx>,
-) -> InterpResult<'tcx> {
+) -> Result<(), ErrorHandled> {
+    let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();
     let mut ref_tracking = RefTracking::new(mplace.clone());
     let mut inner = false;
     while let Some((mplace, path)) = ref_tracking.todo.pop() {
@@ -418,7 +401,10 @@ pub fn const_validate_mplace<'mir, 'tcx>(
                 CtfeValidationMode::Const { allow_immutable_unsafe_cell: !inner }
             }
         };
-        ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)?;
+        ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)
+            // Instead of just reporting the `InterpError` via the usual machinery, we give a more targetted
+            // error about the validation failure.
+            .map_err(|error| report_validation_error(&ecx, error, alloc_id))?;
         inner = true;
     }
 
@@ -426,7 +412,7 @@ pub fn const_validate_mplace<'mir, 'tcx>(
 }
 
 #[inline(always)]
-pub fn const_report_error<'mir, 'tcx>(
+fn report_validation_error<'mir, 'tcx>(
     ecx: &InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>,
     error: InterpErrorInfo<'tcx>,
     alloc_id: AllocId,
@@ -444,7 +430,7 @@ pub fn const_report_error<'mir, 'tcx>(
         *ecx.tcx,
         error,
         None,
-        || crate::const_eval::get_span_and_frames(ecx.tcx, &ecx.machine),
-        move |span, frames| errors::UndefinedBehavior { span, ub_note, frames, raw_bytes },
+        || crate::const_eval::get_span_and_frames(ecx.tcx, ecx.stack()),
+        move |span, frames| errors::ValidationFailure { span, ub_note, frames, raw_bytes },
     )
 }

compiler/rustc_const_eval/src/const_eval/mod.rs

@@ -5,7 +5,7 @@
 use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::{self, Ty};
 
-use crate::interpret::{format_interp_error, InterpCx};
+use crate::interpret::format_interp_error;
 
 mod error;
 mod eval_queries;

compiler/rustc_const_eval/src/errors.rs

@@ -412,11 +412,11 @@ pub struct NullaryIntrinsicError {
 }
 
 #[derive(Diagnostic)]
-#[diag(const_eval_undefined_behavior, code = E0080)]
-pub struct UndefinedBehavior {
+#[diag(const_eval_validation_failure, code = E0080)]
+pub struct ValidationFailure {
     #[primary_span]
     pub span: Span,
-    #[note(const_eval_undefined_behavior_note)]
+    #[note(const_eval_validation_failure_note)]
     pub ub_note: Option<()>,
     #[subdiagnostic]
     pub frames: Vec<FrameNote>,

compiler/rustc_const_eval/src/interpret/eval_context.rs

@@ -220,9 +220,6 @@ pub(super) fn access(&self) -> InterpResult<'tcx, &Operand<Prov>> {
 
     /// Overwrite the local. If the local can be overwritten in place, return a reference
     /// to do so; otherwise return the `MemPlace` to consult instead.
-    ///
-    /// Note: Before calling this, call the `before_access_local_mut` machine hook! You may be
-    /// invalidating machine invariants otherwise!
     #[inline(always)]
     pub(super) fn access_mut(&mut self) -> InterpResult<'tcx, &mut Operand<Prov>> {
         match &mut self.value {
@@ -279,6 +276,39 @@ pub fn lint_root(&self) -> Option<hir::HirId> {
             }
         })
     }
+
+    /// Returns the address of the buffer where the locals are stored. This is used by `Place` as a
+    /// sanity check to detect bugs where we mix up which stack frame a place refers to.
+    #[inline(always)]
+    pub(super) fn locals_addr(&self) -> usize {
+        self.locals.raw.as_ptr().addr()
+    }
+
+    #[must_use]
+    pub fn generate_stacktrace_from_stack(stack: &[Self]) -> Vec<FrameInfo<'tcx>> {
+        let mut frames = Vec::new();
+        // This deliberately does *not* honor `requires_caller_location` since it is used for much
+        // more than just panics.
+        for frame in stack.iter().rev() {
+            let span = match frame.loc {
+                Left(loc) => {
+                    // If the stacktrace passes through MIR-inlined source scopes, add them.
+                    let mir::SourceInfo { mut span, scope } = *frame.body.source_info(loc);
+                    let mut scope_data = &frame.body.source_scopes[scope];
+                    while let Some((instance, call_span)) = scope_data.inlined {
+                        frames.push(FrameInfo { span, instance });
+                        span = call_span;
+                        scope_data = &frame.body.source_scopes[scope_data.parent_scope.unwrap()];
+                    }
+                    span
+                }
+                Right(span) => span,
+            };
+            frames.push(FrameInfo { span, instance: frame.instance });
+        }
+        trace!("generate stacktrace: {:#?}", frames);
+        frames
+    }
 }
 
 // FIXME: only used by miri, should be removed once translatable.
@@ -645,7 +675,7 @@ pub(crate) fn find_closest_untracked_caller_location(&self) -> Span {
     }
 
     #[inline(always)]
-    pub fn layout_of_local(
+    pub(super) fn layout_of_local(
         &self,
         frame: &Frame<'mir, 'tcx, M::Provenance, M::FrameExtra>,
         local: mir::Local,
@@ -896,7 +926,7 @@ pub(super) fn pop_stack_frame(&mut self, unwinding: bool) -> InterpResult<'tcx>
         // Copy return value. Must of course happen *before* we deallocate the locals.
         let copy_ret_result = if !unwinding {
             let op = self
-                .local_to_op(self.frame(), mir::RETURN_PLACE, None)
+                .local_to_op(mir::RETURN_PLACE, None)
                 .expect("return place should always be live");
             let dest = self.frame().return_place.clone();
             let err = if self.stack().len() == 1 {
@@ -1166,37 +1196,9 @@ pub fn dump_place(
         PlacePrinter { ecx: self, place: *place.place() }
     }
 
-    #[must_use]
-    pub fn generate_stacktrace_from_stack(
-        stack: &[Frame<'mir, 'tcx, M::Provenance, M::FrameExtra>],
-    ) -> Vec<FrameInfo<'tcx>> {
-        let mut frames = Vec::new();
-        // This deliberately does *not* honor `requires_caller_location` since it is used for much
-        // more than just panics.
-        for frame in stack.iter().rev() {
-            let span = match frame.loc {
-                Left(loc) => {
-                    // If the stacktrace passes through MIR-inlined source scopes, add them.
-                    let mir::SourceInfo { mut span, scope } = *frame.body.source_info(loc);
-                    let mut scope_data = &frame.body.source_scopes[scope];
-                    while let Some((instance, call_span)) = scope_data.inlined {
-                        frames.push(FrameInfo { span, instance });
-                        span = call_span;
-                        scope_data = &frame.body.source_scopes[scope_data.parent_scope.unwrap()];
-                    }
-                    span
-                }
-                Right(span) => span,
-            };
-            frames.push(FrameInfo { span, instance: frame.instance });
-        }
-        trace!("generate stacktrace: {:#?}", frames);
-        frames
-    }
-
     #[must_use]
     pub fn generate_stacktrace(&self) -> Vec<FrameInfo<'tcx>> {
-        Self::generate_stacktrace_from_stack(self.stack())
+        Frame::generate_stacktrace_from_stack(self.stack())
     }
 }
 
@@ -1212,18 +1214,16 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> std::fmt::Debug
 {
     fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self.place {
-            Place::Local { frame, local, offset } => {
+            Place::Local { local, offset, locals_addr } => {
+                debug_assert_eq!(locals_addr, self.ecx.frame().locals_addr());
                 let mut allocs = Vec::new();
                 write!(fmt, "{local:?}")?;
                 if let Some(offset) = offset {
                     write!(fmt, "+{:#x}", offset.bytes())?;
                 }
-                if frame != self.ecx.frame_idx() {
-                    write!(fmt, " ({} frames up)", self.ecx.frame_idx() - frame)?;
-                }
                 write!(fmt, ":")?;
 
-                match self.ecx.stack()[frame].locals[local].value {
+                match self.ecx.frame().locals[local].value {
                     LocalValue::Dead => write!(fmt, " is dead")?,
                     LocalValue::Live(Operand::Immediate(Immediate::Uninit)) => {
                         write!(fmt, " is uninitialized")?

compiler/rustc_const_eval/src/interpret/intern.rs

@@ -176,7 +176,7 @@ pub fn intern_const_alloc_recursive<
     // This gives us the initial set of nested allocations, which will then all be processed
     // recursively in the loop below.
     let mut todo: Vec<_> = if is_static {
-        // Do not steal the root allocation, we need it later for `take_static_root_alloc`
+        // Do not steal the root allocation, we need it later to create the return value of `eval_static_initializer`.
         // But still change its mutability to match the requested one.
         let alloc = ecx.memory.alloc_map.get_mut(&base_alloc_id).unwrap();
         alloc.1.mutability = base_mutability;

compiler/rustc_const_eval/src/interpret/machine.rs

@@ -260,24 +260,6 @@ fn generate_nan<F1: Float + FloatConvert<F2>, F2: Float>(
         F2::NAN
     }
 
-    /// Called before writing the specified `local` of the `frame`.
-    /// Since writing a ZST is not actually accessing memory or locals, this is never invoked
-    /// for ZST reads.
-    ///
-    /// Due to borrow checker trouble, we indicate the `frame` as an index rather than an `&mut
-    /// Frame`.
-    #[inline(always)]
-    fn before_access_local_mut<'a>(
-        _ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
-        _frame: usize,
-        _local: mir::Local,
-    ) -> InterpResult<'tcx>
-    where
-        'tcx: 'mir,
-    {
-        Ok(())
-    }
-
     /// Called before a basic block terminator is executed.
     #[inline]
     fn before_terminator(_ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> {
@@ -531,7 +513,6 @@ fn after_stack_pop(
     #[inline(always)]
     fn after_local_allocated(
         _ecx: &mut InterpCx<'mir, 'tcx, Self>,
-        _frame: usize,
         _local: mir::Local,
         _mplace: &MPlaceTy<'tcx, Self::Provenance>,
     ) -> InterpResult<'tcx> {

compiler/rustc_const_eval/src/interpret/mod.rs

@@ -39,5 +39,5 @@
 };
 
 pub(crate) use self::intrinsics::eval_nullary_intrinsic;
-pub(crate) use self::util::{create_static_alloc, take_static_root_alloc};
+pub(crate) use self::util::create_static_alloc;
 use eval_context::{from_known_layout, mir_assign_valid_types};

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -13,9 +13,9 @@
 use rustc_target::abi::{self, Abi, HasDataLayout, Size};
 
 use super::{
-    alloc_range, from_known_layout, mir_assign_valid_types, CtfeProvenance, Frame, InterpCx,
-    InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, OffsetMode, PlaceTy, Pointer,
-    Projectable, Provenance, Scalar,
+    alloc_range, from_known_layout, mir_assign_valid_types, CtfeProvenance, InterpCx, InterpResult,
+    MPlaceTy, Machine, MemPlace, MemPlaceMeta, OffsetMode, PlaceTy, Pointer, Projectable,
+    Provenance, Scalar,
 };
 
 /// An `Immediate` represents a single immediate self-contained Rust value.
@@ -633,17 +633,17 @@ pub fn operand_to_simd(
         }
     }
 
-    /// Read from a local.
+    /// Read from a local of the current frame.
     /// Will not access memory, instead an indirect `Operand` is returned.
     ///
     /// This is public because it is used by [priroda](https://github.com/oli-obk/priroda) to get an
     /// OpTy from a local.
     pub fn local_to_op(
         &self,
-        frame: &Frame<'mir, 'tcx, M::Provenance, M::FrameExtra>,
         local: mir::Local,
         layout: Option<TyAndLayout<'tcx>>,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
+        let frame = self.frame();
         let layout = self.layout_of_local(frame, local, layout)?;
         let op = *frame.locals[local].access()?;
         if matches!(op, Operand::Immediate(_)) {
@@ -661,9 +661,10 @@ pub fn place_to_op(
     ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
         match place.as_mplace_or_local() {
             Left(mplace) => Ok(mplace.into()),
-            Right((frame, local, offset)) => {
+            Right((local, offset, locals_addr)) => {
                 debug_assert!(place.layout.is_sized()); // only sized locals can ever be `Place::Local`.
-                let base = self.local_to_op(&self.stack()[frame], local, None)?;
+                debug_assert_eq!(locals_addr, self.frame().locals_addr());
+                let base = self.local_to_op(local, None)?;
                 Ok(match offset {
                     Some(offset) => base.offset(offset, place.layout, self)?,
                     None => {
@@ -687,7 +688,7 @@ pub fn eval_place_to_op(
         // here is not the entire place.
         let layout = if mir_place.projection.is_empty() { layout } else { None };
 
-        let mut op = self.local_to_op(self.frame(), mir_place.local, layout)?;
+        let mut op = self.local_to_op(mir_place.local, layout)?;
         // Using `try_fold` turned out to be bad for performance, hence the loop.
         for elem in mir_place.projection.iter() {
             op = self.project(&op, elem)?

compiler/rustc_const_eval/src/interpret/place.rs

@@ -187,11 +187,13 @@ pub(super) enum Place<Prov: Provenance = CtfeProvenance> {
     /// where in the local this place is located; if it is `None`, no projection has been applied.
     /// Such projections are meaningful even if the offset is 0, since they can change layouts.
     /// (Without that optimization, we'd just always be a `MemPlace`.)
-    /// Note that this only stores the frame index, not the thread this frame belongs to -- that is
-    /// implicit. This means a `Place` must never be moved across interpreter thread boundaries!
+    /// `Local` places always refer to the current stack frame, so they are unstable under
+    /// function calls/returns and switching betweens stacks of different threads!
+    /// We carry around the address of the `locals` buffer of the correct stack frame as a sanity
+    /// chec to be able to catch some cases of using a dangling `Place`.
     ///
     /// This variant shall not be used for unsized types -- those must always live in memory.
-    Local { frame: usize, local: mir::Local, offset: Option<Size> },
+    Local { local: mir::Local, offset: Option<Size>, locals_addr: usize },
 }
 
 /// An evaluated place, together with its type.
@@ -233,10 +235,10 @@ pub(super) fn place(&self) -> &Place<Prov> {
     #[inline(always)]
     pub fn as_mplace_or_local(
         &self,
-    ) -> Either<MPlaceTy<'tcx, Prov>, (usize, mir::Local, Option<Size>)> {
+    ) -> Either<MPlaceTy<'tcx, Prov>, (mir::Local, Option<Size>, usize)> {
         match self.place {
             Place::Ptr(mplace) => Left(MPlaceTy { mplace, layout: self.layout }),
-            Place::Local { frame, local, offset } => Right((frame, local, offset)),
+            Place::Local { local, offset, locals_addr } => Right((local, offset, locals_addr)),
         }
     }
 
@@ -279,7 +281,7 @@ fn offset_with_meta<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
     ) -> InterpResult<'tcx, Self> {
         Ok(match self.as_mplace_or_local() {
             Left(mplace) => mplace.offset_with_meta(offset, mode, meta, layout, ecx)?.into(),
-            Right((frame, local, old_offset)) => {
+            Right((local, old_offset, locals_addr)) => {
                 debug_assert!(layout.is_sized(), "unsized locals should live in memory");
                 assert_matches!(meta, MemPlaceMeta::None); // we couldn't store it anyway...
                 // `Place::Local` are always in-bounds of their surrounding local, so we can just
@@ -292,7 +294,10 @@ fn offset_with_meta<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
                         .offset(old_offset.unwrap_or(Size::ZERO).bytes(), offset.bytes())?,
                 );
 
-                PlaceTy { place: Place::Local { frame, local, offset: Some(new_offset) }, layout }
+                PlaceTy {
+                    place: Place::Local { local, offset: Some(new_offset), locals_addr },
+                    layout,
+                }
             }
         })
     }
@@ -331,7 +336,7 @@ pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> {
 pub trait Writeable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> {
     fn as_mplace_or_local(
         &self,
-    ) -> Either<MPlaceTy<'tcx, Prov>, (usize, mir::Local, Option<Size>, TyAndLayout<'tcx>)>;
+    ) -> Either<MPlaceTy<'tcx, Prov>, (mir::Local, Option<Size>, usize, TyAndLayout<'tcx>)>;
 
     fn force_mplace<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
         &self,
@@ -343,9 +348,9 @@ impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
     #[inline(always)]
     fn as_mplace_or_local(
         &self,
-    ) -> Either<MPlaceTy<'tcx, Prov>, (usize, mir::Local, Option<Size>, TyAndLayout<'tcx>)> {
+    ) -> Either<MPlaceTy<'tcx, Prov>, (mir::Local, Option<Size>, usize, TyAndLayout<'tcx>)> {
         self.as_mplace_or_local()
-            .map_right(|(frame, local, offset)| (frame, local, offset, self.layout))
+            .map_right(|(local, offset, locals_addr)| (local, offset, locals_addr, self.layout))
     }
 
     #[inline(always)]
@@ -361,7 +366,7 @@ impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
     #[inline(always)]
     fn as_mplace_or_local(
         &self,
-    ) -> Either<MPlaceTy<'tcx, Prov>, (usize, mir::Local, Option<Size>, TyAndLayout<'tcx>)> {
+    ) -> Either<MPlaceTy<'tcx, Prov>, (mir::Local, Option<Size>, usize, TyAndLayout<'tcx>)> {
         Left(self.clone())
     }
 
@@ -501,21 +506,21 @@ pub fn mplace_to_simd(
         Ok((mplace, len))
     }
 
+    /// Turn a local in the current frame into a place.
     pub fn local_to_place(
         &self,
-        frame: usize,
         local: mir::Local,
     ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
         // Other parts of the system rely on `Place::Local` never being unsized.
         // So we eagerly check here if this local has an MPlace, and if yes we use it.
-        let frame_ref = &self.stack()[frame];
-        let layout = self.layout_of_local(frame_ref, local, None)?;
+        let frame = self.frame();
+        let layout = self.layout_of_local(frame, local, None)?;
         let place = if layout.is_sized() {
             // We can just always use the `Local` for sized values.
-            Place::Local { frame, local, offset: None }
+            Place::Local { local, offset: None, locals_addr: frame.locals_addr() }
         } else {
             // Unsized `Local` isn't okay (we cannot store the metadata).
-            match frame_ref.locals[local].access()? {
+            match frame.locals[local].access()? {
                 Operand::Immediate(_) => bug!(),
                 Operand::Indirect(mplace) => Place::Ptr(*mplace),
             }
@@ -530,7 +535,7 @@ pub fn eval_place(
         &self,
         mir_place: mir::Place<'tcx>,
     ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
-        let mut place = self.local_to_place(self.frame_idx(), mir_place.local)?;
+        let mut place = self.local_to_place(mir_place.local)?;
         // Using `try_fold` turned out to be bad for performance, hence the loop.
         for elem in mir_place.projection.iter() {
             place = self.project(&place, elem)?
@@ -611,15 +616,15 @@ fn write_immediate_no_validate(
         // See if we can avoid an allocation. This is the counterpart to `read_immediate_raw`,
         // but not factored as a separate function.
         let mplace = match dest.as_mplace_or_local() {
-            Right((frame, local, offset, layout)) => {
+            Right((local, offset, locals_addr, layout)) => {
                 if offset.is_some() {
                     // This has been projected to a part of this local. We could have complicated
                     // logic to still keep this local as an `Operand`... but it's much easier to
                     // just fall back to the indirect path.
                     dest.force_mplace(self)?
                 } else {
-                    M::before_access_local_mut(self, frame, local)?;
-                    match self.stack_mut()[frame].locals[local].access_mut()? {
+                    debug_assert_eq!(locals_addr, self.frame().locals_addr());
+                    match self.frame_mut().locals[local].access_mut()? {
                         Operand::Immediate(local_val) => {
                             // Local can be updated in-place.
                             *local_val = src;
@@ -627,7 +632,7 @@ fn write_immediate_no_validate(
                             // (*After* doing the update for borrow checker reasons.)
                             if cfg!(debug_assertions) {
                                 let local_layout =
-                                    self.layout_of_local(&self.stack()[frame], local, None)?;
+                                    self.layout_of_local(&self.frame(), local, None)?;
                                 match (src, local_layout.abi) {
                                     (Immediate::Scalar(scalar), Abi::Scalar(s)) => {
                                         assert_eq!(scalar.size(), s.size(self))
@@ -725,7 +730,7 @@ pub fn write_uninit(
     ) -> InterpResult<'tcx> {
         let mplace = match dest.as_mplace_or_local() {
             Left(mplace) => mplace,
-            Right((frame, local, offset, layout)) => {
+            Right((local, offset, locals_addr, layout)) => {
                 if offset.is_some() {
                     // This has been projected to a part of this local. We could have complicated
                     // logic to still keep this local as an `Operand`... but it's much easier to
@@ -733,8 +738,8 @@ pub fn write_uninit(
                     // FIXME: share the logic with `write_immediate_no_validate`.
                     dest.force_mplace(self)?
                 } else {
-                    M::before_access_local_mut(self, frame, local)?;
-                    match self.stack_mut()[frame].locals[local].access_mut()? {
+                    debug_assert_eq!(locals_addr, self.frame().locals_addr());
+                    match self.frame_mut().locals[local].access_mut()? {
                         Operand::Immediate(local) => {
                             *local = Immediate::Uninit;
                             return Ok(());
@@ -912,17 +917,16 @@ pub fn force_allocation(
         place: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
         let mplace = match place.place {
-            Place::Local { frame, local, offset } => {
-                M::before_access_local_mut(self, frame, local)?;
-                let whole_local = match self.stack_mut()[frame].locals[local].access_mut()? {
+            Place::Local { local, offset, locals_addr } => {
+                debug_assert_eq!(locals_addr, self.frame().locals_addr());
+                let whole_local = match self.frame_mut().locals[local].access_mut()? {
                     &mut Operand::Immediate(local_val) => {
                         // We need to make an allocation.
 
                         // We need the layout of the local. We can NOT use the layout we got,
                         // that might e.g., be an inner field of a struct with `Scalar` layout,
                         // that has different alignment than the outer field.
-                        let local_layout =
-                            self.layout_of_local(&self.stack()[frame], local, None)?;
+                        let local_layout = self.layout_of_local(&self.frame(), local, None)?;
                         assert!(local_layout.is_sized(), "unsized locals cannot be immediate");
                         let mplace = self.allocate(local_layout, MemoryKind::Stack)?;
                         // Preserve old value. (As an optimization, we can skip this if it was uninit.)
@@ -936,11 +940,11 @@ pub fn force_allocation(
                                 mplace.mplace,
                             )?;
                         }
-                        M::after_local_allocated(self, frame, local, &mplace)?;
+                        M::after_local_allocated(self, local, &mplace)?;
                         // Now we can call `access_mut` again, asserting it goes well, and actually
                         // overwrite things. This points to the entire allocation, not just the part
                         // the place refers to, i.e. we do this before we apply `offset`.
-                        *self.stack_mut()[frame].locals[local].access_mut().unwrap() =
+                        *self.frame_mut().locals[local].access_mut().unwrap() =
                             Operand::Indirect(mplace.mplace);
                         mplace.mplace
                     }

compiler/rustc_const_eval/src/interpret/projection.rs

@@ -357,7 +357,7 @@ pub fn project<P>(&self, base: &P, proj_elem: mir::PlaceElem<'tcx>) -> InterpRes
             Deref => self.deref_pointer(&base.to_op(self)?)?.into(),
             Index(local) => {
                 let layout = self.layout_of(self.tcx.types.usize)?;
-                let n = self.local_to_op(self.frame(), local, Some(layout))?;
+                let n = self.local_to_op(local, Some(layout))?;
                 let n = self.read_target_usize(&n)?;
                 self.project_index(base, n)?
             }

compiler/rustc_const_eval/src/interpret/terminator.rs

@@ -631,7 +631,7 @@ pub(crate) fn eval_fn_call(
                         body.args_iter()
                             .map(|local| (
                                 local,
-                                self.layout_of_local(self.frame(), local, None).unwrap().ty
+                                self.layout_of_local(self.frame(), local, None).unwrap().ty,
                             ))
                             .collect::<Vec<_>>()
                     );

compiler/rustc_const_eval/src/interpret/util.rs

@@ -1,14 +1,15 @@
-use crate::const_eval::CompileTimeEvalContext;
+use crate::const_eval::{CompileTimeEvalContext, CompileTimeInterpreter, InterpretationResult};
 use crate::interpret::{MemPlaceMeta, MemoryKind};
 use rustc_hir::def_id::LocalDefId;
-use rustc_middle::mir::interpret::{AllocId, Allocation, InterpResult, Pointer};
+use rustc_middle::mir;
+use rustc_middle::mir::interpret::{Allocation, InterpResult, Pointer};
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{
     self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor,
 };
 use std::ops::ControlFlow;
 
-use super::MPlaceTy;
+use super::{InterpCx, MPlaceTy};
 
 /// Checks whether a type contains generic parameters which must be instantiated.
 ///
@@ -80,11 +81,15 @@ fn visit_const(&mut self, c: ty::Const<'tcx>) -> Self::Result {
     }
 }
 
-pub(crate) fn take_static_root_alloc<'mir, 'tcx: 'mir>(
-    ecx: &mut CompileTimeEvalContext<'mir, 'tcx>,
-    alloc_id: AllocId,
-) -> Allocation {
-    ecx.memory.alloc_map.swap_remove(&alloc_id).unwrap().1
+impl<'tcx> InterpretationResult<'tcx> for mir::interpret::ConstAllocation<'tcx> {
+    fn make_result<'mir>(
+        mplace: MPlaceTy<'tcx>,
+        ecx: &mut InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>,
+    ) -> Self {
+        let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();
+        let alloc = ecx.memory.alloc_map.swap_remove(&alloc_id).unwrap().1;
+        ecx.tcx.mk_const_alloc(alloc)
+    }
 }
 
 pub(crate) fn create_static_alloc<'mir, 'tcx: 'mir>(

compiler/rustc_const_eval/src/lib.rs

@@ -14,6 +14,7 @@
 #![feature(generic_nonzero)]
 #![feature(let_chains)]
 #![feature(slice_ptr_get)]
+#![feature(strict_provenance)]
 #![feature(never_type)]
 #![feature(trait_alias)]
 #![feature(try_blocks)]

compiler/rustc_driver_impl/src/lib.rs

@@ -28,7 +28,7 @@
     markdown, ColorConfig, DiagCtxt, ErrCode, ErrorGuaranteed, FatalError, PResult,
 };
 use rustc_feature::find_gated_cfg;
-use rustc_interface::util::{self, collect_crate_types, get_codegen_backend};
+use rustc_interface::util::{self, get_codegen_backend};
 use rustc_interface::{interface, Queries};
 use rustc_lint::unerased_lint_store;
 use rustc_metadata::creader::MetadataLoader;
@@ -37,6 +37,7 @@
 use rustc_session::config::{ErrorOutputType, Input, OutFileName, OutputType};
 use rustc_session::getopts::{self, Matches};
 use rustc_session::lint::{Lint, LintId};
+use rustc_session::output::collect_crate_types;
 use rustc_session::{config, filesearch, EarlyDiagCtxt, Session};
 use rustc_span::def_id::LOCAL_CRATE;
 use rustc_span::source_map::FileLoader;

compiler/rustc_errors/src/diagnostic.rs

@@ -189,7 +189,8 @@ fn add_to_diag_with<G: EmissionGuarantee, F: SubdiagMessageOp<G>>(
     );
 }
 
-pub trait SubdiagMessageOp<G> = Fn(&mut Diag<'_, G>, SubdiagMessage) -> SubdiagMessage;
+pub trait SubdiagMessageOp<G: EmissionGuarantee> =
+    Fn(&mut Diag<'_, G>, SubdiagMessage) -> SubdiagMessage;
 
 /// Trait implemented by lint types. This should not be implemented manually. Instead, use
 /// `#[derive(LintDiagnostic)]` -- see [rustc_macros::LintDiagnostic].

compiler/rustc_errors/src/emitter.rs

@@ -2086,7 +2086,7 @@ fn emit_messages_default(
                 }
                 if !self.short_message {
                     for child in children {
-                        assert!(child.level.can_be_top_or_sub().1);
+                        assert!(child.level.can_be_subdiag());
                         let span = &child.span;
                         if let Err(err) = self.emit_messages_default_inner(
                             span,

compiler/rustc_errors/src/lib.rs

@@ -526,12 +526,15 @@ pub enum StashKey {
     UndeterminedMacroResolution,
 }
 
-fn default_track_diagnostic(diag: DiagInner, f: &mut dyn FnMut(DiagInner)) {
+fn default_track_diagnostic<R>(diag: DiagInner, f: &mut dyn FnMut(DiagInner) -> R) -> R {
     (*f)(diag)
 }
 
-pub static TRACK_DIAGNOSTIC: AtomicRef<fn(DiagInner, &mut dyn FnMut(DiagInner))> =
-    AtomicRef::new(&(default_track_diagnostic as _));
+/// Diagnostics emitted by `DiagCtxtInner::emit_diagnostic` are passed through this function. Used
+/// for tracking by incremental, to replay diagnostics as necessary.
+pub static TRACK_DIAGNOSTIC: AtomicRef<
+    fn(DiagInner, &mut dyn FnMut(DiagInner) -> Option<ErrorGuaranteed>) -> Option<ErrorGuaranteed>,
+> = AtomicRef::new(&(default_track_diagnostic as _));
 
 #[derive(Copy, Clone, Default)]
 pub struct DiagCtxtFlags {
@@ -1422,74 +1425,103 @@ fn emit_stashed_diagnostics(&mut self) -> Option<ErrorGuaranteed> {
 
     // Return value is only `Some` if the level is `Error` or `DelayedBug`.
     fn emit_diagnostic(&mut self, mut diagnostic: DiagInner) -> Option<ErrorGuaranteed> {
-        assert!(diagnostic.level.can_be_top_or_sub().0);
-
-        if let Some(expectation_id) = diagnostic.level.get_expectation_id() {
-            // The `LintExpectationId` can be stable or unstable depending on when it was created.
-            // Diagnostics created before the definition of `HirId`s are unstable and can not yet
-            // be stored. Instead, they are buffered until the `LintExpectationId` is replaced by
-            // a stable one by the `LintLevelsBuilder`.
-            if let LintExpectationId::Unstable { .. } = expectation_id {
-                self.unstable_expect_diagnostics.push(diagnostic);
-                return None;
-            }
-            self.suppressed_expected_diag = true;
-            self.fulfilled_expectations.insert(expectation_id.normalize());
-        }
-
         if diagnostic.has_future_breakage() {
             // Future breakages aren't emitted if they're `Level::Allow`,
             // but they still need to be constructed and stashed below,
             // so they'll trigger the must_produce_diag check.
-            self.suppressed_expected_diag = true;
+            assert!(matches!(diagnostic.level, Error | Warning | Allow));
             self.future_breakage_diagnostics.push(diagnostic.clone());
         }
 
-        // Note that because this comes before the `match` below,
-        // `-Zeagerly-emit-delayed-bugs` continues to work even after we've
-        // issued an error and stopped recording new delayed bugs.
-        if diagnostic.level == DelayedBug && self.flags.eagerly_emit_delayed_bugs {
-            diagnostic.level = Error;
-        }
-
+        // We call TRACK_DIAGNOSTIC with an empty closure for the cases that
+        // return early *and* have some kind of side-effect, except where
+        // noted.
         match diagnostic.level {
-            // This must come after the possible promotion of `DelayedBug` to
-            // `Error` above.
-            Fatal | Error if self.treat_next_err_as_bug() => {
-                diagnostic.level = Bug;
+            Bug => {}
+            Fatal | Error => {
+                if self.treat_next_err_as_bug() {
+                    // `Fatal` and `Error` can be promoted to `Bug`.
+                    diagnostic.level = Bug;
+                }
             }
             DelayedBug => {
-                // If we have already emitted at least one error, we don't need
-                // to record the delayed bug, because it'll never be used.
-                return if let Some(guar) = self.has_errors() {
-                    Some(guar)
+                // Note that because we check these conditions first,
+                // `-Zeagerly-emit-delayed-bugs` and `-Ztreat-err-as-bug`
+                // continue to work even after we've issued an error and
+                // stopped recording new delayed bugs.
+                if self.flags.eagerly_emit_delayed_bugs {
+                    // `DelayedBug` can be promoted to `Error` or `Bug`.
+                    if self.treat_next_err_as_bug() {
+                        diagnostic.level = Bug;
+                    } else {
+                        diagnostic.level = Error;
+                    }
                 } else {
-                    let backtrace = std::backtrace::Backtrace::capture();
-                    // This `unchecked_error_guaranteed` is valid. It is where the
-                    // `ErrorGuaranteed` for delayed bugs originates. See
-                    // `DiagCtxtInner::drop`.
-                    #[allow(deprecated)]
-                    let guar = ErrorGuaranteed::unchecked_error_guaranteed();
-                    self.delayed_bugs
-                        .push((DelayedDiagInner::with_backtrace(diagnostic, backtrace), guar));
-                    Some(guar)
-                };
+                    // If we have already emitted at least one error, we don't need
+                    // to record the delayed bug, because it'll never be used.
+                    return if let Some(guar) = self.has_errors() {
+                        Some(guar)
+                    } else {
+                        // No `TRACK_DIAGNOSTIC` call is needed, because the
+                        // incremental session is deleted if there is a delayed
+                        // bug. This also saves us from cloning the diagnostic.
+                        let backtrace = std::backtrace::Backtrace::capture();
+                        // This `unchecked_error_guaranteed` is valid. It is where the
+                        // `ErrorGuaranteed` for delayed bugs originates. See
+                        // `DiagCtxtInner::drop`.
+                        #[allow(deprecated)]
+                        let guar = ErrorGuaranteed::unchecked_error_guaranteed();
+                        self.delayed_bugs
+                            .push((DelayedDiagInner::with_backtrace(diagnostic, backtrace), guar));
+                        Some(guar)
+                    };
+                }
             }
-            Warning if !self.flags.can_emit_warnings => {
+            ForceWarning(None) => {} // `ForceWarning(Some(...))` is below, with `Expect`
+            Warning => {
+                if !self.flags.can_emit_warnings {
+                    // We are not emitting warnings.
+                    if diagnostic.has_future_breakage() {
+                        // The side-effect is at the top of this method.
+                        TRACK_DIAGNOSTIC(diagnostic, &mut |_| None);
+                    }
+                    return None;
+                }
+            }
+            Note | Help | FailureNote => {}
+            OnceNote | OnceHelp => panic!("bad level: {:?}", diagnostic.level),
+            Allow => {
+                // Nothing emitted for allowed lints.
                 if diagnostic.has_future_breakage() {
-                    (*TRACK_DIAGNOSTIC)(diagnostic, &mut |_| {});
+                    // The side-effect is at the top of this method.
+                    TRACK_DIAGNOSTIC(diagnostic, &mut |_| None);
+                    self.suppressed_expected_diag = true;
                 }
                 return None;
             }
-            Allow | Expect(_) => {
-                (*TRACK_DIAGNOSTIC)(diagnostic, &mut |_| {});
-                return None;
+            Expect(expect_id) | ForceWarning(Some(expect_id)) => {
+                // Diagnostics created before the definition of `HirId`s are
+                // unstable and can not yet be stored. Instead, they are
+                // buffered until the `LintExpectationId` is replaced by a
+                // stable one by the `LintLevelsBuilder`.
+                if let LintExpectationId::Unstable { .. } = expect_id {
+                    // We don't call TRACK_DIAGNOSTIC because we wait for the
+                    // unstable ID to be updated, whereupon the diagnostic will
+                    // be passed into this method again.
+                    self.unstable_expect_diagnostics.push(diagnostic);
+                    return None;
+                }
+                self.fulfilled_expectations.insert(expect_id.normalize());
+                if let Expect(_) = diagnostic.level {
+                    // Nothing emitted here for expected lints.
+                    TRACK_DIAGNOSTIC(diagnostic, &mut |_| None);
+                    self.suppressed_expected_diag = true;
+                    return None;
+                }
             }
-            _ => {}
         }
 
-        let mut guaranteed = None;
-        (*TRACK_DIAGNOSTIC)(diagnostic, &mut |mut diagnostic| {
+        TRACK_DIAGNOSTIC(diagnostic, &mut |mut diagnostic| {
             if let Some(code) = diagnostic.code {
                 self.emitted_diagnostic_codes.insert(code);
             }
@@ -1552,17 +1584,17 @@ fn emit_diagnostic(&mut self, mut diagnostic: DiagInner) -> Option<ErrorGuarante
                 // `ErrorGuaranteed` for errors and lint errors originates.
                 #[allow(deprecated)]
                 let guar = ErrorGuaranteed::unchecked_error_guaranteed();
-                guaranteed = Some(guar);
                 if is_lint {
                     self.lint_err_guars.push(guar);
                 } else {
                     self.err_guars.push(guar);
                 }
                 self.panic_if_treat_err_as_bug();
+                Some(guar)
+            } else {
+                None
             }
-        });
-
-        guaranteed
+        })
     }
 
     fn treat_err_as_bug(&self) -> bool {
@@ -1863,23 +1895,13 @@ pub fn is_failure_note(&self) -> bool {
         matches!(*self, FailureNote)
     }
 
-    pub fn get_expectation_id(&self) -> Option<LintExpectationId> {
-        match self {
-            Expect(id) | ForceWarning(Some(id)) => Some(*id),
-            _ => None,
-        }
-    }
-
-    // Can this level be used in a top-level diagnostic message and/or a
-    // subdiagnostic message?
-    fn can_be_top_or_sub(&self) -> (bool, bool) {
+    // Can this level be used in a subdiagnostic message?
+    fn can_be_subdiag(&self) -> bool {
         match self {
             Bug | DelayedBug | Fatal | Error | ForceWarning(_) | FailureNote | Allow
-            | Expect(_) => (true, false),
+            | Expect(_) => false,
 
-            Warning | Note | Help => (true, true),
-
-            OnceNote | OnceHelp => (false, true),
+            Warning | Note | Help | OnceNote | OnceHelp => true,
         }
     }
 }

compiler/rustc_feature/src/builtin_attrs.rs

@@ -894,56 +894,93 @@ pub struct BuiltinAttribute {
     ),
     rustc_attr!(TEST, rustc_insignificant_dtor, Normal, template!(Word), WarnFollowing),
     rustc_attr!(TEST, rustc_strict_coherence, Normal, template!(Word), WarnFollowing),
-    rustc_attr!(TEST, rustc_variance, Normal, template!(Word), WarnFollowing),
-    rustc_attr!(TEST, rustc_variance_of_opaques, Normal, template!(Word), WarnFollowing),
-    rustc_attr!(TEST, rustc_hidden_type_of_opaques, Normal, template!(Word), WarnFollowing),
+    rustc_attr!(TEST, rustc_variance, Normal, template!(Word), WarnFollowing, @only_local: true),
+    rustc_attr!(
+        TEST, rustc_variance_of_opaques, Normal, template!(Word),
+        WarnFollowing, @only_local: true
+    ),
+    rustc_attr!(
+        TEST, rustc_hidden_type_of_opaques, Normal, template!(Word),
+        WarnFollowing, @only_local: true),
     rustc_attr!(TEST, rustc_layout, Normal, template!(List: "field1, field2, ..."), WarnFollowing),
-    rustc_attr!(TEST, rustc_abi, Normal, template!(List: "field1, field2, ..."), WarnFollowing),
-    rustc_attr!(TEST, rustc_regions, Normal, template!(Word), WarnFollowing),
+    rustc_attr!(
+        TEST, rustc_abi, Normal, template!(List: "field1, field2, ..."),
+        WarnFollowing, @only_local: true
+    ),
+    rustc_attr!(
+        TEST, rustc_regions, Normal, template!(Word),
+        WarnFollowing, @only_local: true
+    ),
     rustc_attr!(
         TEST, rustc_error, Normal,
         template!(Word, List: "delayed_bug_from_inside_query"), WarnFollowingWordOnly
     ),
-    rustc_attr!(TEST, rustc_dump_user_args, Normal, template!(Word), WarnFollowing),
+    rustc_attr!(
+        TEST, rustc_dump_user_args, Normal, template!(Word), WarnFollowing,
+        @only_local: true
+    ),
     rustc_attr!(TEST, rustc_evaluate_where_clauses, Normal, template!(Word), WarnFollowing),
     rustc_attr!(
-        TEST, rustc_if_this_changed, Normal, template!(Word, List: "DepNode"), DuplicatesOk
+        TEST, rustc_if_this_changed, Normal, template!(Word, List: "DepNode"),
+        DuplicatesOk, @only_local: true
     ),
     rustc_attr!(
-        TEST, rustc_then_this_would_need, Normal, template!(List: "DepNode"), DuplicatesOk
+        TEST, rustc_then_this_would_need, Normal, template!(List: "DepNode"),
+        DuplicatesOk, @only_local: true
     ),
     rustc_attr!(
         TEST, rustc_clean, Normal,
         template!(List: r#"cfg = "...", /*opt*/ label = "...", /*opt*/ except = "...""#),
-        DuplicatesOk,
+        DuplicatesOk, @only_local: true
     ),
     rustc_attr!(
         TEST, rustc_partition_reused, Normal,
-        template!(List: r#"cfg = "...", module = "...""#), DuplicatesOk,
+        template!(List: r#"cfg = "...", module = "...""#), DuplicatesOk, @only_local: true
     ),
     rustc_attr!(
         TEST, rustc_partition_codegened, Normal,
-        template!(List: r#"cfg = "...", module = "...""#), DuplicatesOk,
+        template!(List: r#"cfg = "...", module = "...""#), DuplicatesOk, @only_local: true
     ),
     rustc_attr!(
         TEST, rustc_expected_cgu_reuse, Normal,
         template!(List: r#"cfg = "...", module = "...", kind = "...""#), DuplicatesOk,
+        @only_local: true
+    ),
+    rustc_attr!(
+        TEST, rustc_symbol_name, Normal, template!(Word), WarnFollowing,
+        @only_local: true
     ),
-    rustc_attr!(TEST, rustc_symbol_name, Normal, template!(Word), WarnFollowing),
     rustc_attr!(TEST, rustc_polymorphize_error, Normal, template!(Word), WarnFollowing),
-    rustc_attr!(TEST, rustc_def_path, Normal, template!(Word), WarnFollowing),
+    rustc_attr!(
+        TEST, rustc_def_path, Normal, template!(Word), WarnFollowing,
+        @only_local: true
+    ),
     rustc_attr!(TEST, rustc_mir, Normal, template!(List: "arg1, arg2, ..."), DuplicatesOk),
     gated!(
         custom_mir, Normal, template!(List: r#"dialect = "...", phase = "...""#),
-        ErrorFollowing, "the `#[custom_mir]` attribute is just used for the Rust test suite",
+        ErrorFollowing, @only_local: true,
+        "the `#[custom_mir]` attribute is just used for the Rust test suite",
+    ),
+    rustc_attr!(
+        TEST, rustc_dump_program_clauses, Normal, template!(Word), WarnFollowing,
+        @only_local: true
+    ),
+    rustc_attr!(
+        TEST, rustc_dump_env_program_clauses, Normal, template!(Word), WarnFollowing,
+        @only_local: true
+    ),
+    rustc_attr!(
+        TEST, rustc_object_lifetime_default, Normal, template!(Word), WarnFollowing,
+        @only_local: true
     ),
-    rustc_attr!(TEST, rustc_dump_program_clauses, Normal, template!(Word), WarnFollowing),
-    rustc_attr!(TEST, rustc_dump_env_program_clauses, Normal, template!(Word), WarnFollowing),
-    rustc_attr!(TEST, rustc_object_lifetime_default, Normal, template!(Word), WarnFollowing),
     rustc_attr!(TEST, rustc_dump_vtable, Normal, template!(Word), WarnFollowing),
-    rustc_attr!(TEST, rustc_dummy, Normal, template!(Word /* doesn't matter*/), DuplicatesOk),
+    rustc_attr!(
+        TEST, rustc_dummy, Normal, template!(Word /* doesn't matter*/), DuplicatesOk,
+        @only_local: true
+    ),
     gated!(
         omit_gdb_pretty_printer_section, Normal, template!(Word), WarnFollowing,
+        @only_local: true,
         "the `#[omit_gdb_pretty_printer_section]` attribute is just used for the Rust test suite",
     ),
     rustc_attr!(

compiler/rustc_hir/src/hir.rs

@@ -2553,6 +2553,11 @@ pub struct OpaqueTy<'hir> {
     pub in_trait: bool,
 }
 
+#[derive(Copy, Clone, Debug, HashStable_Generic)]
+pub struct AssocOpaqueTy {
+    // Add some data if necessary
+}
+
 /// From whence the opaque type came.
 #[derive(Copy, Clone, PartialEq, Eq, Debug, HashStable_Generic)]
 pub enum OpaqueTyOrigin {
@@ -3363,6 +3368,7 @@ pub enum OwnerNode<'hir> {
     TraitItem(&'hir TraitItem<'hir>),
     ImplItem(&'hir ImplItem<'hir>),
     Crate(&'hir Mod<'hir>),
+    AssocOpaqueTy(&'hir AssocOpaqueTy),
 }
 
 impl<'hir> OwnerNode<'hir> {
@@ -3372,7 +3378,7 @@ pub fn ident(&self) -> Option<Ident> {
             | OwnerNode::ForeignItem(ForeignItem { ident, .. })
             | OwnerNode::ImplItem(ImplItem { ident, .. })
             | OwnerNode::TraitItem(TraitItem { ident, .. }) => Some(*ident),
-            OwnerNode::Crate(..) => None,
+            OwnerNode::Crate(..) | OwnerNode::AssocOpaqueTy(..) => None,
         }
     }
 
@@ -3385,6 +3391,7 @@ pub fn span(&self) -> &'hir Span {
             | OwnerNode::ImplItem(ImplItem { span, .. })
             | OwnerNode::TraitItem(TraitItem { span, .. }) => span,
             OwnerNode::Crate(Mod { spans: ModSpans { inner_span, .. }, .. }) => inner_span,
+            OwnerNode::AssocOpaqueTy(..) => unreachable!(),
         }
     }
 
@@ -3443,6 +3450,7 @@ pub fn def_id(self) -> OwnerId {
             | OwnerNode::ImplItem(ImplItem { owner_id, .. })
             | OwnerNode::ForeignItem(ForeignItem { owner_id, .. }) => *owner_id,
             OwnerNode::Crate(..) => crate::CRATE_HIR_ID.owner,
+            OwnerNode::AssocOpaqueTy(..) => unreachable!(),
         }
     }
 
@@ -3486,6 +3494,7 @@ fn into(self) -> Node<'hir> {
             OwnerNode::ImplItem(n) => Node::ImplItem(n),
             OwnerNode::TraitItem(n) => Node::TraitItem(n),
             OwnerNode::Crate(n) => Node::Crate(n),
+            OwnerNode::AssocOpaqueTy(n) => Node::AssocOpaqueTy(n),
         }
     }
 }
@@ -3523,6 +3532,7 @@ pub enum Node<'hir> {
     WhereBoundPredicate(&'hir WhereBoundPredicate<'hir>),
     // FIXME: Merge into `Node::Infer`.
     ArrayLenInfer(&'hir InferArg),
+    AssocOpaqueTy(&'hir AssocOpaqueTy),
     // Span by reference to minimize `Node`'s size
     #[allow(rustc::pass_by_value)]
     Err(&'hir Span),
@@ -3573,6 +3583,7 @@ pub fn ident(&self) -> Option<Ident> {
             | Node::Infer(..)
             | Node::WhereBoundPredicate(..)
             | Node::ArrayLenInfer(..)
+            | Node::AssocOpaqueTy(..)
             | Node::Err(..) => None,
         }
     }
@@ -3678,6 +3689,7 @@ pub fn as_owner(self) -> Option<OwnerNode<'hir>> {
             Node::TraitItem(i) => Some(OwnerNode::TraitItem(i)),
             Node::ImplItem(i) => Some(OwnerNode::ImplItem(i)),
             Node::Crate(i) => Some(OwnerNode::Crate(i)),
+            Node::AssocOpaqueTy(i) => Some(OwnerNode::AssocOpaqueTy(i)),
             _ => None,
         }
     }

compiler/rustc_hir_analysis/src/check/wfcheck.rs

@@ -196,6 +196,7 @@ fn check_well_formed(tcx: TyCtxt<'_>, def_id: hir::OwnerId) -> Result<(), ErrorG
         hir::OwnerNode::TraitItem(item) => check_trait_item(tcx, item),
         hir::OwnerNode::ImplItem(item) => check_impl_item(tcx, item),
         hir::OwnerNode::ForeignItem(item) => check_foreign_item(tcx, item),
+        hir::OwnerNode::AssocOpaqueTy(..) => unreachable!(),
     };
 
     if let Some(generics) = node.generics() {

compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs

@@ -262,6 +262,7 @@ fn resolve_bound_vars(tcx: TyCtxt<'_>, local_def_id: hir::OwnerId) -> ResolveBou
             visitor.visit_impl_item(item)
         }
         hir::OwnerNode::Crate(_) => {}
+        hir::OwnerNode::AssocOpaqueTy(..) => unreachable!(),
     }
 
     let mut rl = ResolveBoundVars::default();

compiler/rustc_hir_pretty/src/lib.rs

@@ -121,6 +121,7 @@ fn print_node(&mut self, node: Node<'_>) {
                 self.print_bounds(":", pred.bounds);
             }
             Node::ArrayLenInfer(_) => self.word("_"),
+            Node::AssocOpaqueTy(..) => unreachable!(),
             Node::Err(_) => self.word("/*ERROR*/"),
         }
     }

compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs

@@ -2174,7 +2174,9 @@ fn label_fn_like(
                 let mut call_finder = FindClosureArg { tcx: self.tcx, calls: vec![] };
                 let node = self
                     .tcx
-                    .opt_local_def_id_to_hir_id(self.tcx.hir().get_parent_item(call_expr.hir_id))
+                    .opt_local_def_id_to_hir_id(
+                        self.tcx.hir().get_parent_item(call_expr.hir_id).def_id,
+                    )
                     .map(|hir_id| self.tcx.hir_node(hir_id));
                 match node {
                     Some(hir::Node::Item(item)) => call_finder.visit_item(item),

compiler/rustc_incremental/src/persist/fs.rs

@@ -110,8 +110,9 @@
 use rustc_data_structures::{base_n, flock};
 use rustc_errors::ErrorGuaranteed;
 use rustc_fs_util::{link_or_copy, try_canonicalize, LinkOrCopy};
+use rustc_session::config::CrateType;
+use rustc_session::output::{collect_crate_types, find_crate_name};
 use rustc_session::{Session, StableCrateId};
-use rustc_span::Symbol;
 
 use std::fs as std_fs;
 use std::io::{self, ErrorKind};
@@ -205,11 +206,7 @@ pub fn in_incr_comp_dir(incr_comp_session_dir: &Path, file_name: &str) -> PathBu
 /// The garbage collection will take care of it.
 ///
 /// [`rustc_interface::queries::dep_graph`]: ../../rustc_interface/struct.Queries.html#structfield.dep_graph
-pub(crate) fn prepare_session_directory(
-    sess: &Session,
-    crate_name: Symbol,
-    stable_crate_id: StableCrateId,
-) -> Result<(), ErrorGuaranteed> {
+pub(crate) fn prepare_session_directory(sess: &Session) -> Result<(), ErrorGuaranteed> {
     if sess.opts.incremental.is_none() {
         return Ok(());
     }
@@ -219,7 +216,7 @@ pub(crate) fn prepare_session_directory(
     debug!("prepare_session_directory");
 
     // {incr-comp-dir}/{crate-name-and-disambiguator}
-    let crate_dir = crate_path(sess, crate_name, stable_crate_id);
+    let crate_dir = crate_path(sess);
     debug!("crate-dir: {}", crate_dir.display());
     create_dir(sess, &crate_dir, "crate")?;
 
@@ -604,9 +601,18 @@ fn string_to_timestamp(s: &str) -> Result<SystemTime, &'static str> {
     Ok(UNIX_EPOCH + duration)
 }
 
-fn crate_path(sess: &Session, crate_name: Symbol, stable_crate_id: StableCrateId) -> PathBuf {
+fn crate_path(sess: &Session) -> PathBuf {
     let incr_dir = sess.opts.incremental.as_ref().unwrap().clone();
 
+    let crate_name = find_crate_name(sess, &[]);
+    let crate_types = collect_crate_types(sess, &[]);
+    let stable_crate_id = StableCrateId::new(
+        crate_name,
+        crate_types.contains(&CrateType::Executable),
+        sess.opts.cg.metadata.clone(),
+        sess.cfg_version,
+    );
+
     let stable_crate_id = base_n::encode(stable_crate_id.as_u64() as u128, INT_ENCODE_BASE);
 
     let crate_name = format!("{crate_name}-{stable_crate_id}");

compiler/rustc_incremental/src/persist/load.rs

@@ -8,8 +8,8 @@
 use rustc_serialize::opaque::MemDecoder;
 use rustc_serialize::Decodable;
 use rustc_session::config::IncrementalStateAssertion;
-use rustc_session::{Session, StableCrateId};
-use rustc_span::{ErrorGuaranteed, Symbol};
+use rustc_session::Session;
+use rustc_span::ErrorGuaranteed;
 use std::path::{Path, PathBuf};
 
 use super::data::*;
@@ -190,13 +190,9 @@ pub fn load_query_result_cache(sess: &Session) -> Option<OnDiskCache<'_>> {
 
 /// Setups the dependency graph by loading an existing graph from disk and set up streaming of a
 /// new graph to an incremental session directory.
-pub fn setup_dep_graph(
-    sess: &Session,
-    crate_name: Symbol,
-    stable_crate_id: StableCrateId,
-) -> Result<DepGraph, ErrorGuaranteed> {
+pub fn setup_dep_graph(sess: &Session) -> Result<DepGraph, ErrorGuaranteed> {
     // `load_dep_graph` can only be called after `prepare_session_directory`.
-    prepare_session_directory(sess, crate_name, stable_crate_id)?;
+    prepare_session_directory(sess)?;
 
     let res = sess.opts.build_dep_graph().then(|| load_dep_graph(sess));
 

compiler/rustc_infer/src/infer/error_reporting/mod.rs

@@ -2554,6 +2554,7 @@ fn visit_ty(&mut self, ty: &'hir hir::Ty<'hir>) {
             hir::OwnerNode::ImplItem(i) => visitor.visit_impl_item(i),
             hir::OwnerNode::TraitItem(i) => visitor.visit_trait_item(i),
             hir::OwnerNode::Crate(_) => bug!("OwnerNode::Crate doesn't not have generics"),
+            hir::OwnerNode::AssocOpaqueTy(..) => unreachable!(),
         }
 
         let ast_generics = self.tcx.hir().get_generics(lifetime_scope).unwrap();

compiler/rustc_interface/messages.ftl

@@ -48,6 +48,3 @@ interface_rustc_error_unexpected_annotation =
 
 interface_temps_dir_error =
     failed to find or create the directory specified by `--temps-dir`
-
-interface_unsupported_crate_type_for_target =
-    dropping unsupported crate type `{$crate_type}` for target `{$target_triple}`

compiler/rustc_interface/src/callbacks.rs

@@ -29,7 +29,7 @@ fn track_span_parent(def_id: rustc_span::def_id::LocalDefId) {
 /// This is a callback from `rustc_errors` as it cannot access the implicit state
 /// in `rustc_middle` otherwise. It is used when diagnostic messages are
 /// emitted and stores them in the current query, if there is one.
-fn track_diagnostic(diagnostic: DiagInner, f: &mut dyn FnMut(DiagInner)) {
+fn track_diagnostic<R>(diagnostic: DiagInner, f: &mut dyn FnMut(DiagInner) -> R) -> R {
     tls::with_context_opt(|icx| {
         if let Some(icx) = icx {
             if let Some(diagnostics) = icx.diagnostics {
@@ -38,11 +38,11 @@ fn track_diagnostic(diagnostic: DiagInner, f: &mut dyn FnMut(DiagInner)) {
 
             // Diagnostics are tracked, we can ignore the dependency.
             let icx = tls::ImplicitCtxt { task_deps: TaskDepsRef::Ignore, ..icx.clone() };
-            return tls::enter_context(&icx, move || (*f)(diagnostic));
+            tls::enter_context(&icx, move || (*f)(diagnostic))
+        } else {
+            // In any other case, invoke diagnostics anyway.
+            (*f)(diagnostic)
         }
-
-        // In any other case, invoke diagnostics anyway.
-        (*f)(diagnostic);
     })
 }
 

compiler/rustc_interface/src/errors.rs

@@ -1,7 +1,5 @@
 use rustc_macros::Diagnostic;
-use rustc_session::config::CrateType;
 use rustc_span::{Span, Symbol};
-use rustc_target::spec::TargetTriple;
 
 use std::io;
 use std::path::Path;
@@ -90,13 +88,6 @@ pub struct FailedWritingFile<'a> {
 #[diag(interface_proc_macro_crate_panic_abort)]
 pub struct ProcMacroCratePanicAbort;
 
-#[derive(Diagnostic)]
-#[diag(interface_unsupported_crate_type_for_target)]
-pub struct UnsupportedCrateTypeForTarget<'a> {
-    pub crate_type: CrateType,
-    pub target_triple: &'a TargetTriple,
-}
-
 #[derive(Diagnostic)]
 #[diag(interface_multiple_output_types_adaption)]
 pub struct MultipleOutputTypesAdaption;

compiler/rustc_interface/src/queries.rs

@@ -18,7 +18,7 @@
 use rustc_serialize::opaque::FileEncodeResult;
 use rustc_session::config::{self, CrateType, OutputFilenames, OutputType};
 use rustc_session::cstore::Untracked;
-use rustc_session::output::find_crate_name;
+use rustc_session::output::{collect_crate_types, find_crate_name};
 use rustc_session::Session;
 use rustc_span::symbol::sym;
 use std::any::Any;
@@ -128,7 +128,7 @@ pub fn global_ctxt(&'tcx self) -> Result<QueryResult<'_, &'tcx GlobalCtxt<'tcx>>
 
             // parse `#[crate_name]` even if `--crate-name` was passed, to make sure it matches.
             let crate_name = find_crate_name(sess, &pre_configured_attrs);
-            let crate_types = util::collect_crate_types(sess, &pre_configured_attrs);
+            let crate_types = collect_crate_types(sess, &pre_configured_attrs);
             let stable_crate_id = StableCrateId::new(
                 crate_name,
                 crate_types.contains(&CrateType::Executable),
@@ -136,7 +136,7 @@ pub fn global_ctxt(&'tcx self) -> Result<QueryResult<'_, &'tcx GlobalCtxt<'tcx>>
                 sess.cfg_version,
             );
             let outputs = util::build_output_filenames(&pre_configured_attrs, sess);
-            let dep_graph = setup_dep_graph(sess, crate_name, stable_crate_id)?;
+            let dep_graph = setup_dep_graph(sess)?;
 
             let cstore = FreezeLock::new(Box::new(CStore::new(
                 self.compiler.codegen_backend.metadata_loader(),

compiler/rustc_interface/src/util.rs

@@ -7,14 +7,15 @@
 use rustc_metadata::{load_symbol_from_dylib, DylibError};
 use rustc_parse::validate_attr;
 use rustc_session as session;
-use rustc_session::config::{self, Cfg, CrateType, OutFileName, OutputFilenames, OutputTypes};
+use rustc_session::config::{Cfg, OutFileName, OutputFilenames, OutputTypes};
 use rustc_session::filesearch::sysroot_candidates;
 use rustc_session::lint::{self, BuiltinLintDiag, LintBuffer};
-use rustc_session::{filesearch, output, Session};
+use rustc_session::{filesearch, Session};
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::edition::Edition;
-use rustc_span::symbol::{sym, Symbol};
+use rustc_span::symbol::sym;
 use rustc_target::spec::Target;
+use session::output::{categorize_crate_type, CRATE_TYPES};
 use session::EarlyDiagCtxt;
 use std::env::consts::{DLL_PREFIX, DLL_SUFFIX};
 use std::path::{Path, PathBuf};
@@ -399,67 +400,6 @@ pub(crate) fn check_attr_crate_type(
     }
 }
 
-const CRATE_TYPES: &[(Symbol, CrateType)] = &[
-    (sym::rlib, CrateType::Rlib),
-    (sym::dylib, CrateType::Dylib),
-    (sym::cdylib, CrateType::Cdylib),
-    (sym::lib, config::default_lib_output()),
-    (sym::staticlib, CrateType::Staticlib),
-    (sym::proc_dash_macro, CrateType::ProcMacro),
-    (sym::bin, CrateType::Executable),
-];
-
-fn categorize_crate_type(s: Symbol) -> Option<CrateType> {
-    Some(CRATE_TYPES.iter().find(|(key, _)| *key == s)?.1)
-}
-
-pub fn collect_crate_types(session: &Session, attrs: &[ast::Attribute]) -> Vec<CrateType> {
-    // If we're generating a test executable, then ignore all other output
-    // styles at all other locations
-    if session.opts.test {
-        return vec![CrateType::Executable];
-    }
-
-    // Only check command line flags if present. If no types are specified by
-    // command line, then reuse the empty `base` Vec to hold the types that
-    // will be found in crate attributes.
-    // JUSTIFICATION: before wrapper fn is available
-    #[allow(rustc::bad_opt_access)]
-    let mut base = session.opts.crate_types.clone();
-    if base.is_empty() {
-        let attr_types = attrs.iter().filter_map(|a| {
-            if a.has_name(sym::crate_type)
-                && let Some(s) = a.value_str()
-            {
-                categorize_crate_type(s)
-            } else {
-                None
-            }
-        });
-        base.extend(attr_types);
-        if base.is_empty() {
-            base.push(output::default_output_for_target(session));
-        } else {
-            base.sort();
-            base.dedup();
-        }
-    }
-
-    base.retain(|crate_type| {
-        if output::invalid_output_for_target(session, *crate_type) {
-            session.dcx().emit_warn(errors::UnsupportedCrateTypeForTarget {
-                crate_type: *crate_type,
-                target_triple: &session.opts.target_triple,
-            });
-            false
-        } else {
-            true
-        }
-    });
-
-    base
-}
-
 fn multiple_output_types_to_stdout(
     output_types: &OutputTypes,
     single_output_file_is_stdout: bool,

compiler/rustc_lint/src/late.rs

@@ -356,7 +356,7 @@ pub fn late_lint_mod<'tcx, T: LateLintPass<'tcx> + 'tcx>(
         cached_typeck_results: Cell::new(None),
         param_env: ty::ParamEnv::empty(),
         effective_visibilities: tcx.effective_visibilities(()),
-        last_node_with_lint_attrs: tcx.local_def_id_to_hir_id(module_def_id.into()),
+        last_node_with_lint_attrs: tcx.local_def_id_to_hir_id(module_def_id),
         generics: None,
         only_module: true,
     };

compiler/rustc_lint/src/levels.rs

@@ -103,11 +103,12 @@ fn raw_lint_id_level(
         mut idx: LintStackIndex,
         aux: Option<&FxIndexMap<LintId, LevelAndSource>>,
     ) -> (Option<Level>, LintLevelSource) {
-        if let Some(specs) = aux {
-            if let Some(&(level, src)) = specs.get(&id) {
-                return (Some(level), src);
-            }
+        if let Some(specs) = aux
+            && let Some(&(level, src)) = specs.get(&id)
+        {
+            return (Some(level), src);
         }
+
         loop {
             let LintSet { ref specs, parent } = self.list[idx];
             if let Some(&(level, src)) = specs.get(&id) {
@@ -177,7 +178,7 @@ fn shallow_lint_levels_on(tcx: TyCtxt<'_>, owner: hir::OwnerId) -> ShallowLintLe
         // There is only something to do if there are attributes at all.
         [] => {}
         // Most of the time, there is only one attribute. Avoid fetching HIR in that case.
-        [(local_id, _)] => levels.add_id(HirId { owner, local_id: *local_id }),
+        &[(local_id, _)] => levels.add_id(HirId { owner, local_id }),
         // Otherwise, we need to visit the attributes in source code order, so we fetch HIR and do
         // a standard visit.
         // FIXME(#102522) Just iterate on attrs once that iteration order matches HIR's.
@@ -190,6 +191,7 @@ fn shallow_lint_levels_on(tcx: TyCtxt<'_>, owner: hir::OwnerId) -> ShallowLintLe
                 levels.add_id(hir::CRATE_HIR_ID);
                 levels.visit_mod(mod_, mod_.spans.inner_span, hir::CRATE_HIR_ID)
             }
+            hir::OwnerNode::AssocOpaqueTy(..) => unreachable!(),
         },
     }
 
@@ -643,63 +645,61 @@ fn insert_spec(&mut self, id: LintId, (mut level, src): LevelAndSource) {
         //
         // This means that this only errors if we're truly lowering the lint
         // level from forbid.
-        if self.lint_added_lints && level != Level::Forbid {
-            if let Level::Forbid = old_level {
-                // Backwards compatibility check:
-                //
-                // We used to not consider `forbid(lint_group)`
-                // as preventing `allow(lint)` for some lint `lint` in
-                // `lint_group`. For now, issue a future-compatibility
-                // warning for this case.
-                let id_name = id.lint.name_lower();
-                let fcw_warning = match old_src {
-                    LintLevelSource::Default => false,
-                    LintLevelSource::Node { name, .. } => self.store.is_lint_group(name),
-                    LintLevelSource::CommandLine(symbol, _) => self.store.is_lint_group(symbol),
-                };
-                debug!(
-                    "fcw_warning={:?}, specs.get(&id) = {:?}, old_src={:?}, id_name={:?}",
-                    fcw_warning,
-                    self.current_specs(),
-                    old_src,
-                    id_name
-                );
-                let sub = match old_src {
-                    LintLevelSource::Default => {
-                        OverruledAttributeSub::DefaultSource { id: id.to_string() }
-                    }
-                    LintLevelSource::Node { span, reason, .. } => {
-                        OverruledAttributeSub::NodeSource { span, reason }
-                    }
-                    LintLevelSource::CommandLine(_, _) => OverruledAttributeSub::CommandLineSource,
-                };
-                if !fcw_warning {
-                    self.sess.dcx().emit_err(OverruledAttribute {
-                        span: src.span(),
+        if self.lint_added_lints && level != Level::Forbid && old_level == Level::Forbid {
+            // Backwards compatibility check:
+            //
+            // We used to not consider `forbid(lint_group)`
+            // as preventing `allow(lint)` for some lint `lint` in
+            // `lint_group`. For now, issue a future-compatibility
+            // warning for this case.
+            let id_name = id.lint.name_lower();
+            let fcw_warning = match old_src {
+                LintLevelSource::Default => false,
+                LintLevelSource::Node { name, .. } => self.store.is_lint_group(name),
+                LintLevelSource::CommandLine(symbol, _) => self.store.is_lint_group(symbol),
+            };
+            debug!(
+                "fcw_warning={:?}, specs.get(&id) = {:?}, old_src={:?}, id_name={:?}",
+                fcw_warning,
+                self.current_specs(),
+                old_src,
+                id_name
+            );
+            let sub = match old_src {
+                LintLevelSource::Default => {
+                    OverruledAttributeSub::DefaultSource { id: id.to_string() }
+                }
+                LintLevelSource::Node { span, reason, .. } => {
+                    OverruledAttributeSub::NodeSource { span, reason }
+                }
+                LintLevelSource::CommandLine(_, _) => OverruledAttributeSub::CommandLineSource,
+            };
+            if !fcw_warning {
+                self.sess.dcx().emit_err(OverruledAttribute {
+                    span: src.span(),
+                    overruled: src.span(),
+                    lint_level: level.as_str(),
+                    lint_source: src.name(),
+                    sub,
+                });
+            } else {
+                self.emit_span_lint(
+                    FORBIDDEN_LINT_GROUPS,
+                    src.span().into(),
+                    OverruledAttributeLint {
                         overruled: src.span(),
                         lint_level: level.as_str(),
                         lint_source: src.name(),
                         sub,
-                    });
-                } else {
-                    self.emit_span_lint(
-                        FORBIDDEN_LINT_GROUPS,
-                        src.span().into(),
-                        OverruledAttributeLint {
-                            overruled: src.span(),
-                            lint_level: level.as_str(),
-                            lint_source: src.name(),
-                            sub,
-                        },
-                    );
-                }
+                    },
+                );
+            }
 
-                // Retain the forbid lint level, unless we are
-                // issuing a FCW. In the FCW case, we want to
-                // respect the new setting.
-                if !fcw_warning {
-                    return;
-                }
+            // Retain the forbid lint level, unless we are
+            // issuing a FCW. In the FCW case, we want to
+            // respect the new setting.
+            if !fcw_warning {
+                return;
             }
         }
 
@@ -770,15 +770,15 @@ fn add(&mut self, attrs: &[ast::Attribute], is_crate_node: bool, source_hir_id:
 
             let Some(mut metas) = attr.meta_item_list() else { continue };
 
-            if metas.is_empty() {
+            // Check whether `metas` is empty, and get its last element.
+            let Some(tail_li) = metas.last() else {
                 // This emits the unused_attributes lint for `#[level()]`
                 continue;
-            }
+            };
 
             // Before processing the lint names, look for a reason (RFC 2383)
             // at the end.
             let mut reason = None;
-            let tail_li = &metas[metas.len() - 1];
             if let Some(item) = tail_li.meta_item() {
                 match item.kind {
                     ast::MetaItemKind::Word => {} // actual lint names handled later
@@ -834,21 +834,16 @@ fn add(&mut self, attrs: &[ast::Attribute], is_crate_node: bool, source_hir_id:
                 let meta_item = match li {
                     ast::NestedMetaItem::MetaItem(meta_item) if meta_item.is_word() => meta_item,
                     _ => {
-                        if let Some(item) = li.meta_item() {
-                            if let ast::MetaItemKind::NameValue(_) = item.kind {
-                                if item.path == sym::reason {
-                                    sess.dcx().emit_err(MalformedAttribute {
-                                        span: sp,
-                                        sub: MalformedAttributeSub::ReasonMustComeLast(sp),
-                                    });
-                                    continue;
-                                }
-                            }
-                        }
-                        sess.dcx().emit_err(MalformedAttribute {
-                            span: sp,
-                            sub: MalformedAttributeSub::BadAttributeArgument(sp),
-                        });
+                        let sub = if let Some(item) = li.meta_item()
+                            && let ast::MetaItemKind::NameValue(_) = item.kind
+                            && item.path == sym::reason
+                        {
+                            MalformedAttributeSub::ReasonMustComeLast(sp)
+                        } else {
+                            MalformedAttributeSub::BadAttributeArgument(sp)
+                        };
+
+                        sess.dcx().emit_err(MalformedAttribute { span: sp, sub });
                         continue;
                     }
                 };
@@ -987,11 +982,7 @@ fn add(&mut self, attrs: &[ast::Attribute], is_crate_node: bool, source_hir_id:
                     }
 
                     CheckLintNameResult::NoLint(suggestion) => {
-                        let name = if let Some(tool_ident) = tool_ident {
-                            format!("{}::{}", tool_ident.name, name)
-                        } else {
-                            name.to_string()
-                        };
+                        let name = tool_ident.map(|tool| format!("{tool}::{name}")).unwrap_or(name);
                         let suggestion = suggestion.map(|(replace, from_rustc)| {
                             UnknownLintSuggestion::WithSpan { suggestion: sp, replace, from_rustc }
                         });
@@ -1005,27 +996,24 @@ fn add(&mut self, attrs: &[ast::Attribute], is_crate_node: bool, source_hir_id:
                 if let CheckLintNameResult::Renamed(new_name) = lint_result {
                     // Ignore any errors or warnings that happen because the new name is inaccurate
                     // NOTE: `new_name` already includes the tool name, so we don't have to add it again.
-                    if let CheckLintNameResult::Ok(ids) =
+                    let CheckLintNameResult::Ok(ids) =
                         self.store.check_lint_name(&new_name, None, self.registered_tools)
-                    {
-                        let src = LintLevelSource::Node {
-                            name: Symbol::intern(&new_name),
-                            span: sp,
-                            reason,
-                        };
-                        for &id in ids {
-                            if self.check_gated_lint(id, attr.span, false) {
-                                self.insert_spec(id, (level, src));
-                            }
-                        }
-                        if let Level::Expect(expect_id) = level {
-                            self.provider.push_expectation(
-                                expect_id,
-                                LintExpectation::new(reason, sp, false, tool_name),
-                            );
-                        }
-                    } else {
+                    else {
                         panic!("renamed lint does not exist: {new_name}");
+                    };
+
+                    let src =
+                        LintLevelSource::Node { name: Symbol::intern(&new_name), span: sp, reason };
+                    for &id in ids {
+                        if self.check_gated_lint(id, attr.span, false) {
+                            self.insert_spec(id, (level, src));
+                        }
+                    }
+                    if let Level::Expect(expect_id) = level {
+                        self.provider.push_expectation(
+                            expect_id,
+                            LintExpectation::new(reason, sp, false, tool_name),
+                        );
                     }
                 }
             }
@@ -1058,38 +1046,44 @@ fn add(&mut self, attrs: &[ast::Attribute], is_crate_node: bool, source_hir_id:
     /// Returns `true` if the lint's feature is enabled.
     #[track_caller]
     fn check_gated_lint(&self, lint_id: LintId, span: Span, lint_from_cli: bool) -> bool {
-        if let Some(feature) = lint_id.lint.feature_gate {
-            if !self.features.active(feature) {
-                if self.lint_added_lints {
-                    let lint = builtin::UNKNOWN_LINTS;
-                    let (level, src) = self.lint_level(builtin::UNKNOWN_LINTS);
-                    // FIXME: make this translatable
-                    #[allow(rustc::diagnostic_outside_of_impl)]
-                    #[allow(rustc::untranslatable_diagnostic)]
-                    lint_level(
-                        self.sess,
+        let feature = if let Some(feature) = lint_id.lint.feature_gate
+            && !self.features.active(feature)
+        {
+            // Lint is behind a feature that is not enabled; eventually return false.
+            feature
+        } else {
+            // Lint is ungated or its feature is enabled; exit early.
+            return true;
+        };
+
+        if self.lint_added_lints {
+            let lint = builtin::UNKNOWN_LINTS;
+            let (level, src) = self.lint_level(builtin::UNKNOWN_LINTS);
+            // FIXME: make this translatable
+            #[allow(rustc::diagnostic_outside_of_impl)]
+            #[allow(rustc::untranslatable_diagnostic)]
+            lint_level(
+                self.sess,
+                lint,
+                level,
+                src,
+                Some(span.into()),
+                fluent::lint_unknown_gated_lint,
+                |lint| {
+                    lint.arg("name", lint_id.lint.name_lower());
+                    lint.note(fluent::lint_note);
+                    rustc_session::parse::add_feature_diagnostics_for_issue(
                         lint,
-                        level,
-                        src,
-                        Some(span.into()),
-                        fluent::lint_unknown_gated_lint,
-                        |lint| {
-                            lint.arg("name", lint_id.lint.name_lower());
-                            lint.note(fluent::lint_note);
-                            rustc_session::parse::add_feature_diagnostics_for_issue(
-                                lint,
-                                &self.sess,
-                                feature,
-                                GateIssue::Language,
-                                lint_from_cli,
-                            );
-                        },
+                        &self.sess,
+                        feature,
+                        GateIssue::Language,
+                        lint_from_cli,
                     );
-                }
-                return false;
-            }
+                },
+            );
         }
-        true
+
+        false
     }
 
     /// Find the lint level for a lint.

compiler/rustc_lint_defs/src/builtin.rs

@@ -557,6 +557,7 @@
     /// fn main() {
     ///     use foo::bar;
     ///     foo::bar();
+    ///     bar();
     /// }
     /// ```
     ///
@@ -704,6 +705,20 @@
     /// `PhantomData`.
     ///
     /// Otherwise consider removing the unused code.
+    ///
+    /// ### Limitations
+    ///
+    /// Removing fields that are only used for side-effects and never
+    /// read will result in behavioral changes. Examples of this
+    /// include:
+    ///
+    /// - If a field's value performs an action when it is dropped.
+    /// - If a field's type does not implement an auto trait
+    ///   (e.g. `Send`, `Sync`, `Unpin`).
+    ///
+    /// For side-effects from dropping field values, this lint should
+    /// be allowed on those fields. For side-effects from containing
+    /// field types, `PhantomData` should be used.
     pub DEAD_CODE,
     Warn,
     "detect unused, unexported items"
@@ -4341,7 +4356,6 @@
     pub UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES,
     Warn,
     "unrecognized or malformed diagnostic attribute",
-    @feature_gate = sym::diagnostic_namespace;
 }
 
 declare_lint! {

compiler/rustc_middle/src/arena.rs

@@ -115,6 +115,7 @@ macro_rules! arena_types {
             [] features: rustc_feature::Features,
             [decode] specialization_graph: rustc_middle::traits::specialization_graph::Graph,
             [] crate_inherent_impls: rustc_middle::ty::CrateInherentImpls,
+            [] hir_owner_nodes: rustc_hir::OwnerNodes<'tcx>,
         ]);
     )
 }

compiler/rustc_middle/src/hir/map/mod.rs

@@ -161,7 +161,7 @@ pub fn hir_owner_node(self, owner_id: OwnerId) -> OwnerNode<'tcx> {
     /// Retrieves the `hir::Node` corresponding to `id`, returning `None` if cannot be found.
     #[inline]
     pub fn opt_hir_node_by_def_id(self, id: LocalDefId) -> Option<Node<'tcx>> {
-        Some(self.hir_node(self.opt_local_def_id_to_hir_id(id)?))
+        Some(self.hir_node_by_def_id(id))
     }
 
     /// Retrieves the `hir::Node` corresponding to `id`.
@@ -169,12 +169,10 @@ pub fn hir_node(self, id: HirId) -> Node<'tcx> {
         self.hir_owner_nodes(id.owner).nodes[id.local_id].node
     }
 
-    /// Retrieves the `hir::Node` corresponding to `id`, panicking if it cannot be found.
+    /// Retrieves the `hir::Node` corresponding to `id`.
     #[inline]
-    #[track_caller]
     pub fn hir_node_by_def_id(self, id: LocalDefId) -> Node<'tcx> {
-        self.opt_hir_node_by_def_id(id)
-            .unwrap_or_else(|| bug!("couldn't find HIR node for def id {id:?}"))
+        self.hir_node(self.local_def_id_to_hir_id(id))
     }
 
     /// Returns `HirId` of the parent HIR node of node with this `hir_id`.
@@ -963,6 +961,7 @@ pub fn span_with_body(self, hir_id: HirId) -> Span {
             Node::Crate(item) => item.spans.inner_span,
             Node::WhereBoundPredicate(pred) => pred.span,
             Node::ArrayLenInfer(inf) => inf.span,
+            Node::AssocOpaqueTy(..) => unreachable!(),
             Node::Err(span) => *span,
         }
     }
@@ -1227,6 +1226,7 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
         Node::Crate(..) => String::from("(root_crate)"),
         Node::WhereBoundPredicate(_) => node_str("where bound predicate"),
         Node::ArrayLenInfer(_) => node_str("array len infer"),
+        Node::AssocOpaqueTy(..) => unreachable!(),
         Node::Err(_) => node_str("error"),
     }
 }

compiler/rustc_middle/src/hir/mod.rs

@@ -8,6 +8,9 @@
 
 use crate::query::Providers;
 use crate::ty::{EarlyBinder, ImplSubject, TyCtxt};
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::sorted_map::SortedMap;
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::{try_par_for_each_in, DynSend, DynSync};
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId};
@@ -121,18 +124,40 @@ pub fn is_foreign_item(self, def_id: impl Into<DefId>) -> bool {
         self.opt_parent(def_id.into())
             .is_some_and(|parent| matches!(self.def_kind(parent), DefKind::ForeignMod))
     }
+
+    pub fn hash_owner_nodes(
+        self,
+        node: OwnerNode<'_>,
+        bodies: &SortedMap<ItemLocalId, &Body<'_>>,
+        attrs: &SortedMap<ItemLocalId, &[rustc_ast::Attribute]>,
+    ) -> (Option<Fingerprint>, Option<Fingerprint>) {
+        if self.needs_crate_hash() {
+            self.with_stable_hashing_context(|mut hcx| {
+                let mut stable_hasher = StableHasher::new();
+                node.hash_stable(&mut hcx, &mut stable_hasher);
+                // Bodies are stored out of line, so we need to pull them explicitly in the hash.
+                bodies.hash_stable(&mut hcx, &mut stable_hasher);
+                let h1 = stable_hasher.finish();
+
+                let mut stable_hasher = StableHasher::new();
+                attrs.hash_stable(&mut hcx, &mut stable_hasher);
+                let h2 = stable_hasher.finish();
+                (Some(h1), Some(h2))
+            })
+        } else {
+            (None, None)
+        }
+    }
 }
 
 pub fn provide(providers: &mut Providers) {
     providers.hir_crate_items = map::hir_crate_items;
     providers.crate_hash = map::crate_hash;
     providers.hir_module_items = map::hir_module_items;
-    providers.opt_local_def_id_to_hir_id = |tcx, def_id| {
-        Some(match tcx.hir_crate(()).owners[def_id] {
-            MaybeOwner::Owner(_) => HirId::make_owner(def_id),
-            MaybeOwner::NonOwner(hir_id) => hir_id,
-            MaybeOwner::Phantom => bug!("No HirId for {:?}", def_id),
-        })
+    providers.local_def_id_to_hir_id = |tcx, def_id| match tcx.hir_crate(()).owners[def_id] {
+        MaybeOwner::Owner(_) => HirId::make_owner(def_id),
+        MaybeOwner::NonOwner(hir_id) => hir_id,
+        MaybeOwner::Phantom => bug!("No HirId for {:?}", def_id),
     };
     providers.opt_hir_owner_nodes =
         |tcx, id| tcx.hir_crate(()).owners.get(id)?.as_owner().map(|i| &i.nodes);

compiler/rustc_middle/src/hooks/mod.rs

@@ -6,6 +6,7 @@
 use crate::mir;
 use crate::query::TyCtxtAt;
 use crate::ty::{Ty, TyCtxt};
+use rustc_span::def_id::LocalDefId;
 use rustc_span::DUMMY_SP;
 
 macro_rules! declare_hooks {
@@ -70,4 +71,10 @@ fn clone(&self) -> Self { *self }
 
     /// Getting a &core::panic::Location referring to a span.
     hook const_caller_location(file: rustc_span::Symbol, line: u32, col: u32) -> mir::ConstValue<'tcx>;
+
+    /// Returns `true` if this def is a function-like thing that is eligible for
+    /// coverage instrumentation under `-Cinstrument-coverage`.
+    ///
+    /// (Eligible functions might nevertheless be skipped for other reasons.)
+    hook is_eligible_for_coverage(key: LocalDefId) -> bool;
 }

compiler/rustc_middle/src/mir/coverage.rs

@@ -2,10 +2,19 @@
 
 use rustc_index::IndexVec;
 use rustc_macros::HashStable;
-use rustc_span::Symbol;
+use rustc_span::{Span, Symbol};
 
 use std::fmt::{self, Debug, Formatter};
 
+rustc_index::newtype_index! {
+    /// Used by [`CoverageKind::BlockMarker`] to mark blocks during THIR-to-MIR
+    /// lowering, so that those blocks can be identified later.
+    #[derive(HashStable)]
+    #[encodable]
+    #[debug_format = "BlockMarkerId({})"]
+    pub struct BlockMarkerId {}
+}
+
 rustc_index::newtype_index! {
     /// ID of a coverage counter. Values ascend from 0.
     ///
@@ -83,6 +92,12 @@ pub enum CoverageKind {
     /// codegen.
     SpanMarker,
 
+    /// Marks its enclosing basic block with an ID that can be referred to by
+    /// side data in [`BranchInfo`].
+    ///
+    /// Has no effect during codegen.
+    BlockMarker { id: BlockMarkerId },
+
     /// Marks the point in MIR control flow represented by a coverage counter.
     ///
     /// This is eventually lowered to `llvm.instrprof.increment` in LLVM IR.
@@ -107,6 +122,7 @@ fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
         use CoverageKind::*;
         match self {
             SpanMarker => write!(fmt, "SpanMarker"),
+            BlockMarker { id } => write!(fmt, "BlockMarker({:?})", id.index()),
             CounterIncrement { id } => write!(fmt, "CounterIncrement({:?})", id.index()),
             ExpressionUsed { id } => write!(fmt, "ExpressionUsed({:?})", id.index()),
         }
@@ -163,14 +179,18 @@ pub struct Expression {
 pub enum MappingKind {
     /// Associates a normal region of code with a counter/expression/zero.
     Code(CovTerm),
+    /// Associates a branch region with separate counters for true and false.
+    Branch { true_term: CovTerm, false_term: CovTerm },
 }
 
 impl MappingKind {
     /// Iterator over all coverage terms in this mapping kind.
     pub fn terms(&self) -> impl Iterator<Item = CovTerm> {
-        let one = |a| std::iter::once(a);
+        let one = |a| std::iter::once(a).chain(None);
+        let two = |a, b| std::iter::once(a).chain(Some(b));
         match *self {
             Self::Code(term) => one(term),
+            Self::Branch { true_term, false_term } => two(true_term, false_term),
         }
     }
 
@@ -179,6 +199,9 @@ pub fn terms(&self) -> impl Iterator<Item = CovTerm> {
     pub fn map_terms(&self, map_fn: impl Fn(CovTerm) -> CovTerm) -> Self {
         match *self {
             Self::Code(term) => Self::Code(map_fn(term)),
+            Self::Branch { true_term, false_term } => {
+                Self::Branch { true_term: map_fn(true_term), false_term: map_fn(false_term) }
+            }
         }
     }
 }
@@ -202,3 +225,22 @@ pub struct FunctionCoverageInfo {
     pub expressions: IndexVec<ExpressionId, Expression>,
     pub mappings: Vec<Mapping>,
 }
+
+/// Branch information recorded during THIR-to-MIR lowering, and stored in MIR.
+#[derive(Clone, Debug)]
+#[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
+pub struct BranchInfo {
+    /// 1 more than the highest-numbered [`CoverageKind::BlockMarker`] that was
+    /// injected into the MIR body. This makes it possible to allocate per-ID
+    /// data structures without having to scan the entire body first.
+    pub num_block_markers: usize,
+    pub branch_spans: Vec<BranchSpan>,
+}
+
+#[derive(Clone, Debug)]
+#[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
+pub struct BranchSpan {
+    pub span: Span,
+    pub true_marker: BlockMarkerId,
+    pub false_marker: BlockMarkerId,
+}

compiler/rustc_middle/src/mir/mod.rs

@@ -403,6 +403,12 @@ pub struct Body<'tcx> {
 
     pub tainted_by_errors: Option<ErrorGuaranteed>,
 
+    /// Branch coverage information collected during MIR building, to be used by
+    /// the `InstrumentCoverage` pass.
+    ///
+    /// Only present if branch coverage is enabled and this function is eligible.
+    pub coverage_branch_info: Option<Box<coverage::BranchInfo>>,
+
     /// Per-function coverage information added by the `InstrumentCoverage`
     /// pass, to be used in conjunction with the coverage statements injected
     /// into this body's blocks.
@@ -450,6 +456,7 @@ pub fn new(
             is_polymorphic: false,
             injection_phase: None,
             tainted_by_errors,
+            coverage_branch_info: None,
             function_coverage_info: None,
         };
         body.is_polymorphic = body.has_non_region_param();
@@ -479,6 +486,7 @@ pub fn new_cfg_only(basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>) ->
             is_polymorphic: false,
             injection_phase: None,
             tainted_by_errors: None,
+            coverage_branch_info: None,
             function_coverage_info: None,
         };
         body.is_polymorphic = body.has_non_region_param();

compiler/rustc_middle/src/mir/pretty.rs

@@ -461,6 +461,9 @@ pub fn write_mir_intro<'tcx>(
     // Add an empty line before the first block is printed.
     writeln!(w)?;
 
+    if let Some(branch_info) = &body.coverage_branch_info {
+        write_coverage_branch_info(branch_info, w)?;
+    }
     if let Some(function_coverage_info) = &body.function_coverage_info {
         write_function_coverage_info(function_coverage_info, w)?;
     }
@@ -468,6 +471,25 @@ pub fn write_mir_intro<'tcx>(
     Ok(())
 }
 
+fn write_coverage_branch_info(
+    branch_info: &coverage::BranchInfo,
+    w: &mut dyn io::Write,
+) -> io::Result<()> {
+    let coverage::BranchInfo { branch_spans, .. } = branch_info;
+
+    for coverage::BranchSpan { span, true_marker, false_marker } in branch_spans {
+        writeln!(
+            w,
+            "{INDENT}coverage branch {{ true: {true_marker:?}, false: {false_marker:?} }} => {span:?}",
+        )?;
+    }
+    if !branch_spans.is_empty() {
+        writeln!(w)?;
+    }
+
+    Ok(())
+}
+
 fn write_function_coverage_info(
     function_coverage_info: &coverage::FunctionCoverageInfo,
     w: &mut dyn io::Write,

compiler/rustc_middle/src/query/mod.rs

@@ -174,10 +174,8 @@
         cache_on_disk_if { true }
     }
 
-    /// Gives access to the HIR ID for the given `LocalDefId` owner `key` if any.
-    ///
-    /// Definitions that were generated with no HIR, would be fed to return `None`.
-    query opt_local_def_id_to_hir_id(key: LocalDefId) -> Option<hir::HirId>{
+    /// Returns HIR ID for the given `LocalDefId`.
+    query local_def_id_to_hir_id(key: LocalDefId) -> hir::HirId {
         desc { |tcx| "getting HIR ID of `{}`", tcx.def_path_str(key) }
         feedable
     }
@@ -196,6 +194,7 @@
     /// Avoid calling this query directly.
     query opt_hir_owner_nodes(key: LocalDefId) -> Option<&'tcx hir::OwnerNodes<'tcx>> {
         desc { |tcx| "getting HIR owner items in `{}`", tcx.def_path_str(key) }
+        feedable
     }
 
     /// Gives access to the HIR attributes inside the HIR owner `key`.
@@ -204,6 +203,7 @@
     /// Avoid calling this query directly.
     query hir_attrs(key: hir::OwnerId) -> &'tcx hir::AttributeMap<'tcx> {
         desc { |tcx| "getting HIR owner attributes in `{}`", tcx.def_path_str(key) }
+        feedable
     }
 
     /// Given the def_id of a const-generic parameter, computes the associated default const

compiler/rustc_middle/src/traits/mod.rs

@@ -618,6 +618,7 @@ pub enum SelectionError<'tcx> {
     OpaqueTypeAutoTraitLeakageUnknown(DefId),
 }
 
+// FIXME(@lcnr): The `Binder` here should be unnecessary. Just use `TraitRef` instead.
 #[derive(Clone, Debug, TypeVisitable)]
 pub struct SignatureMismatchData<'tcx> {
     pub found_trait_ref: ty::PolyTraitRef<'tcx>,

compiler/rustc_middle/src/ty/context.rs

@@ -589,6 +589,11 @@ impl<'tcx> TyCtxtFeed<'tcx, LocalDefId> {
     pub fn def_id(&self) -> LocalDefId {
         self.key
     }
+
+    // Caller must ensure that `self.key` ID is indeed an owner.
+    pub fn feed_owner_id(&self) -> TyCtxtFeed<'tcx, hir::OwnerId> {
+        TyCtxtFeed { tcx: self.tcx, key: hir::OwnerId { def_id: self.key } }
+    }
 }
 
 /// The central data structure of the compiler. It stores references
@@ -2350,8 +2355,8 @@ pub fn intrinsic(self, def_id: impl IntoQueryParam<DefId> + Copy) -> Option<ty::
         self.intrinsic_raw(def_id)
     }
 
-    pub fn local_def_id_to_hir_id(self, local_def_id: LocalDefId) -> HirId {
-        self.opt_local_def_id_to_hir_id(local_def_id).unwrap()
+    pub fn opt_local_def_id_to_hir_id(self, local_def_id: LocalDefId) -> Option<HirId> {
+        Some(self.local_def_id_to_hir_id(local_def_id))
     }
 
     pub fn next_trait_solver_globally(self) -> bool {

compiler/rustc_middle/src/ty/structural_impls.rs

@@ -405,6 +405,7 @@ fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
     ::rustc_hir::HirId,
     ::rustc_hir::MatchSource,
     ::rustc_target::asm::InlineAsmRegOrRegClass,
+    crate::mir::coverage::BlockMarkerId,
     crate::mir::coverage::CounterId,
     crate::mir::coverage::ExpressionId,
     crate::mir::Local,

compiler/rustc_middle/src/ty/sty.rs

@@ -2436,8 +2436,9 @@ pub fn to_opt_closure_kind(self) -> Option<ty::ClosureKind> {
             },
 
             // "Bound" types appear in canonical queries when the
-            // closure type is not yet known
-            Bound(..) | Param(_) | Infer(_) => None,
+            // closure type is not yet known, and `Placeholder` and `Param`
+            // may be encountered in generic `AsyncFnKindHelper` goals.
+            Bound(..) | Placeholder(_) | Param(_) | Infer(_) => None,
 
             Error(_) => Some(ty::ClosureKind::Fn),
 

compiler/rustc_mir_build/src/build/coverageinfo.rs

@@ -0,0 +1,148 @@
+use std::assert_matches::assert_matches;
+use std::collections::hash_map::Entry;
+
+use rustc_data_structures::fx::FxHashMap;
+use rustc_middle::mir::coverage::{BlockMarkerId, BranchSpan, CoverageKind};
+use rustc_middle::mir::{self, BasicBlock, UnOp};
+use rustc_middle::thir::{ExprId, ExprKind, Thir};
+use rustc_middle::ty::TyCtxt;
+use rustc_span::def_id::LocalDefId;
+
+use crate::build::Builder;
+
+pub(crate) struct BranchInfoBuilder {
+    /// Maps condition expressions to their enclosing `!`, for better instrumentation.
+    nots: FxHashMap<ExprId, NotInfo>,
+
+    num_block_markers: usize,
+    branch_spans: Vec<BranchSpan>,
+}
+
+#[derive(Clone, Copy)]
+struct NotInfo {
+    /// When visiting the associated expression as a branch condition, treat this
+    /// enclosing `!` as the branch condition instead.
+    enclosing_not: ExprId,
+    /// True if the associated expression is nested within an odd number of `!`
+    /// expressions relative to `enclosing_not` (inclusive of `enclosing_not`).
+    is_flipped: bool,
+}
+
+impl BranchInfoBuilder {
+    /// Creates a new branch info builder, but only if branch coverage instrumentation
+    /// is enabled and `def_id` represents a function that is eligible for coverage.
+    pub(crate) fn new_if_enabled(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<Self> {
+        if tcx.sess.instrument_coverage_branch() && tcx.is_eligible_for_coverage(def_id) {
+            Some(Self { nots: FxHashMap::default(), num_block_markers: 0, branch_spans: vec![] })
+        } else {
+            None
+        }
+    }
+
+    /// Unary `!` expressions inside an `if` condition are lowered by lowering
+    /// their argument instead, and then reversing the then/else arms of that `if`.
+    ///
+    /// That's awkward for branch coverage instrumentation, so to work around that
+    /// we pre-emptively visit any affected `!` expressions, and record extra
+    /// information that [`Builder::visit_coverage_branch_condition`] can use to
+    /// synthesize branch instrumentation for the enclosing `!`.
+    pub(crate) fn visit_unary_not(&mut self, thir: &Thir<'_>, unary_not: ExprId) {
+        assert_matches!(thir[unary_not].kind, ExprKind::Unary { op: UnOp::Not, .. });
+
+        self.visit_with_not_info(
+            thir,
+            unary_not,
+            // Set `is_flipped: false` for the `!` itself, so that its enclosed
+            // expression will have `is_flipped: true`.
+            NotInfo { enclosing_not: unary_not, is_flipped: false },
+        );
+    }
+
+    fn visit_with_not_info(&mut self, thir: &Thir<'_>, expr_id: ExprId, not_info: NotInfo) {
+        match self.nots.entry(expr_id) {
+            // This expression has already been marked by an enclosing `!`.
+            Entry::Occupied(_) => return,
+            Entry::Vacant(entry) => entry.insert(not_info),
+        };
+
+        match thir[expr_id].kind {
+            ExprKind::Unary { op: UnOp::Not, arg } => {
+                // Invert the `is_flipped` flag for the contents of this `!`.
+                let not_info = NotInfo { is_flipped: !not_info.is_flipped, ..not_info };
+                self.visit_with_not_info(thir, arg, not_info);
+            }
+            ExprKind::Scope { value, .. } => self.visit_with_not_info(thir, value, not_info),
+            ExprKind::Use { source } => self.visit_with_not_info(thir, source, not_info),
+            // All other expressions (including `&&` and `||`) don't need any
+            // special handling of their contents, so stop visiting.
+            _ => {}
+        }
+    }
+
+    fn next_block_marker_id(&mut self) -> BlockMarkerId {
+        let id = BlockMarkerId::from_usize(self.num_block_markers);
+        self.num_block_markers += 1;
+        id
+    }
+
+    pub(crate) fn into_done(self) -> Option<Box<mir::coverage::BranchInfo>> {
+        let Self { nots: _, num_block_markers, branch_spans } = self;
+
+        if num_block_markers == 0 {
+            assert!(branch_spans.is_empty());
+            return None;
+        }
+
+        Some(Box::new(mir::coverage::BranchInfo { num_block_markers, branch_spans }))
+    }
+}
+
+impl Builder<'_, '_> {
+    /// If branch coverage is enabled, inject marker statements into `then_block`
+    /// and `else_block`, and record their IDs in the table of branch spans.
+    pub(crate) fn visit_coverage_branch_condition(
+        &mut self,
+        mut expr_id: ExprId,
+        mut then_block: BasicBlock,
+        mut else_block: BasicBlock,
+    ) {
+        // Bail out if branch coverage is not enabled for this function.
+        let Some(branch_info) = self.coverage_branch_info.as_ref() else { return };
+
+        // If this condition expression is nested within one or more `!` expressions,
+        // replace it with the enclosing `!` collected by `visit_unary_not`.
+        if let Some(&NotInfo { enclosing_not, is_flipped }) = branch_info.nots.get(&expr_id) {
+            expr_id = enclosing_not;
+            if is_flipped {
+                std::mem::swap(&mut then_block, &mut else_block);
+            }
+        }
+        let source_info = self.source_info(self.thir[expr_id].span);
+
+        // Now that we have `source_info`, we can upgrade to a &mut reference.
+        let branch_info = self.coverage_branch_info.as_mut().expect("upgrading & to &mut");
+
+        let mut inject_branch_marker = |block: BasicBlock| {
+            let id = branch_info.next_block_marker_id();
+
+            let marker_statement = mir::Statement {
+                source_info,
+                kind: mir::StatementKind::Coverage(Box::new(mir::Coverage {
+                    kind: CoverageKind::BlockMarker { id },
+                })),
+            };
+            self.cfg.push(block, marker_statement);
+
+            id
+        };
+
+        let true_marker = inject_branch_marker(then_block);
+        let false_marker = inject_branch_marker(else_block);
+
+        branch_info.branch_spans.push(BranchSpan {
+            span: source_info.span,
+            true_marker,
+            false_marker,
+        });
+    }
+}

compiler/rustc_mir_build/src/build/custom/mod.rs

@@ -60,6 +60,7 @@ pub(super) fn build_custom_mir<'tcx>(
         tainted_by_errors: None,
         injection_phase: None,
         pass_count: 0,
+        coverage_branch_info: None,
         function_coverage_info: None,
     };
 

compiler/rustc_mir_build/src/build/matches/mod.rs

@@ -105,6 +105,13 @@ fn then_else_break_inner(
                 success_block.unit()
             }
             ExprKind::Unary { op: UnOp::Not, arg } => {
+                // Improve branch coverage instrumentation by noting conditions
+                // nested within one or more `!` expressions.
+                // (Skipped if branch coverage is not enabled.)
+                if let Some(branch_info) = this.coverage_branch_info.as_mut() {
+                    branch_info.visit_unary_not(this.thir, expr_id);
+                }
+
                 let local_scope = this.local_scope();
                 let (success_block, failure_block) =
                     this.in_if_then_scope(local_scope, expr_span, |this| {
@@ -149,6 +156,10 @@ fn then_else_break_inner(
                 let else_block = this.cfg.start_new_block();
                 let term = TerminatorKind::if_(operand, then_block, else_block);
 
+                // Record branch coverage info for this condition.
+                // (Does nothing if branch coverage is not enabled.)
+                this.visit_coverage_branch_condition(expr_id, then_block, else_block);
+
                 let source_info = this.source_info(expr_span);
                 this.cfg.terminate(block, source_info, term);
                 this.break_for_else(else_block, source_info);

compiler/rustc_mir_build/src/build/mod.rs

@@ -234,6 +234,10 @@ struct Builder<'a, 'tcx> {
     // the root (most of them do) and saves us from retracing many sub-paths
     // many times, and rechecking many nodes.
     lint_level_roots_cache: GrowableBitSet<hir::ItemLocalId>,
+
+    /// Collects additional coverage information during MIR building.
+    /// Only present if branch coverage is enabled and this function is eligible.
+    coverage_branch_info: Option<coverageinfo::BranchInfoBuilder>,
 }
 
 type CaptureMap<'tcx> = SortedIndexMultiMap<usize, hir::HirId, Capture<'tcx>>;
@@ -807,6 +811,7 @@ fn new(
             unit_temp: None,
             var_debug_info: vec![],
             lint_level_roots_cache: GrowableBitSet::new_empty(),
+            coverage_branch_info: coverageinfo::BranchInfoBuilder::new_if_enabled(tcx, def),
         };
 
         assert_eq!(builder.cfg.start_new_block(), START_BLOCK);
@@ -826,7 +831,7 @@ fn finish(self) -> Body<'tcx> {
             }
         }
 
-        Body::new(
+        let mut body = Body::new(
             MirSource::item(self.def_id.to_def_id()),
             self.cfg.basic_blocks,
             self.source_scopes,
@@ -837,7 +842,9 @@ fn finish(self) -> Body<'tcx> {
             self.fn_span,
             self.coroutine,
             None,
-        )
+        );
+        body.coverage_branch_info = self.coverage_branch_info.and_then(|b| b.into_done());
+        body
     }
 
     fn insert_upvar_arg(&mut self) {
@@ -1111,6 +1118,7 @@ pub(crate) fn parse_float_into_scalar(
 
 mod block;
 mod cfg;
+mod coverageinfo;
 mod custom;
 mod expr;
 mod matches;

compiler/rustc_mir_transform/src/add_retag.rs

@@ -118,7 +118,7 @@ fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         }
 
         // PART 3
-        // Add retag after assignments where data "enters" this function: the RHS is behind a deref and the LHS is not.
+        // Add retag after assignments.
         for block_data in basic_blocks {
             // We want to insert statements as we iterate. To this end, we
             // iterate backwards using indices.

compiler/rustc_mir_transform/src/coverage/mod.rs

@@ -14,7 +14,6 @@
 use crate::MirPass;
 
 use rustc_middle::hir;
-use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::coverage::*;
 use rustc_middle::mir::{
     self, BasicBlock, BasicBlockData, Coverage, SourceInfo, Statement, StatementKind, Terminator,
@@ -44,7 +43,7 @@ fn run_pass(&self, tcx: TyCtxt<'tcx>, mir_body: &mut mir::Body<'tcx>) {
 
         let def_id = mir_source.def_id().expect_local();
 
-        if !is_eligible_for_coverage(tcx, def_id) {
+        if !tcx.is_eligible_for_coverage(def_id) {
             trace!("InstrumentCoverage skipped for {def_id:?} (not eligible)");
             return;
         }
@@ -140,6 +139,10 @@ fn create_mappings<'tcx>(
         .filter_map(|&BcbMapping { kind: bcb_mapping_kind, span }| {
             let kind = match bcb_mapping_kind {
                 BcbMappingKind::Code(bcb) => MappingKind::Code(term_for_bcb(bcb)),
+                BcbMappingKind::Branch { true_bcb, false_bcb } => MappingKind::Branch {
+                    true_term: term_for_bcb(true_bcb),
+                    false_term: term_for_bcb(false_bcb),
+                },
             };
             let code_region = make_code_region(source_map, file_name, span, body_span)?;
             Some(Mapping { kind, code_region })
@@ -349,37 +352,6 @@ fn check_code_region(code_region: CodeRegion) -> Option<CodeRegion> {
     }
 }
 
-fn is_eligible_for_coverage(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
-    // Only instrument functions, methods, and closures (not constants since they are evaluated
-    // at compile time by Miri).
-    // FIXME(#73156): Handle source code coverage in const eval, but note, if and when const
-    // expressions get coverage spans, we will probably have to "carve out" space for const
-    // expressions from coverage spans in enclosing MIR's, like we do for closures. (That might
-    // be tricky if const expressions have no corresponding statements in the enclosing MIR.
-    // Closures are carved out by their initial `Assign` statement.)
-    if !tcx.def_kind(def_id).is_fn_like() {
-        trace!("InstrumentCoverage skipped for {def_id:?} (not an fn-like)");
-        return false;
-    }
-
-    // Don't instrument functions with `#[automatically_derived]` on their
-    // enclosing impl block, on the assumption that most users won't care about
-    // coverage for derived impls.
-    if let Some(impl_of) = tcx.impl_of_method(def_id.to_def_id())
-        && tcx.is_automatically_derived(impl_of)
-    {
-        trace!("InstrumentCoverage skipped for {def_id:?} (automatically derived)");
-        return false;
-    }
-
-    if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::NO_COVERAGE) {
-        trace!("InstrumentCoverage skipped for {def_id:?} (`#[coverage(off)]`)");
-        return false;
-    }
-
-    true
-}
-
 /// Function information extracted from HIR by the coverage instrumentor.
 #[derive(Debug)]
 struct ExtractedHirInfo {

compiler/rustc_mir_transform/src/coverage/query.rs

@@ -1,14 +1,49 @@
-use super::*;
-
 use rustc_data_structures::captures::Captures;
-use rustc_middle::mir::coverage::*;
-use rustc_middle::mir::{Body, CoverageIdsInfo};
-use rustc_middle::query::Providers;
-use rustc_middle::ty::{self};
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
+use rustc_middle::mir::coverage::{CounterId, CoverageKind};
+use rustc_middle::mir::{Body, Coverage, CoverageIdsInfo, Statement, StatementKind};
+use rustc_middle::query::TyCtxtAt;
+use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::util::Providers;
+use rustc_span::def_id::LocalDefId;
 
-/// A `query` provider for retrieving coverage information injected into MIR.
+/// Registers query/hook implementations related to coverage.
 pub(crate) fn provide(providers: &mut Providers) {
-    providers.coverage_ids_info = |tcx, def_id| coverage_ids_info(tcx, def_id);
+    providers.hooks.is_eligible_for_coverage =
+        |TyCtxtAt { tcx, .. }, def_id| is_eligible_for_coverage(tcx, def_id);
+    providers.queries.coverage_ids_info = coverage_ids_info;
+}
+
+/// Hook implementation for [`TyCtxt::is_eligible_for_coverage`].
+fn is_eligible_for_coverage(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
+    // Only instrument functions, methods, and closures (not constants since they are evaluated
+    // at compile time by Miri).
+    // FIXME(#73156): Handle source code coverage in const eval, but note, if and when const
+    // expressions get coverage spans, we will probably have to "carve out" space for const
+    // expressions from coverage spans in enclosing MIR's, like we do for closures. (That might
+    // be tricky if const expressions have no corresponding statements in the enclosing MIR.
+    // Closures are carved out by their initial `Assign` statement.)
+    if !tcx.def_kind(def_id).is_fn_like() {
+        trace!("InstrumentCoverage skipped for {def_id:?} (not an fn-like)");
+        return false;
+    }
+
+    // Don't instrument functions with `#[automatically_derived]` on their
+    // enclosing impl block, on the assumption that most users won't care about
+    // coverage for derived impls.
+    if let Some(impl_of) = tcx.impl_of_method(def_id.to_def_id())
+        && tcx.is_automatically_derived(impl_of)
+    {
+        trace!("InstrumentCoverage skipped for {def_id:?} (automatically derived)");
+        return false;
+    }
+
+    if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::NO_COVERAGE) {
+        trace!("InstrumentCoverage skipped for {def_id:?} (`#[coverage(off)]`)");
+        return false;
+    }
+
+    true
 }
 
 /// Query implementation for `coverage_ids_info`.

compiler/rustc_mir_transform/src/coverage/spans.rs

@@ -13,6 +13,8 @@
 pub(super) enum BcbMappingKind {
     /// Associates an ordinary executable code span with its corresponding BCB.
     Code(BasicCoverageBlock),
+    /// Associates a branch span with BCBs for its true and false arms.
+    Branch { true_bcb: BasicCoverageBlock, false_bcb: BasicCoverageBlock },
 }
 
 #[derive(Debug)]
@@ -66,6 +68,12 @@ pub(super) fn generate_coverage_spans(
             // Each span produced by the generator represents an ordinary code region.
             BcbMapping { kind: BcbMappingKind::Code(bcb), span }
         }));
+
+        mappings.extend(from_mir::extract_branch_mappings(
+            mir_body,
+            hir_info.body_span,
+            basic_coverage_blocks,
+        ));
     }
 
     if mappings.is_empty() {
@@ -80,6 +88,10 @@ pub(super) fn generate_coverage_spans(
     for &BcbMapping { kind, span: _ } in &mappings {
         match kind {
             BcbMappingKind::Code(bcb) => insert(bcb),
+            BcbMappingKind::Branch { true_bcb, false_bcb } => {
+                insert(true_bcb);
+                insert(false_bcb);
+            }
         }
     }
 

compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs

@@ -1,7 +1,9 @@
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::FxHashSet;
+use rustc_index::IndexVec;
+use rustc_middle::mir::coverage::{BlockMarkerId, BranchSpan, CoverageKind};
 use rustc_middle::mir::{
-    self, AggregateKind, FakeReadCause, Rvalue, Statement, StatementKind, Terminator,
+    self, AggregateKind, BasicBlock, FakeReadCause, Rvalue, Statement, StatementKind, Terminator,
     TerminatorKind,
 };
 use rustc_span::{ExpnKind, MacroKind, Span, Symbol};
@@ -9,6 +11,7 @@
 use crate::coverage::graph::{
     BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph, START_BCB,
 };
+use crate::coverage::spans::{BcbMapping, BcbMappingKind};
 use crate::coverage::ExtractedHirInfo;
 
 /// Traverses the MIR body to produce an initial collection of coverage-relevant
@@ -179,8 +182,6 @@ fn is_closure_like(statement: &Statement<'_>) -> bool {
 /// If the MIR `Statement` has a span contributive to computing coverage spans,
 /// return it; otherwise return `None`.
 fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> {
-    use mir::coverage::CoverageKind;
-
     match statement.kind {
         // These statements have spans that are often outside the scope of the executed source code
         // for their parent `BasicBlock`.
@@ -225,6 +226,11 @@ fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> {
             Some(statement.source_info.span)
         }
 
+        StatementKind::Coverage(box mir::Coverage {
+            // Block markers are used for branch coverage, so ignore them here.
+            kind: CoverageKind::BlockMarker {..}
+        }) => None,
+
         StatementKind::Coverage(box mir::Coverage {
             // These coverage statements should not exist prior to coverage instrumentation.
             kind: CoverageKind::CounterIncrement { .. } | CoverageKind::ExpressionUsed { .. }
@@ -358,3 +364,51 @@ fn new(
         Self { span, visible_macro, bcb, is_hole }
     }
 }
+
+pub(super) fn extract_branch_mappings(
+    mir_body: &mir::Body<'_>,
+    body_span: Span,
+    basic_coverage_blocks: &CoverageGraph,
+) -> Vec<BcbMapping> {
+    let Some(branch_info) = mir_body.coverage_branch_info.as_deref() else {
+        return vec![];
+    };
+
+    let mut block_markers = IndexVec::<BlockMarkerId, Option<BasicBlock>>::from_elem_n(
+        None,
+        branch_info.num_block_markers,
+    );
+
+    // Fill out the mapping from block marker IDs to their enclosing blocks.
+    for (bb, data) in mir_body.basic_blocks.iter_enumerated() {
+        for statement in &data.statements {
+            if let StatementKind::Coverage(coverage) = &statement.kind
+                && let CoverageKind::BlockMarker { id } = coverage.kind
+            {
+                block_markers[id] = Some(bb);
+            }
+        }
+    }
+
+    branch_info
+        .branch_spans
+        .iter()
+        .filter_map(|&BranchSpan { span: raw_span, true_marker, false_marker }| {
+            // For now, ignore any branch span that was introduced by
+            // expansion. This makes things like assert macros less noisy.
+            if !raw_span.ctxt().outer_expn_data().is_root() {
+                return None;
+            }
+            let (span, _) = unexpand_into_body_span_with_visible_macro(raw_span, body_span)?;
+
+            let bcb_from_marker = |marker: BlockMarkerId| {
+                Some(basic_coverage_blocks.bcb_from_bb(block_markers[marker]?)?)
+            };
+
+            let true_bcb = bcb_from_marker(true_marker)?;
+            let false_bcb = bcb_from_marker(false_marker)?;
+
+            Some(BcbMapping { kind: BcbMappingKind::Branch { true_bcb, false_bcb }, span })
+        })
+        .collect::<Vec<_>>()
+}

compiler/rustc_mir_transform/src/lib.rs

@@ -37,8 +37,9 @@
     LocalDecl, MirPass, MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue,
     SourceInfo, Statement, StatementKind, TerminatorKind, START_BLOCK,
 };
-use rustc_middle::query::Providers;
+use rustc_middle::query;
 use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
+use rustc_middle::util::Providers;
 use rustc_span::{source_map::Spanned, sym, DUMMY_SP};
 use rustc_trait_selection::traits;
 
@@ -124,7 +125,7 @@ pub fn provide(providers: &mut Providers) {
     ffi_unwind_calls::provide(providers);
     shim::provide(providers);
     cross_crate_inline::provide(providers);
-    *providers = Providers {
+    providers.queries = query::Providers {
         mir_keys,
         mir_const,
         mir_const_qualif,
@@ -139,7 +140,7 @@ pub fn provide(providers: &mut Providers) {
         mir_inliner_callees: inline::cycle::mir_inliner_callees,
         promoted_mir,
         deduced_param_attrs: deduce_param_attrs::deduced_param_attrs,
-        ..*providers
+        ..providers.queries
     };
 }
 

compiler/rustc_monomorphize/src/collector.rs

@@ -818,13 +818,16 @@ fn visit_rvalue(&mut self, rvalue: &mir::Rvalue<'tcx>, location: Location) {
         self.super_rvalue(rvalue, location);
     }
 
-    /// This does not walk the constant, as it has been handled entirely here and trying
-    /// to walk it would attempt to evaluate the `ty::Const` inside, which doesn't necessarily
-    /// work, as some constants cannot be represented in the type system.
+    /// This does not walk the MIR of the constant as that is not needed for codegen, all we need is
+    /// to ensure that the constant evaluates successfully and walk the result.
     #[instrument(skip(self), level = "debug")]
     fn visit_constant(&mut self, constant: &mir::ConstOperand<'tcx>, location: Location) {
         let const_ = self.monomorphize(constant.const_);
         let param_env = ty::ParamEnv::reveal_all();
+        // Evaluate the constant. This makes const eval failure a collection-time error (rather than
+        // a codegen-time error). rustc stops after collection if there was an error, so this
+        // ensures codegen never has to worry about failing consts.
+        // (codegen relies on this and ICEs will happen if this is violated.)
         let val = match const_.eval(self.tcx, param_env, None) {
             Ok(v) => v,
             Err(ErrorHandled::Reported(..)) => return,

compiler/rustc_monomorphize/src/partitioning.rs

@@ -1112,6 +1112,9 @@ fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[Co
 
     let (items, usage_map) = collector::collect_crate_mono_items(tcx, collection_mode);
 
+    // If there was an error during collection (e.g. from one of the constants we evaluated),
+    // then we stop here. This way codegen does not have to worry about failing constants.
+    // (codegen relies on this and ICEs will happen if this is violated.)
     tcx.dcx().abort_if_errors();
 
     let (codegen_units, _) = tcx.sess.time("partition_and_assert_distinct_symbols", || {

compiler/rustc_passes/src/reachable.rs

@@ -270,7 +270,8 @@ fn propagate_node(&mut self, node: &Node<'tcx>, search_item: LocalDefId) {
             | Node::Ctor(..)
             | Node::Field(_)
             | Node::Ty(_)
-            | Node::Crate(_) => {}
+            | Node::Crate(_)
+            | Node::AssocOpaqueTy(..) => {}
             _ => {
                 bug!(
                     "found unexpected node kind in worklist: {} ({:?})",

compiler/rustc_resolve/src/check_unused.rs

@@ -23,18 +23,19 @@
 //  - `check_unused` finally emits the diagnostics based on the data generated
 //    in the last step
 
-use crate::imports::ImportKind;
+use crate::imports::{Import, ImportKind};
 use crate::module_to_string;
 use crate::Resolver;
 
-use crate::NameBindingKind;
+use crate::{LexicalScopeBinding, NameBindingKind};
 use rustc_ast as ast;
 use rustc_ast::visit::{self, Visitor};
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::{pluralize, MultiSpan};
 use rustc_hir::def::{DefKind, Res};
-use rustc_session::lint::builtin::{MACRO_USE_EXTERN_CRATE, UNUSED_EXTERN_CRATES, UNUSED_IMPORTS};
+use rustc_session::lint::builtin::{MACRO_USE_EXTERN_CRATE, UNUSED_EXTERN_CRATES};
+use rustc_session::lint::builtin::{UNUSED_IMPORTS, UNUSED_QUALIFICATIONS};
 use rustc_session::lint::BuiltinLintDiag;
 use rustc_span::symbol::{kw, Ident};
 use rustc_span::{Span, DUMMY_SP};
@@ -514,8 +515,59 @@ pub(crate) fn check_unused(&mut self, krate: &ast::Crate) {
             }
         }
 
+        let mut redundant_imports = UnordSet::default();
         for import in check_redundant_imports {
-            self.check_for_redundant_imports(import);
+            if self.check_for_redundant_imports(import)
+                && let Some(id) = import.id()
+            {
+                redundant_imports.insert(id);
+            }
+        }
+
+        // The lint fixes for unused_import and unnecessary_qualification may conflict.
+        // Deleting both unused imports and unnecessary segments of an item may result
+        // in the item not being found.
+        for unn_qua in &self.potentially_unnecessary_qualifications {
+            if let LexicalScopeBinding::Item(name_binding) = unn_qua.binding
+                && let NameBindingKind::Import { import, .. } = name_binding.kind
+                && (is_unused_import(import, &unused_imports)
+                    || is_redundant_import(import, &redundant_imports))
+            {
+                continue;
+            }
+
+            self.lint_buffer.buffer_lint_with_diagnostic(
+                UNUSED_QUALIFICATIONS,
+                unn_qua.node_id,
+                unn_qua.path_span,
+                "unnecessary qualification",
+                BuiltinLintDiag::UnusedQualifications { removal_span: unn_qua.removal_span },
+            );
+        }
+
+        fn is_redundant_import(
+            import: Import<'_>,
+            redundant_imports: &UnordSet<ast::NodeId>,
+        ) -> bool {
+            if let Some(id) = import.id()
+                && redundant_imports.contains(&id)
+            {
+                return true;
+            }
+            false
+        }
+
+        fn is_unused_import(
+            import: Import<'_>,
+            unused_imports: &FxIndexMap<ast::NodeId, UnusedImport>,
+        ) -> bool {
+            if let Some(unused_import) = unused_imports.get(&import.root_id)
+                && let Some(id) = import.id()
+                && unused_import.unused.contains(&id)
+            {
+                return true;
+            }
+            false
         }
     }
 }

compiler/rustc_resolve/src/imports.rs

@@ -1306,7 +1306,7 @@ fn finalize_import(&mut self, import: Import<'a>) -> Option<UnresolvedImportErro
         None
     }
 
-    pub(crate) fn check_for_redundant_imports(&mut self, import: Import<'a>) {
+    pub(crate) fn check_for_redundant_imports(&mut self, import: Import<'a>) -> bool {
         // This function is only called for single imports.
         let ImportKind::Single {
             source, target, ref source_bindings, ref target_bindings, id, ..
@@ -1317,12 +1317,12 @@ pub(crate) fn check_for_redundant_imports(&mut self, import: Import<'a>) {
 
         // Skip if the import is of the form `use source as target` and source != target.
         if source != target {
-            return;
+            return false;
         }
 
         // Skip if the import was produced by a macro.
         if import.parent_scope.expansion != LocalExpnId::ROOT {
-            return;
+            return false;
         }
 
         // Skip if we are inside a named module (in contrast to an anonymous
@@ -1332,7 +1332,7 @@ pub(crate) fn check_for_redundant_imports(&mut self, import: Import<'a>) {
         if import.used.get() == Some(Used::Other)
             || self.effective_visibilities.is_exported(self.local_def_id(id))
         {
-            return;
+            return false;
         }
 
         let mut is_redundant = true;
@@ -1375,7 +1375,10 @@ pub(crate) fn check_for_redundant_imports(&mut self, import: Import<'a>) {
                 format!("the item `{source}` is imported redundantly"),
                 BuiltinLintDiag::RedundantImport(redundant_spans, source),
             );
+            return true;
         }
+
+        false
     }
 
     fn resolve_glob_import(&mut self, import: Import<'a>) {

compiler/rustc_resolve/src/late.rs

@@ -580,6 +580,15 @@ fn eval(self, r: &Resolver<'_, '_>) -> bool {
     }
 }
 
+/// Used for recording UnnecessaryQualification.
+#[derive(Debug)]
+pub(crate) struct UnnecessaryQualification<'a> {
+    pub binding: LexicalScopeBinding<'a>,
+    pub node_id: NodeId,
+    pub path_span: Span,
+    pub removal_span: Span,
+}
+
 #[derive(Default)]
 struct DiagMetadata<'ast> {
     /// The current trait's associated items' ident, used for diagnostic suggestions.
@@ -4654,20 +4663,16 @@ fn lint_unused_qualifications(&mut self, path: &[Segment], ns: Namespace, finali
             let ns = if i + 1 == path.len() { ns } else { TypeNS };
             let res = self.r.partial_res_map.get(&seg.id?)?.full_res()?;
             let binding = self.resolve_ident_in_lexical_scope(seg.ident, ns, None, None)?;
-
-            (res == binding.res()).then_some(seg)
+            (res == binding.res()).then_some((seg, binding))
         });
 
-        if let Some(unqualified) = unqualified {
-            self.r.lint_buffer.buffer_lint_with_diagnostic(
-                lint::builtin::UNUSED_QUALIFICATIONS,
-                finalize.node_id,
-                finalize.path_span,
-                "unnecessary qualification",
-                lint::BuiltinLintDiag::UnusedQualifications {
-                    removal_span: path[0].ident.span.until(unqualified.ident.span),
-                },
-            );
+        if let Some((seg, binding)) = unqualified {
+            self.r.potentially_unnecessary_qualifications.push(UnnecessaryQualification {
+                binding,
+                node_id: finalize.node_id,
+                path_span: finalize.path_span,
+                removal_span: path[0].ident.span.until(seg.ident.span),
+            });
         }
     }
 }

compiler/rustc_resolve/src/lib.rs

@@ -68,7 +68,7 @@
 
 use diagnostics::{ImportSuggestion, LabelSuggestion, Suggestion};
 use imports::{Import, ImportData, ImportKind, NameResolution};
-use late::{HasGenericParams, PathSource, PatternSource};
+use late::{HasGenericParams, PathSource, PatternSource, UnnecessaryQualification};
 use macros::{MacroRulesBinding, MacroRulesScope, MacroRulesScopeRef};
 
 use crate::effective_visibilities::EffectiveVisibilitiesVisitor;
@@ -372,7 +372,7 @@ fn from(seg: &'a ast::PathSegment) -> Segment {
 /// This refers to the thing referred by a name. The difference between `Res` and `Item` is that
 /// items are visible in their whole block, while `Res`es only from the place they are defined
 /// forward.
-#[derive(Debug)]
+#[derive(Debug, Copy, Clone)]
 enum LexicalScopeBinding<'a> {
     Item(NameBinding<'a>),
     Res(Res),
@@ -1105,6 +1105,8 @@ pub struct Resolver<'a, 'tcx> {
 
     potentially_unused_imports: Vec<Import<'a>>,
 
+    potentially_unnecessary_qualifications: Vec<UnnecessaryQualification<'a>>,
+
     /// Table for mapping struct IDs into struct constructor IDs,
     /// it's not used during normal resolution, only for better error reporting.
     /// Also includes of list of each fields visibility
@@ -1464,6 +1466,7 @@ pub fn new(
             local_macro_def_scopes: FxHashMap::default(),
             name_already_seen: FxHashMap::default(),
             potentially_unused_imports: Vec::new(),
+            potentially_unnecessary_qualifications: Default::default(),
             struct_constructors: Default::default(),
             unused_macros: Default::default(),
             unused_macro_rules: Default::default(),

compiler/rustc_session/messages.ftl

@@ -111,4 +111,7 @@ session_unleashed_feature_help_unnamed = skipping check that does not even have
 
 session_unstable_virtual_function_elimination = `-Zvirtual-function-elimination` requires `-Clto`
 
+session_unsupported_crate_type_for_target =
+    dropping unsupported crate type `{$crate_type}` for target `{$target_triple}`
+
 session_unsupported_dwarf_version = requested DWARF version {$dwarf_version} is greater than 5

compiler/rustc_session/src/config.rs

@@ -146,7 +146,7 @@ pub enum InstrumentCoverage {
 /// Individual flag values controlled by `-Z coverage-options`.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub struct CoverageOptions {
-    /// Add branch coverage instrumentation (placeholder flag; not yet implemented).
+    /// Add branch coverage instrumentation.
     pub branch: bool,
 }
 

compiler/rustc_session/src/errors.rs

@@ -10,7 +10,7 @@
 use rustc_span::{Span, Symbol};
 use rustc_target::spec::{SplitDebuginfo, StackProtector, TargetTriple};
 
-use crate::parse::ParseSess;
+use crate::{config::CrateType, parse::ParseSess};
 
 pub struct FeatureGateError {
     pub span: MultiSpan,
@@ -345,6 +345,13 @@ pub(crate) struct BinaryFloatLiteralNotSupported {
     pub span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(session_unsupported_crate_type_for_target)]
+pub struct UnsupportedCrateTypeForTarget<'a> {
+    pub crate_type: CrateType,
+    pub target_triple: &'a TargetTriple,
+}
+
 pub fn report_lit_error(
     psess: &ParseSess,
     err: LitError,

compiler/rustc_session/src/output.rs

@@ -1,7 +1,7 @@
 //! Related to out filenames of compilation (e.g. binaries).
-use crate::config::{CrateType, Input, OutFileName, OutputFilenames, OutputType};
+use crate::config::{self, CrateType, Input, OutFileName, OutputFilenames, OutputType};
 use crate::errors::{
-    CrateNameDoesNotMatch, CrateNameEmpty, CrateNameInvalid, FileIsNotWriteable,
+    self, CrateNameDoesNotMatch, CrateNameEmpty, CrateNameInvalid, FileIsNotWriteable,
     InvalidCharacterInCrateName, InvalidCrateNameHelp,
 };
 use crate::Session;
@@ -200,3 +200,64 @@ pub fn invalid_output_for_target(sess: &Session, crate_type: CrateType) -> bool
 
     false
 }
+
+pub const CRATE_TYPES: &[(Symbol, CrateType)] = &[
+    (sym::rlib, CrateType::Rlib),
+    (sym::dylib, CrateType::Dylib),
+    (sym::cdylib, CrateType::Cdylib),
+    (sym::lib, config::default_lib_output()),
+    (sym::staticlib, CrateType::Staticlib),
+    (sym::proc_dash_macro, CrateType::ProcMacro),
+    (sym::bin, CrateType::Executable),
+];
+
+pub fn categorize_crate_type(s: Symbol) -> Option<CrateType> {
+    Some(CRATE_TYPES.iter().find(|(key, _)| *key == s)?.1)
+}
+
+pub fn collect_crate_types(session: &Session, attrs: &[ast::Attribute]) -> Vec<CrateType> {
+    // If we're generating a test executable, then ignore all other output
+    // styles at all other locations
+    if session.opts.test {
+        return vec![CrateType::Executable];
+    }
+
+    // Only check command line flags if present. If no types are specified by
+    // command line, then reuse the empty `base` Vec to hold the types that
+    // will be found in crate attributes.
+    // JUSTIFICATION: before wrapper fn is available
+    #[allow(rustc::bad_opt_access)]
+    let mut base = session.opts.crate_types.clone();
+    if base.is_empty() {
+        let attr_types = attrs.iter().filter_map(|a| {
+            if a.has_name(sym::crate_type)
+                && let Some(s) = a.value_str()
+            {
+                categorize_crate_type(s)
+            } else {
+                None
+            }
+        });
+        base.extend(attr_types);
+        if base.is_empty() {
+            base.push(default_output_for_target(session));
+        } else {
+            base.sort();
+            base.dedup();
+        }
+    }
+
+    base.retain(|crate_type| {
+        if invalid_output_for_target(session, *crate_type) {
+            session.dcx().emit_warn(errors::UnsupportedCrateTypeForTarget {
+                crate_type: *crate_type,
+                target_triple: &session.opts.target_triple,
+            });
+            false
+        } else {
+            true
+        }
+    });
+
+    base
+}

compiler/rustc_smir/src/rustc_smir/context.rs

@@ -23,7 +23,8 @@
 use stable_mir::target::{MachineInfo, MachineSize};
 use stable_mir::ty::{
     AdtDef, AdtKind, Allocation, ClosureDef, ClosureKind, Const, FieldDef, FnDef, ForeignDef,
-    ForeignItemKind, GenericArgs, LineInfo, PolyFnSig, RigidTy, Span, Ty, TyKind, VariantDef,
+    ForeignItemKind, GenericArgs, LineInfo, PolyFnSig, RigidTy, Span, Ty, TyKind, UintTy,
+    VariantDef,
 };
 use stable_mir::{Crate, CrateDef, CrateItem, CrateNum, DefId, Error, Filename, ItemKind, Symbol};
 use std::cell::RefCell;
@@ -341,15 +342,56 @@ fn eval_target_usize(&self, cnst: &Const) -> Result<u64, Error> {
             .ok_or_else(|| Error::new(format!("Const `{cnst:?}` cannot be encoded as u64")))
     }
 
-    fn usize_to_const(&self, val: u64) -> Result<Const, Error> {
+    fn try_new_const_zst(&self, ty: Ty) -> Result<Const, Error> {
         let mut tables = self.0.borrow_mut();
-        let ty = tables.tcx.types.usize;
+        let tcx = tables.tcx;
+        let ty_internal = ty.internal(&mut *tables, tcx);
+        let size = tables
+            .tcx
+            .layout_of(ParamEnv::empty().and(ty_internal))
+            .map_err(|err| {
+                Error::new(format!(
+                    "Cannot create a zero-sized constant for type `{ty_internal}`: {err}"
+                ))
+            })?
+            .size;
+        if size.bytes() != 0 {
+            return Err(Error::new(format!(
+                "Cannot create a zero-sized constant for type `{ty_internal}`: \
+                 Type `{ty_internal}` has {} bytes",
+                size.bytes()
+            )));
+        }
+
+        Ok(ty::Const::zero_sized(tables.tcx, ty_internal).stable(&mut *tables))
+    }
+
+    fn new_const_str(&self, value: &str) -> Const {
+        let mut tables = self.0.borrow_mut();
+        let tcx = tables.tcx;
+        let ty = ty::Ty::new_static_str(tcx);
+        let bytes = value.as_bytes();
+        let val_tree = ty::ValTree::from_raw_bytes(tcx, bytes);
+
+        ty::Const::new_value(tcx, val_tree, ty).stable(&mut *tables)
+    }
+
+    fn new_const_bool(&self, value: bool) -> Const {
+        let mut tables = self.0.borrow_mut();
+        ty::Const::from_bool(tables.tcx, value).stable(&mut *tables)
+    }
+
+    fn try_new_const_uint(&self, value: u128, uint_ty: UintTy) -> Result<Const, Error> {
+        let mut tables = self.0.borrow_mut();
+        let tcx = tables.tcx;
+        let ty = ty::Ty::new_uint(tcx, uint_ty.internal(&mut *tables, tcx));
         let size = tables.tcx.layout_of(ParamEnv::empty().and(ty)).unwrap().size;
 
-        let scalar = ScalarInt::try_from_uint(val, size).ok_or_else(|| {
-            Error::new(format!("Value overflow: cannot convert `{val}` to usize."))
+        // We don't use Const::from_bits since it doesn't have any error checking.
+        let scalar = ScalarInt::try_from_uint(value, size).ok_or_else(|| {
+            Error::new(format!("Value overflow: cannot convert `{value}` to `{ty}`."))
         })?;
-        Ok(rustc_middle::ty::Const::new_value(tables.tcx, ValTree::from_scalar_int(scalar), ty)
+        Ok(ty::Const::new_value(tables.tcx, ValTree::from_scalar_int(scalar), ty)
             .stable(&mut *tables))
     }
 
@@ -556,7 +598,9 @@ fn vtable_allocation(
         global_alloc: &GlobalAlloc,
     ) -> Option<stable_mir::mir::alloc::AllocId> {
         let mut tables = self.0.borrow_mut();
-        let GlobalAlloc::VTable(ty, trait_ref) = global_alloc else { return None };
+        let GlobalAlloc::VTable(ty, trait_ref) = global_alloc else {
+            return None;
+        };
         let tcx = tables.tcx;
         let alloc_id = tables.tcx.vtable_allocation((
             ty.internal(&mut *tables, tcx),

compiler/rustc_target/src/abi/call/mod.rs

@@ -633,10 +633,8 @@ pub fn make_indirect(&mut self) {
     /// If the resulting alignment differs from the type's alignment,
     /// the argument will be copied to an alloca with sufficient alignment,
     /// either in the caller (if the type's alignment is lower than the byval alignment)
-    /// or in the callee† (if the type's alignment is higher than the byval alignment),
+    /// or in the callee (if the type's alignment is higher than the byval alignment),
     /// to ensure that Rust code never sees an underaligned pointer.
-    ///
-    /// † This is currently broken, see <https://github.com/rust-lang/rust/pull/122212>.
     pub fn make_indirect_byval(&mut self, byval_align: Option<Align>) {
         assert!(!self.layout.is_unsized(), "used byval ABI for unsized layout");
         self.make_indirect();

compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs

@@ -120,6 +120,8 @@ pub(in crate::solve) fn instantiate_constituent_tys_for_sized_trait<'tcx>(
     ty: Ty<'tcx>,
 ) -> Result<Vec<ty::Binder<'tcx, Ty<'tcx>>>, NoSolution> {
     match *ty.kind() {
+        // impl Sized for u*, i*, bool, f*, FnDef, FnPtr, *(const/mut) T, char, &mut? T, [T; N], dyn* Trait, !
+        // impl Sized for Coroutine, CoroutineWitness, Closure, CoroutineClosure
         ty::Infer(ty::IntVar(_) | ty::FloatVar(_))
         | ty::Uint(_)
         | ty::Int(_)
@@ -152,8 +154,10 @@ pub(in crate::solve) fn instantiate_constituent_tys_for_sized_trait<'tcx>(
             bug!("unexpected type `{ty}`")
         }
 
+        // impl Sized for (T1, T2, .., Tn) where T1: Sized, T2: Sized, .. Tn: Sized
         ty::Tuple(tys) => Ok(tys.iter().map(ty::Binder::dummy).collect()),
 
+        // impl Sized for Adt where T: Sized forall T in field types
         ty::Adt(def, args) => {
             let sized_crit = def.sized_constraint(ecx.tcx());
             Ok(sized_crit.iter_instantiated(ecx.tcx(), args).map(ty::Binder::dummy).collect())
@@ -167,6 +171,7 @@ pub(in crate::solve) fn instantiate_constituent_tys_for_copy_clone_trait<'tcx>(
     ty: Ty<'tcx>,
 ) -> Result<Vec<ty::Binder<'tcx, Ty<'tcx>>>, NoSolution> {
     match *ty.kind() {
+        // impl Copy/Clone for FnDef, FnPtr
         ty::FnDef(..) | ty::FnPtr(_) | ty::Error(_) => Ok(vec![]),
 
         // Implementations are provided in core
@@ -196,12 +201,16 @@ pub(in crate::solve) fn instantiate_constituent_tys_for_copy_clone_trait<'tcx>(
             bug!("unexpected type `{ty}`")
         }
 
+        // impl Copy/Clone for (T1, T2, .., Tn) where T1: Copy/Clone, T2: Copy/Clone, .. Tn: Copy/Clone
         ty::Tuple(tys) => Ok(tys.iter().map(ty::Binder::dummy).collect()),
 
+        // impl Copy/Clone for Closure where Self::TupledUpvars: Copy/Clone
         ty::Closure(_, args) => Ok(vec![ty::Binder::dummy(args.as_closure().tupled_upvars_ty())]),
 
         ty::CoroutineClosure(..) => Err(NoSolution),
 
+        // only when `coroutine_clone` is enabled and the coroutine is movable
+        // impl Copy/Clone for Coroutine where T: Copy/Clone forall T in (upvars, witnesses)
         ty::Coroutine(def_id, args) => match ecx.tcx().coroutine_movability(def_id) {
             Movability::Static => Err(NoSolution),
             Movability::Movable => {
@@ -217,6 +226,7 @@ pub(in crate::solve) fn instantiate_constituent_tys_for_copy_clone_trait<'tcx>(
             }
         },
 
+        // impl Copy/Clone for CoroutineWitness where T: Copy/Clone forall T in coroutine_hidden_types
         ty::CoroutineWitness(def_id, args) => Ok(ecx
             .tcx()
             .coroutine_hidden_types(def_id)

compiler/rustc_trait_selection/src/solve/trait_goals.rs

@@ -250,6 +250,7 @@ fn consider_builtin_fn_ptr_trait_candidate(
     ) -> QueryResult<'tcx> {
         let self_ty = goal.predicate.self_ty();
         match goal.predicate.polarity {
+            // impl FnPtr for FnPtr {}
             ty::ImplPolarity::Positive => {
                 if self_ty.is_fn_ptr() {
                     ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
@@ -257,6 +258,7 @@ fn consider_builtin_fn_ptr_trait_candidate(
                     Err(NoSolution)
                 }
             }
+            //  impl !FnPtr for T where T != FnPtr && T is rigid {}
             ty::ImplPolarity::Negative => {
                 // If a type is rigid and not a fn ptr, then we know for certain
                 // that it does *not* implement `FnPtr`.
@@ -374,6 +376,12 @@ fn consider_builtin_async_fn_kind_helper_candidate(
         }
     }
 
+    /// ```rust, ignore (not valid rust syntax)
+    /// impl Tuple for () {}
+    /// impl Tuple for (T1,) {}
+    /// impl Tuple for (T1, T2) {}
+    /// impl Tuple for (T1, .., Tn) {}
+    /// ```
     fn consider_builtin_tuple_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,

compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs

@@ -3409,6 +3409,8 @@ fn report_opaque_type_auto_trait_leakage(
         self.dcx().try_steal_replace_and_emit_err(self.tcx.def_span(def_id), StashKey::Cycle, err)
     }
 
+    // FIXME(@lcnr): This function could be changed to trait `TraitRef` directly
+    // instead of using a `Binder`.
     fn report_signature_mismatch_error(
         &self,
         obligation: &PredicateObligation<'tcx>,

compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs

@@ -165,7 +165,6 @@ fn assemble_candidates_from_projected_tys(
             let poly_trait_predicate = self.infcx.resolve_vars_if_possible(obligation.predicate);
             let placeholder_trait_predicate =
                 self.infcx.enter_forall_and_leak_universe(poly_trait_predicate);
-            debug!(?placeholder_trait_predicate);
 
             // The bounds returned by `item_bounds` may contain duplicates after
             // normalization, so try to deduplicate when possible to avoid
@@ -184,8 +183,8 @@ fn assemble_candidates_from_projected_tys(
                     selcx.infcx.probe(|_| {
                         match selcx.match_normalize_trait_ref(
                             obligation,
-                            bound.to_poly_trait_ref(),
                             placeholder_trait_predicate.trait_ref,
+                            bound.to_poly_trait_ref(),
                         ) {
                             Ok(None) => {
                                 candidates.vec.push(ProjectionCandidate(idx));
@@ -881,8 +880,8 @@ fn assemble_candidates_from_object_ty(
                         self.infcx.probe(|_| {
                             self.match_normalize_trait_ref(
                                 obligation,
-                                upcast_trait_ref,
                                 placeholder_trait_predicate.trait_ref,
+                                upcast_trait_ref,
                             )
                             .is_ok()
                         })

compiler/rustc_trait_selection/src/traits/select/confirmation.rs

@@ -9,7 +9,7 @@
 use rustc_ast::Mutability;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir::lang_items::LangItem;
-use rustc_infer::infer::BoundRegionConversionTime::HigherRankedType;
+use rustc_infer::infer::HigherRankedType;
 use rustc_infer::infer::{DefineOpaqueTypes, InferOk};
 use rustc_middle::traits::{BuiltinImplSource, SignatureMismatchData};
 use rustc_middle::ty::{
@@ -161,8 +161,6 @@ fn confirm_projection_candidate(
         let placeholder_trait_predicate =
             self.infcx.enter_forall_and_leak_universe(trait_predicate).trait_ref;
         let placeholder_self_ty = placeholder_trait_predicate.self_ty();
-        let placeholder_trait_predicate = ty::Binder::dummy(placeholder_trait_predicate);
-
         let candidate_predicate = self
             .for_each_item_bound(
                 placeholder_self_ty,
@@ -182,6 +180,11 @@ fn confirm_projection_candidate(
             .expect("projection candidate is not a trait predicate")
             .map_bound(|t| t.trait_ref);
 
+        let candidate = self.infcx.instantiate_binder_with_fresh_vars(
+            obligation.cause.span,
+            HigherRankedType,
+            candidate,
+        );
         let mut obligations = Vec::new();
         let candidate = normalize_with_depth_to(
             self,
@@ -195,7 +198,7 @@ fn confirm_projection_candidate(
         obligations.extend(
             self.infcx
                 .at(&obligation.cause, obligation.param_env)
-                .sup(DefineOpaqueTypes::No, placeholder_trait_predicate, candidate)
+                .eq(DefineOpaqueTypes::No, placeholder_trait_predicate, candidate)
                 .map(|InferOk { obligations, .. }| obligations)
                 .map_err(|_| Unimplemented)?,
         );
@@ -499,7 +502,6 @@ fn confirm_object_candidate(
 
         let trait_predicate = self.infcx.enter_forall_and_leak_universe(obligation.predicate);
         let self_ty = self.infcx.shallow_resolve(trait_predicate.self_ty());
-        let obligation_trait_ref = ty::Binder::dummy(trait_predicate.trait_ref);
         let ty::Dynamic(data, ..) = *self_ty.kind() else {
             span_bug!(obligation.cause.span, "object candidate with non-object");
         };
@@ -520,19 +522,24 @@ fn confirm_object_candidate(
         let unnormalized_upcast_trait_ref =
             supertraits.nth(index).expect("supertraits iterator no longer has as many elements");
 
+        let upcast_trait_ref = self.infcx.instantiate_binder_with_fresh_vars(
+            obligation.cause.span,
+            HigherRankedType,
+            unnormalized_upcast_trait_ref,
+        );
         let upcast_trait_ref = normalize_with_depth_to(
             self,
             obligation.param_env,
             obligation.cause.clone(),
             obligation.recursion_depth + 1,
-            unnormalized_upcast_trait_ref,
+            upcast_trait_ref,
             &mut nested,
         );
 
         nested.extend(
             self.infcx
                 .at(&obligation.cause, obligation.param_env)
-                .sup(DefineOpaqueTypes::No, obligation_trait_ref, upcast_trait_ref)
+                .eq(DefineOpaqueTypes::No, trait_predicate.trait_ref, upcast_trait_ref)
                 .map(|InferOk { obligations, .. }| obligations)
                 .map_err(|_| Unimplemented)?,
         );
@@ -1021,7 +1028,13 @@ fn confirm_poly_trait_refs(
         obligation: &PolyTraitObligation<'tcx>,
         self_ty_trait_ref: ty::PolyTraitRef<'tcx>,
     ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
-        let obligation_trait_ref = obligation.predicate.to_poly_trait_ref();
+        let obligation_trait_ref =
+            self.infcx.enter_forall_and_leak_universe(obligation.predicate.to_poly_trait_ref());
+        let self_ty_trait_ref = self.infcx.instantiate_binder_with_fresh_vars(
+            obligation.cause.span,
+            HigherRankedType,
+            self_ty_trait_ref,
+        );
         // Normalize the obligation and expected trait refs together, because why not
         let Normalized { obligations: nested, value: (obligation_trait_ref, expected_trait_ref) } =
             ensure_sufficient_stack(|| {
@@ -1037,15 +1050,15 @@ fn confirm_poly_trait_refs(
         // needed to define opaque types for tests/ui/type-alias-impl-trait/assoc-projection-ice.rs
         self.infcx
             .at(&obligation.cause, obligation.param_env)
-            .sup(DefineOpaqueTypes::Yes, obligation_trait_ref, expected_trait_ref)
+            .eq(DefineOpaqueTypes::Yes, obligation_trait_ref, expected_trait_ref)
             .map(|InferOk { mut obligations, .. }| {
                 obligations.extend(nested);
                 obligations
             })
             .map_err(|terr| {
                 SignatureMismatch(Box::new(SignatureMismatchData {
-                    expected_trait_ref: obligation_trait_ref,
-                    found_trait_ref: expected_trait_ref,
+                    expected_trait_ref: ty::Binder::dummy(obligation_trait_ref),
+                    found_trait_ref: ty::Binder::dummy(expected_trait_ref),
                     terr,
                 }))
             })

compiler/rustc_trait_selection/src/traits/select/mod.rs

@@ -33,6 +33,7 @@
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_infer::infer::BoundRegionConversionTime;
+use rustc_infer::infer::BoundRegionConversionTime::HigherRankedType;
 use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_infer::traits::TraitObligation;
 use rustc_middle::dep_graph::dep_kinds;
@@ -42,7 +43,7 @@
 use rustc_middle::ty::abstract_const::NotConstEvaluatable;
 use rustc_middle::ty::relate::TypeRelation;
 use rustc_middle::ty::GenericArgsRef;
-use rustc_middle::ty::{self, PolyProjectionPredicate, ToPolyTraitRef, ToPredicate};
+use rustc_middle::ty::{self, PolyProjectionPredicate, ToPredicate};
 use rustc_middle::ty::{Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_span::symbol::sym;
 use rustc_span::Symbol;
@@ -1651,15 +1652,20 @@ pub(super) fn for_each_item_bound<T>(
     fn match_normalize_trait_ref(
         &mut self,
         obligation: &PolyTraitObligation<'tcx>,
-        trait_bound: ty::PolyTraitRef<'tcx>,
         placeholder_trait_ref: ty::TraitRef<'tcx>,
-    ) -> Result<Option<ty::PolyTraitRef<'tcx>>, ()> {
+        trait_bound: ty::PolyTraitRef<'tcx>,
+    ) -> Result<Option<ty::TraitRef<'tcx>>, ()> {
         debug_assert!(!placeholder_trait_ref.has_escaping_bound_vars());
         if placeholder_trait_ref.def_id != trait_bound.def_id() {
             // Avoid unnecessary normalization
             return Err(());
         }
 
+        let trait_bound = self.infcx.instantiate_binder_with_fresh_vars(
+            obligation.cause.span,
+            HigherRankedType,
+            trait_bound,
+        );
         let Normalized { value: trait_bound, obligations: _ } = ensure_sufficient_stack(|| {
             normalize_with_depth(
                 self,
@@ -1671,7 +1677,7 @@ fn match_normalize_trait_ref(
         });
         self.infcx
             .at(&obligation.cause, obligation.param_env)
-            .sup(DefineOpaqueTypes::No, ty::Binder::dummy(placeholder_trait_ref), trait_bound)
+            .eq(DefineOpaqueTypes::No, placeholder_trait_ref, trait_bound)
             .map(|InferOk { obligations: _, value: () }| {
                 // This method is called within a probe, so we can't have
                 // inference variables and placeholders escape.
@@ -1683,7 +1689,6 @@ fn match_normalize_trait_ref(
             })
             .map_err(|_| ())
     }
-
     fn where_clause_may_apply<'o>(
         &mut self,
         stack: &TraitObligationStack<'o, 'tcx>,
@@ -1733,7 +1738,7 @@ pub(super) fn match_projection_projections(
         let is_match = self
             .infcx
             .at(&obligation.cause, obligation.param_env)
-            .sup(DefineOpaqueTypes::No, obligation.predicate, infer_projection)
+            .eq(DefineOpaqueTypes::No, obligation.predicate, infer_projection)
             .is_ok_and(|InferOk { obligations, value: () }| {
                 self.evaluate_predicates_recursively(
                     TraitObligationStackList::empty(&ProvisionalEvaluationCache::default()),
@@ -2533,7 +2538,7 @@ fn match_upcast_principal(
                     nested.extend(
                         self.infcx
                             .at(&obligation.cause, obligation.param_env)
-                            .sup(
+                            .eq(
                                 DefineOpaqueTypes::No,
                                 upcast_principal.map_bound(|trait_ref| {
                                     ty::ExistentialTraitRef::erase_self_ty(tcx, trait_ref)
@@ -2571,7 +2576,7 @@ fn match_upcast_principal(
                     nested.extend(
                         self.infcx
                             .at(&obligation.cause, obligation.param_env)
-                            .sup(DefineOpaqueTypes::No, source_projection, target_projection)
+                            .eq(DefineOpaqueTypes::No, source_projection, target_projection)
                             .map_err(|_| SelectionError::Unimplemented)?
                             .into_obligations(),
                     );
@@ -2615,9 +2620,15 @@ fn match_poly_trait_ref(
         obligation: &PolyTraitObligation<'tcx>,
         poly_trait_ref: ty::PolyTraitRef<'tcx>,
     ) -> Result<Vec<PredicateObligation<'tcx>>, ()> {
+        let predicate = self.infcx.enter_forall_and_leak_universe(obligation.predicate);
+        let trait_ref = self.infcx.instantiate_binder_with_fresh_vars(
+            obligation.cause.span,
+            HigherRankedType,
+            poly_trait_ref,
+        );
         self.infcx
             .at(&obligation.cause, obligation.param_env)
-            .sup(DefineOpaqueTypes::No, obligation.predicate.to_poly_trait_ref(), poly_trait_ref)
+            .eq(DefineOpaqueTypes::No, predicate.trait_ref, trait_ref)
             .map(|InferOk { obligations, .. }| obligations)
             .map_err(|_| ())
     }

compiler/rustc_trait_selection/src/traits/vtable.rs

@@ -320,6 +320,7 @@ fn vtable_entries<'tcx>(
 }
 
 /// Find slot base for trait methods within vtable entries of another trait
+// FIXME(@lcnr): This isn't a query, so why does it take a tuple as its argument.
 pub(super) fn vtable_trait_first_method_offset<'tcx>(
     tcx: TyCtxt<'tcx>,
     key: (

compiler/rustc_ty_utils/src/assoc.rs

@@ -1,10 +1,11 @@
 use rustc_data_structures::fx::FxIndexSet;
-use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, DefIdMap, LocalDefId};
 use rustc_hir::intravisit::{self, Visitor};
+use rustc_hir::{self as hir, HirId};
+use rustc_index::IndexVec;
 use rustc_middle::query::Providers;
-use rustc_middle::ty::{self, ImplTraitInTraitData, TyCtxt};
+use rustc_middle::ty::{self, ImplTraitInTraitData, TyCtxt, TyCtxtFeed};
 use rustc_span::symbol::kw;
 
 pub(crate) fn provide(providers: &mut Providers) {
@@ -237,6 +238,28 @@ fn visit_ty(&mut self, ty: &'tcx hir::Ty<'tcx>) {
     }
 }
 
+fn feed_hir(feed: &TyCtxtFeed<'_, LocalDefId>) {
+    feed.local_def_id_to_hir_id(HirId::make_owner(feed.def_id()));
+
+    let node = hir::OwnerNode::AssocOpaqueTy(&hir::AssocOpaqueTy {});
+    let bodies = Default::default();
+    let attrs = hir::AttributeMap::EMPTY;
+
+    let (opt_hash_including_bodies, _) = feed.tcx.hash_owner_nodes(node, &bodies, &attrs.map);
+    feed.opt_hir_owner_nodes(Some(feed.tcx.arena.alloc(hir::OwnerNodes {
+        opt_hash_including_bodies,
+        nodes: IndexVec::from_elem_n(
+            hir::ParentedNode {
+                parent: hir::ItemLocalId::INVALID,
+                node: hir::Node::AssocOpaqueTy(&hir::AssocOpaqueTy {}),
+            },
+            1,
+        ),
+        bodies,
+    })));
+    feed.feed_owner_id().hir_attrs(attrs);
+}
+
 /// Given an `opaque_ty_def_id` corresponding to an `impl Trait` in an associated
 /// function from a trait, synthesize an associated type for that `impl Trait`
 /// that inherits properties that we infer from the method and the opaque type.
@@ -258,9 +281,7 @@ fn associated_type_for_impl_trait_in_trait(
     let local_def_id = trait_assoc_ty.def_id();
     let def_id = local_def_id.to_def_id();
 
-    // There's no HIR associated with this new synthesized `def_id`, so feed
-    // `opt_local_def_id_to_hir_id` with `None`.
-    trait_assoc_ty.opt_local_def_id_to_hir_id(None);
+    feed_hir(&trait_assoc_ty);
 
     // Copy span of the opaque.
     trait_assoc_ty.def_ident_span(Some(span));
@@ -318,9 +339,7 @@ fn associated_type_for_impl_trait_in_impl(
     let local_def_id = impl_assoc_ty.def_id();
     let def_id = local_def_id.to_def_id();
 
-    // There's no HIR associated with this new synthesized `def_id`, so feed
-    // `opt_local_def_id_to_hir_id` with `None`.
-    impl_assoc_ty.opt_local_def_id_to_hir_id(None);
+    feed_hir(&impl_assoc_ty);
 
     // Copy span of the opaque.
     impl_assoc_ty.def_ident_span(Some(span));

compiler/stable_mir/src/compiler_interface.rs

@@ -14,7 +14,7 @@
     AdtDef, AdtKind, Allocation, ClosureDef, ClosureKind, Const, FieldDef, FnDef, ForeignDef,
     ForeignItemKind, ForeignModule, ForeignModuleDef, GenericArgs, GenericPredicates, Generics,
     ImplDef, ImplTrait, LineInfo, PolyFnSig, RigidTy, Span, TraitDecl, TraitDef, Ty, TyKind,
-    VariantDef,
+    UintTy, VariantDef,
 };
 use crate::{
     mir, Crate, CrateItem, CrateItems, CrateNum, DefId, Error, Filename, ImplTraitDecls, ItemKind,
@@ -101,8 +101,17 @@ pub trait Context {
     /// Evaluate constant as a target usize.
     fn eval_target_usize(&self, cnst: &Const) -> Result<u64, Error>;
 
-    /// Create a target usize constant for the given value.
-    fn usize_to_const(&self, val: u64) -> Result<Const, Error>;
+    /// Create a new zero-sized constant.
+    fn try_new_const_zst(&self, ty: Ty) -> Result<Const, Error>;
+
+    /// Create a new constant that represents the given string value.
+    fn new_const_str(&self, value: &str) -> Const;
+
+    /// Create a new constant that represents the given boolean value.
+    fn new_const_bool(&self, value: bool) -> Const;
+
+    /// Create a new constant that represents the given value.
+    fn try_new_const_uint(&self, value: u128, uint_ty: UintTy) -> Result<Const, Error>;
 
     /// Create a new type from the given kind.
     fn new_rigid_ty(&self, kind: RigidTy) -> Ty;
@@ -200,7 +209,7 @@ fn resolve_closure(
 
 // A thread local variable that stores a pointer to the tables mapping between TyCtxt
 // datastructures and stable MIR datastructures
-scoped_thread_local! (static TLV: Cell<*const ()>);
+scoped_thread_local!(static TLV: Cell<*const ()>);
 
 pub fn run<F, T>(context: &dyn Context, f: F) -> Result<T, Error>
 where

compiler/stable_mir/src/ty.rs

@@ -128,13 +128,38 @@ pub fn ty(&self) -> Ty {
 
     /// Creates an interned usize constant.
     fn try_from_target_usize(val: u64) -> Result<Self, Error> {
-        with(|cx| cx.usize_to_const(val))
+        with(|cx| cx.try_new_const_uint(val.into(), UintTy::Usize))
     }
 
     /// Try to evaluate to a target `usize`.
     pub fn eval_target_usize(&self) -> Result<u64, Error> {
         with(|cx| cx.eval_target_usize(self))
     }
+
+    /// Create a constant that represents a new zero-sized constant of type T.
+    /// Fails if the type is not a ZST or if it doesn't have a known size.
+    pub fn try_new_zero_sized(ty: Ty) -> Result<Const, Error> {
+        with(|cx| cx.try_new_const_zst(ty))
+    }
+
+    /// Build a new constant that represents the given string.
+    ///
+    /// Note that there is no guarantee today about duplication of the same constant.
+    /// I.e.: Calling this function multiple times with the same argument may or may not return
+    /// the same allocation.
+    pub fn from_str(value: &str) -> Const {
+        with(|cx| cx.new_const_str(value))
+    }
+
+    /// Build a new constant that represents the given boolean value.
+    pub fn from_bool(value: bool) -> Const {
+        with(|cx| cx.new_const_bool(value))
+    }
+
+    /// Build a new constant that represents the given unsigned integer.
+    pub fn try_from_uint(value: u128, uint_ty: UintTy) -> Result<Const, Error> {
+        with(|cx| cx.try_new_const_uint(value, uint_ty))
+    }
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]

library/core/src/ffi/c_str.rs

@@ -5,8 +5,11 @@
 use crate::ffi::c_char;
 use crate::fmt;
 use crate::intrinsics;
+use crate::iter::FusedIterator;
+use crate::marker::PhantomData;
 use crate::ops;
 use crate::ptr::addr_of;
+use crate::ptr::NonNull;
 use crate::slice;
 use crate::slice::memchr;
 use crate::str;
@@ -504,6 +507,13 @@ pub const fn as_ptr(&self) -> *const c_char {
         self.inner.as_ptr()
     }
 
+    /// We could eventually expose this publicly, if we wanted.
+    #[inline]
+    #[must_use]
+    const fn as_non_null_ptr(&self) -> NonNull<c_char> {
+        NonNull::from(&self.inner).as_non_null_ptr()
+    }
+
     /// Returns the length of `self`. Like C's `strlen`, this does not include the nul terminator.
     ///
     /// > **Note**: This method is currently implemented as a constant-time
@@ -617,6 +627,26 @@ pub const fn to_bytes_with_nul(&self) -> &[u8] {
         unsafe { &*(addr_of!(self.inner) as *const [u8]) }
     }
 
+    /// Iterates over the bytes in this C string.
+    ///
+    /// The returned iterator will **not** contain the trailing nul terminator
+    /// that this C string has.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(cstr_bytes)]
+    /// use std::ffi::CStr;
+    ///
+    /// let cstr = CStr::from_bytes_with_nul(b"foo\0").expect("CStr::from_bytes_with_nul failed");
+    /// assert!(cstr.bytes().eq(*b"foo"));
+    /// ```
+    #[inline]
+    #[unstable(feature = "cstr_bytes", issue = "112115")]
+    pub fn bytes(&self) -> Bytes<'_> {
+        Bytes::new(self)
+    }
+
     /// Yields a <code>&[str]</code> slice if the `CStr` contains valid UTF-8.
     ///
     /// If the contents of the `CStr` are valid UTF-8 data, this
@@ -735,3 +765,64 @@ fn strlen_rt(s: *const c_char) -> usize {
         intrinsics::const_eval_select((ptr,), strlen_ct, strlen_rt)
     }
 }
+
+/// An iterator over the bytes of a [`CStr`], without the nul terminator.
+///
+/// This struct is created by the [`bytes`] method on [`CStr`].
+/// See its documentation for more.
+///
+/// [`bytes`]: CStr::bytes
+#[must_use = "iterators are lazy and do nothing unless consumed"]
+#[unstable(feature = "cstr_bytes", issue = "112115")]
+#[derive(Clone, Debug)]
+pub struct Bytes<'a> {
+    // since we know the string is nul-terminated, we only need one pointer
+    ptr: NonNull<u8>,
+    phantom: PhantomData<&'a u8>,
+}
+impl<'a> Bytes<'a> {
+    #[inline]
+    fn new(s: &'a CStr) -> Self {
+        Self { ptr: s.as_non_null_ptr().cast(), phantom: PhantomData }
+    }
+
+    #[inline]
+    fn is_empty(&self) -> bool {
+        // SAFETY: We uphold that the pointer is always valid to dereference
+        // by starting with a valid C string and then never incrementing beyond
+        // the nul terminator.
+        unsafe { self.ptr.read() == 0 }
+    }
+}
+
+#[unstable(feature = "cstr_bytes", issue = "112115")]
+impl Iterator for Bytes<'_> {
+    type Item = u8;
+
+    #[inline]
+    fn next(&mut self) -> Option<u8> {
+        // SAFETY: We only choose a pointer from a valid C string, which must
+        // be non-null and contain at least one value. Since we always stop at
+        // the nul terminator, which is guaranteed to exist, we can assume that
+        // the pointer is non-null and valid. This lets us safely dereference
+        // it and assume that adding 1 will create a new, non-null, valid
+        // pointer.
+        unsafe {
+            let ret = self.ptr.read();
+            if ret == 0 {
+                None
+            } else {
+                self.ptr = self.ptr.offset(1);
+                Some(ret)
+            }
+        }
+    }
+
+    #[inline]
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        if self.is_empty() { (0, Some(0)) } else { (1, None) }
+    }
+}
+
+#[unstable(feature = "cstr_bytes", issue = "112115")]
+impl FusedIterator for Bytes<'_> {}

library/core/src/iter/range.rs

@@ -22,7 +22,7 @@ unsafe impl TrustedStep for $type {}
 ///
 /// The *successor* operation moves towards values that compare greater.
 /// The *predecessor* operation moves towards values that compare lesser.
-#[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")]
+#[unstable(feature = "step_trait", issue = "42168")]
 pub trait Step: Clone + PartialOrd + Sized {
     /// Returns the number of *successor* steps required to get from `start` to `end`.
     ///
@@ -52,15 +52,12 @@ pub trait Step: Clone + PartialOrd + Sized {
     /// For any `a`, `n`, and `m`:
     ///
     /// * `Step::forward_checked(a, n).and_then(|x| Step::forward_checked(x, m)) == Step::forward_checked(a, m).and_then(|x| Step::forward_checked(x, n))`
-    ///
-    /// For any `a`, `n`, and `m` where `n + m` does not overflow:
-    ///
-    /// * `Step::forward_checked(a, n).and_then(|x| Step::forward_checked(x, m)) == Step::forward_checked(a, n + m)`
+    /// * `Step::forward_checked(a, n).and_then(|x| Step::forward_checked(x, m)) == try { Step::forward_checked(a, n.checked_add(m)) }`
     ///
     /// For any `a` and `n`:
     ///
     /// * `Step::forward_checked(a, n) == (0..n).try_fold(a, |x, _| Step::forward_checked(&x, 1))`
-    ///   * Corollary: `Step::forward_checked(&a, 0) == Some(a)`
+    ///   * Corollary: `Step::forward_checked(a, 0) == Some(a)`
     fn forward_checked(start: Self, count: usize) -> Option<Self>;
 
     /// Returns the value that would be obtained by taking the *successor*
@@ -106,6 +103,7 @@ fn forward(start: Self, count: usize) -> Self {
     /// * if there exists `b` such that `b > a`, it is safe to call `Step::forward_unchecked(a, 1)`
     /// * if there exists `b`, `n` such that `steps_between(&a, &b) == Some(n)`,
     ///   it is safe to call `Step::forward_unchecked(a, m)` for any `m <= n`.
+    ///   * Corollary: `Step::forward_unchecked(a, 0)` is always safe.
     ///
     /// For any `a` and `n`, where no overflow occurs:
     ///
@@ -128,8 +126,8 @@ unsafe fn forward_unchecked(start: Self, count: usize) -> Self {
     ///
     /// For any `a` and `n`:
     ///
-    /// * `Step::backward_checked(a, n) == (0..n).try_fold(a, |x, _| Step::backward_checked(&x, 1))`
-    ///   * Corollary: `Step::backward_checked(&a, 0) == Some(a)`
+    /// * `Step::backward_checked(a, n) == (0..n).try_fold(a, |x, _| Step::backward_checked(x, 1))`
+    ///   * Corollary: `Step::backward_checked(a, 0) == Some(a)`
     fn backward_checked(start: Self, count: usize) -> Option<Self>;
 
     /// Returns the value that would be obtained by taking the *predecessor*
@@ -175,6 +173,7 @@ fn backward(start: Self, count: usize) -> Self {
     /// * if there exists `b` such that `b < a`, it is safe to call `Step::backward_unchecked(a, 1)`
     /// * if there exists `b`, `n` such that `steps_between(&b, &a) == Some(n)`,
     ///   it is safe to call `Step::backward_unchecked(a, m)` for any `m <= n`.
+    ///   * Corollary: `Step::backward_unchecked(a, 0)` is always safe.
     ///
     /// For any `a` and `n`, where no overflow occurs:
     ///
@@ -184,8 +183,25 @@ unsafe fn backward_unchecked(start: Self, count: usize) -> Self {
     }
 }
 
-// These are still macro-generated because the integer literals resolve to different types.
-macro_rules! step_identical_methods {
+// Separate impls for signed ranges because the distance within a signed range can be larger
+// than the signed::MAX value. Therefore `as` casting to the signed type would be incorrect.
+macro_rules! step_signed_methods {
+    ($unsigned: ty) => {
+        #[inline]
+        unsafe fn forward_unchecked(start: Self, n: usize) -> Self {
+            // SAFETY: the caller has to guarantee that `start + n` doesn't overflow.
+            unsafe { start.checked_add_unsigned(n as $unsigned).unwrap_unchecked() }
+        }
+
+        #[inline]
+        unsafe fn backward_unchecked(start: Self, n: usize) -> Self {
+            // SAFETY: the caller has to guarantee that `start - n` doesn't overflow.
+            unsafe { start.checked_sub_unsigned(n as $unsigned).unwrap_unchecked() }
+        }
+    };
+}
+
+macro_rules! step_unsigned_methods {
     () => {
         #[inline]
         unsafe fn forward_unchecked(start: Self, n: usize) -> Self {
@@ -198,7 +214,12 @@ unsafe fn backward_unchecked(start: Self, n: usize) -> Self {
             // SAFETY: the caller has to guarantee that `start - n` doesn't overflow.
             unsafe { start.unchecked_sub(n as Self) }
         }
+    };
+}
 
+// These are still macro-generated because the integer literals resolve to different types.
+macro_rules! step_identical_methods {
+    () => {
         #[inline]
         #[allow(arithmetic_overflow)]
         #[rustc_inherit_overflow_checks]
@@ -239,6 +260,7 @@ macro_rules! step_integer_impls {
             #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")]
             impl Step for $u_narrower {
                 step_identical_methods!();
+                step_unsigned_methods!();
 
                 #[inline]
                 fn steps_between(start: &Self, end: &Self) -> Option<usize> {
@@ -271,6 +293,7 @@ fn backward_checked(start: Self, n: usize) -> Option<Self> {
             #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")]
             impl Step for $i_narrower {
                 step_identical_methods!();
+                step_signed_methods!($u_narrower);
 
                 #[inline]
                 fn steps_between(start: &Self, end: &Self) -> Option<usize> {
@@ -335,6 +358,7 @@ fn backward_checked(start: Self, n: usize) -> Option<Self> {
             #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")]
             impl Step for $u_wider {
                 step_identical_methods!();
+                step_unsigned_methods!();
 
                 #[inline]
                 fn steps_between(start: &Self, end: &Self) -> Option<usize> {
@@ -360,6 +384,7 @@ fn backward_checked(start: Self, n: usize) -> Option<Self> {
             #[unstable(feature = "step_trait", reason = "recently redesigned", issue = "42168")]
             impl Step for $i_wider {
                 step_identical_methods!();
+                step_signed_methods!($u_wider);
 
                 #[inline]
                 fn steps_between(start: &Self, end: &Self) -> Option<usize> {

library/core/src/time.rs

@@ -856,6 +856,48 @@ pub const fn as_secs_f32(&self) -> f32 {
         (self.secs as f32) + (self.nanos.0 as f32) / (NANOS_PER_SEC as f32)
     }
 
+    /// Returns the number of milliseconds contained by this `Duration` as `f64`.
+    ///
+    /// The returned value does include the fractional (nanosecond) part of the duration.
+    ///
+    /// # Examples
+    /// ```
+    /// #![feature(duration_millis_float)]
+    /// use std::time::Duration;
+    ///
+    /// let dur = Duration::new(2, 345_678_000);
+    /// assert_eq!(dur.as_millis_f64(), 2345.678);
+    /// ```
+    #[unstable(feature = "duration_millis_float", issue = "122451")]
+    #[must_use]
+    #[inline]
+    #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")]
+    pub const fn as_millis_f64(&self) -> f64 {
+        (self.secs as f64) * (MILLIS_PER_SEC as f64)
+            + (self.nanos.0 as f64) / (NANOS_PER_MILLI as f64)
+    }
+
+    /// Returns the number of milliseconds contained by this `Duration` as `f32`.
+    ///
+    /// The returned value does include the fractional (nanosecond) part of the duration.
+    ///
+    /// # Examples
+    /// ```
+    /// #![feature(duration_millis_float)]
+    /// use std::time::Duration;
+    ///
+    /// let dur = Duration::new(2, 345_678_000);
+    /// assert_eq!(dur.as_millis_f32(), 2345.678);
+    /// ```
+    #[unstable(feature = "duration_millis_float", issue = "122451")]
+    #[must_use]
+    #[inline]
+    #[rustc_const_unstable(feature = "duration_consts_float", issue = "72440")]
+    pub const fn as_millis_f32(&self) -> f32 {
+        (self.secs as f32) * (MILLIS_PER_SEC as f32)
+            + (self.nanos.0 as f32) / (NANOS_PER_MILLI as f32)
+    }
+
     /// Creates a new `Duration` from the specified number of seconds represented
     /// as `f64`.
     ///

library/core/tests/iter/range.rs

@@ -325,6 +325,11 @@ fn test_range_advance_by() {
     assert_eq!(Ok(()), r.advance_back_by(usize::MAX));
 
     assert_eq!((r.start, r.end), (0u128 + usize::MAX as u128, u128::MAX - usize::MAX as u128));
+
+    // issue 122420, Step::forward_unchecked was unsound for signed integers
+    let mut r = -128i8..127;
+    assert_eq!(Ok(()), r.advance_by(200));
+    assert_eq!(r.next(), Some(72));
 }
 
 #[test]

library/profiler_builtins/Cargo.toml

@@ -13,4 +13,4 @@ core = { path = "../core" }
 compiler_builtins = { version = "0.1.0", features = ['rustc-dep-of-std'] }
 
 [build-dependencies]
-cc = "1.0.69"
+cc = "1.0.90"

library/std/src/os/unix/process.rs

@@ -39,6 +39,13 @@ pub trait CommandExt: Sealed {
     /// Sets the child process's user ID. This translates to a
     /// `setuid` call in the child process. Failure in the `setuid`
     /// call will cause the spawn to fail.
+    ///
+    /// # Notes
+    ///
+    /// This will also trigger a call to `setgroups(0, NULL)` in the child
+    /// process if no groups have been specified.
+    /// This removes supplementary groups that might have given the child
+    /// unwanted permissions.
     #[stable(feature = "rust1", since = "1.0.0")]
     fn uid(&mut self, id: UserId) -> &mut process::Command;
 

library/std/src/os/windows/io/handle.rs

@@ -7,7 +7,7 @@
 use crate::fs;
 use crate::io;
 use crate::marker::PhantomData;
-use crate::mem::forget;
+use crate::mem::{forget, ManuallyDrop};
 use crate::ptr;
 use crate::sys;
 use crate::sys::cvt;
@@ -91,7 +91,7 @@ pub struct OwnedHandle {
 #[repr(transparent)]
 #[stable(feature = "io_safety", since = "1.63.0")]
 #[derive(Debug)]
-pub struct HandleOrNull(OwnedHandle);
+pub struct HandleOrNull(RawHandle);
 
 /// FFI type for handles in return values or out parameters, where `INVALID_HANDLE_VALUE` is used
 /// as a sentry value to indicate errors, such as in the return value of `CreateFileW`. This uses
@@ -110,7 +110,7 @@ pub struct OwnedHandle {
 #[repr(transparent)]
 #[stable(feature = "io_safety", since = "1.63.0")]
 #[derive(Debug)]
-pub struct HandleOrInvalid(OwnedHandle);
+pub struct HandleOrInvalid(RawHandle);
 
 // The Windows [`HANDLE`] type may be transferred across and shared between
 // thread boundaries (despite containing a `*mut void`, which in general isn't
@@ -163,15 +163,24 @@ impl TryFrom<HandleOrNull> for OwnedHandle {
 
     #[inline]
     fn try_from(handle_or_null: HandleOrNull) -> Result<Self, NullHandleError> {
-        let owned_handle = handle_or_null.0;
-        if owned_handle.handle.is_null() {
-            // Don't call `CloseHandle`; it'd be harmless, except that it could
-            // overwrite the `GetLastError` error.
-            forget(owned_handle);
-
-            Err(NullHandleError(()))
+        let handle_or_null = ManuallyDrop::new(handle_or_null);
+        if handle_or_null.is_valid() {
+            // SAFETY: The handle is not null.
+            Ok(unsafe { OwnedHandle::from_raw_handle(handle_or_null.0) })
         } else {
-            Ok(owned_handle)
+            Err(NullHandleError(()))
+        }
+    }
+}
+
+#[stable(feature = "io_safety", since = "1.63.0")]
+impl Drop for HandleOrNull {
+    #[inline]
+    fn drop(&mut self) {
+        if self.is_valid() {
+            unsafe {
+                let _ = sys::c::CloseHandle(self.0);
+            }
         }
     }
 }
@@ -232,15 +241,24 @@ impl TryFrom<HandleOrInvalid> for OwnedHandle {
 
     #[inline]
     fn try_from(handle_or_invalid: HandleOrInvalid) -> Result<Self, InvalidHandleError> {
-        let owned_handle = handle_or_invalid.0;
-        if owned_handle.handle == sys::c::INVALID_HANDLE_VALUE {
-            // Don't call `CloseHandle`; it'd be harmless, except that it could
-            // overwrite the `GetLastError` error.
-            forget(owned_handle);
-
-            Err(InvalidHandleError(()))
+        let handle_or_invalid = ManuallyDrop::new(handle_or_invalid);
+        if handle_or_invalid.is_valid() {
+            // SAFETY: The handle is not invalid.
+            Ok(unsafe { OwnedHandle::from_raw_handle(handle_or_invalid.0) })
         } else {
-            Ok(owned_handle)
+            Err(InvalidHandleError(()))
+        }
+    }
+}
+
+#[stable(feature = "io_safety", since = "1.63.0")]
+impl Drop for HandleOrInvalid {
+    #[inline]
+    fn drop(&mut self) {
+        if self.is_valid() {
+            unsafe {
+                let _ = sys::c::CloseHandle(self.0);
+            }
         }
     }
 }
@@ -333,7 +351,11 @@ impl HandleOrNull {
     #[stable(feature = "io_safety", since = "1.63.0")]
     #[inline]
     pub unsafe fn from_raw_handle(handle: RawHandle) -> Self {
-        Self(OwnedHandle::from_raw_handle(handle))
+        Self(handle)
+    }
+
+    fn is_valid(&self) -> bool {
+        !self.0.is_null()
     }
 }
 
@@ -356,7 +378,11 @@ impl HandleOrInvalid {
     #[stable(feature = "io_safety", since = "1.63.0")]
     #[inline]
     pub unsafe fn from_raw_handle(handle: RawHandle) -> Self {
-        Self(OwnedHandle::from_raw_handle(handle))
+        Self(handle)
+    }
+
+    fn is_valid(&self) -> bool {
+        self.0 != sys::c::INVALID_HANDLE_VALUE
     }
 }
 

library/std/src/sys/pal/unix/fs.rs

@@ -463,15 +463,15 @@ pub fn file_type(&self) -> FileType {
 #[cfg(target_os = "netbsd")]
 impl FileAttr {
     pub fn modified(&self) -> io::Result<SystemTime> {
-        Ok(SystemTime::new(self.stat.st_mtime as i64, self.stat.st_mtimensec as i64))
+        SystemTime::new(self.stat.st_mtime as i64, self.stat.st_mtimensec as i64)
     }
 
     pub fn accessed(&self) -> io::Result<SystemTime> {
-        Ok(SystemTime::new(self.stat.st_atime as i64, self.stat.st_atimensec as i64))
+        SystemTime::new(self.stat.st_atime as i64, self.stat.st_atimensec as i64)
     }
 
     pub fn created(&self) -> io::Result<SystemTime> {
-        Ok(SystemTime::new(self.stat.st_birthtime as i64, self.stat.st_birthtimensec as i64))
+        SystemTime::new(self.stat.st_birthtime as i64, self.stat.st_birthtimensec as i64)
     }
 }
 
@@ -503,16 +503,16 @@ pub fn modified(&self) -> io::Result<SystemTime> {
         #[cfg(target_pointer_width = "32")]
         cfg_has_statx! {
             if let Some(mtime) = self.stx_mtime() {
-                return Ok(SystemTime::new(mtime.tv_sec, mtime.tv_nsec as i64));
+                return SystemTime::new(mtime.tv_sec, mtime.tv_nsec as i64);
             }
         }
 
-        Ok(SystemTime::new(self.stat.st_mtime as i64, self.stat.st_mtime_nsec as i64))
+        SystemTime::new(self.stat.st_mtime as i64, self.stat.st_mtime_nsec as i64)
     }
 
     #[cfg(any(target_os = "vxworks", target_os = "espidf", target_os = "vita"))]
     pub fn modified(&self) -> io::Result<SystemTime> {
-        Ok(SystemTime::new(self.stat.st_mtime as i64, 0))
+        SystemTime::new(self.stat.st_mtime as i64, 0)
     }
 
     #[cfg(any(target_os = "horizon", target_os = "hurd"))]
@@ -531,16 +531,16 @@ pub fn accessed(&self) -> io::Result<SystemTime> {
         #[cfg(target_pointer_width = "32")]
         cfg_has_statx! {
             if let Some(atime) = self.stx_atime() {
-                return Ok(SystemTime::new(atime.tv_sec, atime.tv_nsec as i64));
+                return SystemTime::new(atime.tv_sec, atime.tv_nsec as i64);
             }
         }
 
-        Ok(SystemTime::new(self.stat.st_atime as i64, self.stat.st_atime_nsec as i64))
+        SystemTime::new(self.stat.st_atime as i64, self.stat.st_atime_nsec as i64)
     }
 
     #[cfg(any(target_os = "vxworks", target_os = "espidf", target_os = "vita"))]
     pub fn accessed(&self) -> io::Result<SystemTime> {
-        Ok(SystemTime::new(self.stat.st_atime as i64, 0))
+        SystemTime::new(self.stat.st_atime as i64, 0)
     }
 
     #[cfg(any(target_os = "horizon", target_os = "hurd"))]
@@ -557,7 +557,7 @@ pub fn accessed(&self) -> io::Result<SystemTime> {
         target_os = "watchos",
     ))]
     pub fn created(&self) -> io::Result<SystemTime> {
-        Ok(SystemTime::new(self.stat.st_birthtime as i64, self.stat.st_birthtime_nsec as i64))
+        SystemTime::new(self.stat.st_birthtime as i64, self.stat.st_birthtime_nsec as i64)
     }
 
     #[cfg(not(any(
@@ -573,7 +573,7 @@ pub fn created(&self) -> io::Result<SystemTime> {
         cfg_has_statx! {
             if let Some(ext) = &self.statx_extra_fields {
                 return if (ext.stx_mask & libc::STATX_BTIME) != 0 {
-                    Ok(SystemTime::new(ext.stx_btime.tv_sec, ext.stx_btime.tv_nsec as i64))
+                    SystemTime::new(ext.stx_btime.tv_sec, ext.stx_btime.tv_nsec as i64)
                 } else {
                     Err(io::const_io_error!(
                         io::ErrorKind::Unsupported,
@@ -592,22 +592,22 @@ pub fn created(&self) -> io::Result<SystemTime> {
 
     #[cfg(target_os = "vita")]
     pub fn created(&self) -> io::Result<SystemTime> {
-        Ok(SystemTime::new(self.stat.st_ctime as i64, 0))
+        SystemTime::new(self.stat.st_ctime as i64, 0)
     }
 }
 
 #[cfg(target_os = "nto")]
 impl FileAttr {
     pub fn modified(&self) -> io::Result<SystemTime> {
-        Ok(SystemTime::new(self.stat.st_mtim.tv_sec, self.stat.st_mtim.tv_nsec))
+        SystemTime::new(self.stat.st_mtim.tv_sec, self.stat.st_mtim.tv_nsec)
     }
 
     pub fn accessed(&self) -> io::Result<SystemTime> {
-        Ok(SystemTime::new(self.stat.st_atim.tv_sec, self.stat.st_atim.tv_nsec))
+        SystemTime::new(self.stat.st_atim.tv_sec, self.stat.st_atim.tv_nsec)
     }
 
     pub fn created(&self) -> io::Result<SystemTime> {
-        Ok(SystemTime::new(self.stat.st_ctim.tv_sec, self.stat.st_ctim.tv_nsec))
+        SystemTime::new(self.stat.st_ctim.tv_sec, self.stat.st_ctim.tv_nsec)
     }
 }